diff --git a/.env.example b/.env.example index e2a78fd..c192f93 100644 --- a/.env.example +++ b/.env.example @@ -49,6 +49,8 @@ TENDWIRE_ACKNOWLEDGED_FINAL_RETENTION_COUNT=4096 # malformed-owner, known-incomplete, and automation holds are inspection-only. TENDWIRE_MAX_OUTBOX_ATTEMPTS=10 TENDWIRE_CONNECTOR_CLAIM_TTL_SECONDS=60 +TENDWIRE_CONNECTOR_MAX_CLAIM_TTL_SECONDS=300 +TENDWIRE_CONNECTOR_ACK_TTL_SECONDS=300 # Automatic snapshot cleanup defaults to 14 days and 4096 rows per host. # Each host's latest row is always retained. At a 5-minute snapshot interval, @@ -81,6 +83,7 @@ TENDWIRE_HERDR_TIMEOUT_SECONDS=1.0 # TENDWIRE_MAX_WORKERS. The internal refresh queue is fixed at 64. TENDWIRE_TURN_REFRESH_INTERVAL_SECONDS=2.0 TENDWIRE_TURN_REFRESH_WORKERS=4 +TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS=86400 # Backend prompts remain visible as stale after a transient adapter failure for # one fixed, non-sliding grace window. Repeated failures do not extend it. diff --git a/INSTALL.md b/INSTALL.md index 4a8ace7..1d213ae 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -45,6 +45,7 @@ Environment=TENDWIRE_HERDR_BACKEND=socket Environment=TENDWIRE_DB_PATH=%h/.local/share/tendwire/tendwire.db Environment=TENDWIRE_TURN_REFRESH_INTERVAL_SECONDS=2.0 Environment=TENDWIRE_TURN_REFRESH_WORKERS=4 +Environment=TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS=86400 ExecStart=%h/.local/bin/tendwire daemon --db-path %h/.local/share/tendwire/tendwire.db Restart=always RestartSec=5s @@ -71,6 +72,8 @@ reconciles. `TENDWIRE_TURN_REFRESH_WORKERS` defaults to `4`, must be from 1 through 32, and cannot exceed `TENDWIRE_MAX_WORKERS`. Every adapter uses `TENDWIRE_HERDR_TIMEOUT_SECONDS`; the queue is fixed at 64. One private target is serialized with itself while distinct targets can use the worker pool. +Unobserved command-turn claims become terminal tombstones after +`TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS` (default `86400`). OMP JSONL cache/IPC state is coordinate-only: parse/EOF and replay offsets, observed file identity/size/timestamps, an open-turn flag, and validated project @@ -628,10 +631,11 @@ following together: - `turn.list` schema v2 with descriptor schema v1, 1,000-character previews and insertion-stable paging; schema-v1 `turn.content.get` with a 49,152-byte UTF-8 page ceiling; -- range-only schema-v1 `connector.prepare` begin/part/commit/recover, root-wide - leases, independent part ACKs, and immutable schema-v2 source-less route - lineage. Cleanup retains bounded delivered tombstones so repeated snapshots - cannot recreate/repost removed acknowledged roots; and +- range-only schema-v1 `connector.prepare` begin/part/commit/recover, per-worker + ordered roots and parts, bounded renewable/releasable leases, deadline-based + awaiting-ACK recovery, independent part ACKs, and immutable schema-v2 + source-less route lineage. Cleanup retains bounded delivered tombstones so + repeated snapshots cannot recreate/repost removed acknowledged roots; and - fair dead-letter inspection, exact root/failed-plan retry including source-less recovery, retained ACK prefix, cumulative attempts, request-ID idempotency, immutable recovery audit, and no provider-perfect exactly-once diff --git a/README.md b/README.md index 5581acd..c8302c8 100644 --- a/README.md +++ b/README.md @@ -534,6 +534,8 @@ variables: | `max_workers` | `TENDWIRE_MAX_WORKERS` | `512` | integer >= 1 | | `max_outbox_attempts` | `TENDWIRE_MAX_OUTBOX_ATTEMPTS` | `10` | integer >= 1 | | `connector_claim_ttl_seconds` | `TENDWIRE_CONNECTOR_CLAIM_TTL_SECONDS` | `60` | integer >= 1 | +| `connector_max_claim_ttl_seconds` | `TENDWIRE_CONNECTOR_MAX_CLAIM_TTL_SECONDS` | `300` | integer >= 1; caps `turn-final` poll and renew requests | +| `connector_ack_ttl_seconds` | `TENDWIRE_CONNECTOR_ACK_TTL_SECONDS` | `300` | integer >= 1; grace period for committed plans awaiting ACK completion | | `command_retry_horizon_seconds` | `TENDWIRE_COMMAND_RETRY_HORIZON_SECONDS` | `604800` | positive integer no greater than 604800 | | `command_receipt_retention_seconds` | `TENDWIRE_COMMAND_RECEIPT_RETENTION_SECONDS` | `2592000` | integer at least 691200 and strictly greater than the retry horizon | | `command_receipt_retention_count` | `TENDWIRE_COMMAND_RECEIPT_RETENTION_COUNT` | `4096` | positive integer; newest bounded inactive receipts per host | @@ -545,6 +547,7 @@ variables: | `store_maintenance_cadence_seconds` | `TENDWIRE_STORE_MAINTENANCE_CADENCE_SECONDS` | `3600` | positive integer | | `turn_refresh_interval_seconds` | `TENDWIRE_TURN_REFRESH_INTERVAL_SECONDS` | `2.0` | finite positive float | | `turn_refresh_workers` | `TENDWIRE_TURN_REFRESH_WORKERS` | `4` | integer from 1 through 32 and no greater than `max_workers` | +| `turn_claim_hard_ttl_seconds` | `TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS` | `86400` | positive integer; unobserved command claims become terminal after this interval | The socket/event backend uses `event_debounce_seconds` for event batching and `reconcile_interval_seconds` for bounded periodic full reconciles. Set @@ -1134,15 +1137,17 @@ See `INSTALL.md` for the required maintenance and rollback order. Tendwire exposes a Tendwire-only connector delivery boundary above the SQLite store. The public daemon methods are `connector.prepare`, `connector.poll`, -`connector.ack`, `connector.fail`, `connector.defer`, `connector.inspect`, -`connector.retry`, and the operational helper `connector.reclaim`. Matching -JSON-only CLI hooks include: +`connector.ack`, `connector.fail`, `connector.defer`, `connector.renew`, +`connector.release`, `connector.inspect`, `connector.retry`, and the operational +helper `connector.reclaim`. Matching JSON-only CLI hooks include: ```bash tendwire connector poll --name attention --limit 10 --lease-seconds 60 --db-path /path/to/tendwire.db tendwire connector ack --name attention --ref '' --response-json '{"delivered":true}' --db-path /path/to/tendwire.db tendwire connector fail --name attention --ref '' --reason temporary --delay-seconds 60 --db-path /path/to/tendwire.db tendwire connector defer --name attention --ref '' --reason scheduled --available-at 2026-01-01T00:10:00+00:00 --db-path /path/to/tendwire.db +tendwire connector renew --name turn-final --ref '' --lease-seconds 120 --db-path /path/to/tendwire.db +tendwire connector release --name turn-final --ref '' --db-path /path/to/tendwire.db tendwire connector inspect --name turn-final --status dead_letter --limit 100 --db-path /path/to/tendwire.db tendwire connector retry --name turn-final --final-identity 'twfinal1.' --db-path /path/to/tendwire.db ``` @@ -1247,20 +1252,44 @@ effect. `connector.poll` atomically leases due `connector_outbox` rows for one `name` and returns opaque per-attempt refs. A live lease prevents duplicate polling. Expired leases are reclaimed before polling and before ref-mutating operations; -`connector.reclaim` can also be called directly. `connector.ack` validates the -host, name, attempt, lease, and ref before marking the delivery and outbox item -delivered. `connector.fail` records sanitized failure data and schedules retry -availability. `connector.defer` records sanitized defer data and schedules future -availability without treating the item as delivered. Stale, expired, -wrong-host, wrong-name, and superseded refs fail closed with neutral errors. +the daemon first performs a read-only due check on its periodic tick and takes +the reclaim write transaction only when work is actually expired. +`connector.reclaim` can be called directly. `connector.renew` extends a live +lease without creating a delivery attempt. `connector.release` records the live +attempt as released and makes the row immediately pollable again. `connector.ack` +validates the host, name, attempt, lease, and ref before marking the delivery and +outbox item delivered. `connector.fail` records sanitized failure data and +schedules retry availability. `connector.defer` records sanitized defer data and +schedules future availability without treating the item as delivered. Stale, +expired, wrong-host, wrong-name, and superseded refs fail closed with neutral +errors. When callers omit `lease_seconds`, the daemon and CLI use `connector_claim_ttl_seconds` from config, defaulting to 60 seconds; an explicit -public `lease_seconds` value still wins. `max_outbox_attempts` prevents +public `lease_seconds` value still wins. `turn-final` poll and renew requests are +capped by `connector_max_claim_ttl_seconds`, defaulting to 300 seconds. +`max_outbox_attempts` prevents unbounded retry loops. Once a failed job reaches the configured cap, Tendwire moves the outbox item to a neutral terminal `dead_letter` state and returns the public status `attempts_exhausted` without exposing private outbox or delivery state. +Final-root FIFO is partitioned by the turn's immutable stable worker key, with +the persisted worker ID as the enqueue-time fallback. Legacy backfill prefers +the enqueue-era stable key carried in the outbox payload; rows that predate that +metadata still fall back to the turn's current persisted worker ID. If neither +identity can be resolved, each row receives its own `orphan:` partition so +unrelated degraded rows cannot block one another. A blocked worker therefore +cannot starve another worker, while roots and plan parts for the same worker stay +strictly ordered. `dead_letter`, `superseded`, and `delivered` roots are terminal +for this gate. A committed source in `awaiting_ack` no longer blocks by itself; +its plan jobs carry the ordering obligation, including source-less recovery +plans. Commit stamps an ACK deadline, each acknowledged plan job extends it, +and a valid part lease prevents deadline reclaim from interrupting in-flight +delivery. ACK-deadline recovery does not consume the connector failure-attempt +budget. An expired incomplete plan is otherwise failed and its source requeued. +Missing or unrecoverable plan state terminates the source instead of leaving it +pending. + ### Delivery-aware final roots and acknowledged history Persisting an authoritative, owner-authenticated complete final atomically diff --git a/scripts/turn_ingestion_benchmark.py b/scripts/turn_ingestion_benchmark.py index 5b9d2bf..13ec651 100755 --- a/scripts/turn_ingestion_benchmark.py +++ b/scripts/turn_ingestion_benchmark.py @@ -1241,7 +1241,16 @@ def submit_command(_config: Config, payload: str) -> Mapping[str, Any]: ) ) - baseline_threads source_calls_final = len(adapter_records) - forbidden_values.extend(str(process_id) for process_id in process_ids) + forbidden_values.extend( + marker + for process_id in process_ids + for marker in ( + f'"process_id":{process_id}', + f'"process_id":"{process_id}"', + f'"pid":{process_id}', + f'"pid":"{process_id}"', + ) + ) overlap_ns = _first_call_overlap_ns(adapter_records, args.blocked_workers) response_bytes_max = max( metric["response_bytes_max"] for metric in latency.values() diff --git a/src/tendwire/backends/herdr_turns.py b/src/tendwire/backends/herdr_turns.py index f575f7b..4549e20 100644 --- a/src/tendwire/backends/herdr_turns.py +++ b/src/tendwire/backends/herdr_turns.py @@ -36,9 +36,11 @@ redact_private_prompt_text, ) from ..store.sqlite import ( + TURN_CLAIM_SWEEP_MIN_GRACE_SECONDS, apply_turn_refresh, list_worker_bindings, prune_backend_pending, + sweep_turn_claims, ) @@ -4409,6 +4411,20 @@ def _scan_bindings(self) -> None: except Exception: pass + try: + sweep_turn_claims( + self.config.db_path, + self.config.host_id, + grace_seconds=max( + TURN_CLAIM_SWEEP_MIN_GRACE_SECONDS, + 10.0 * self.refresh_interval_seconds, + ), + hard_ttl_seconds=self.config.turn_claim_hard_ttl_seconds, + now=self._utc_clock(), + ) + except Exception: + pass + def _binding_for_item(self, item: _TurnRefreshItem) -> WorkerBinding | None: if self.config.db_path is None: return None diff --git a/src/tendwire/cli.py b/src/tendwire/cli.py index 48e9ce7..f17fd92 100644 --- a/src/tendwire/cli.py +++ b/src/tendwire/cli.py @@ -497,15 +497,18 @@ def add_common(action_parser: argparse.ArgumentParser) -> None: reclaim_parser = actions.add_parser("reclaim", help="Expire stale connector leases.") add_common(reclaim_parser) - for action in ("ack", "fail", "defer"): + for action in ("ack", "fail", "defer", "renew", "release"): action_parser = actions.add_parser(action, help=f"Apply connector.{action} to a live ref.") add_common(action_parser) action_parser.add_argument("--ref", required=True) - action_parser.add_argument("--response-json", dest="response_json", default=None) + if action in {"ack", "fail", "defer"}: + action_parser.add_argument("--response-json", dest="response_json", default=None) if action in {"fail", "defer"}: action_parser.add_argument("--reason", default="") action_parser.add_argument("--available-at", dest="available_at", default=None) action_parser.add_argument("--delay-seconds", dest="delay_seconds", type=int, default=None) + if action == "renew": + action_parser.add_argument("--lease-seconds", dest="lease_seconds", type=int, default=None) def _load_worker_bindings(config: Config) -> list[WorkerBinding]: @@ -1035,6 +1038,8 @@ def cmd_turns( limit=limit, cursor=cursor, since=since, + turn_refresh_interval_seconds=config.turn_refresh_interval_seconds, + claim_hard_ttl_seconds=config.turn_claim_hard_ttl_seconds, ) elif daemon_attempt.error_kind == "timeout": payload = { @@ -1103,7 +1108,10 @@ def cmd_turn_content_get(config: Config, args: argparse.Namespace) -> int: else: from .store.sqlite import get_turn_content, init_store - init_store(config.db_path) + init_store( + config.db_path, + connector_ack_ttl_seconds=config.connector_ack_ttl_seconds, + ) payload = get_turn_content( config.db_path, config.host_id, @@ -1425,9 +1433,9 @@ def _connector_params_from_args(args: argparse.Namespace) -> dict[str, Any]: params["limit"] = args.limit if args.lease_seconds is not None: params["lease_seconds"] = args.lease_seconds - if args.connector_action in {"ack", "fail", "defer"}: + if args.connector_action in {"ack", "fail", "defer", "renew", "release"}: params["ref"] = args.ref - if args.response_json: + if getattr(args, "response_json", None): try: parsed = json.loads(args.response_json) except json.JSONDecodeError: @@ -1440,6 +1448,8 @@ def _connector_params_from_args(args: argparse.Namespace) -> dict[str, Any]: params["available_at"] = args.available_at if args.delay_seconds is not None: params["delay_seconds"] = args.delay_seconds + if args.connector_action == "renew" and args.lease_seconds is not None: + params["lease_seconds"] = args.lease_seconds return params @@ -1507,11 +1517,16 @@ def cmd_connector(config: Config, args: argparse.Namespace) -> int: from .connectors import ConnectorOutboxAPI from .store.sqlite import init_store - init_store(config.db_path) + init_store( + config.db_path, + connector_ack_ttl_seconds=config.connector_ack_ttl_seconds, + ) payload = ConnectorOutboxAPI( config.db_path, config.host_id, default_lease_seconds=config.connector_claim_ttl_seconds, + max_lease_seconds=config.connector_max_claim_ttl_seconds, + ack_ttl_seconds=config.connector_ack_ttl_seconds, max_attempts=config.max_outbox_attempts, ).dispatch(method, params) print(_connector_payload_json(payload, indent=2)) diff --git a/src/tendwire/command_submission.py b/src/tendwire/command_submission.py index 70476b6..b1b7405 100644 --- a/src/tendwire/command_submission.py +++ b/src/tendwire/command_submission.py @@ -58,6 +58,7 @@ claim_backend_pending_choice, claim_backend_pending_decision, command_pending_turn_terminal_effect, + delete_command_pending_turn_claim_effect, command_reservation_is_live, envelope_to_receipt_json, finish_command_request, @@ -69,6 +70,7 @@ reserve_terminal_command_replay, start_backend_pending_choice_send, start_backend_pending_decision_send, + upsert_command_pending_turn, ) @@ -349,13 +351,16 @@ def _submit_private_pane_input(client: Any, pane_id: str, instruction_text: str, # A single ctrl+u is not reliable across all foreground TUIs. Clear stale # input first, then submit text and Enter in one Herdr operation so the # foreground application cannot observe a staged prompt between requests. - for keys in _PRIVATE_PANE_CLEAR_KEY_SEQUENCES: - _socket_request( - client, - "pane.send_keys", - {"pane_id": pane_id, "keys": list(keys)}, - timeout=timeout, - ) + try: + for keys in _PRIVATE_PANE_CLEAR_KEY_SEQUENCES: + _socket_request( + client, + "pane.send_keys", + {"pane_id": pane_id, "keys": list(keys)}, + timeout=timeout, + ) + except Exception as exc: + raise _PaneInputNotStartedError from exc _socket_request( client, "pane.send_input", @@ -364,6 +369,10 @@ def _submit_private_pane_input(client: Any, pane_id: str, instruction_text: str, ) +class _PaneInputNotStartedError(RuntimeError): + """The instruction input operation was never attempted.""" + + def _target_state_at_send(worker: Worker) -> str: status = str(worker.status or "").strip().lower().replace("-", "_") return status or "unknown" @@ -1253,10 +1262,22 @@ def _mark_request_send_started( reservation: ReservedCommandMutation, *, binding_fingerprint: str, -) -> CommandEnvelope | None: + worker: Worker | None = None, + instruction_text: str | None = None, +) -> CommandEnvelope | Mapping[str, Any] | None: if config.db_path is None: return _backend_uncertain(request, "command receipt store is unavailable") try: + send_started_effect = ( + command_pending_turn_terminal_effect( + host_id=config.host_id, + worker=worker, + request_id=request.request_id or "", + instruction_text=instruction_text, + ) + if worker is not None and instruction_text is not None + else None + ) started = mark_command_send_started( config.db_path, host_id=config.host_id, @@ -1264,6 +1285,7 @@ def _mark_request_send_started( canonical_fingerprint=reservation.canonical.fingerprint, owner_token=reservation.owner_token, binding_fingerprint=binding_fingerprint, + send_started_effect=send_started_effect, event_payload=_transition_payload( request, worker_id=reservation.canonical.public_worker_id, @@ -1283,7 +1305,12 @@ def _mark_request_send_started( and started["receipt"].get("state") == "send_started" and _receipt_is_canonical(request, reservation.canonical, started["receipt"]) ): - return None + if send_started_effect is None: + return None + effect_result = started.get("effect_result") + if isinstance(effect_result, Mapping): + return effect_result + return _recover_request(config, request, reservation.canonical) if isinstance(started, Mapping) and isinstance(started.get("receipt"), Mapping): embedded = _envelope_from_receipt( request, @@ -1304,7 +1331,13 @@ def _mark_request_send_started( def _accepted_send_envelope( request: CommandRequest, worker: Worker, + turn: Mapping[str, Any], ) -> CommandEnvelope: + observed_turn_state = "pending_observation" + if str(turn.get("source_turn_id") or "").strip(): + observed_turn_state = ( + "complete" if turn.get("complete") is True else "observed" + ) return CommandEnvelope.from_result( request, ok=True, @@ -1315,7 +1348,8 @@ def _accepted_send_envelope( "delivery_state": "submitted", "transport_state": "submitted", "target_state_at_send": _target_state_at_send(worker), - "observed_turn_state": "pending_observation", + "turn_id": str(turn.get("id") or ""), + "observed_turn_state": observed_turn_state, }, ) @@ -1329,14 +1363,23 @@ def _submit_instruction( ) -> CommandEnvelope: assert config.db_path is not None try: - send_start_error = _mark_request_send_started( + send_started = _mark_request_send_started( config, request, reservation, binding_fingerprint=prepared.binding_fingerprint, + worker=worker, + instruction_text=_instruction_text(request), ) - if send_start_error is not None: - return send_start_error + if isinstance(send_started, CommandEnvelope): + return send_started + if not isinstance(send_started, Mapping): + return _recover_request( + config, + request, + reservation.canonical, + ) + pending_turn = send_started try: _submit_private_pane_input( @@ -1345,6 +1388,23 @@ def _submit_instruction( _instruction_text(request), timeout=config.herdr_timeout_seconds, ) + except _PaneInputNotStartedError: + envelope = _backend_uncertain( + request, + "Herdr socket pane input did not start after send start", + ) + return _finish_request( + config, + request, + reservation, + envelope, + expected_state="send_started", + terminal_state="uncertain", + terminal_effect=delete_command_pending_turn_claim_effect( + host_id=config.host_id, + request_id=request.request_id or "", + ), + ) except Exception: # noqa: BLE001 envelope = _backend_uncertain( request, @@ -1358,23 +1418,27 @@ def _submit_instruction( expected_state="send_started", terminal_state="uncertain", ) + + # The ingestion scheduler can adopt the write-early claim while the + # pane call is in flight. Re-read through the idempotent upsert so the + # accepted envelope reports the canonical row's observed state rather + # than the pre-send snapshot of that row. + try: + refreshed_turn = upsert_command_pending_turn( + config.db_path, + config.host_id, + worker, + request_id=request.request_id or "", + instruction_text=_instruction_text(request), + ) + except Exception: # noqa: BLE001 + refreshed_turn = None + if isinstance(refreshed_turn, Mapping): + pending_turn = refreshed_turn finally: _close_socket_client(prepared.client) - accepted = _accepted_send_envelope(request, worker) - try: - effect = command_pending_turn_terminal_effect( - host_id=config.host_id, - worker=worker, - request_id=request.request_id or "", - instruction_text=_instruction_text(request), - ) - except Exception: - return _recover_request( - config, - request, - reservation.canonical, - ) + accepted = _accepted_send_envelope(request, worker, pending_turn) return _finish_request( config, request, @@ -1382,7 +1446,6 @@ def _submit_instruction( accepted, expected_state="send_started", terminal_state="accepted", - terminal_effect=effect, ) @@ -1674,6 +1737,13 @@ def _submit_decision_calibration( {"pane_id": pane_id, "keys": list(step.keys)}, timeout=timeout, ) + elif step.operation == "text": + _socket_request( + client, + "pane.send_text", + {"pane_id": pane_id, "text": step.text}, + timeout=timeout, + ) else: _socket_request( client, diff --git a/src/tendwire/config.py b/src/tendwire/config.py index 52ea3e6..ccbfad9 100644 --- a/src/tendwire/config.py +++ b/src/tendwire/config.py @@ -21,9 +21,12 @@ DEFAULT_MAX_WORKERS = 512 DEFAULT_TURN_REFRESH_INTERVAL_SECONDS = 2.0 DEFAULT_TURN_REFRESH_WORKERS = 4 +DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS = 86_400 DEFAULT_PENDING_STALE_GRACE_SECONDS = 30.0 DEFAULT_MAX_OUTBOX_ATTEMPTS = 10 DEFAULT_CONNECTOR_CLAIM_TTL_SECONDS = 60 +DEFAULT_CONNECTOR_MAX_CLAIM_TTL_SECONDS = 300 +DEFAULT_CONNECTOR_ACK_TTL_SECONDS = 300 DEFAULT_ACKNOWLEDGED_FINAL_RETENTION_DAYS = 30 DEFAULT_ACKNOWLEDGED_FINAL_RETENTION_COUNT = 4096 DEFAULT_COMMAND_RETRY_HORIZON_SECONDS = 604_800 @@ -59,9 +62,12 @@ class Config: max_workers: int = DEFAULT_MAX_WORKERS turn_refresh_interval_seconds: float = DEFAULT_TURN_REFRESH_INTERVAL_SECONDS turn_refresh_workers: int = DEFAULT_TURN_REFRESH_WORKERS + turn_claim_hard_ttl_seconds: int = DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS pending_stale_grace_seconds: float = DEFAULT_PENDING_STALE_GRACE_SECONDS max_outbox_attempts: int = DEFAULT_MAX_OUTBOX_ATTEMPTS connector_claim_ttl_seconds: int = DEFAULT_CONNECTOR_CLAIM_TTL_SECONDS + connector_max_claim_ttl_seconds: int = DEFAULT_CONNECTOR_MAX_CLAIM_TTL_SECONDS + connector_ack_ttl_seconds: int = DEFAULT_CONNECTOR_ACK_TTL_SECONDS acknowledged_final_retention_days: int = DEFAULT_ACKNOWLEDGED_FINAL_RETENTION_DAYS acknowledged_final_retention_count: int = DEFAULT_ACKNOWLEDGED_FINAL_RETENTION_COUNT command_retry_horizon_seconds: int = DEFAULT_COMMAND_RETRY_HORIZON_SECONDS @@ -145,6 +151,15 @@ def __post_init__(self) -> None: ) if self.turn_refresh_workers > self.max_workers: raise ValueError("turn_refresh_workers must be <= max_workers") + object.__setattr__( + self, + "turn_claim_hard_ttl_seconds", + _bounded_positive_int( + self.turn_claim_hard_ttl_seconds, + "turn_claim_hard_ttl_seconds", + maximum=MAX_MAINTENANCE_CADENCE_SECONDS, + ), + ) object.__setattr__( self, "pending_stale_grace_seconds", @@ -167,6 +182,24 @@ def __post_init__(self) -> None: minimum=1, ), ) + object.__setattr__( + self, + "connector_max_claim_ttl_seconds", + _positive_int( + self.connector_max_claim_ttl_seconds, + "connector_max_claim_ttl_seconds", + minimum=1, + ), + ) + object.__setattr__( + self, + "connector_ack_ttl_seconds", + _positive_int( + self.connector_ack_ttl_seconds, + "connector_ack_ttl_seconds", + minimum=1, + ), + ) object.__setattr__( self, "acknowledged_final_retention_days", @@ -353,9 +386,12 @@ def load_config( max_workers: int | str | None = None, turn_refresh_interval_seconds: float | str | None = None, turn_refresh_workers: int | str | None = None, + turn_claim_hard_ttl_seconds: int | str | None = None, pending_stale_grace_seconds: float | str | None = None, max_outbox_attempts: int | str | None = None, connector_claim_ttl_seconds: int | str | None = None, + connector_max_claim_ttl_seconds: int | str | None = None, + connector_ack_ttl_seconds: int | str | None = None, acknowledged_final_retention_days: int | str | None = None, acknowledged_final_retention_count: int | str | None = None, command_retry_horizon_seconds: int | str | None = None, @@ -464,6 +500,11 @@ def load_config( "TENDWIRE_TURN_REFRESH_WORKERS", DEFAULT_TURN_REFRESH_WORKERS, ), + turn_claim_hard_ttl_seconds=_resolve_value( + turn_claim_hard_ttl_seconds, + "TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS", + DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS, + ), pending_stale_grace_seconds=_resolve_value( pending_stale_grace_seconds, "TENDWIRE_PENDING_STALE_GRACE_SECONDS", @@ -479,6 +520,16 @@ def load_config( "TENDWIRE_CONNECTOR_CLAIM_TTL_SECONDS", DEFAULT_CONNECTOR_CLAIM_TTL_SECONDS, ), + connector_max_claim_ttl_seconds=_resolve_value( + connector_max_claim_ttl_seconds, + "TENDWIRE_CONNECTOR_MAX_CLAIM_TTL_SECONDS", + DEFAULT_CONNECTOR_MAX_CLAIM_TTL_SECONDS, + ), + connector_ack_ttl_seconds=_resolve_value( + connector_ack_ttl_seconds, + "TENDWIRE_CONNECTOR_ACK_TTL_SECONDS", + DEFAULT_CONNECTOR_ACK_TTL_SECONDS, + ), acknowledged_final_retention_days=_resolve_value( acknowledged_final_retention_days, "TENDWIRE_ACKNOWLEDGED_FINAL_RETENTION_DAYS", diff --git a/src/tendwire/connectors/outbox.py b/src/tendwire/connectors/outbox.py index b1583f6..dd94796 100644 --- a/src/tendwire/connectors/outbox.py +++ b/src/tendwire/connectors/outbox.py @@ -22,6 +22,8 @@ prepare_connector_plan_recover, prepare_connector_plan_part, reclaim_expired_connector_leases, + release_connector_delivery, + renew_connector_delivery, retry_final_ready_delivery, ) @@ -198,7 +200,7 @@ def _request_id(value: Any) -> str: class ConnectorOutboxAPI: - """Public-neutral facade for connector.poll/ack/fail/defer.""" + """Public-neutral facade for connector delivery and lease operations.""" def __init__( self, @@ -206,11 +208,15 @@ def __init__( host_id: str, *, default_lease_seconds: int = 60, + max_lease_seconds: int = 300, + ack_ttl_seconds: int = 300, max_attempts: int = 10, ) -> None: self.db_path = Path(db_path) if db_path is not None else None self.host_id = str(host_id) self.default_lease_seconds = max(1, int(default_lease_seconds)) + self.max_lease_seconds = max(1, int(max_lease_seconds)) + self.ack_ttl_seconds = max(1, int(ack_ttl_seconds)) self.max_attempts = max(1, int(max_attempts)) def _require_store(self, name: str = "") -> dict[str, Any] | None: @@ -327,6 +333,7 @@ def prepare(self, params: Mapping[str, Any] | None = None) -> dict[str, Any]: name=name, plan_token=token, source_ref=source_ref, + ack_ttl_seconds=self.ack_ttl_seconds, ) if set(data) != { @@ -405,7 +412,11 @@ def poll(self, params: Mapping[str, Any] | None = None) -> dict[str, Any]: data.get("lease_seconds"), self.default_lease_seconds, minimum=1, - maximum=86400, + maximum=( + self.max_lease_seconds + if name == _PREPARE_NAME + else 86400 + ), ), max_attempts=self.max_attempts, ) @@ -479,6 +490,7 @@ def ack(self, params: Mapping[str, Any] | None = None) -> dict[str, Any]: name=name, ref=live_ref, response=_clean_mapping(data.get("response")), + ack_ttl_seconds=self.ack_ttl_seconds, ) def fail(self, params: Mapping[str, Any] | None = None) -> dict[str, Any]: @@ -487,6 +499,51 @@ def fail(self, params: Mapping[str, Any] | None = None) -> dict[str, Any]: def defer(self, params: Mapping[str, Any] | None = None) -> dict[str, Any]: return self._schedule("defer", params) + def renew(self, params: Mapping[str, Any] | None = None) -> dict[str, Any]: + data, name, live_ref = self._mutation_parts(params) + if not name: + return _error("invalid_params", host_id=self.host_id) + if live_ref is None: + return _error("invalid_ref", host_id=self.host_id, name=name) + unavailable = self._require_store(name) + if unavailable is not None: + return unavailable + assert self.db_path is not None + lease_seconds = _int( + data.get("lease_seconds"), + self.default_lease_seconds, + minimum=1, + maximum=( + self.max_lease_seconds + if name == _PREPARE_NAME + else 86400 + ), + ) + return renew_connector_delivery( + self.db_path, + host_id=self.host_id, + name=name, + ref=live_ref, + lease_seconds=lease_seconds, + ) + + def release(self, params: Mapping[str, Any] | None = None) -> dict[str, Any]: + _data, name, live_ref = self._mutation_parts(params) + if not name: + return _error("invalid_params", host_id=self.host_id) + if live_ref is None: + return _error("invalid_ref", host_id=self.host_id, name=name) + unavailable = self._require_store(name) + if unavailable is not None: + return unavailable + assert self.db_path is not None + return release_connector_delivery( + self.db_path, + host_id=self.host_id, + name=name, + ref=live_ref, + ) + def _schedule(self, action: str, params: Mapping[str, Any] | None) -> dict[str, Any]: data, name, live_ref = self._mutation_parts(params) if not name: @@ -600,6 +657,10 @@ def dispatch(self, method: str, params: Mapping[str, Any] | None = None) -> dict return self.fail(params) if method == "connector.defer": return self.defer(params) + if method == "connector.renew": + return self.renew(params) + if method == "connector.release": + return self.release(params) if method == "connector.reclaim": return self.reclaim(params) if method == "connector.inspect": diff --git a/src/tendwire/core/models.py b/src/tendwire/core/models.py index 33fdc05..3bb05b5 100644 --- a/src/tendwire/core/models.py +++ b/src/tendwire/core/models.py @@ -443,7 +443,9 @@ "id", "max_outbox_attempts", "origin_command_id", + "outbox_ack_ttl_seconds", "outbox_claim_ttl_seconds", + "outbox_max_claim_ttl_seconds", "output_excerpt_chars", "raw_status", "request_id", diff --git a/src/tendwire/core/turns.py b/src/tendwire/core/turns.py index 089b2f4..cda5cfe 100644 --- a/src/tendwire/core/turns.py +++ b/src/tendwire/core/turns.py @@ -1784,6 +1784,8 @@ class Turn: completed_at: str | None = None origin_command_id: str | None = None source_turn_id: str | None = None + superseded_by_turn_id: str | None = None + superseded_at: str | None = None meta: dict[str, Any] = field(default_factory=dict) id: str = "" fingerprint: str = "" @@ -1808,6 +1810,10 @@ def __post_init__(self) -> None: completed_at = _optional_timestamp(self.completed_at) origin_command_id = _optional_public_text(self.origin_command_id) raw_source_turn_id = _optional_public_text(self.source_turn_id) + superseded_by_turn_id = _optional_public_text( + self.superseded_by_turn_id + ) + superseded_at = _optional_timestamp(self.superseded_at) meta = _clean_meta(self.meta) source_id_candidates = turn_source_id_candidates( raw_source_turn_id, @@ -1891,6 +1897,12 @@ def __post_init__(self) -> None: object.__setattr__(self, "source", source) object.__setattr__(self, "origin_command_id", origin_command_id) object.__setattr__(self, "source_turn_id", source_turn_id) + object.__setattr__( + self, + "superseded_by_turn_id", + superseded_by_turn_id, + ) + object.__setattr__(self, "superseded_at", superseded_at) object.__setattr__(self, "fingerprint", fingerprint) object.__setattr__(self, "meta", meta) @@ -1921,6 +1933,8 @@ def to_dict(self) -> dict[str, Any]: "source": self.source, "origin_command_id": self.origin_command_id, "source_turn_id": self.source_turn_id, + "superseded_by_turn_id": self.superseded_by_turn_id, + "superseded_at": self.superseded_at, "fingerprint": self.fingerprint, "meta": _clean_meta(self.meta), } @@ -1955,6 +1969,8 @@ def from_dict(cls, data: "Turn | Mapping[str, Any]") -> "Turn": source=clean.get("source", "snapshot"), origin_command_id=clean.get("origin_command_id"), source_turn_id=clean.get("source_turn_id"), + superseded_by_turn_id=clean.get("superseded_by_turn_id"), + superseded_at=clean.get("superseded_at"), fingerprint=_string_value(clean.get("fingerprint")), meta=clean.get("meta", {}), ) diff --git a/src/tendwire/daemon.py b/src/tendwire/daemon.py index de7c9da..d8b1234 100644 --- a/src/tendwire/daemon.py +++ b/src/tendwire/daemon.py @@ -431,10 +431,19 @@ def default_socket_path(config: Config) -> Path: return Path(config.data_dir) / "tendwire.sock" -def _default_init_store(db_path: Path) -> None: +def _default_init_store( + db_path: Path, + *, + connector_ack_ttl_seconds: int | None = None, +) -> None: from .store.sqlite import init_store - init_store(db_path) + kwargs = ( + {"connector_ack_ttl_seconds": connector_ack_ttl_seconds} + if connector_ack_ttl_seconds is not None + else {} + ) + init_store(db_path, **kwargs) def _default_observe_initial_snapshot(config: Config) -> Snapshot: @@ -517,7 +526,16 @@ def start(self) -> None: self.config.installation_key_sentinel_path, ), ) - self.hooks.init_store(Path(self.config.db_path)) + if self.hooks.init_store is _default_init_store: + _default_init_store( + Path(self.config.db_path), + connector_ack_ttl_seconds=( + self.config.connector_ack_ttl_seconds + ), + ) + else: + self.hooks.init_store(Path(self.config.db_path)) + self._connector_periodic_tick() if self.config.herdr_backend == "socket": self._snapshot = self._start_socket_event_backend() else: @@ -543,6 +561,7 @@ def start(self) -> None: stop_event=self.stop_event, socket_group=self.config.socket_group, prepare_parent=self._prepare_socket_parent, + periodic_callback=self._connector_periodic_tick, ) self._server = server # Bind before ingestion starts. Managed store connections and the @@ -954,6 +973,10 @@ def get_health(self) -> dict[str, Any]: "max_workers": self.config.max_workers, "max_outbox_attempts": self.config.max_outbox_attempts, "outbox_claim_ttl_seconds": self.config.connector_claim_ttl_seconds, + "outbox_max_claim_ttl_seconds": ( + self.config.connector_max_claim_ttl_seconds + ), + "outbox_ack_ttl_seconds": self.config.connector_ack_ttl_seconds, "acknowledged_final_retention_days": ( self.config.acknowledged_final_retention_days ), @@ -1026,6 +1049,8 @@ def get_turns( limit=limit, cursor=cursor, since=since, + turn_refresh_interval_seconds=self.config.turn_refresh_interval_seconds, + claim_hard_ttl_seconds=self.config.turn_claim_hard_ttl_seconds, ) def get_turn_content(self, params: Mapping[str, Any]) -> Mapping[str, Any]: @@ -1071,9 +1096,37 @@ def connector_call(self, method: str, params: Mapping[str, Any]) -> Mapping[str, Path(self.config.db_path), self.config.host_id, default_lease_seconds=self.config.connector_claim_ttl_seconds, + max_lease_seconds=self.config.connector_max_claim_ttl_seconds, + ack_ttl_seconds=self.config.connector_ack_ttl_seconds, max_attempts=self.config.max_outbox_attempts, ).dispatch(method, params) + def _connector_periodic_tick(self) -> None: + """Eagerly reclaim expired connector work without waiting for a poll.""" + if self.config.db_path is None: + return + from .store.sqlite import ( + connector_reclaim_due, + reclaim_expired_connector_leases, + ) + + try: + if not connector_reclaim_due( + Path(self.config.db_path), + self.config.host_id, + None, + ): + return + reclaim_expired_connector_leases( + Path(self.config.db_path), + self.config.host_id, + None, + ) + except Exception: + # Startup and periodic maintenance remain best-effort; store health + # is reported through the normal daemon health surface. + return + def submit_command(self, params: Mapping[str, Any]) -> CommandEnvelope | Mapping[str, Any]: # Preserve the submitted keys exactly so the existing command parser can # reject private/connector fields instead of receiving sanitized input. diff --git a/src/tendwire/daemon_api.py b/src/tendwire/daemon_api.py index c553a4e..a8e1373 100644 --- a/src/tendwire/daemon_api.py +++ b/src/tendwire/daemon_api.py @@ -152,6 +152,8 @@ "connector.ack", "connector.fail", "connector.defer", + "connector.renew", + "connector.release", "connector.reclaim", "connector.retry", "connector.inspect", @@ -935,6 +937,8 @@ def __init__( request_workers: int = 8, max_in_flight_requests: int = 32, shutdown_grace_seconds: float = 6.0, + periodic_callback: Callable[[], Any] | None = None, + periodic_interval_seconds: float = 1.0, ) -> None: if ( not isinstance(request_workers, int) @@ -969,6 +973,12 @@ def __init__( self.request_workers = request_workers self.max_in_flight_requests = max_in_flight_requests self.shutdown_grace_seconds = float(shutdown_grace_seconds) + self.periodic_callback = periodic_callback + self.periodic_interval_seconds = max( + self.accept_timeout_seconds, + float(periodic_interval_seconds), + ) + self._next_periodic_at = time.monotonic() + self.periodic_interval_seconds self._listener: socket.socket | None = None self._identity: EntryIdentity | None = None self._pin_fd: int | None = None @@ -1142,6 +1152,7 @@ def serve_forever(self) -> None: self.start() try: while not self.stop_event.is_set(): + self._run_periodic_callback_if_due() listener = self._listener if listener is None: break @@ -1159,6 +1170,17 @@ def serve_forever(self) -> None: finally: self.close() + def _run_periodic_callback_if_due(self) -> None: + callback = self.periodic_callback + if callback is None or time.monotonic() < self._next_periodic_at: + return + self._next_periodic_at = time.monotonic() + self.periodic_interval_seconds + try: + callback() + except Exception: + # Periodic maintenance is best-effort and must not stop the API loop. + return + def _submit_connection(self, conn: socket.socket) -> None: if not self._admission.acquire(blocking=False): self._reject_connection(conn, "server_busy") diff --git a/src/tendwire/store/sqlite.py b/src/tendwire/store/sqlite.py index 950703c..7d437d6 100644 --- a/src/tendwire/store/sqlite.py +++ b/src/tendwire/store/sqlite.py @@ -9,6 +9,7 @@ import fcntl import hashlib import json +import logging import math import os import secrets @@ -25,7 +26,11 @@ from typing import Any, Literal from urllib.parse import parse_qsl, quote, urlsplit -from ..config import DEFAULT_PENDING_STALE_GRACE_SECONDS +from ..config import ( + DEFAULT_CONNECTOR_ACK_TTL_SECONDS, + DEFAULT_PENDING_STALE_GRACE_SECONDS, + DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS, +) from ..local_state import ( EntryIdentity, EntryType, @@ -107,7 +112,11 @@ FINGERPRINT_HEX_LENGTH = FINGERPRINT_HEX_CHARS -STORE_SCHEMA_VERSION = 14 +STORE_SCHEMA_VERSION = 17 +CONNECTOR_ACK_TTL_SECONDS = DEFAULT_CONNECTOR_ACK_TTL_SECONDS +TURN_CLAIM_HARD_TTL_SECONDS = DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS +TURN_CLAIM_SWEEP_MIN_GRACE_SECONDS = 60.0 +TURN_SUBMISSION_OBSERVATION_ADOPTION_WINDOW_SECONDS = 60.0 ACKNOWLEDGED_FINAL_RETENTION_DAYS = 30 ACKNOWLEDGED_FINAL_RETENTION_COUNT = 4096 COMMAND_RETRY_HORIZON_SECONDS = 604_800 @@ -137,6 +146,9 @@ ATTENTION_MISSING_REQUIRED = 2 ATTENTION_MISSING_GRACE_SECONDS = 120 _ATTENTION_SEVERITY_RANK = {"info": 0, "warning": 1, "critical": 2} +_LOGGER = logging.getLogger(__name__) +_TURN_CLAIM_SWEEP_LAST_AT: dict[tuple[str, str], float] = {} +_TURN_CLAIM_SWEEP_LOCK = threading.Lock() class StoreSchemaError(RuntimeError): @@ -147,6 +159,19 @@ def __init__(self, status: str) -> None: super().__init__(self.status) +def _configured_turn_claim_hard_ttl_seconds() -> int: + raw = os.environ.get("TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS") + if raw is None: + return TURN_CLAIM_HARD_TTL_SECONDS + try: + configured = int(raw) + except (TypeError, ValueError) as exc: + raise StoreSchemaError("turn_claim_hard_ttl_invalid") from exc + if configured <= 0: + raise StoreSchemaError("turn_claim_hard_ttl_invalid") + return configured + + @dataclass(frozen=True) class Migration: """One exact, transaction-external schema transition.""" @@ -722,6 +747,11 @@ def _record_response_size( ), ) +CREATE_CONNECTOR_ORDERING_INDEX = ( + "CREATE INDEX IF NOT EXISTS idx_connector_outbox_final_ordering " + "ON connector_outbox(host_id, connector, ordering_key, delivery_kind, status, id)" +) + CREATE_TURN_PRESENTATION_PLANS_TABLE = """ CREATE TABLE IF NOT EXISTS turn_presentation_plans ( @@ -977,6 +1007,7 @@ def _record_response_size( delivery_kind TEXT NOT NULL DEFAULT 'generic', turn_id TEXT, content_revision TEXT, + ordering_key TEXT NOT NULL DEFAULT '', status TEXT NOT NULL, payload_json TEXT NOT NULL, private_state_json TEXT NOT NULL DEFAULT '{}', @@ -1876,6 +1907,7 @@ def _connector_response( key: str | None = None, attempt: int | None = None, available_at: str | None = None, + leased_until: str | None = None, ) -> dict[str, Any]: payload: dict[str, Any] = { "schema_version": 1, @@ -1892,6 +1924,8 @@ def _connector_response( payload["attempt"] = int(attempt) if available_at is not None: payload["available_at"] = str(available_at) + if leased_until is not None: + payload["leased_until"] = str(leased_until) return sanitize_public_value(payload) @@ -1996,9 +2030,241 @@ def _connector_reclaim_expired_leases_conn( now=now, ) reclaimed += 1 + reclaimed += _connector_reclaim_expired_awaiting_ack_conn( + conn, + host_id=str(host_id), + name=str(name) if name is not None else None, + now=str(now), + ) + return reclaimed + + +def _connector_reclaim_expired_awaiting_ack_conn( + conn: sqlite3.Connection, + *, + host_id: str, + name: str | None, + now: str, +) -> int: + clauses = ["outbox.host_id = ?", "outbox.status = 'awaiting_ack'"] + params: list[Any] = [str(host_id)] + if name is not None: + clauses.append("outbox.connector = ?") + params.append(str(name)) + rows = conn.execute( + f""" + SELECT outbox.id, outbox.connector, outbox.private_state_json + FROM connector_outbox AS outbox + WHERE {" AND ".join(clauses)} + ORDER BY outbox.id + """, + params, + ).fetchall() + reclaimed = 0 + now_dt = _connector_datetime(now) + for outbox_id, connector, private_state_json in rows: + outbox_state = _json_object(private_state_json) + deadline = str(outbox_state.get("ack_deadline_at") or "") + delivery = conn.execute( + """ + SELECT id, private_state_json + FROM connector_deliveries + WHERE outbox_id = ? AND status = 'awaiting_ack' + ORDER BY id DESC + LIMIT 1 + """, + (int(outbox_id),), + ).fetchone() + if not deadline and delivery is not None: + deadline = str( + _json_object(delivery[1]).get("ack_deadline_at") or "" + ) + if not deadline or _connector_datetime(deadline) > now_dt: + continue + + plan = conn.execute( + """ + SELECT id, state, generation + FROM turn_presentation_plans + WHERE source_outbox_id = ? + ORDER BY id DESC + LIMIT 1 + """, + (int(outbox_id),), + ).fetchone() + if plan is not None and str(plan[1]) == "completed": + if delivery is not None: + conn.execute( + """ + UPDATE connector_deliveries + SET status = 'delivered', delivered_at = COALESCE(delivered_at, ?) + WHERE id = ? AND status = 'awaiting_ack' + """, + (str(now), int(delivery[0])), + ) + conn.execute( + """ + UPDATE connector_outbox + SET status = 'delivered', next_attempt_at = NULL, + updated_at = ?, private_state_json = ? + WHERE id = ? AND status = 'awaiting_ack' + """, + ( + str(now), + _connector_private_clear_current(private_state_json), + int(outbox_id), + ), + ) + reclaimed += 1 + continue + + recoverable = plan is not None and str(plan[1]) in { + "active", + "waiting_predecessor", + "failed", + } + if recoverable: + plan_id = int(plan[0]) + if _connector_plan_has_live_job_lease_conn( + conn, + plan_id=plan_id, + now_dt=now_dt, + ): + continue + conn.execute( + "UPDATE turn_presentation_plans SET state = 'failed' WHERE id = ?", + (plan_id,), + ) + job_rows = conn.execute( + """ + SELECT outbox.id, outbox.private_state_json + FROM turn_presentation_jobs AS jobs + JOIN connector_outbox AS outbox ON outbox.id = jobs.outbox_id + WHERE jobs.plan_id = ? + AND outbox.status NOT IN ('delivered', 'superseded', 'dead_letter') + """, + (plan_id,), + ).fetchall() + for job_outbox_id, job_private in job_rows: + conn.execute( + """ + UPDATE connector_deliveries + SET status = 'failed', response_json = ?, delivered_at = ? + WHERE outbox_id = ? AND status = 'leased' + """, + ( + _canonical_json( + {"schema_version": 1, "status": "ack_deadline_expired"} + ), + str(now), + int(job_outbox_id), + ), + ) + conn.execute( + """ + UPDATE connector_outbox + SET status = 'dead_letter', next_attempt_at = NULL, + updated_at = ?, private_state_json = ? + WHERE id = ? + """, + ( + str(now), + _connector_private_clear_current(job_private), + int(job_outbox_id), + ), + ) + source_state = _json_object( + _connector_private_clear_current(private_state_json) + ) + source_state["presentation_generation"] = max( + int(source_state.get("presentation_generation") or 1), + int(plan[2] or 1) + 1, + ) + source_status = "queued" + else: + source_state = _json_object( + _connector_private_clear_current(private_state_json) + ) + source_status = _CONNECTOR_EXHAUSTED_OUTBOX_STATUS + + if delivery is not None: + conn.execute( + """ + UPDATE connector_deliveries + SET status = 'failed', response_json = ?, delivered_at = ? + WHERE id = ? AND status = 'awaiting_ack' + """, + ( + _canonical_json( + { + "schema_version": 1, + "status": ( + "ack_deadline_expired" + if recoverable + else "plan_unrecoverable" + ), + } + ), + str(now), + int(delivery[0]), + ), + ) + conn.execute( + """ + UPDATE connector_outbox + SET status = ?, next_attempt_at = ?, updated_at = ?, + private_state_json = ? + WHERE id = ? AND status = 'awaiting_ack' + """, + ( + source_status, + str(now) if source_status == "queued" else None, + str(now), + _canonical_json(source_state), + int(outbox_id), + ), + ) + _activate_waiting_presentation_plans_conn( + conn, + host_id=str(host_id), + name=str(connector), + now=str(now), + ) + reclaimed += 1 return reclaimed +def _connector_plan_has_live_job_lease_conn( + conn: sqlite3.Connection, + *, + plan_id: int, + now_dt: datetime, +) -> bool: + live_job_leases = conn.execute( + """ + SELECT deliveries.private_state_json + FROM turn_presentation_jobs AS jobs + JOIN connector_outbox AS outbox + ON outbox.id = jobs.outbox_id + JOIN connector_deliveries AS deliveries + ON deliveries.outbox_id = outbox.id + WHERE jobs.plan_id = ? + AND outbox.status = 'leased' + AND deliveries.status = 'leased' + """, + (int(plan_id),), + ).fetchall() + return any( + ( + lease_expires_at := str( + _json_object(row[0]).get("lease_expires_at") or "" + ) + ) + and _connector_datetime(lease_expires_at) > now_dt + for row in live_job_leases + ) + + def _connector_exhaust_retryable_conn( conn: sqlite3.Connection, *, @@ -2013,7 +2279,22 @@ def _connector_exhaust_retryable_conn( "status IN ('queued', 'deferred', 'retry')", """ ( - SELECT COALESCE(MAX(d.attempt), 0) + SELECT + COALESCE(MAX(d.attempt), 0) + - COALESCE( + SUM( + CASE + WHEN d.status = 'failed' + AND COALESCE( + json_extract(d.response_json, '$.status'), + '' + ) = 'ack_deadline_expired' + THEN 1 + ELSE 0 + END + ), + 0 + ) FROM connector_deliveries d WHERE d.outbox_id = connector_outbox.id ) >= ? @@ -2056,6 +2337,93 @@ def _connector_exhaust_retryable_conn( return int(cursor.rowcount or 0) +def connector_reclaim_due( + db_path: Path, + host_id: str, + name: str | None = None, + *, + now: str | None = None, +) -> bool: + """Return whether connector work is due for reclaim without taking a write lock.""" + if not _sqlite_store_exists(db_path): + return False + current_time = _connector_now(now) + now_dt = _connector_datetime(current_time) + delivery_clauses = ["host_id = ?", "status = 'leased'"] + outbox_clauses = ["host_id = ?", "status = 'awaiting_ack'"] + params: list[Any] = [str(host_id)] + if name is not None: + delivery_clauses.append("connector = ?") + outbox_clauses.append("connector = ?") + params.append(str(name)) + with _connect(db_path) as conn: + _ensure_schema(conn) + lease_rows = conn.execute( + f""" + SELECT private_state_json + FROM connector_deliveries + WHERE {' AND '.join(delivery_clauses)} + """, + params, + ).fetchall() + for (private_state_json,) in lease_rows: + expires_at = str( + _json_object(private_state_json).get("lease_expires_at") or "" + ) + if expires_at and _connector_datetime(expires_at) <= now_dt: + return True + awaiting_rows = conn.execute( + f""" + SELECT + outbox.id, + outbox.private_state_json, + ( + SELECT deliveries.private_state_json + FROM connector_deliveries AS deliveries + WHERE deliveries.outbox_id = outbox.id + AND deliveries.status = 'awaiting_ack' + ORDER BY deliveries.id DESC + LIMIT 1 + ) + FROM connector_outbox AS outbox + WHERE {' AND '.join(outbox_clauses)} + """, + params, + ).fetchall() + for outbox_id, outbox_private, delivery_private in awaiting_rows: + deadline = str( + _json_object(outbox_private).get("ack_deadline_at") or "" + ) + if not deadline: + deadline = str( + _json_object(delivery_private).get("ack_deadline_at") or "" + ) + if not deadline or _connector_datetime(deadline) > now_dt: + continue + plan = conn.execute( + """ + SELECT id, state + FROM turn_presentation_plans + WHERE source_outbox_id = ? + ORDER BY id DESC + LIMIT 1 + """, + (int(outbox_id),), + ).fetchone() + if ( + plan is not None + and str(plan[1]) in {"active", "waiting_predecessor", "failed"} + and _connector_plan_has_live_job_lease_conn( + conn, + plan_id=int(plan[0]), + now_dt=now_dt, + ) + ): + continue + return True + return False + + def reclaim_expired_connector_leases( db_path: Path, host_id: str, @@ -2194,6 +2562,30 @@ def _valid_final_stable_key(value: Any) -> bool: ) +def _turn_ordering_key_conn( + conn: sqlite3.Connection, + *, + host_id: str, + turn_id: str, +) -> str: + row = conn.execute( + "SELECT worker_id, payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (str(host_id), str(turn_id)), + ).fetchone() + if row is None: + return f"orphan:{turn_id}" + payload = _json_object(row[1]) + meta = _json_object(payload.get("meta")) + stable_key = meta.get("stable_key") + if ( + _valid_final_stable_key(stable_key) + and type(meta.get("stable_key_version")) is int + and meta.get("stable_key_version") == 1 + ): + return str(stable_key) + return str(row[0] or f"orphan:{turn_id}") + + def _final_content_field_descriptor( *, revision: str, @@ -2868,6 +3260,11 @@ def _ensure_final_ready_anchor_conn( else "queued" ) payload_json = _canonical_json(payload) + ordering_key = _turn_ordering_key_conn( + conn, + host_id=str(host_id), + turn_id=str(turn_id), + ) if existing is not None: existing_state = _json_object(existing[2]) activated_after = ( @@ -2916,13 +3313,14 @@ def _ensure_final_ready_anchor_conn( delivery_kind, turn_id, content_revision, + ordering_key, status, payload_json, private_state_json, created_at, updated_at, next_attempt_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, '{}', ?, ?, NULL) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, '{}', ?, ?, NULL) ON CONFLICT(host_id, connector, delivery_key) DO NOTHING """, ( @@ -2932,6 +3330,7 @@ def _ensure_final_ready_anchor_conn( delivery_kind, str(turn_id), str(content_revision_value), + ordering_key, initial_status, payload_json, str(now), @@ -3653,14 +4052,25 @@ def _materialize_connector_plan_job_conn( ) -> int: plan_identity = conn.execute( """ - SELECT turn_id, content_revision - FROM turn_presentation_plans - WHERE id = ? + SELECT plans.turn_id, plans.content_revision, source.ordering_key + FROM turn_presentation_plans AS plans + LEFT JOIN connector_outbox AS source + ON source.id = plans.source_outbox_id + WHERE plans.id = ? """, (int(plan_id),), ).fetchone() if plan_identity is None: raise StoreSchemaError("presentation_plan_not_found") + ordering_key = ( + str(plan_identity[2]) + if plan_identity[2] is not None + else _turn_ordering_key_conn( + conn, + host_id=str(host_id), + turn_id=str(plan_identity[0]), + ) + ) cursor = conn.execute( """ INSERT INTO connector_outbox ( @@ -3670,13 +4080,14 @@ def _materialize_connector_plan_job_conn( delivery_kind, turn_id, content_revision, + ordering_key, status, payload_json, private_state_json, created_at, updated_at, next_attempt_at - ) VALUES (?, ?, ?, 'final_part', ?, ?, 'queued', ?, '{}', ?, ?, NULL) + ) VALUES (?, ?, ?, 'final_part', ?, ?, ?, 'queued', ?, '{}', ?, ?, NULL) """, ( str(host_id), @@ -3684,6 +4095,7 @@ def _materialize_connector_plan_job_conn( str(delivery_key), str(plan_identity[0]), str(plan_identity[1]), + ordering_key, _canonical_json(dict(payload)), str(created_at), str(created_at), @@ -4004,6 +4416,7 @@ def _update_presentation_plan_after_outbox_conn( outbox_id: int, outbox_status: str, now: str, + ack_ttl_seconds: int | None = None, ) -> None: plan = conn.execute( """ @@ -4045,28 +4458,85 @@ def _update_presentation_plan_after_outbox_conn( """, (plan_id,), ).fetchone() - if int(remaining[0] or 0) == 0: - conn.execute( + remaining_count = int(remaining[0] or 0) + source_outbox_id = ( + int(plan[4]) if plan[4] is not None else None + ) + if ( + remaining_count > 0 + and source_outbox_id is not None + and ack_ttl_seconds is not None + ): + deadline = _connector_add_seconds( + str(now), + max(1, int(ack_ttl_seconds)), + ) + source_row = conn.execute( """ - UPDATE turn_presentation_plans - SET state = 'completed', completed_at = COALESCE(completed_at, ?) - WHERE id = ? AND state = 'active' + SELECT private_state_json + FROM connector_outbox + WHERE id = ? AND status = 'awaiting_ack' """, - (str(now), plan_id), - ) - source_outbox_id = ( - int(plan[4]) if plan[4] is not None else None - ) - if source_outbox_id is not None: + (source_outbox_id,), + ).fetchone() + if source_row is not None: + source_state = _json_object(source_row[0]) + source_state["ack_deadline_at"] = deadline conn.execute( """ - UPDATE connector_deliveries - SET status = 'delivered', delivered_at = ? - WHERE outbox_id = ? AND status = 'awaiting_ack' + UPDATE connector_outbox + SET private_state_json = ?, updated_at = ? + WHERE id = ? AND status = 'awaiting_ack' """, - (str(now), source_outbox_id), - ) - source_row = conn.execute( + ( + _canonical_json(source_state), + str(now), + source_outbox_id, + ), + ) + delivery_row = conn.execute( + """ + SELECT id, private_state_json + FROM connector_deliveries + WHERE outbox_id = ? AND status = 'awaiting_ack' + ORDER BY id DESC + LIMIT 1 + """, + (source_outbox_id,), + ).fetchone() + if delivery_row is not None: + delivery_state = _json_object(delivery_row[1]) + delivery_state["ack_deadline_at"] = deadline + conn.execute( + """ + UPDATE connector_deliveries + SET private_state_json = ? + WHERE id = ? AND status = 'awaiting_ack' + """, + ( + _canonical_json(delivery_state), + int(delivery_row[0]), + ), + ) + if remaining_count == 0: + conn.execute( + """ + UPDATE turn_presentation_plans + SET state = 'completed', completed_at = COALESCE(completed_at, ?) + WHERE id = ? AND state = 'active' + """, + (str(now), plan_id), + ) + if source_outbox_id is not None: + conn.execute( + """ + UPDATE connector_deliveries + SET status = 'delivered', delivered_at = ? + WHERE outbox_id = ? AND status = 'awaiting_ack' + """, + (str(now), source_outbox_id), + ) + source_row = conn.execute( """ SELECT private_state_json FROM connector_outbox @@ -4239,6 +4709,7 @@ def prepare_connector_plan_commit( name: str, plan_token: str, source_ref: str | None = None, + ack_ttl_seconds: int = CONNECTOR_ACK_TTL_SECONDS, now: str | None = None, ) -> dict[str, Any]: """Atomically validate exact coverage and materialize one plan's ordered jobs.""" @@ -4263,6 +4734,10 @@ def prepare_connector_plan_commit( name=name, ) current_time = _connector_now(now) + ack_deadline_at = _connector_add_seconds( + current_time, + max(1, int(ack_ttl_seconds)), + ) with _connect(db_path, isolation_level=None) as conn: _ensure_schema(conn) conn.execute("BEGIN IMMEDIATE") @@ -4611,11 +5086,18 @@ def prepare_connector_plan_commit( ).fetchone() if source_private is None: raise StoreSchemaError("presentation_source_not_live") + source_delivery_private = conn.execute( + "SELECT private_state_json FROM connector_deliveries WHERE id = ?", + (int(source_delivery[0]),), + ).fetchone() + if source_delivery_private is None: + raise StoreSchemaError("presentation_source_not_live") delivery_cursor = conn.execute( """ UPDATE connector_deliveries SET status = 'awaiting_ack', response_json = ?, + private_state_json = ?, delivered_at = NULL WHERE id = ? AND outbox_id = ? AND status = 'leased' """, @@ -4626,6 +5108,12 @@ def prepare_connector_plan_commit( "status": "prepared", } ), + _canonical_json( + { + **_json_object(source_delivery_private[0]), + "ack_deadline_at": ack_deadline_at, + } + ), int(source_delivery[0]), source_outbox_id, ), @@ -4634,6 +5122,7 @@ def prepare_connector_plan_commit( _connector_private_clear_current(source_private[0]) ) next_source_state["presentation_generation"] = int(plan[7]) + next_source_state["ack_deadline_at"] = ack_deadline_at next_source_state["presentation_max_part_count"] = max( int( next_source_state.get( @@ -5279,24 +5768,13 @@ def poll_connector_outbox( WHERE earlier_final.host_id = outbox.host_id AND earlier_final.connector = outbox.connector AND earlier_final.delivery_kind = 'final_ready' + AND earlier_final.ordering_key = outbox.ordering_key AND earlier_final.id < outbox.id - AND json_extract( - earlier_final.payload_json, - '$.stable_key' - ) = json_extract( - outbox.payload_json, - '$.stable_key' - ) - AND json_extract( - earlier_final.payload_json, - '$.stable_key_version' - ) = json_extract( - outbox.payload_json, - '$.stable_key_version' - ) AND earlier_final.status NOT IN ( 'delivered', - 'superseded' + 'superseded', + 'dead_letter', + 'awaiting_ack' ) ) AND NOT EXISTS ( @@ -5316,6 +5794,74 @@ def poll_connector_outbox( OR active_outbox.status != 'delivered' ) ) + AND NOT EXISTS ( + SELECT 1 + FROM turn_presentation_plans AS earlier_plan + LEFT JOIN connector_outbox AS earlier_source + ON earlier_source.id = earlier_plan.source_outbox_id + WHERE earlier_plan.host_id = outbox.host_id + AND earlier_plan.name = outbox.connector + AND earlier_plan.state IN ( + 'active', + 'waiting_predecessor' + ) + AND ( + ( + earlier_source.ordering_key + = outbox.ordering_key + AND earlier_source.id < outbox.id + ) + OR ( + earlier_plan.source_outbox_id IS NULL + AND EXISTS ( + SELECT 1 + FROM turn_presentation_jobs + AS ordering_job + JOIN connector_outbox + AS ordering_outbox + ON ordering_outbox.id + = ordering_job.outbox_id + WHERE ordering_job.plan_id + = earlier_plan.id + AND ordering_outbox.ordering_key + = outbox.ordering_key + AND ordering_outbox.id < outbox.id + ) + ) + ) + AND EXISTS ( + SELECT 1 + FROM turn_presentation_jobs AS earlier_job + LEFT JOIN connector_outbox AS earlier_job_outbox + ON earlier_job_outbox.id = earlier_job.outbox_id + WHERE earlier_job.plan_id = earlier_plan.id + AND ( + earlier_job_outbox.id IS NULL + OR earlier_job_outbox.status NOT IN ( + 'delivered', + 'superseded', + 'dead_letter' + ) + ) + ) + ) + ) + ) + AND ( + outbox.delivery_kind != 'final_part' + OR NOT EXISTS ( + SELECT 1 + FROM connector_outbox AS earlier_part + WHERE earlier_part.host_id = outbox.host_id + AND earlier_part.connector = outbox.connector + AND earlier_part.delivery_kind = 'final_part' + AND earlier_part.ordering_key = outbox.ordering_key + AND earlier_part.id < outbox.id + AND earlier_part.status NOT IN ( + 'delivered', + 'superseded', + 'dead_letter' + ) ) ) AND ( @@ -5544,6 +6090,188 @@ def _connector_validate_live_ref_conn( return None, "invalid_ref" +def renew_connector_delivery( + db_path: Path, + *, + host_id: str, + name: str, + ref: str, + lease_seconds: int, + now: str | None = None, +) -> dict[str, Any]: + """Extend one live connector lease without creating another attempt.""" + if not _sqlite_store_exists(db_path): + return _connector_error_response( + status="store_unavailable", host_id=host_id, name=name, ref=ref + ) + current_time = _connector_now(now) + with _connect(db_path, isolation_level=None) as conn: + _ensure_schema(conn) + conn.execute("BEGIN IMMEDIATE") + try: + _connector_reclaim_expired_leases_conn( + conn, + host_id=str(host_id), + name=str(name), + now=current_time, + ) + row, error = _connector_validate_live_ref_conn( + conn, + host_id=str(host_id), + name=str(name), + ref=str(ref), + now=current_time, + ) + if error is not None or row is None: + conn.rollback() + return _connector_error_response( + status=error or "invalid_ref", + host_id=host_id, + name=name, + ref=ref, + ) + delivery_state = _json_object(row[7]) + outbox_state = _json_object(row[9]) + current_expiry = str(delivery_state.get("lease_expires_at") or current_time) + requested_expiry = _connector_datetime( + _connector_add_seconds(current_time, max(1, int(lease_seconds))) + ) + leased_until = _connector_iso( + max( + _connector_datetime(current_expiry), + requested_expiry, + ) + ) + delivery_state["lease_expires_at"] = leased_until + outbox_state["lease_expires_at"] = leased_until + conn.execute( + "UPDATE connector_deliveries SET private_state_json = ? WHERE id = ?", + (_canonical_json(delivery_state), int(row[0])), + ) + conn.execute( + """ + UPDATE connector_outbox + SET private_state_json = ?, updated_at = ? + WHERE id = ? AND status = 'leased' + """, + (_canonical_json(outbox_state), current_time, int(row[1])), + ) + conn.commit() + return _connector_response( + ok=True, + status="renewed", + host_id=host_id, + name=name, + ref=ref, + key=str(row[4]), + attempt=int(row[5] or 0), + leased_until=leased_until, + ) + except Exception: + conn.rollback() + raise + + +def release_connector_delivery( + db_path: Path, + *, + host_id: str, + name: str, + ref: str, + now: str | None = None, +) -> dict[str, Any]: + """Release one live lease and make its outbox row immediately available.""" + if not _sqlite_store_exists(db_path): + return _connector_error_response( + status="store_unavailable", host_id=host_id, name=name, ref=ref + ) + current_time = _connector_now(now) + with _connect(db_path, isolation_level=None) as conn: + _ensure_schema(conn) + conn.execute("BEGIN IMMEDIATE") + try: + _connector_reclaim_expired_leases_conn( + conn, + host_id=str(host_id), + name=str(name), + now=current_time, + ) + row, error = _connector_validate_live_ref_conn( + conn, + host_id=str(host_id), + name=str(name), + ref=str(ref), + now=current_time, + ) + if error is not None or row is None: + conn.rollback() + return _connector_error_response( + status=error or "invalid_ref", + host_id=host_id, + name=name, + ref=ref, + ) + terminal_after_lease = bool( + _json_object(row[9]).get("terminal_after_lease") + ) + next_status = ( + _CONNECTOR_SUPERSEDED_OUTBOX_STATUS + if terminal_after_lease + else "queued" + ) + result_status = "superseded" if terminal_after_lease else "released" + conn.execute( + """ + UPDATE connector_deliveries + SET status = ?, response_json = ?, delivered_at = ? + WHERE id = ? AND status = 'leased' + """, + ( + result_status, + _canonical_json( + {"schema_version": 1, "status": result_status} + ), + current_time, + int(row[0]), + ), + ) + conn.execute( + """ + UPDATE connector_outbox + SET status = ?, next_attempt_at = ?, updated_at = ?, + private_state_json = ? + WHERE id = ? AND status = 'leased' + """, + ( + next_status, + None if terminal_after_lease else current_time, + current_time, + _connector_private_clear_current(row[9]), + int(row[1]), + ), + ) + _update_presentation_plan_after_outbox_conn( + conn, + outbox_id=int(row[1]), + outbox_status=next_status, + now=current_time, + ) + conn.commit() + return _connector_response( + ok=True, + status=result_status, + host_id=host_id, + name=name, + ref=ref, + key=str(row[4]), + attempt=int(row[5] or 0), + available_at=None if terminal_after_lease else current_time, + ) + except Exception: + conn.rollback() + raise + + def _connector_update_ref( db_path: Path, *, @@ -5556,6 +6284,7 @@ def _connector_update_ref( available_at: str | None = None, delay_seconds: int | None = None, max_attempts: int | None = None, + ack_ttl_seconds: int | None = None, now: str | None = None, ) -> dict[str, Any]: if not _sqlite_store_exists(db_path): @@ -5690,6 +6419,7 @@ def _connector_update_ref( outbox_id=outbox_id, outbox_status=_CONNECTOR_TERMINAL_OUTBOX_STATUS, now=current_time, + ack_ttl_seconds=ack_ttl_seconds, ) conn.commit() return _connector_response( @@ -5710,7 +6440,40 @@ def _connector_update_ref( else: available_at = _connector_iso(available_at) attempt_limit = max(1, int(max_attempts)) if max_attempts is not None else None - exhausted = action == "fail" and attempt_limit is not None and attempt >= attempt_limit + attempts_used = int( + conn.execute( + """ + SELECT + COALESCE(MAX(attempt), 0) + - COALESCE( + SUM( + CASE + WHEN status = 'failed' + AND COALESCE( + json_extract( + response_json, + '$.status' + ), + '' + ) = 'ack_deadline_expired' + THEN 1 + ELSE 0 + END + ), + 0 + ) + FROM connector_deliveries + WHERE outbox_id = ? + """, + (outbox_id,), + ).fetchone()[0] + or 0 + ) + exhausted = ( + action == "fail" + and attempt_limit is not None + and attempts_used >= attempt_limit + ) result_status = "attempts_exhausted" if exhausted else ("retry_scheduled" if action == "fail" else "deferred") delivery_status = "failed" if action == "fail" else "deferred" outbox_status = ( @@ -5784,6 +6547,7 @@ def ack_connector_delivery( name: str, ref: str, response: Mapping[str, Any] | None = None, + ack_ttl_seconds: int = CONNECTOR_ACK_TTL_SECONDS, now: str | None = None, ) -> dict[str, Any]: """Acknowledge a live connector lease and make the outbox item terminal.""" @@ -5794,6 +6558,7 @@ def ack_connector_delivery( name=name, ref=ref, response=response, + ack_ttl_seconds=max(1, int(ack_ttl_seconds)), now=now, ) @@ -10843,53 +11608,346 @@ def _migrate_v13_to_v14_conn(conn: sqlite3.Connection) -> None: conn.execute(statement) -MIGRATIONS: tuple[Migration, ...] = ( - Migration(0, 1, _migrate_v0_to_v1_conn), - Migration(1, 2, _migrate_v1_to_v2_conn), - Migration(2, 3, _migrate_v2_to_v3_conn), - Migration(3, 4, _migrate_v3_to_v4_conn), - Migration(4, 5, _migrate_v4_to_v5_conn), - Migration(5, 6, _migrate_v5_to_v6_conn), - Migration(6, 7, _migrate_v6_to_v7_conn), - Migration(7, 8, _migrate_v7_to_v8_conn), - Migration(8, 9, _migrate_v8_to_v9_conn), - Migration(9, 10, _migrate_v9_to_v10_conn), - Migration(10, 11, _migrate_v10_to_v11_conn), - Migration(11, 12, _migrate_v11_to_v12_conn), - Migration(12, 13, _migrate_v12_to_v13_conn), - Migration(13, 14, _migrate_v13_to_v14_conn), -) - - -def _validate_migration_registry( - migrations: tuple[Migration, ...] | None = None, - *, - target_version: int = STORE_SCHEMA_VERSION, -) -> None: - registry = MIGRATIONS if migrations is None else migrations - expected = 0 - for migration in registry: - if ( - migration.from_version != expected - or migration.to_version != expected + 1 - ): - raise RuntimeError("invalid migration registry") - expected = migration.to_version - if expected != STORE_SCHEMA_VERSION: - raise RuntimeError("invalid migration registry target") - if not 0 <= int(target_version) <= STORE_SCHEMA_VERSION: - raise RuntimeError("unsupported migration target") - - -def _create_current_schema_conn(conn: sqlite3.Connection) -> None: - """Create an empty database directly at the current schema.""" - if conn.in_transaction: - raise StoreSchemaError("schema_migration_in_transaction") - conn.execute("BEGIN IMMEDIATE") - try: - conn.execute(CREATE_SNAPSHOTS_TABLE) - conn.execute(CREATE_COMMAND_RECEIPTS_TABLE) - conn.execute(CREATE_WORKER_BINDINGS_TABLE) +def _migrate_v14_to_v15_conn(conn: sqlite3.Connection) -> None: + """Tombstone duplicate and stale command claims without rekeying turns.""" + if not _table_columns(conn, "turns"): + return + now = utc_timestamp() + now_dt = datetime.fromisoformat(now) + rows = conn.execute( + """ + SELECT turns.host_id, turns.turn_id, turns.worker_id, + turns.payload_json, turns.observed_at, + revisions.user_text, revisions.assistant_final_text, + revisions.user_state, revisions.final_state + FROM turns + LEFT JOIN turn_content_revisions AS revisions + ON revisions.host_id = turns.host_id + AND revisions.turn_id = turns.turn_id + AND revisions.is_current = 1 + """ + ).fetchall() + decoded = [] + for row in rows: + payload = _json_object(row[3]) + current = ( + { + "user_text": row[5], + "assistant_final_text": row[6], + "user_state": str(row[7]), + "final_state": str(row[8]), + } + if row[7] is not None + else None + ) + decoded.append( + ( + str(row[0]), + str(row[1]), + str(row[2]), + payload, + current, + str(row[4] or ""), + ) + ) + claims = [ + row + for row in decoded + if str(row[3].get("source") or "") == "command" + and not str(row[3].get("source_turn_id") or "").strip() + and not _turn_is_tombstoned(row[3]) + and row[3].get("complete") is not True + ] + done = [ + row + for row in decoded + if str(row[3].get("source_turn_id") or "").strip() + and not _turn_is_tombstoned(row[3]) + and ( + row[3].get("complete") is True + or row[4] is not None + and str(row[4].get("final_state") or "") == "complete" + ) + ] + affected_hosts: set[str] = set() + used_done = { + str(row[3].get("superseded_by_turn_id") or "") + for row in decoded + if _turn_is_tombstoned(row[3]) + and str(row[3].get("superseded_by_turn_id") or "").strip() + } + for claim in claims: + claim_view = _turn_with_current_content(claim[3], claim[4]) + matches = [ + observed + for observed in done + if observed[0] == claim[0] + and observed[2] == claim[2] + and observed[1] not in used_done + and _turn_content_matches_origin( + _turn_with_current_content(observed[3], observed[4]), + claim_view, + ) + ] + if len(matches) != 1: + continue + matching_claims = [ + candidate + for candidate in claims + if candidate[0] == claim[0] + and candidate[2] == claim[2] + and _turn_content_matches_origin( + _turn_with_current_content(matches[0][3], matches[0][4]), + _turn_with_current_content(candidate[3], candidate[4]), + ) + ] + if len(matching_claims) != 1: + continue + if _tombstone_turn_conn( + conn, + claim[0], + claim[1], + superseded_by_turn_id=matches[0][1], + superseded_at=now, + ): + affected_hosts.add(claim[0]) + used_done.add(matches[0][1]) + + configured_hard_ttl = _configured_turn_claim_hard_ttl_seconds() + for claim in claims: + stored = conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (claim[0], claim[1]), + ).fetchone() + if stored is None or _turn_is_tombstoned(_json_object(stored[0])): + continue + claim_dt = _turn_row_time(claim[3], claim[5]) + if claim_dt is None or (now_dt - claim_dt).total_seconds() < configured_hard_ttl: + continue + if _tombstone_turn_conn( + conn, + claim[0], + claim[1], + superseded_by_turn_id=None, + superseded_at=now, + ): + affected_hosts.add(claim[0]) + for host_id in sorted(affected_hosts): + _increment_turn_list_generation_conn(conn, host_id) + + +def _legacy_outbox_ordering_key( + *, + outbox_id: int, + outbox_payload: Mapping[str, Any], + worker_id: Any, + turn_payload: Mapping[str, Any], +) -> str: + nested_turn = outbox_payload.get("turn") + route = dict(nested_turn) if isinstance(nested_turn, Mapping) else outbox_payload + route_meta = _json_object(route.get("meta")) + route_stable_key = route.get("stable_key") or route_meta.get("stable_key") + route_stable_key_version = ( + route.get("stable_key_version") + if route.get("stable_key") is not None + else route_meta.get("stable_key_version") + ) + if ( + _valid_final_stable_key(route_stable_key) + and type(route_stable_key_version) is int + and route_stable_key_version == 1 + ): + return str(route_stable_key) + meta = _json_object(turn_payload.get("meta")) + stable_key = meta.get("stable_key") + if ( + _valid_final_stable_key(stable_key) + and type(meta.get("stable_key_version")) is int + and meta.get("stable_key_version") == 1 + ): + return str(stable_key) + return str(worker_id or route.get("worker_id") or f"orphan:{outbox_id}") + + +def _migrate_v15_to_v16_conn( + conn: sqlite3.Connection, + *, + connector_ack_ttl_seconds: int = CONNECTOR_ACK_TTL_SECONDS, +) -> None: + """Partition final FIFO order and bound legacy awaiting-ack plans.""" + columns = _table_columns(conn, "connector_outbox") + if not columns: + conn.execute(CREATE_CONNECTOR_OUTBOX_TABLE) + elif "ordering_key" not in columns: + conn.execute( + "ALTER TABLE connector_outbox " + "ADD COLUMN ordering_key TEXT NOT NULL DEFAULT ''" + ) + if _table_columns(conn, "turns"): + rows = conn.execute( + """ + SELECT outbox.id, outbox.payload_json, turns.worker_id, turns.payload_json + FROM connector_outbox AS outbox + LEFT JOIN turns + ON turns.host_id = outbox.host_id + AND turns.turn_id = outbox.turn_id + WHERE outbox.ordering_key = '' + ORDER BY outbox.id + """ + ).fetchall() + else: + rows = [ + (row[0], row[1], None, None) + for row in conn.execute( + """ + SELECT id, payload_json + FROM connector_outbox + WHERE ordering_key = '' + ORDER BY id + """ + ).fetchall() + ] + for outbox_id, outbox_payload, worker_id, turn_payload in rows: + conn.execute( + "UPDATE connector_outbox SET ordering_key = ? WHERE id = ?", + ( + _legacy_outbox_ordering_key( + outbox_id=int(outbox_id), + outbox_payload=_json_object(outbox_payload), + worker_id=worker_id, + turn_payload=_json_object(turn_payload), + ), + int(outbox_id), + ), + ) + deadline = _connector_add_seconds( + utc_timestamp(), + max(1, int(connector_ack_ttl_seconds)), + ) + awaiting_rows = conn.execute( + "SELECT id, private_state_json FROM connector_outbox WHERE status = 'awaiting_ack'" + ).fetchall() + for outbox_id, private_state_json in awaiting_rows: + state = _json_object(private_state_json) + state["ack_deadline_at"] = deadline + conn.execute( + "UPDATE connector_outbox SET private_state_json = ? WHERE id = ?", + (_canonical_json(state), int(outbox_id)), + ) + delivery_rows = ( + conn.execute( + """ + SELECT id, private_state_json + FROM connector_deliveries + WHERE outbox_id = ? AND status = 'awaiting_ack' + """, + (int(outbox_id),), + ).fetchall() + if _table_columns(conn, "connector_deliveries") + else [] + ) + for delivery_id, delivery_private in delivery_rows: + delivery_state = _json_object(delivery_private) + delivery_state["ack_deadline_at"] = deadline + conn.execute( + "UPDATE connector_deliveries SET private_state_json = ? WHERE id = ?", + (_canonical_json(delivery_state), int(delivery_id)), + ) + conn.execute(CREATE_CONNECTOR_ORDERING_INDEX) + + +def _migrate_v16_to_v17_conn(conn: sqlite3.Connection) -> None: + """Give unresolved legacy outbox rows independent FIFO partitions.""" + if _table_columns(conn, "turns"): + rows = conn.execute( + """ + SELECT outbox.id, outbox.payload_json, + turns.worker_id, turns.payload_json + FROM connector_outbox AS outbox + LEFT JOIN turns + ON turns.host_id = outbox.host_id + AND turns.turn_id = outbox.turn_id + WHERE outbox.ordering_key = '' + ORDER BY outbox.id + """ + ).fetchall() + else: + rows = [ + (row[0], row[1], None, None) + for row in conn.execute( + """ + SELECT id, payload_json + FROM connector_outbox + WHERE ordering_key = '' + ORDER BY id + """ + ).fetchall() + ] + for outbox_id, outbox_payload, worker_id, turn_payload in rows: + conn.execute( + "UPDATE connector_outbox SET ordering_key = ? WHERE id = ?", + ( + _legacy_outbox_ordering_key( + outbox_id=int(outbox_id), + outbox_payload=_json_object(outbox_payload), + worker_id=worker_id, + turn_payload=_json_object(turn_payload), + ), + int(outbox_id), + ), + ) + + +MIGRATIONS: tuple[Migration, ...] = ( + Migration(0, 1, _migrate_v0_to_v1_conn), + Migration(1, 2, _migrate_v1_to_v2_conn), + Migration(2, 3, _migrate_v2_to_v3_conn), + Migration(3, 4, _migrate_v3_to_v4_conn), + Migration(4, 5, _migrate_v4_to_v5_conn), + Migration(5, 6, _migrate_v5_to_v6_conn), + Migration(6, 7, _migrate_v6_to_v7_conn), + Migration(7, 8, _migrate_v7_to_v8_conn), + Migration(8, 9, _migrate_v8_to_v9_conn), + Migration(9, 10, _migrate_v9_to_v10_conn), + Migration(10, 11, _migrate_v10_to_v11_conn), + Migration(11, 12, _migrate_v11_to_v12_conn), + Migration(12, 13, _migrate_v12_to_v13_conn), + Migration(13, 14, _migrate_v13_to_v14_conn), + Migration(14, 15, _migrate_v14_to_v15_conn), + Migration(15, 16, _migrate_v15_to_v16_conn), + Migration(16, 17, _migrate_v16_to_v17_conn), +) + + +def _validate_migration_registry( + migrations: tuple[Migration, ...] | None = None, + *, + target_version: int = STORE_SCHEMA_VERSION, +) -> None: + registry = MIGRATIONS if migrations is None else migrations + expected = 0 + for migration in registry: + if ( + migration.from_version != expected + or migration.to_version != expected + 1 + ): + raise RuntimeError("invalid migration registry") + expected = migration.to_version + if expected != STORE_SCHEMA_VERSION: + raise RuntimeError("invalid migration registry target") + if not 0 <= int(target_version) <= STORE_SCHEMA_VERSION: + raise RuntimeError("unsupported migration target") + + +def _create_current_schema_conn(conn: sqlite3.Connection) -> None: + """Create an empty database directly at the current schema.""" + if conn.in_transaction: + raise StoreSchemaError("schema_migration_in_transaction") + conn.execute("BEGIN IMMEDIATE") + try: + conn.execute(CREATE_SNAPSHOTS_TABLE) + conn.execute(CREATE_COMMAND_RECEIPTS_TABLE) + conn.execute(CREATE_WORKER_BINDINGS_TABLE) for statement in CREATE_CURRENT_PR6_TABLES: conn.execute(statement) conn.execute(CREATE_LEGACY_BACKEND_PENDING_TABLE) @@ -10923,6 +11981,7 @@ def _create_current_schema_conn(conn: sqlite3.Connection) -> None: conn.execute(statement) for statement in CREATE_FINAL_DELIVERY_INDEXES: conn.execute(statement) + conn.execute(CREATE_CONNECTOR_ORDERING_INDEX) for statement in CREATE_SNAPSHOT_INDEXES: conn.execute(statement) conn.execute(INSERT_STORE_MAINTENANCE_STATE) @@ -10953,6 +12012,7 @@ def _run_migrations( conn: sqlite3.Connection, *, target_version: int = STORE_SCHEMA_VERSION, + connector_ack_ttl_seconds: int = CONNECTOR_ACK_TTL_SECONDS, ) -> None: """Run exact ordered transitions with one transaction per version.""" if conn.in_transaction: @@ -10965,7 +12025,15 @@ def _run_migrations( raise RuntimeError("invalid migration registry dispatch") conn.execute("BEGIN IMMEDIATE") try: - migration.apply(conn) + if migration.apply is _migrate_v15_to_v16_conn: + _migrate_v15_to_v16_conn( + conn, + connector_ack_ttl_seconds=max( + 1, int(connector_ack_ttl_seconds) + ), + ) + else: + migration.apply(conn) conn.execute(f"PRAGMA user_version = {migration.to_version}") conn.commit() except Exception: @@ -10976,7 +12044,11 @@ def _run_migrations( raise StoreSchemaError("schema_version_not_advanced") -def ensure_schema(conn: sqlite3.Connection) -> None: +def ensure_schema( + conn: sqlite3.Connection, + *, + connector_ack_ttl_seconds: int = CONNECTOR_ACK_TTL_SECONDS, +) -> None: """Gate the current schema cheaply, or initialize/migrate older stores.""" version = int(conn.execute("PRAGMA user_version").fetchone()[0]) if version == STORE_SCHEMA_VERSION: @@ -11002,16 +12074,30 @@ def ensure_schema(conn: sqlite3.Connection) -> None: if version == 0 and not _database_has_application_objects(conn): _create_current_schema_conn(conn) return - _run_migrations(conn) + _run_migrations( + conn, + connector_ack_ttl_seconds=max( + 1, int(connector_ack_ttl_seconds) + ), + ) _ensure_schema = ensure_schema -def init_store(db_path: Path) -> None: +def init_store( + db_path: Path, + *, + connector_ack_ttl_seconds: int = CONNECTOR_ACK_TTL_SECONDS, +) -> None: """Initialize or migrate the sqlite store to the current schema.""" with _connect(db_path, prepare=True) as conn: - ensure_schema(conn) + ensure_schema( + conn, + connector_ack_ttl_seconds=max( + 1, int(connector_ack_ttl_seconds) + ), + ) def _normalized_command_request_policy( @@ -15473,7 +16559,8 @@ def _snapshot_owned_turn_candidate( source_rows = [ row for row in rows - if str(row[1].get("source_turn_id") or "").strip() + if not _turn_is_tombstoned(row[1]) + and str(row[1].get("source_turn_id") or "").strip() and str(row[1].get("origin_command_id") or "").strip() == origin_command_id ] @@ -15484,7 +16571,8 @@ def _snapshot_owned_turn_candidate( command_rows = [ row for row in rows - if not str(row[1].get("source_turn_id") or "").strip() + if not _turn_is_tombstoned(row[1]) + and not str(row[1].get("source_turn_id") or "").strip() and str(row[1].get("origin_command_id") or "").strip() == origin_command_id ] @@ -15554,6 +16642,9 @@ def _normalized_persisted_turn_payload( turn_id: str, payload: Mapping[str, Any], stored_payload: Mapping[str, Any], + *, + adopt_source_turn_id: bool = False, + adopt_origin_command_id: bool = False, ) -> dict[str, Any]: """Sanitize a row update while retaining every published identity.""" stored_source_turn_id = str( @@ -15562,26 +16653,38 @@ def _normalized_persisted_turn_payload( stored_origin_command_id = str( stored_payload.get("origin_command_id") or "" ).strip() + incoming_origin_command_id = str( + payload.get("origin_command_id") or "" + ).strip() + origin_command_id = ( + stored_origin_command_id + or (incoming_origin_command_id if adopt_origin_command_id else "") + ) stored_kind = str(stored_payload.get("kind") or "").strip() normalized = Turn.from_dict(payload).to_dict() - if stored_source_turn_id: - normalized["source_turn_id"] = stored_source_turn_id + source_turn_id = stored_source_turn_id or ( + str(normalized.get("source_turn_id") or "").strip() + if adopt_source_turn_id + else "" + ) + if source_turn_id: + normalized["source_turn_id"] = source_turn_id else: normalized.pop("source_turn_id", None) - if stored_origin_command_id: - normalized["origin_command_id"] = stored_origin_command_id + if origin_command_id: + normalized["origin_command_id"] = origin_command_id else: normalized.pop("origin_command_id", None) # Recompute the fingerprint from the preserved compatibility token and # provenance, then restore the published row identity after normalization. normalized = Turn.from_dict(normalized).to_dict() normalized["id"] = str(turn_id) - if stored_source_turn_id: - normalized["source_turn_id"] = stored_source_turn_id + if source_turn_id: + normalized["source_turn_id"] = source_turn_id else: normalized.pop("source_turn_id", None) - if stored_origin_command_id: - normalized["origin_command_id"] = stored_origin_command_id + if origin_command_id: + normalized["origin_command_id"] = origin_command_id else: normalized.pop("origin_command_id", None) if stored_kind: @@ -15601,12 +16704,16 @@ def _update_persisted_turn_row( current_time: str, *, snapshot_content_fingerprint: str | None = None, + adopt_source_turn_id: bool = False, + adopt_origin_command_id: bool = False, ) -> tuple[bool, dict[str, Any]]: """Update a current projection without rekeying its persisted turn.""" item = _normalized_persisted_turn_payload( str(turn_id), payload, stored_payload, + adopt_source_turn_id=adopt_source_turn_id, + adopt_origin_command_id=adopt_origin_command_id, ) encoded = _canonical_json(item) row = conn.execute( @@ -17698,7 +18805,11 @@ def _current_turn_content_rows_conn( """, (str(host_id), str(worker_id)), ).fetchall() - return _decode_turn_content_rows(rows) + return [ + row + for row in _decode_turn_content_rows(rows) + if not _turn_is_tombstoned(row[1]) + ] def _current_owned_turn_content_rows_conn( @@ -17731,7 +18842,8 @@ def _current_owned_turn_content_rows_conn( return [ row for row in _decode_turn_content_rows(rows) - if _turn_continuity_identity(row[1]) == owner_identity + if not _turn_is_tombstoned(row[1]) + and _turn_continuity_identity(row[1]) == owner_identity ] @@ -17746,54 +18858,477 @@ def _turn_with_current_content( return merged -def _source_turn_matches(payload: Mapping[str, Any], incoming_source_turn: str) -> bool: - stored = str(payload.get("source_turn_id") or "").strip() - if not stored or not incoming_source_turn: - return False - candidate = Turn.from_dict({**dict(payload), "source_turn_id": incoming_source_turn}) - return candidate.source_turn_id == stored +def _turn_is_tombstoned(payload: Mapping[str, Any]) -> bool: + return bool(str(payload.get("superseded_at") or "").strip()) -def _owned_source_turn_matches( - payload: Mapping[str, Any], - incoming_source_turn: str, +def _tombstone_turn_conn( + conn: sqlite3.Connection, + host_id: str, + turn_id: str, + *, + superseded_by_turn_id: str | None, + superseded_at: str, ) -> bool: - stored = str(payload.get("source_turn_id") or "").strip() - meta = payload.get("meta") - if ( - not stored - or not incoming_source_turn - or not isinstance(meta, Mapping) - ): + row = conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (str(host_id), str(turn_id)), + ).fetchone() + if row is None: return False - candidates = turn_source_id_candidates( - incoming_source_turn, - meta=meta, - source=payload.get("source"), - kind=payload.get("kind"), + stored = _json_object(row[0]) + if _turn_is_tombstoned(stored): + return False + payload = dict(stored) + payload.update( + { + "status": "closed", + "complete": True, + "has_open_turn": False, + "assistant_stream_text": None, + "completed_at": payload.get("completed_at") or superseded_at, + "updated_at": superseded_at, + "superseded_by_turn_id": ( + str(superseded_by_turn_id) + if str(superseded_by_turn_id or "").strip() + else None + ), + "superseded_at": superseded_at, + } ) - return stored in candidates - - -def _merge_canonical_field( - incoming: str | None, - current_text: Any, - current_state: Any, -) -> tuple[str | None, str]: - if incoming is None or incoming == "": - state = str(current_state or "absent") - if state not in {"absent", "complete", "known_incomplete"}: - state = "absent" - return ( - str(current_text) if current_text is not None and state != "absent" else None, - state, - ) - return incoming, "complete" + _update_persisted_turn_row( + conn, + str(host_id), + str(turn_id), + payload, + stored, + superseded_at, + ) + return True -def _retain_authoritative_completion( - metadata: Mapping[str, Any], - current: Mapping[str, Any] | None, +def _superseding_turn_content_conn( + conn: sqlite3.Connection, + host_id: str, + turn_id: str, +) -> dict[str, Any] | None: + """Follow a tombstone chain to the current list-visible successor.""" + current_turn_id = str(turn_id) + seen: set[str] = set() + first = True + while current_turn_id and current_turn_id not in seen: + seen.add(current_turn_id) + row = conn.execute( + """ + SELECT turns.payload_json, + revisions.user_text, + revisions.assistant_final_text, + revisions.user_state, + revisions.final_state + FROM turns + LEFT JOIN turn_content_revisions AS revisions + ON revisions.host_id = turns.host_id + AND revisions.turn_id = turns.turn_id + AND revisions.is_current = 1 + WHERE turns.host_id = ? AND turns.turn_id = ? + """, + (str(host_id), current_turn_id), + ).fetchone() + if row is None: + return None + payload = _json_object(row[0]) + if not _turn_is_tombstoned(payload): + if first: + return None + current = ( + { + "user_text": row[1], + "assistant_final_text": row[2], + "user_state": str(row[3]), + "final_state": str(row[4]), + } + if row[3] is not None + else None + ) + return _turn_with_current_content(payload, current) + first = False + current_turn_id = str( + payload.get("superseded_by_turn_id") or "" + ).strip() + return None + + +def _tombstone_matching_command_sibling_conn( + conn: sqlite3.Connection, + host_id: str, + worker_id: str, + completing_turn_id: str, + *, + observed_at: str, +) -> bool: + rows = _current_turn_content_rows_conn(conn, host_id, worker_id) + completing = next( + (row for row in rows if str(row[0]) == str(completing_turn_id)), + None, + ) + if completing is None: + return False + completing_view = _turn_with_current_content(completing[1], completing[2]) + if not ( + completing_view.get("complete") is True + or completing[2] is not None + and str(completing[2].get("final_state") or "") == "complete" + ): + return False + candidates = [ + row + for row in rows + if str(row[0]) != str(completing_turn_id) + and str(row[1].get("source") or "") == "command" + and not str(row[1].get("source_turn_id") or "").strip() + and not _turn_is_tombstoned(row[1]) + and row[1].get("has_open_turn") is True + and _turn_content_matches_origin( + _turn_with_current_content(row[1], row[2]), + completing_view, + ) + ] + if len(candidates) != 1: + return False + return _tombstone_turn_conn( + conn, + str(host_id), + str(candidates[0][0]), + superseded_by_turn_id=str(completing_turn_id), + superseded_at=str(observed_at), + ) + + +def _turn_row_time(payload: Mapping[str, Any], observed_at: str) -> datetime | None: + for value in ( + payload.get("started_at"), + payload.get("updated_at"), + observed_at, + ): + timestamp = _strict_utc_timestamp(value) + if timestamp is not None: + return datetime.fromisoformat(timestamp) + return None + + +def _turn_is_open_or_incomplete( + payload: Mapping[str, Any], + current: Mapping[str, Any] | None, +) -> bool: + return bool( + payload.get("has_open_turn") is True + or payload.get("complete") is not True + or current is not None + and str(current.get("final_state") or "") != "complete" + ) + + +def _turn_observed_near_submission( + observed_at: str, + send_started_at: str, +) -> bool: + observed = _strict_utc_timestamp(observed_at) + started = _strict_utc_timestamp(send_started_at) + if observed is None or started is None: + return False + delta = abs( + ( + datetime.fromisoformat(started) + - datetime.fromisoformat(observed) + ).total_seconds() + ) + return delta <= TURN_SUBMISSION_OBSERVATION_ADOPTION_WINDOW_SECONDS + + +def _sweep_turn_claims_conn( + conn: sqlite3.Connection, + host_id: str, + *, + grace_seconds: float, + hard_ttl_seconds: float, + now: str, +) -> int: + current_timestamp = _strict_utc_timestamp(now) + if current_timestamp is None: + raise ValueError("invalid sweep timestamp") + current_dt = datetime.fromisoformat(current_timestamp) + has_claim = conn.execute( + """ + SELECT 1 + FROM turns + WHERE host_id = ? + AND json_extract(payload_json, '$.source') = 'command' + AND COALESCE(json_extract(payload_json, '$.origin_command_id'), '') != '' + AND COALESCE(json_extract(payload_json, '$.source_turn_id'), '') = '' + AND COALESCE(json_extract(payload_json, '$.superseded_at'), '') = '' + AND json_extract(payload_json, '$.has_open_turn') = 1 + LIMIT 1 + """, + (str(host_id),), + ).fetchone() + if has_claim is None: + return 0 + rows = conn.execute( + """ + SELECT turns.turn_id, turns.worker_id, turns.payload_json, + turns.observed_at, revisions.user_text, + revisions.assistant_final_text, revisions.user_state, + revisions.final_state + FROM turns + LEFT JOIN turn_content_revisions AS revisions + ON revisions.host_id = turns.host_id + AND revisions.turn_id = turns.turn_id + AND revisions.is_current = 1 + WHERE turns.host_id = ? + """, + (str(host_id),), + ).fetchall() + decoded: list[tuple[str, str, dict[str, Any], dict[str, Any] | None, str]] = [] + for row in rows: + payload = _json_object(row[2]) + current = ( + { + "user_text": row[4], + "assistant_final_text": row[5], + "user_state": str(row[6]), + "final_state": str(row[7]), + } + if row[6] is not None + else None + ) + decoded.append((str(row[0]), str(row[1]), payload, current, str(row[3] or ""))) + + claims = [ + row + for row in decoded + if str(row[2].get("source") or "") == "command" + and str(row[2].get("origin_command_id") or "").strip() + and not str(row[2].get("source_turn_id") or "").strip() + and not _turn_is_tombstoned(row[2]) + and row[2].get("has_open_turn") is True + ] + claims.sort( + key=lambda row: ( + _turn_row_time(row[2], row[4]) or current_dt, + row[0], + ) + ) + done = [ + row + for row in decoded + if str(row[2].get("source_turn_id") or "").strip() + and not _turn_is_tombstoned(row[2]) + and ( + row[2].get("complete") is True + or row[3] is not None + and str(row[3].get("final_state") or "") == "complete" + ) + ] + used_done = { + str(row[2].get("superseded_by_turn_id") or "") + for row in decoded + if _turn_is_tombstoned(row[2]) + } + changed = 0 + unresolved: list[tuple[str, str, dict[str, Any], dict[str, Any] | None, str]] = [] + for claim in claims: + claim_dt = _turn_row_time(claim[2], claim[4]) + if claim_dt is None or (current_dt - claim_dt).total_seconds() < grace_seconds: + unresolved.append(claim) + continue + claim_view = _turn_with_current_content(claim[2], claim[3]) + matches = [ + candidate + for candidate in done + if candidate[1] == claim[1] + and candidate[0] not in used_done + and _turn_content_matches_origin( + _turn_with_current_content(candidate[2], candidate[3]), + claim_view, + ) + ] + if len(matches) == 1: + if _tombstone_turn_conn( + conn, + str(host_id), + claim[0], + superseded_by_turn_id=matches[0][0], + superseded_at=current_timestamp, + ): + changed += 1 + used_done.add(matches[0][0]) + continue + unresolved.append(claim) + + for worker_id in sorted({row[1] for row in unresolved}): + worker_claims = sorted( + (row for row in unresolved if row[1] == worker_id), + key=lambda row: (_turn_row_time(row[2], row[4]) or current_dt, row[0]), + ) + if len(worker_claims) != 1: + continue + claim = worker_claims[0] + claim_dt = _turn_row_time(claim[2], claim[4]) + if claim_dt is None or (current_dt - claim_dt).total_seconds() < grace_seconds: + continue + newer_done = sorted( + ( + row + for row in done + if row[1] == worker_id + and row[0] not in used_done + and not str(row[2].get("origin_command_id") or "").strip() + and (_turn_row_time(row[2], row[4]) or current_dt) > claim_dt + ), + key=lambda row: (_turn_row_time(row[2], row[4]) or current_dt, row[0]), + ) + if len(newer_done) == 1 and _tombstone_turn_conn( + conn, + str(host_id), + claim[0], + superseded_by_turn_id=newer_done[0][0], + superseded_at=current_timestamp, + ): + changed += 1 + used_done.add(newer_done[0][0]) + + for claim in claims: + row = conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (str(host_id), claim[0]), + ).fetchone() + if row is None or _turn_is_tombstoned(_json_object(row[0])): + continue + claim_dt = _turn_row_time(claim[2], claim[4]) + if claim_dt is None or (current_dt - claim_dt).total_seconds() < hard_ttl_seconds: + continue + if _tombstone_turn_conn( + conn, + str(host_id), + claim[0], + superseded_by_turn_id=None, + superseded_at=current_timestamp, + ): + changed += 1 + return changed + + +def sweep_turn_claims( + db_path: Path | str, + host_id: str, + *, + grace_seconds: float, + hard_ttl_seconds: float = TURN_CLAIM_HARD_TTL_SECONDS, + now: str | None = None, +) -> int: + """Resolve or expire durable command claims without deleting referenced rows.""" + if grace_seconds <= 0 or hard_ttl_seconds <= 0: + raise ValueError("turn claim TTL values must be positive") + if not _sqlite_store_exists(db_path): + return 0 + current = now or utc_timestamp() + with _connect(db_path, isolation_level=None) as conn: + _ensure_schema(conn) + conn.execute("PRAGMA busy_timeout=50") + conn.execute("BEGIN IMMEDIATE") + try: + changed = _sweep_turn_claims_conn( + conn, + str(host_id), + grace_seconds=float(grace_seconds), + hard_ttl_seconds=float(hard_ttl_seconds), + now=current, + ) + conn.commit() + return changed + except Exception: + conn.rollback() + raise + + +def _reserve_lazy_turn_claim_sweep( + db_path: Path | str, + host_id: str, + *, + current_clock: float, + refresh_interval_seconds: float, +) -> tuple[tuple[str, str], bool]: + key = (str(Path(db_path).absolute()), str(host_id)) + with _TURN_CLAIM_SWEEP_LOCK: + last = _TURN_CLAIM_SWEEP_LAST_AT.get(key) + if ( + last is not None + and current_clock >= last + and current_clock - last < refresh_interval_seconds + ): + return key, False + _TURN_CLAIM_SWEEP_LAST_AT[key] = current_clock + return key, True + + +def _release_failed_lazy_turn_claim_sweep( + key: tuple[str, str], + *, + current_clock: float, +) -> None: + with _TURN_CLAIM_SWEEP_LOCK: + if _TURN_CLAIM_SWEEP_LAST_AT.get(key) == current_clock: + _TURN_CLAIM_SWEEP_LAST_AT.pop(key, None) + + +def _source_turn_matches(payload: Mapping[str, Any], incoming_source_turn: str) -> bool: + stored = str(payload.get("source_turn_id") or "").strip() + if not stored or not incoming_source_turn: + return False + candidate = Turn.from_dict({**dict(payload), "source_turn_id": incoming_source_turn}) + return candidate.source_turn_id == stored + + +def _owned_source_turn_matches( + payload: Mapping[str, Any], + incoming_source_turn: str, +) -> bool: + stored = str(payload.get("source_turn_id") or "").strip() + meta = payload.get("meta") + if ( + not stored + or not incoming_source_turn + or not isinstance(meta, Mapping) + ): + return False + candidates = turn_source_id_candidates( + incoming_source_turn, + meta=meta, + source=payload.get("source"), + kind=payload.get("kind"), + ) + return stored in candidates + + +def _merge_canonical_field( + incoming: str | None, + current_text: Any, + current_state: Any, +) -> tuple[str | None, str]: + if incoming is None or incoming == "": + state = str(current_state or "absent") + if state not in {"absent", "complete", "known_incomplete"}: + state = "absent" + return ( + str(current_text) if current_text is not None and state != "absent" else None, + state, + ) + return incoming, "complete" + + +def _retain_authoritative_completion( + metadata: Mapping[str, Any], + current: Mapping[str, Any] | None, existing: Mapping[str, Any], *, incoming_final: str | None, @@ -17936,7 +19471,8 @@ def _owned_command_candidates( payload = row[1] row_origin = str(payload.get("origin_command_id") or "").strip() if ( - str(payload.get("source_turn_id") or "").strip() + _turn_is_tombstoned(payload) + or str(payload.get("source_turn_id") or "").strip() or not row_origin or expected_origin and row_origin != expected_origin or not _turn_content_matches_origin( @@ -17957,7 +19493,8 @@ def _owned_placeholder_candidates( return [ row for row in rows - if not str(row[1].get("source_turn_id") or "").strip() + if not _turn_is_tombstoned(row[1]) + and not str(row[1].get("source_turn_id") or "").strip() and not str(row[1].get("origin_command_id") or "").strip() ] @@ -18125,8 +19662,20 @@ def _merge_owned_turn_content_conn( else: command_rows = _owned_command_candidates(rows, automation_probe) if len(command_rows) > 1: - raise StoreSchemaError("turn_owner_command_ambiguous") - if command_rows: + _LOGGER.warning( + "turn_ingestion_ambiguity_fallthrough", + extra={ + "tendwire_diagnostic": { + "code": "turn_ingestion_ambiguity_fallthrough", + "host_id": str(host_id), + "worker_id": str( + current_projection.get("worker_id") or "" + ), + "candidate_count": len(command_rows), + } + }, + ) + if len(command_rows) == 1: base_row = command_rows[0] else: placeholder_rows = _owned_placeholder_candidates(rows) @@ -18151,80 +19700,126 @@ def _merge_owned_turn_content_conn( ) if incoming_source_turn: - seed = { - key: base_payload.get(key) - for key in _TURN_IDENTITY_SEED_FIELDS - if base_payload.get(key) is not None - } - if base_row is None or base_row not in command_rows: + if len(command_rows) == 1: + ( + command_turn_id, + command_payload, + command_current, + _command_observed_at, + ) = command_rows[0] + payload = _adopt_turn_projection( + command_payload, + current_projection, + ) + payload.update( + _retain_authoritative_completion( + clean_content, + command_current, + payload, + incoming_final=incoming_final, + observed_at=observed_at, + ) + ) + payload["source_turn_id"] = incoming_source_turn + metadata_changed, persisted_item = _update_persisted_turn_row( + conn, + str(host_id), + str(command_turn_id), + payload, + command_payload, + observed_at, + snapshot_content_fingerprint=snapshot_content_fingerprint, + adopt_source_turn_id=True, + ) + revision_changed = _replace_current_turn_content_conn( + conn, + host_id=str(host_id), + turn_id=str(command_turn_id), + current=command_current, + incoming_user=incoming_user, + incoming_final=incoming_final, + current_time=observed_at, + ) + revision_repaired = _ensure_absent_turn_content_revision_conn( + conn, + host_id=str(host_id), + turn_id=str(command_turn_id), + observed_at=observed_at, + ) + selected_turn_id = str(command_turn_id) + changed = ( + metadata_changed + or revision_changed + or revision_repaired + ) + else: + seed = { + key: base_payload.get(key) + for key in _TURN_IDENTITY_SEED_FIELDS + if base_payload.get(key) is not None + } seed.pop("origin_command_id", None) if str(seed.get("source") or "") == "command": seed["source"] = "snapshot" - seed.update( - _retain_authoritative_completion( - clean_content, - None, - {}, + seed.update( + _retain_authoritative_completion( + clean_content, + None, + {}, + incoming_final=incoming_final, + observed_at=observed_at, + ) + ) + item = _strip_canonical_turn_payload( + Turn.from_dict(seed).to_dict() + ) + turn_id = str(item.get("id") or "") + collision = conn.execute( + """ + SELECT payload_json + FROM turns + WHERE host_id = ? AND turn_id = ? + """, + (str(host_id), turn_id), + ).fetchone() + if collision is not None: + collision_payload = _json_object(collision[0]) + if ( + _turn_continuity_identity(collision_payload) + != _turn_continuity_identity(item) + or not _owned_source_turn_matches( + collision_payload, + incoming_source_turn, + ) + ): + raise StoreSchemaError( + "turn_owner_source_identity_conflict" + ) + raise StoreSchemaError("turn_owner_source_ambiguous") + turn_id = _insert_owned_turn_conn( + conn, + host_id=str(host_id), + item=item, + snapshot_content_fingerprint=snapshot_content_fingerprint, + observed_at=observed_at, + ) + _replace_current_turn_content_conn( + conn, + host_id=str(host_id), + turn_id=turn_id, + current=None, + incoming_user=incoming_user, incoming_final=incoming_final, + current_time=observed_at, + ) + _ensure_absent_turn_content_revision_conn( + conn, + host_id=str(host_id), + turn_id=turn_id, observed_at=observed_at, ) - ) - item = _strip_canonical_turn_payload( - Turn.from_dict(seed).to_dict() - ) - turn_id = str(item.get("id") or "") - collision = conn.execute( - """ - SELECT payload_json - FROM turns - WHERE host_id = ? AND turn_id = ? - """, - (str(host_id), turn_id), - ).fetchone() - if collision is not None: - collision_payload = _json_object(collision[0]) - if ( - _turn_continuity_identity(collision_payload) - != _turn_continuity_identity(item) - or not _owned_source_turn_matches( - collision_payload, - incoming_source_turn, - ) - ): - raise StoreSchemaError( - "turn_owner_source_identity_conflict" - ) - # A matching physical row would have appeared in the exact - # tier above. Treat its absence as corruption, never overwrite. - raise StoreSchemaError("turn_owner_source_ambiguous") - turn_id = _insert_owned_turn_conn( - conn, - host_id=str(host_id), - item=item, - snapshot_content_fingerprint=snapshot_content_fingerprint, - observed_at=observed_at, - ) - _replace_current_turn_content_conn( - conn, - host_id=str(host_id), - turn_id=turn_id, - current=None, - incoming_user=incoming_user, - incoming_final=incoming_final, - current_time=observed_at, - ) - _ensure_absent_turn_content_revision_conn( - conn, - host_id=str(host_id), - turn_id=turn_id, - observed_at=observed_at, - ) - selected_turn_id = turn_id - if command_rows: - predecessor_id = str(command_rows[0][0]) - if predecessor_id != selected_turn_id: - command_predecessor_turn_id = predecessor_id - changed = True + selected_turn_id = turn_id + changed = True elif base_row is not None: authoritative_older = ( _turn_has_authoritative_observation( @@ -18346,14 +19941,20 @@ def _merge_owned_turn_content_conn( if current_revision is not None else None ) - if ( - anchor_id is not None - and command_predecessor_turn_id is not None - ): - _delete_turn_if_unreferenced_conn( + _tombstone_matching_command_sibling_conn( + conn, + str(host_id), + str(current_projection.get("worker_id") or ""), + selected_turn_id, + observed_at=str(observed_at), + ) + if anchor_id is not None and command_predecessor_turn_id is not None: + _tombstone_turn_conn( conn, str(host_id), command_predecessor_turn_id, + superseded_by_turn_id=selected_turn_id, + superseded_at=str(observed_at), ) return int(changed) @@ -18674,11 +20275,20 @@ def _merge_turn_content_conn( if current_revision is not None else None ) + _tombstone_matching_command_sibling_conn( + conn, + str(host_id), + str(worker_id), + selected_turn_id, + observed_at=str(observed_at), + ) if anchor_id is not None and command_predecessor_turn_id is not None: - _delete_turn_if_unreferenced_conn( + _tombstone_turn_conn( conn, str(host_id), command_predecessor_turn_id, + superseded_by_turn_id=selected_turn_id, + superseded_at=str(observed_at), ) return int(changed) @@ -19019,6 +20629,15 @@ def _upsert_command_pending_turn_impl( _ensure_schema(conn) conn.execute("BEGIN IMMEDIATE") try: + superseding_turn = _superseding_turn_content_conn( + conn, + str(host_id), + turn_id, + ) + if superseding_turn is not None: + if owns_transaction: + conn.commit() + return sanitize_public_mapping(superseding_turn) owner_identity = _turn_continuity_identity(item) if owner_identity is not None: owned_rows = _current_owned_turn_content_rows_conn( @@ -19074,6 +20693,58 @@ def _upsert_command_pending_turn_impl( if owns_transaction: conn.commit() return sanitize_public_mapping(existing_item) + observation_rows = [ + row + for row in owned_rows + if str(row[1].get("source_turn_id") or "").strip() + and not str( + row[1].get("origin_command_id") or "" + ).strip() + and not _turn_is_tombstoned(row[1]) + and ( + _turn_is_open_or_incomplete(row[1], row[2]) + or _turn_observed_near_submission( + row[3], + current_time, + ) + ) + and _turn_content_matches_origin( + _turn_with_current_content(row[1], row[2]), + {"user_text": clean_text}, + ) + ] + if len(observation_rows) == 1: + ( + persisted_turn_id, + stored_payload, + current, + _stored_observed_at, + ) = observation_rows[0] + adopted_payload = _adopt_turn_projection( + stored_payload, + item, + ) + adopted_payload["origin_command_id"] = clean_request_id + adopted_payload["source"] = "command" + _metadata_changed, persisted_item = ( + _update_persisted_turn_row( + conn, + str(host_id), + str(persisted_turn_id), + adopted_payload, + stored_payload, + current_time, + snapshot_content_fingerprint=content_fingerprint, + adopt_origin_command_id=True, + ) + ) + existing_item = _turn_with_current_content( + persisted_item, + current, + ) + if owns_transaction: + conn.commit() + return sanitize_public_mapping(existing_item) command_rows = [ row for row in owned_rows @@ -19294,9 +20965,9 @@ def command_pending_turn_terminal_effect( worker: Any, request_id: str, instruction_text: str, -) -> Callable[[sqlite3.Connection], None]: - """Build a pending-turn effect for a command terminal transaction.""" - def effect(conn: sqlite3.Connection) -> None: +) -> Callable[[sqlite3.Connection], dict[str, Any]]: + """Build a transactional pending-turn upsert effect.""" + def effect(conn: sqlite3.Connection) -> dict[str, Any]: item = _upsert_command_pending_turn_conn( conn, host_id, @@ -19307,6 +20978,44 @@ def effect(conn: sqlite3.Connection) -> None: ) if item is None: raise StoreSchemaError("command_pending_turn_terminal_effect_failed") + return item + + return effect + + +def delete_command_pending_turn_claim_effect( + *, + host_id: str, + request_id: str, +) -> Callable[[sqlite3.Connection], bool]: + """Build a transactional delete for a claim proven not to have been sent.""" + clean_request_id = str(request_id or "").strip() + + def effect(conn: sqlite3.Connection) -> bool: + rows = conn.execute( + """ + SELECT turn_id, payload_json + FROM turns + WHERE host_id = ? + AND json_extract(payload_json, '$.source') = 'command' + AND json_extract(payload_json, '$.origin_command_id') = ? + AND COALESCE( + json_extract(payload_json, '$.source_turn_id'), + '' + ) = '' + """, + (str(host_id), clean_request_id), + ).fetchall() + if not clean_request_id or len(rows) != 1: + return False + payload = _json_object(rows[0][1]) + if _turn_is_tombstoned(payload): + return False + return _delete_turn_if_unreferenced_conn( + conn, + str(host_id), + str(rows[0][0]), + ) return effect @@ -19322,6 +21031,8 @@ def turns_payload_from_store( since: str | None = None, now: float | int | None = None, work_counters: TurnContentWorkCounters | None = None, + turn_refresh_interval_seconds: float = 2.0, + claim_hard_ttl_seconds: float = TURN_CLAIM_HARD_TTL_SECONDS, ) -> dict[str, Any]: """Return one insertion-stable, byte-bounded turn-list page.""" requested_schema = int(schema_version) @@ -19353,6 +21064,35 @@ def turns_payload_from_store( "status": "store_unavailable", } current_clock = time.time() if now is None else float(now) + refresh_interval = float(turn_refresh_interval_seconds) + sweep_key, sweep_due = _reserve_lazy_turn_claim_sweep( + db_path, + str(host_id), + current_clock=current_clock, + refresh_interval_seconds=refresh_interval, + ) + if sweep_due: + try: + sweep_turn_claims( + db_path, + str(host_id), + grace_seconds=max( + TURN_CLAIM_SWEEP_MIN_GRACE_SECONDS, + 10.0 * refresh_interval, + ), + hard_ttl_seconds=float(claim_hard_ttl_seconds), + now=datetime.fromtimestamp( + current_clock, + tz=timezone.utc, + ).isoformat(), + ) + except Exception: + _release_failed_lazy_turn_claim_sweep( + sweep_key, + current_clock=current_clock, + ) + # Listing remains available if opportunistic maintenance is contended. + pass try: decoded_cursor = ( decode_turn_list_cursor( @@ -19553,6 +21293,10 @@ def turns_payload_from_store( WHERE turns.host_id = ? AND turns.list_sequence > ? AND turns.list_sequence <= ? + AND COALESCE( + json_extract(turns.payload_json, '$.superseded_at'), + '' + ) = '' {continuation} ORDER BY turns.worker_id ASC, @@ -21474,6 +23218,7 @@ def mark_command_send_started( owner_token: str, binding_fingerprint: str, event_payload: Mapping[str, Any] | None = None, + send_started_effect: Callable[[sqlite3.Connection], Any] | None = None, now: str | None = None, ) -> dict[str, Any]: """CAS the exact reserved owner to send_started before external mutation.""" @@ -21526,6 +23271,11 @@ def mark_command_send_started( row = _command_request_row(conn, host_id, request_id) conn.commit() return _command_request_response("not_owner", row) + effect_result = ( + send_started_effect(conn) + if send_started_effect is not None + else None + ) row = _command_request_row(conn, host_id, request_id) if row is None: raise RuntimeError("command request send-start disappeared") @@ -21537,9 +23287,12 @@ def mark_command_send_started( event_payload=event_payload, ) conn.commit() - return _command_request_response( + response = _command_request_response( "send_started", row, owner_token=str(owner_token) ) + if send_started_effect is not None: + response["effect_result"] = effect_result + return response except Exception: if conn.in_transaction: conn.rollback() diff --git a/tests/test_backend_pending.py b/tests/test_backend_pending.py index 4283a7f..4cfd409 100644 --- a/tests/test_backend_pending.py +++ b/tests/test_backend_pending.py @@ -1449,7 +1449,7 @@ def test_current_schema_creation_has_exact_binding_and_claim_state(tmp_path: Pat db = tmp_path / "current-schema.db" init_store(db) with sqlite3.connect(db) as conn: - assert conn.execute("PRAGMA user_version").fetchone()[0] == STORE_SCHEMA_VERSION == 14 + assert conn.execute("PRAGMA user_version").fetchone()[0] == STORE_SCHEMA_VERSION == 17 columns = { str(row[1]) for row in conn.execute("PRAGMA table_info(backend_pending)").fetchall() diff --git a/tests/test_cli.py b/tests/test_cli.py index 14600af..70dff35 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -480,6 +480,8 @@ def forbidden_snapshot(*_args: Any, **_kwargs: Any) -> Any: "limit": 7, "cursor": None, "since": None, + "turn_refresh_interval_seconds": 2.0, + "claim_hard_ttl_seconds": 86400, }, ) ] @@ -555,6 +557,8 @@ def read_page(_db_path: Path, host_id: str, **kwargs: Any) -> dict[str, Any]: "limit": 9, "cursor": position_value if position_flag == "--cursor" else None, "since": position_value if position_flag == "--since" else None, + "turn_refresh_interval_seconds": 2.0, + "claim_hard_ttl_seconds": 86400, } ] diff --git a/tests/test_command_submission.py b/tests/test_command_submission.py index 59e455b..171aceb 100644 --- a/tests/test_command_submission.py +++ b/tests/test_command_submission.py @@ -478,12 +478,14 @@ def test_submit_command_uses_socket_pane_input_once_and_caches_result(tmp_path: assert first.status == STATUS_ACCEPTED assert first.disposition == DISPOSITION_TERMINAL_ACCEPTED + assert str(first.result["turn_id"]).startswith("turn-") assert first.result == { "target": {"worker_id": "w-1"}, "delivery_state": "submitted", "transport_state": "submitted", "target_state_at_send": "active", "observed_turn_state": "pending_observation", + "turn_id": first.result["turn_id"], } assert second.to_dict() == first.to_dict() assert duplicate.status == STATUS_DUPLICATE_REQUEST @@ -580,13 +582,323 @@ def test_submit_command_uses_socket_pane_input_once_and_caches_result(tmp_path: _assert_no_private_json(surface) -def test_submit_command_sends_identical_100_character_instructions_with_distinct_ids( +def test_submit_command_writes_turn_claim_before_pane_send(tmp_path: Path) -> None: + config = _config(tmp_path) + assert config.db_path is not None + worker = Worker( + id="w-1", + name="Alpha", + status="active", + meta={ + "stable_key": "wsk1_" + ("a" * 64), + "stable_key_version": 1, + }, + ) + _seed(config, [worker], [_binding(worker)]) + calls: list[dict[str, Any]] = [] + claim_ids: list[str] = [] + + class InspectingClient(_FakeSocketClient): + def request(self, method, params, *, timeout=None): + if method == "pane.send_input": + with sqlite3.connect(str(config.db_path)) as conn: + claim_ids.extend( + str(row[0]) + for row in conn.execute( + """ + SELECT turn_id FROM turns + WHERE host_id = ? + AND json_extract(payload_json, '$.origin_command_id') = ? + """, + (config.host_id, "write-early-request"), + ).fetchall() + ) + return super().request(method, params, timeout=timeout) + + accepted = submit_command( + config, + _request(request_id="write-early-request"), + socket_client_factory=lambda _config: InspectingClient(calls), + ) + + assert accepted.status == STATUS_ACCEPTED + assert claim_ids == [accepted.result["turn_id"]] + + +def test_submit_command_removes_claim_when_pane_input_never_starts( tmp_path: Path, ) -> None: config = _config(tmp_path) + assert config.db_path is not None worker = Worker(id="w-1", name="Alpha", status="active") _seed(config, [worker], [_binding(worker)]) calls: list[dict[str, Any]] = [] + + class ClearFailsClient(_FakeSocketClient): + def request(self, method, params, *, timeout=None): + if method == "pane.send_keys": + calls.append({"method": method, "params": dict(params)}) + raise HerdrSocketDisconnectedError("clear failed before input") + return super().request(method, params, timeout=timeout) + + envelope = submit_command( + config, + _request(request_id="clear-failure-request"), + socket_client_factory=lambda _config: ClearFailsClient(calls), + ) + + assert envelope.status == STATUS_REQUEST_STATE_UNCERTAIN + assert not any(call["method"] == "pane.send_input" for call in calls) + receipt = get_command_request( + config.db_path, + config.host_id, + "clear-failure-request", + ) + assert receipt is not None + assert receipt["state"] == "uncertain" + with sqlite3.connect(str(config.db_path)) as conn: + claims = conn.execute( + """ + SELECT turn_id + FROM turns + WHERE host_id = ? + AND json_extract(payload_json, '$.origin_command_id') = ? + """, + (config.host_id, "clear-failure-request"), + ).fetchall() + assert claims == [] + + +def test_observation_first_submission_adopts_source_row_in_place( + tmp_path: Path, +) -> None: + config = _config(tmp_path) + assert config.db_path is not None + worker = Worker( + id="w-1", + name="Alpha", + status="active", + meta={ + "stable_key": "wsk1_" + ("b" * 64), + "stable_key_version": 1, + }, + ) + _seed(config, [worker], [_binding(worker)]) + observed_at = store_sqlite.utc_timestamp() + assert merge_turn_content( + config.db_path, + config.host_id, + worker.id, + { + "source_turn_id": "observation-first-source", + "user_text": "hello", + "assistant_final_text": "already observed", + "complete": True, + "has_open_turn": False, + }, + observed_at=observed_at, + ) == 1 + observed = next( + turn + for turn in turns_payload_from_store( + config.db_path, + config.host_id, + schema_version=2, + )["turns"] + if turn.get("assistant_final_text") == "already observed" + ) + with sqlite3.connect(str(config.db_path)) as conn: + observed_list_sequence = conn.execute( + "SELECT list_sequence FROM turns WHERE host_id = ? AND turn_id = ?", + (config.host_id, observed["id"]), + ).fetchone()[0] + + calls: list[dict[str, Any]] = [] + accepted = submit_command( + config, + _request(request_id="observation-first-request"), + socket_client_factory=_factory(calls), + ) + + assert accepted.status == STATUS_ACCEPTED + assert accepted.result["turn_id"] == observed["id"] + assert accepted.result["observed_turn_state"] == "complete" + assert calls == _expected_submit_calls() + assert merge_turn_content( + config.db_path, + config.host_id, + worker.id, + { + "source_turn_id": "observation-first-source", + "user_text": "hello", + "assistant_final_text": "tier one refresh", + "complete": True, + "has_open_turn": False, + }, + observed_at=store_sqlite.utc_timestamp(), + ) == 1 + with sqlite3.connect(str(config.db_path)) as conn: + matching = conn.execute( + """ + SELECT turns.turn_id, turns.list_sequence + FROM turns + JOIN turn_content_revisions AS revisions + ON revisions.host_id = turns.host_id + AND revisions.turn_id = turns.turn_id + AND revisions.is_current = 1 + WHERE turns.host_id = ? AND revisions.user_text = 'hello' + """, + (config.host_id,), + ).fetchall() + assert len(matching) == 1 + assert matching[0][0] == observed["id"] + assert matching[0][1] == observed_list_sequence + refreshed = turns_payload_from_store( + config.db_path, + config.host_id, + schema_version=2, + )["turns"] + assert len([turn for turn in refreshed if turn.get("source_turn_id")]) == 1 + assert next( + turn for turn in refreshed if turn.get("source_turn_id") + )["assistant_final_text"] == "tier one refresh" + + +def test_submission_first_envelope_reflects_observation_during_send( + tmp_path: Path, +) -> None: + config = _config(tmp_path) + assert config.db_path is not None + worker = Worker( + id="w-1", + name="Alpha", + status="active", + meta={ + "stable_key": "wsk1_" + ("c" * 64), + "stable_key_version": 1, + }, + ) + _seed(config, [worker], [_binding(worker)]) + calls: list[dict[str, Any]] = [] + + class ObservingClient(_FakeSocketClient): + def request(self, method, params, *, timeout=None): + result = super().request(method, params, timeout=timeout) + if method == "pane.send_input": + assert merge_turn_content( + config.db_path, + config.host_id, + worker.id, + { + "source_turn_id": "submission-first-source", + "user_text": "hello", + "assistant_final_text": "observed during send", + "complete": True, + "has_open_turn": False, + }, + observed_at="2099-07-19T10:00:00+00:00", + ) == 1 + return result + + accepted = submit_command( + config, + _request(request_id="submission-first-request"), + socket_client_factory=lambda _config: ObservingClient(calls), + ) + + assert accepted.status == STATUS_ACCEPTED + assert accepted.result["observed_turn_state"] == "complete" + turns = turns_payload_from_store( + config.db_path, + config.host_id, + schema_version=2, + )["turns"] + matching = [turn for turn in turns if turn.get("user_text") == "hello"] + assert len(matching) == 1 + assert matching[0]["id"] == accepted.result["turn_id"] + assert matching[0]["assistant_final_text"] == "observed during send" + + +def test_legacy_submission_envelope_follows_tombstoned_claim_successor( + tmp_path: Path, +) -> None: + config = _config(tmp_path) + assert config.db_path is not None + worker = Worker(id="w-1", name="Alpha", status="active") + _seed(config, [worker], [_binding(worker)]) + calls: list[dict[str, Any]] = [] + + class LegacyObservingClient(_FakeSocketClient): + def request(self, method, params, *, timeout=None): + result = super().request(method, params, timeout=timeout) + if method == "pane.send_input": + assert merge_turn_content( + config.db_path, + config.host_id, + worker.id, + { + "source_turn_id": "legacy-envelope-source", + "user_text": "hello", + "assistant_final_text": "legacy observed during send", + "complete": True, + "has_open_turn": False, + }, + observed_at=store_sqlite.utc_timestamp(), + ) == 1 + return result + + accepted = submit_command( + config, + _request(request_id="legacy-envelope-request"), + socket_client_factory=lambda _config: LegacyObservingClient(calls), + ) + + assert accepted.status == STATUS_ACCEPTED + assert accepted.result["observed_turn_state"] == "complete" + turns = turns_payload_from_store( + config.db_path, + config.host_id, + schema_version=2, + )["turns"] + successor = next( + turn + for turn in turns + if turn.get("assistant_final_text") == "legacy observed during send" + ) + assert accepted.result["turn_id"] == successor["id"] + with sqlite3.connect(str(config.db_path)) as conn: + tombstones = [ + json.loads(row[0]) + for row in conn.execute( + """ + SELECT payload_json + FROM turns + WHERE host_id = ? + AND json_extract(payload_json, '$.origin_command_id') = ? + AND COALESCE(json_extract(payload_json, '$.superseded_at'), '') != '' + """, + (config.host_id, "legacy-envelope-request"), + ).fetchall() + ] + assert len(tombstones) == 1 + assert tombstones[0]["superseded_by_turn_id"] == successor["id"] + + +def test_submit_command_sends_identical_100_character_instructions_with_distinct_ids( + tmp_path: Path, +) -> None: + config = _config(tmp_path) + worker = Worker( + id="w-1", + name="Alpha", + status="active", + meta={ + "stable_key": "wsk1_" + ("d" * 64), + "stable_key_version": 1, + }, + ) + _seed(config, [worker], [_binding(worker)]) + calls: list[dict[str, Any]] = [] text = "x" * 100 first = submit_command( @@ -602,6 +914,7 @@ def test_submit_command_sends_identical_100_character_instructions_with_distinct assert first.status == STATUS_ACCEPTED assert second.status == STATUS_ACCEPTED + assert first.result["turn_id"] != second.result["turn_id"] expected_send = [ {"method": "agent.get", "params": {"target": "agent-secret"}}, *_expected_private_clear_calls(), @@ -617,12 +930,64 @@ def test_submit_command_sends_identical_100_character_instructions_with_distinct assert second_receipt is not None assert second_receipt["state"] == "accepted" with sqlite3.connect(str(config.db_path)) as conn: + turn_count = conn.execute( + """ + SELECT COUNT(*) + FROM turns + WHERE host_id = ? + AND json_extract(payload_json, '$.source') = 'command' + AND json_extract(payload_json, '$.has_open_turn') = 1 + """, + (config.host_id,), + ).fetchone()[0] events = [ row[0] for row in conn.execute( "SELECT event_type FROM events WHERE aggregate_type = 'command_request' ORDER BY id" ).fetchall() ] + assert turn_count == 2 + assert merge_turn_content( + config.db_path, + config.host_id, + worker.id, + { + "source_turn_id": "ambiguous-identical-source", + "user_text": text, + "assistant_final_text": "independent observation", + "complete": True, + "has_open_turn": False, + }, + observed_at="2099-07-19T10:00:00+00:00", + ) == 1 + with sqlite3.connect(str(config.db_path)) as conn: + stored = [ + (str(turn_id), json.loads(payload_json)) + for turn_id, payload_json in conn.execute( + "SELECT turn_id, payload_json FROM turns WHERE host_id = ?", + (config.host_id,), + ).fetchall() + ] + relevant = [ + (turn_id, payload) + for turn_id, payload in stored + if payload.get("origin_command_id") + or payload.get("source_turn_id") + ] + assert len(relevant) == 3 + assert {first.result["turn_id"], second.result["turn_id"]}.issubset( + {turn_id for turn_id, _payload in relevant} + ) + observed = [ + (turn_id, payload) + for turn_id, payload in relevant + if payload.get("source_turn_id") + ] + assert len(observed) == 1 + assert observed[0][0] not in { + first.result["turn_id"], + second.result["turn_id"], + } assert events == [ "command.request.reserved", "command.request.send_started", @@ -807,12 +1172,14 @@ def test_submit_command_reports_submitted_transport_and_worker_state(tmp_path: P envelope = submit_command(config, _request(), socket_client_factory=_factory(calls)) assert envelope.status == STATUS_ACCEPTED + assert str(envelope.result["turn_id"]).startswith("turn-") assert envelope.result == { "target": {"worker_id": "w-1"}, "delivery_state": "submitted", "transport_state": "submitted", "target_state_at_send": "active", "observed_turn_state": "pending_observation", + "turn_id": envelope.result["turn_id"], } @@ -825,12 +1192,14 @@ def test_submit_command_marks_idle_worker_delivery_as_submitted(tmp_path: Path) envelope = submit_command(config, _request(), socket_client_factory=_factory(calls)) assert envelope.status == STATUS_ACCEPTED + assert str(envelope.result["turn_id"]).startswith("turn-") assert envelope.result == { "target": {"worker_id": "w-1"}, "delivery_state": "submitted", "transport_state": "submitted", "target_state_at_send": "idle", "observed_turn_state": "pending_observation", + "turn_id": envelope.result["turn_id"], } @@ -3325,7 +3694,7 @@ def test_stable_owner_pending_command_survives_worker_churn_and_source_wins( for turn in completed_payload["turns"] if turn.get("assistant_final_text") == "durable owner answer" ) - assert completed_source["id"] != command_before["id"] + assert completed_source["id"] == command_before["id"] assert completed_source["origin_command_id"] == request_id assert completed_source["source_turn_id"].startswith("turnsrc-") assert completed_source["source_turn_id"] != raw_source @@ -3366,10 +3735,10 @@ def test_stable_owner_pending_command_survives_worker_churn_and_source_wins( """, (config.host_id,), ).fetchone() - assert ( - final_root["working_predecessor_turn_id"] - == command_before["id"] - ) + # Adopt-in-place preserves the command row's turn id, so the final's own + # turn IS the working predecessor; the payload pointer is correctly + # omitted (it is only stamped when the ids differ). + assert "working_predecessor_turn_id" not in final_root source_wins = upsert_command_pending_turn( config.db_path, diff --git a/tests/test_config.py b/tests/test_config.py index 26c4620..ccad0f2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -12,6 +12,7 @@ DEFAULT_COMMAND_RECEIPT_RETENTION_COUNT, DEFAULT_COMMAND_RECEIPT_RETENTION_SECONDS, DEFAULT_COMMAND_RETRY_HORIZON_SECONDS, + DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS, MAX_COMMAND_RETRY_HORIZON_SECONDS, MIN_COMMAND_RECEIPT_RETENTION_SECONDS, DEFAULT_TURN_REFRESH_INTERVAL_SECONDS, @@ -33,6 +34,8 @@ def test_pr16_runtime_knobs_have_documented_defaults(monkeypatch) -> None: "TENDWIRE_MAX_WORKERS", "TENDWIRE_MAX_OUTBOX_ATTEMPTS", "TENDWIRE_CONNECTOR_CLAIM_TTL_SECONDS", + "TENDWIRE_CONNECTOR_MAX_CLAIM_TTL_SECONDS", + "TENDWIRE_CONNECTOR_ACK_TTL_SECONDS", "TENDWIRE_COMMAND_RETRY_HORIZON_SECONDS", "TENDWIRE_COMMAND_RECEIPT_RETENTION_SECONDS", "TENDWIRE_COMMAND_RECEIPT_RETENTION_COUNT", @@ -48,6 +51,8 @@ def test_pr16_runtime_knobs_have_documented_defaults(monkeypatch) -> None: assert config.max_workers == 512 assert config.max_outbox_attempts == 10 assert config.connector_claim_ttl_seconds == 60 + assert config.connector_max_claim_ttl_seconds == 300 + assert config.connector_ack_ttl_seconds == 300 assert config.command_retry_horizon_seconds == 604_800 assert config.command_receipt_retention_seconds == 2_592_000 assert config.command_receipt_retention_count == 4096 @@ -61,6 +66,8 @@ def test_pr16_runtime_knobs_accept_constructor_and_env(monkeypatch) -> None: monkeypatch.setenv("TENDWIRE_MAX_WORKERS", "64") monkeypatch.setenv("TENDWIRE_MAX_OUTBOX_ATTEMPTS", "3") monkeypatch.setenv("TENDWIRE_CONNECTOR_CLAIM_TTL_SECONDS", "45") + monkeypatch.setenv("TENDWIRE_CONNECTOR_MAX_CLAIM_TTL_SECONDS", "240") + monkeypatch.setenv("TENDWIRE_CONNECTOR_ACK_TTL_SECONDS", "180") monkeypatch.setenv("TENDWIRE_COMMAND_RETRY_HORIZON_SECONDS", "120") monkeypatch.setenv("TENDWIRE_COMMAND_RECEIPT_RETENTION_SECONDS", "691200") monkeypatch.setenv("TENDWIRE_COMMAND_RECEIPT_RETENTION_COUNT", "99") @@ -74,6 +81,8 @@ def test_pr16_runtime_knobs_accept_constructor_and_env(monkeypatch) -> None: max_workers="9", max_outbox_attempts="4", connector_claim_ttl_seconds="15", + connector_max_claim_ttl_seconds="120", + connector_ack_ttl_seconds="90", command_retry_horizon_seconds="60", command_receipt_retention_seconds="691200", command_receipt_retention_count="12", @@ -85,6 +94,8 @@ def test_pr16_runtime_knobs_accept_constructor_and_env(monkeypatch) -> None: assert env_config.max_workers == 64 assert env_config.max_outbox_attempts == 3 assert env_config.connector_claim_ttl_seconds == 45 + assert env_config.connector_max_claim_ttl_seconds == 240 + assert env_config.connector_ack_ttl_seconds == 180 assert env_config.command_retry_horizon_seconds == 120 assert env_config.command_receipt_retention_seconds == 691_200 assert env_config.command_receipt_retention_count == 99 @@ -95,6 +106,8 @@ def test_pr16_runtime_knobs_accept_constructor_and_env(monkeypatch) -> None: assert explicit.max_workers == 9 assert explicit.max_outbox_attempts == 4 assert explicit.connector_claim_ttl_seconds == 15 + assert explicit.connector_max_claim_ttl_seconds == 120 + assert explicit.connector_ack_ttl_seconds == 90 assert explicit.command_retry_horizon_seconds == 60 assert explicit.command_receipt_retention_seconds == 691_200 assert explicit.command_receipt_retention_count == 12 @@ -110,6 +123,12 @@ def test_pr16_runtime_knobs_accept_constructor_and_env(monkeypatch) -> None: ("max_workers", 0, "max_workers must be >= 1"), ("max_outbox_attempts", 0, "max_outbox_attempts must be >= 1"), ("connector_claim_ttl_seconds", 0, "connector_claim_ttl_seconds must be >= 1"), + ( + "connector_max_claim_ttl_seconds", + 0, + "connector_max_claim_ttl_seconds must be >= 1", + ), + ("connector_ack_ttl_seconds", 0, "connector_ack_ttl_seconds must be >= 1"), ], ) def test_pr16_runtime_knobs_reject_invalid_values(field: str, value: object, message: str) -> None: @@ -307,6 +326,7 @@ def test_command_receipt_retention_must_strictly_exceed_retry_horizon( TURN_REFRESH_ENV_NAMES = ( + "TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS", "TENDWIRE_TURN_REFRESH_INTERVAL_SECONDS", "TENDWIRE_TURN_REFRESH_WORKERS", ) @@ -320,25 +340,31 @@ def test_turn_refresh_knobs_have_documented_defaults(monkeypatch) -> None: assert DEFAULT_TURN_REFRESH_INTERVAL_SECONDS == 2.0 assert DEFAULT_TURN_REFRESH_WORKERS == 4 + assert DEFAULT_TURN_CLAIM_HARD_TTL_SECONDS == 86_400 assert config.turn_refresh_interval_seconds == 2.0 assert config.turn_refresh_workers == 4 + assert config.turn_claim_hard_ttl_seconds == 86_400 def test_turn_refresh_knobs_use_explicit_before_environment(monkeypatch) -> None: monkeypatch.setenv("TENDWIRE_TURN_REFRESH_INTERVAL_SECONDS", "3.5") monkeypatch.setenv("TENDWIRE_TURN_REFRESH_WORKERS", "8") + monkeypatch.setenv("TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS", "7200") env_config = load_config(max_workers=16) explicit = load_config( max_workers=16, turn_refresh_interval_seconds="0.25", turn_refresh_workers="6", + turn_claim_hard_ttl_seconds="3600", ) assert env_config.turn_refresh_interval_seconds == 3.5 assert env_config.turn_refresh_workers == 8 + assert env_config.turn_claim_hard_ttl_seconds == 7200 assert explicit.turn_refresh_interval_seconds == 0.25 assert explicit.turn_refresh_workers == 6 + assert explicit.turn_claim_hard_ttl_seconds == 3600 @pytest.mark.parametrize("value", [0, -0.01, "nan", "inf", "-inf"]) diff --git a/tests/test_connector_daemon_cli.py b/tests/test_connector_daemon_cli.py index d30ab6c..96b1c65 100644 --- a/tests/test_connector_daemon_cli.py +++ b/tests/test_connector_daemon_cli.py @@ -17,6 +17,7 @@ from tendwire.core.models import Snapshot from tendwire.daemon import TendwireDaemon from tendwire.daemon_api import TendwireDaemonAPI +from tendwire.store import sqlite as store_sqlite from tendwire.store.sqlite import init_store @@ -119,6 +120,101 @@ def test_daemon_api_routes_connector_methods_safely(tmp_path: Path) -> None: _assert_json_only_and_safe(ack) +def test_daemon_api_routes_renew_and_release(tmp_path: Path) -> None: + db_path = tmp_path / "daemon-renew-release.db" + _enqueue(db_path, host_id="daemon-host", key="renew-release") + daemon = TendwireDaemon(Config(host_id="daemon-host", db_path=db_path)) + api = TendwireDaemonAPI( + get_snapshot=lambda: Snapshot(host_id="daemon-host"), + get_health=lambda: {"schema_version": 1, "status": "ok", "host_id": "daemon-host"}, + submit_command=daemon.submit_command, + connector_call=daemon.connector_call, + ) + first = api.dispatch( + {"method": "connector.poll", "params": {"name": "attention"}} + )["result"]["items"][0] + renewed = api.dispatch( + { + "method": "connector.renew", + "params": { + "name": "attention", + "ref": first["ref"], + "lease_seconds": 120, + }, + } + ) + released = api.dispatch( + { + "method": "connector.release", + "params": {"name": "attention", "ref": first["ref"]}, + } + ) + second = api.dispatch( + {"method": "connector.poll", "params": {"name": "attention"}} + )["result"]["items"][0] + + assert renewed["result"]["status"] == "renewed" + assert released["result"]["status"] == "released" + assert second["attempt"] == 2 + _assert_json_only_and_safe(renewed) + _assert_json_only_and_safe(released) + + +def test_daemon_periodic_tick_reclaims_without_a_followup_poll(tmp_path: Path) -> None: + db_path = tmp_path / "daemon-periodic-reclaim.db" + _enqueue(db_path, host_id="daemon-host", key="expired") + daemon = TendwireDaemon(Config(host_id="daemon-host", db_path=db_path)) + item = daemon.connector_call( + "connector.poll", + {"name": "attention", "lease_seconds": 60}, + )["items"][0] + assert item["attempt"] == 1 + with sqlite3.connect(str(db_path)) as conn: + for table in ("connector_outbox", "connector_deliveries"): + rows = conn.execute( + f"SELECT id, private_state_json FROM {table}" + ).fetchall() + for row_id, private_state_json in rows: + private = json.loads(private_state_json) + private["lease_expires_at"] = "2000-01-01T00:00:00+00:00" + conn.execute( + f"UPDATE {table} SET private_state_json = ? WHERE id = ?", + (json.dumps(private), row_id), + ) + + daemon._connector_periodic_tick() + + with sqlite3.connect(str(db_path)) as conn: + outbox_status = conn.execute( + "SELECT status FROM connector_outbox" + ).fetchone()[0] + delivery_status = conn.execute( + "SELECT status FROM connector_deliveries" + ).fetchone()[0] + assert outbox_status == "queued" + assert delivery_status == "expired" + + +def test_daemon_periodic_tick_skips_write_reclaim_when_nothing_is_due( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "daemon-no-reclaim-due.db" + _enqueue(db_path, host_id="daemon-host", key="not-leased") + daemon = TendwireDaemon(Config(host_id="daemon-host", db_path=db_path)) + + def reject_write(*_args: Any, **_kwargs: Any) -> None: + raise AssertionError("write reclaim should not run without due work") + + monkeypatch.setattr( + store_sqlite, + "reclaim_expired_connector_leases", + reject_write, + ) + + daemon._connector_periodic_tick() + + def test_cli_connector_poll_and_ack_print_json_only(tmp_path: Path, capsys) -> None: db_path = tmp_path / "cli-connector.db" _enqueue(db_path) @@ -169,6 +265,72 @@ def test_cli_connector_poll_and_ack_print_json_only(tmp_path: Path, capsys) -> N _assert_json_only_and_safe(ack_payload) +def test_cli_connector_renew_and_release_forward_live_ref_options( + tmp_path: Path, + capsys, +) -> None: + db_path = tmp_path / "cli-renew-release.db" + _enqueue(db_path, key="cli-renew-release") + poll_code = main( + [ + "--host-id", + "host-a", + "connector", + "poll", + "--name", + "attention", + "--db-path", + str(db_path), + ] + ) + poll_payload = json.loads(capsys.readouterr().out) + ref = poll_payload["items"][0]["ref"] + + renew_code = main( + [ + "--host-id", + "host-a", + "connector", + "renew", + "--name", + "attention", + "--ref", + ref, + "--lease-seconds", + "120", + "--db-path", + str(db_path), + ] + ) + renew_capture = capsys.readouterr() + renew_payload = json.loads(renew_capture.out) + release_code = main( + [ + "--host-id", + "host-a", + "connector", + "release", + "--name", + "attention", + "--ref", + ref, + "--db-path", + str(db_path), + ] + ) + release_capture = capsys.readouterr() + release_payload = json.loads(release_capture.out) + + assert poll_code == renew_code == release_code == 0 + assert renew_capture.err == release_capture.err == "" + assert renew_payload["status"] == "renewed" + assert renew_payload["leased_until"] + assert release_payload["status"] == "released" + assert release_payload["available_at"] + _assert_json_only_and_safe(renew_payload) + _assert_json_only_and_safe(release_payload) + + def test_cli_connector_prepare_reads_bounded_action_from_stdin( tmp_path: Path, capsys, diff --git a/tests/test_connector_outbox.py b/tests/test_connector_outbox.py index 54c770e..2d8845d 100644 --- a/tests/test_connector_outbox.py +++ b/tests/test_connector_outbox.py @@ -5,7 +5,7 @@ import json import sqlite3 from concurrent.futures import ThreadPoolExecutor -from datetime import datetime +from datetime import datetime, timedelta, timezone from pathlib import Path from threading import Barrier from typing import Any @@ -18,6 +18,7 @@ from tendwire.store.sqlite import ( SnapshotObservationContext, ack_connector_delivery, + connector_reclaim_due, defer_connector_delivery, fail_connector_delivery, init_store, @@ -106,6 +107,36 @@ def _enqueue(db_path: Path, *, key: str = "job-1", status: str = "queued") -> No ) +def _enqueue_final_root( + db_path: Path, + *, + key_suffix: str, + ordering_key: str, + status: str = "queued", +) -> str: + init_store(db_path) + key = f"turn-final:revision:twfinal1.{key_suffix:0<64}"[:94] + with sqlite3.connect(str(db_path)) as conn: + conn.execute( + """ + INSERT INTO connector_outbox ( + host_id, connector, delivery_key, delivery_kind, + ordering_key, status, payload_json, private_state_json, + created_at, updated_at + ) VALUES (?, 'turn-final', ?, 'final_ready', ?, ?, '{}', '{}', ?, ?) + """, + ( + "host-a", + key, + ordering_key, + status, + "2026-01-01T00:00:00+00:00", + "2026-01-01T00:00:00+00:00", + ), + ) + return key + + def _delivery_rows(db_path: Path) -> list[tuple[Any, ...]]: with sqlite3.connect(str(db_path)) as conn: return conn.execute( @@ -170,6 +201,129 @@ def test_poll_uses_configured_default_lease_and_explicit_lease_wins(tmp_path: Pa assert explicit_delta == 5 +def test_turn_final_lease_request_is_capped_at_server_max(tmp_path: Path) -> None: + db_path = tmp_path / "turn-final-lease-cap.db" + _enqueue_final_root( + db_path, + key_suffix="lease-cap", + ordering_key="worker-a", + ) + api = ConnectorOutboxAPI(db_path, "host-a") + item = api.poll( + {"name": "turn-final", "lease_seconds": 900} + )["items"][0] + delta = ( + datetime.fromisoformat(item["leased_until"]) + - datetime.fromisoformat(item["available_at"]) + ).total_seconds() + assert delta == 300 + renewed = api.renew( + {"name": "turn-final", "ref": item["ref"], "lease_seconds": 900} + ) + renewal_delta = ( + datetime.fromisoformat(renewed["leased_until"]) + - datetime.fromisoformat(item["leased_until"]) + ).total_seconds() + assert 0 <= renewal_delta < 10 + + +def test_renew_extends_live_lease_and_release_requeues_immediately(tmp_path: Path) -> None: + db_path = tmp_path / "renew-release.db" + _enqueue(db_path, key="renew-release") + api = ConnectorOutboxAPI(db_path, "host-a") + first = api.poll({"name": "attention", "lease_seconds": 60})["items"][0] + renewed = api.renew( + {"name": "attention", "ref": first["ref"], "lease_seconds": 120} + ) + released = api.release({"name": "attention", "ref": first["ref"]}) + second = api.poll({"name": "attention"})["items"][0] + + assert renewed["status"] == "renewed" + assert datetime.fromisoformat(renewed["leased_until"]) > datetime.fromisoformat( + first["leased_until"] + ) + assert released["status"] == "released" + assert second["key"] == "renew-release" + assert second["attempt"] == 2 + + +@pytest.mark.parametrize("action", ["renew", "release"]) +def test_renew_and_release_reject_stale_live_refs( + tmp_path: Path, + action: str, +) -> None: + db_path = tmp_path / f"stale-{action}.db" + _enqueue(db_path, key=f"stale-{action}") + api = ConnectorOutboxAPI(db_path, "host-a") + item = api.poll({"name": "attention"})["items"][0] + with sqlite3.connect(str(db_path)) as conn: + row_id, private_state_json = conn.execute( + "SELECT id, private_state_json FROM connector_outbox" + ).fetchone() + private_state = json.loads(private_state_json) + private_state["current_delivery_id"] = -1 + conn.execute( + "UPDATE connector_outbox SET private_state_json = ? WHERE id = ?", + (json.dumps(private_state), row_id), + ) + + params = {"name": "attention", "ref": item["ref"]} + if action == "renew": + params["lease_seconds"] = 120 + result = getattr(api, action)(params) + + assert result["ok"] is False + assert result["status"] == "stale_ref" + _assert_no_forbidden(result) + + +@pytest.mark.parametrize("terminal_status", ["dead_letter", "awaiting_ack"]) +def test_terminal_or_planned_final_head_does_not_block_same_worker( + tmp_path: Path, + terminal_status: str, +) -> None: + db_path = tmp_path / f"nonblocking-{terminal_status}.db" + _enqueue_final_root( + db_path, + key_suffix=f"head-{terminal_status}", + ordering_key="worker-a", + status=terminal_status, + ) + tail_key = _enqueue_final_root( + db_path, + key_suffix=f"tail-{terminal_status}", + ordering_key="worker-a", + ) + items = ConnectorOutboxAPI(db_path, "host-a").poll( + {"name": "turn-final", "limit": 10} + )["items"] + assert [item["key"] for item in items] == [tail_key] + + +def test_final_fifo_is_strict_per_worker_but_isolates_other_workers(tmp_path: Path) -> None: + db_path = tmp_path / "per-worker-fifo.db" + _enqueue_final_root( + db_path, + key_suffix="worker-a-head", + ordering_key="worker-a", + status="leased", + ) + _enqueue_final_root( + db_path, + key_suffix="worker-a-tail", + ordering_key="worker-a", + ) + worker_b_key = _enqueue_final_root( + db_path, + key_suffix="worker-b", + ordering_key="worker-b", + ) + items = ConnectorOutboxAPI(db_path, "host-a").poll( + {"name": "turn-final", "limit": 10} + )["items"] + assert [item["key"] for item in items] == [worker_b_key] + + def test_reclaim_allows_fresh_ref_and_rejects_stale_ref(tmp_path: Path) -> None: db_path = tmp_path / "reclaim.db" _enqueue(db_path) @@ -758,6 +912,7 @@ def _canonical_turn( host_id: str = "host-a", worker_id: str = "worker-prepare", source_turn_id: str = "source-prepare", + stable_key: str = "wsk1_" + ("a" * 64), final_text: str, user_text: str | None = None, ) -> tuple[str, str]: @@ -798,7 +953,7 @@ def _canonical_turn( "source_turn_id": source_turn_id, "complete": True, "meta": { - "stable_key": "wsk1_" + ("a" * 64), + "stable_key": stable_key, "stable_key_version": 1, }, } @@ -943,6 +1098,49 @@ def _stage_final_plan( return _commit_plan(api, token) +def _stage_source_bound_plan( + api: ConnectorOutboxAPI, + *, + turn_id: str, + revision: str, + ranges: list[tuple[int, int]], + version: str, +) -> tuple[dict[str, Any], dict[str, Any]]: + source = api.poll({"name": "turn-final"})["items"][0] + begun = api.prepare( + { + "schema_version": 1, + "action": "begin", + "name": "turn-final", + "turn_id": turn_id, + "content_revision": revision, + "presentation_version": version, + "part_count": len(ranges), + "source_ref": source["ref"], + } + ) + assert begun["ok"] is True + for ordinal, (start, end) in enumerate(ranges): + assert _put_final_part( + api, + plan_token=begun["plan_token"], + ordinal=ordinal, + start=start, + end=end, + )["ok"] is True + committed = api.prepare( + { + "schema_version": 1, + "action": "commit", + "name": "turn-final", + "plan_token": begun["plan_token"], + "source_ref": source["ref"], + } + ) + assert committed["ok"] is True + return source, committed + + def _drain_turn_final(api: ConnectorOutboxAPI) -> list[str]: keys: list[str] = [] while True: @@ -1069,7 +1267,7 @@ def test_v6_to_current_plan_migration_is_bounded_atomic_and_preserves_jobs( ).fetchall() } foreign_keys = conn.execute("PRAGMA foreign_key_check").fetchall() - assert version == store_sqlite.STORE_SCHEMA_VERSION == 14 + assert version == store_sqlite.STORE_SCHEMA_VERSION == 17 assert plan_row == (plan["plan_token"], 1, None, "active") assert job_count == 2 assert outbox_count == 3 @@ -1624,6 +1822,7 @@ def test_unrelated_plans_poll_concurrently_but_never_colease_siblings( db_path, worker_id="worker-two", source_turn_id="source-two", + stable_key="wsk1_" + ("b" * 64), final_text="ijklmnop", ) api = ConnectorOutboxAPI(db_path, "host-a") @@ -2036,3 +2235,829 @@ def test_lost_commit_response_retries_after_supersede_or_failure_without_duplica assert conn.execute( "SELECT COUNT(*) FROM connector_outbox WHERE connector = 'turn-final'" ).fetchone()[0] == before_retry_count + + +@pytest.mark.parametrize( + ("stable_key", "expected_ordering_key"), + [ + ("wsk1_" + ("b" * 64), "wsk1_" + ("b" * 64)), + ("invalid", "worker-ordering-fallback"), + ], +) +def test_final_ready_anchor_materializes_the_turn_ordering_key( + tmp_path: Path, + stable_key: str, + expected_ordering_key: str, +) -> None: + db_path = tmp_path / f"root-ordering-{expected_ordering_key[:12]}.db" + turn_id, revision = _canonical_turn( + db_path, + worker_id="worker-ordering-fallback", + stable_key=stable_key, + final_text="abcdefgh", + ) + with sqlite3.connect(str(db_path)) as conn: + outbox_id = store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=turn_id, + content_revision_value=revision, + now="2026-01-01T00:00:00+00:00", + ) + ordering_key = conn.execute( + "SELECT ordering_key FROM connector_outbox WHERE id = ?", + (outbox_id,), + ).fetchone()[0] + assert ordering_key == expected_ordering_key + + +def test_final_parts_inherit_the_turn_ordering_key(tmp_path: Path) -> None: + db_path = tmp_path / "part-ordering-key.db" + stable_key = "wsk1_" + ("b" * 64) + turn_id, revision = _canonical_turn( + db_path, + stable_key=stable_key, + final_text="abcdefgh", + ) + committed = _stage_final_plan( + ConnectorOutboxAPI(db_path, "host-a"), + turn_id=turn_id, + revision=revision, + ranges=[(0, 4), (4, 8)], + ) + with sqlite3.connect(str(db_path)) as conn: + keys = conn.execute( + """ + SELECT outbox.ordering_key + FROM turn_presentation_jobs AS jobs + JOIN turn_presentation_plans AS plans ON plans.id = jobs.plan_id + JOIN connector_outbox AS outbox ON outbox.id = jobs.outbox_id + WHERE plans.plan_token = ? + ORDER BY jobs.sequence_index + """, + (committed["plan_token"],), + ).fetchall() + assert keys == [(stable_key,), (stable_key,)] + + +def test_final_parts_preserve_enqueued_worker_fallback_after_adoption( + tmp_path: Path, +) -> None: + db_path = tmp_path / "part-ordering-worker-adoption.db" + turn_id, revision = _canonical_turn( + db_path, + worker_id="worker-before-adoption", + final_text="abcdefgh", + ) + with sqlite3.connect(str(db_path)) as conn: + source_id = store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=turn_id, + content_revision_value=revision, + now="2026-01-01T00:00:00+00:00", + ) + assert source_id is not None + # Model a root whose enqueue-time fallback predates an adoption. The + # source row is the authority after enqueue, even if the turn's current + # worker identity later changes. + conn.execute( + "UPDATE connector_outbox SET ordering_key = ? WHERE id = ?", + ("worker-before-adoption", source_id), + ) + conn.execute( + "UPDATE turns SET worker_id = ? WHERE host_id = ? AND turn_id = ?", + ("worker-after-adoption", "host-a", turn_id), + ) + + api = ConnectorOutboxAPI(db_path, "host-a") + source = api.poll({"name": "turn-final"})["items"][0] + begun = api.prepare( + { + "schema_version": 1, + "action": "begin", + "name": "turn-final", + "turn_id": turn_id, + "content_revision": revision, + "presentation_version": "turn-present-adopted-worker-v1", + "part_count": 2, + "source_ref": source["ref"], + } + ) + _put_final_part(api, plan_token=begun["plan_token"], ordinal=0, start=0, end=4) + _put_final_part(api, plan_token=begun["plan_token"], ordinal=1, start=4, end=8) + committed = api.prepare( + { + "schema_version": 1, + "action": "commit", + "name": "turn-final", + "plan_token": begun["plan_token"], + "source_ref": source["ref"], + } + ) + + assert committed["ok"] is True + with sqlite3.connect(str(db_path)) as conn: + keys = conn.execute( + """ + SELECT outbox.ordering_key + FROM turn_presentation_jobs AS jobs + JOIN turn_presentation_plans AS plans ON plans.id = jobs.plan_id + JOIN connector_outbox AS outbox ON outbox.id = jobs.outbox_id + WHERE plans.plan_token = ? + ORDER BY jobs.sequence_index + """, + (committed["plan_token"],), + ).fetchall() + assert keys == [("worker-before-adoption",), ("worker-before-adoption",)] + + +def test_unresolvable_final_rows_receive_independent_ordering_partitions( + tmp_path: Path, +) -> None: + db_path = tmp_path / "orphan-ordering-partitions.db" + init_store(db_path) + with sqlite3.connect(str(db_path)) as conn: + first_key = store_sqlite._turn_ordering_key_conn( + conn, + host_id="host-a", + turn_id="turn-deleted-a", + ) + second_key = store_sqlite._turn_ordering_key_conn( + conn, + host_id="host-a", + turn_id="turn-deleted-b", + ) + _enqueue_final_root( + db_path, + key_suffix="deleted-a", + ordering_key=first_key, + ) + _enqueue_final_root( + db_path, + key_suffix="deleted-b", + ordering_key=second_key, + ) + + items = ConnectorOutboxAPI(db_path, "host-a").poll( + {"name": "turn-final", "limit": 10} + )["items"] + + assert first_key == "orphan:turn-deleted-a" + assert second_key == "orphan:turn-deleted-b" + assert len(items) == 2 + + +def test_source_less_recovery_plan_blocks_later_same_worker_final( + tmp_path: Path, +) -> None: + db_path = tmp_path / "source-less-recovery-plan-ordering.db" + stable_key = "wsk1_" + ("e" * 64) + first_turn, first_revision = _canonical_turn( + db_path, + worker_id="worker-shared", + source_turn_id="source-first", + stable_key=stable_key, + final_text="abcdefgh", + ) + api = ConnectorOutboxAPI(db_path, "host-a", max_attempts=1) + failed_plan = _stage_final_plan( + api, + turn_id=first_turn, + revision=first_revision, + ranges=[(0, 4), (4, 8)], + ) + failed_job = api.poll({"name": "turn-final"})["items"][0] + assert api.fail( + { + "name": "turn-final", + "ref": failed_job["ref"], + "delay_seconds": 0, + } + )["status"] == "attempts_exhausted" + recovery_plan = api.prepare( + { + "schema_version": 1, + "action": "recover", + "name": "turn-final", + "failed_plan_token": failed_plan["plan_token"], + "request_id": "source-less-ordering-recovery-1", + } + ) + assert recovery_plan["status"] == "recovered" + second_turn, second_revision = _canonical_turn( + db_path, + worker_id="worker-shared", + source_turn_id="source-second", + stable_key=stable_key, + final_text="ijklmnop", + ) + with sqlite3.connect(str(db_path)) as conn: + later_source_id = store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=second_turn, + content_revision_value=second_revision, + now="2026-01-01T00:01:00+00:00", + ) + assert later_source_id is not None + with sqlite3.connect(str(db_path)) as conn: + assert conn.execute( + """ + SELECT source_outbox_id, recovers_plan_token + FROM turn_presentation_plans + WHERE plan_token = ? + """, + (recovery_plan["plan_token"],), + ).fetchone() == (None, failed_plan["plan_token"]) + + items = api.poll({"name": "turn-final", "limit": 10})["items"] + + assert len(items) == 1 + assert items[0]["payload"]["plan_token"] == recovery_plan["plan_token"] + + +def test_part_ack_progress_extends_source_deadline( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "ack-progress-deadline.db" + current = [datetime(2026, 1, 1, tzinfo=timezone.utc)] + monkeypatch.setattr( + store_sqlite, + "utc_timestamp", + lambda: current[0].isoformat(), + ) + turn_id, revision = _canonical_turn(db_path, final_text="abcdefgh") + with sqlite3.connect(str(db_path)) as conn: + source_id = store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=turn_id, + content_revision_value=revision, + now=current[0].isoformat(), + ) + assert source_id is not None + api = ConnectorOutboxAPI(db_path, "host-a", ack_ttl_seconds=30) + _source, plan = _stage_source_bound_plan( + api, + turn_id=turn_id, + revision=revision, + ranges=[(0, 4), (4, 8)], + version="turn-present-progress-v1", + ) + first_part = api.poll({"name": "turn-final"})["items"][0] + current[0] += timedelta(seconds=20) + + acknowledged = api.ack( + {"name": "turn-final", "ref": first_part["ref"]} + ) + with sqlite3.connect(str(db_path)) as conn: + source_private = json.loads( + conn.execute( + "SELECT private_state_json FROM connector_outbox WHERE id = ?", + (source_id,), + ).fetchone()[0] + ) + current[0] += timedelta(seconds=11) + reclaimed = reclaim_expired_connector_leases( + db_path, + "host-a", + "turn-final", + now=current[0].isoformat(), + ) + + assert acknowledged["status"] == "acknowledged" + assert datetime.fromisoformat(source_private["ack_deadline_at"]) == ( + datetime(2026, 1, 1, tzinfo=timezone.utc) + timedelta(seconds=50) + ) + assert reclaimed["reclaimed"] == 0 + with sqlite3.connect(str(db_path)) as conn: + assert conn.execute( + "SELECT state FROM turn_presentation_plans WHERE plan_token = ?", + (plan["plan_token"],), + ).fetchone() == ("active",) + + +def test_ack_deadline_reclaim_preserves_validly_leased_part( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "ack-deadline-live-part.db" + current = [datetime(2026, 1, 1, tzinfo=timezone.utc)] + monkeypatch.setattr( + store_sqlite, + "utc_timestamp", + lambda: current[0].isoformat(), + ) + turn_id, revision = _canonical_turn(db_path, final_text="abcdefgh") + with sqlite3.connect(str(db_path)) as conn: + source_id = store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=turn_id, + content_revision_value=revision, + now=current[0].isoformat(), + ) + assert source_id is not None + api = ConnectorOutboxAPI(db_path, "host-a", ack_ttl_seconds=30) + _source, plan = _stage_source_bound_plan( + api, + turn_id=turn_id, + revision=revision, + ranges=[(0, 4), (4, 8)], + version="turn-present-live-lease-v1", + ) + leased_part = api.poll( + {"name": "turn-final", "lease_seconds": 60} + )["items"][0] + current[0] += timedelta(seconds=31) + + assert connector_reclaim_due( + db_path, + "host-a", + "turn-final", + now=current[0].isoformat(), + ) is False + reclaimed = reclaim_expired_connector_leases( + db_path, + "host-a", + "turn-final", + now=current[0].isoformat(), + ) + acknowledged = api.ack( + {"name": "turn-final", "ref": leased_part["ref"]} + ) + + assert reclaimed["reclaimed"] == 0 + assert acknowledged["status"] == "acknowledged" + with sqlite3.connect(str(db_path)) as conn: + assert conn.execute( + "SELECT state FROM turn_presentation_plans WHERE plan_token = ?", + (plan["plan_token"],), + ).fetchone() == ("active",) + assert conn.execute( + "SELECT status FROM connector_outbox WHERE id = ?", + (source_id,), + ).fetchone() == ("awaiting_ack",) + + +def test_repeated_ack_deadline_reclaims_do_not_exhaust_healthy_source( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "repeated-ack-deadline-reclaim.db" + current = [datetime(2026, 1, 1, tzinfo=timezone.utc)] + monkeypatch.setattr( + store_sqlite, + "utc_timestamp", + lambda: current[0].isoformat(), + ) + turn_id, revision = _canonical_turn(db_path, final_text="abcdefgh") + with sqlite3.connect(str(db_path)) as conn: + source_id = store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=turn_id, + content_revision_value=revision, + now=current[0].isoformat(), + ) + assert source_id is not None + api = ConnectorOutboxAPI( + db_path, + "host-a", + ack_ttl_seconds=30, + max_attempts=3, + ) + + for cycle in range(3): + source, _plan = _stage_source_bound_plan( + api, + turn_id=turn_id, + revision=revision, + ranges=[(0, 8)], + version=f"turn-present-slow-ack-v{cycle + 1}", + ) + assert source["attempt"] == cycle + 1 + current[0] += timedelta(seconds=31) + reclaimed = reclaim_expired_connector_leases( + db_path, + "host-a", + "turn-final", + now=current[0].isoformat(), + ) + assert reclaimed["reclaimed"] == 1 + + fourth = api.poll({"name": "turn-final"})["items"][0] + + assert fourth["attempt"] == 4 + with sqlite3.connect(str(db_path)) as conn: + assert conn.execute( + "SELECT status FROM connector_outbox WHERE id = ?", + (source_id,), + ).fetchone() == ("leased",) + + +def test_ack_deadline_reclaims_do_not_inflate_later_failure_budget( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "ack-deadline-failure-budget.db" + current = [datetime(2026, 1, 1, tzinfo=timezone.utc)] + monkeypatch.setattr( + store_sqlite, + "utc_timestamp", + lambda: current[0].isoformat(), + ) + turn_id, revision = _canonical_turn(db_path, final_text="abcdefgh") + with sqlite3.connect(str(db_path)) as conn: + assert store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=turn_id, + content_revision_value=revision, + now=current[0].isoformat(), + ) is not None + api = ConnectorOutboxAPI( + db_path, + "host-a", + ack_ttl_seconds=30, + max_attempts=2, + ) + + _stage_source_bound_plan( + api, + turn_id=turn_id, + revision=revision, + ranges=[(0, 8)], + version="turn-present-timeout-before-failure-v1", + ) + current[0] += timedelta(seconds=31) + assert reclaim_expired_connector_leases( + db_path, + "host-a", + "turn-final", + now=current[0].isoformat(), + )["reclaimed"] == 1 + + first_real_attempt = api.poll({"name": "turn-final"})["items"][0] + first_failure = api.fail( + { + "name": "turn-final", + "ref": first_real_attempt["ref"], + "delay_seconds": 0, + } + ) + second_real_attempt = api.poll({"name": "turn-final"})["items"][0] + second_failure = api.fail( + { + "name": "turn-final", + "ref": second_real_attempt["ref"], + "delay_seconds": 0, + } + ) + + assert first_real_attempt["attempt"] == 2 + assert first_failure["status"] == "retry_scheduled" + assert second_real_attempt["attempt"] == 3 + assert second_failure["status"] == "attempts_exhausted" + + +def test_awaiting_ack_deadline_fails_plan_and_requeues_source(tmp_path: Path) -> None: + db_path = tmp_path / "awaiting-ack-reclaim.db" + turn_id, revision = _canonical_turn(db_path, final_text="abcdefgh") + with sqlite3.connect(str(db_path)) as conn: + source_id = store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=turn_id, + content_revision_value=revision, + now="2026-01-01T00:00:00+00:00", + ) + conn.commit() + assert source_id is not None + api = ConnectorOutboxAPI(db_path, "host-a", ack_ttl_seconds=30) + source = api.poll({"name": "turn-final"})["items"][0] + begun = api.prepare( + { + "schema_version": 1, + "action": "begin", + "name": "turn-final", + "turn_id": turn_id, + "content_revision": revision, + "presentation_version": "turn-present-ack-deadline-v1", + "part_count": 2, + "source_ref": source["ref"], + } + ) + _put_final_part(api, plan_token=begun["plan_token"], ordinal=0, start=0, end=4) + _put_final_part(api, plan_token=begun["plan_token"], ordinal=1, start=4, end=8) + committed = api.prepare( + { + "schema_version": 1, + "action": "commit", + "name": "turn-final", + "plan_token": begun["plan_token"], + "source_ref": source["ref"], + } + ) + reclaimed = reclaim_expired_connector_leases( + db_path, + "host-a", + "turn-final", + now="9999-01-01T00:00:00+00:00", + ) + retry = poll_connector_outbox( + db_path, + "host-a", + "turn-final", + now="9999-01-01T00:00:00+00:00", + )["items"][0] + with sqlite3.connect(str(db_path)) as conn: + plan_state = conn.execute( + "SELECT state FROM turn_presentation_plans WHERE plan_token = ?", + (committed["plan_token"],), + ).fetchone()[0] + source_status = conn.execute( + "SELECT status FROM connector_outbox WHERE id = ?", + (source_id,), + ).fetchone()[0] + job_statuses = conn.execute( + """ + SELECT outbox.status + FROM turn_presentation_jobs AS jobs + JOIN connector_outbox AS outbox ON outbox.id = jobs.outbox_id + JOIN turn_presentation_plans AS plans ON plans.id = jobs.plan_id + WHERE plans.plan_token = ? + ORDER BY jobs.sequence_index + """, + (committed["plan_token"],), + ).fetchall() + assert reclaimed["reclaimed"] == 1 + assert plan_state == "failed" + assert source_status == "leased" + assert job_statuses == [("dead_letter",), ("dead_letter",)] + assert retry["key"] == source["key"] + assert retry["attempt"] == 2 + + +def test_awaiting_ack_deadline_preserves_explicit_suffix_recovery( + tmp_path: Path, +) -> None: + db_path = tmp_path / "awaiting-ack-suffix-recovery.db" + turn_id, revision = _canonical_turn(db_path, final_text="abcdefgh") + with sqlite3.connect(str(db_path)) as conn: + source_id = store_sqlite._ensure_final_ready_anchor_conn( + conn, + host_id="host-a", + turn_id=turn_id, + content_revision_value=revision, + now="2026-01-01T00:00:00+00:00", + ) + conn.commit() + assert source_id is not None + api = ConnectorOutboxAPI(db_path, "host-a", ack_ttl_seconds=30) + source = api.poll({"name": "turn-final"})["items"][0] + begun = api.prepare( + { + "schema_version": 1, + "action": "begin", + "name": "turn-final", + "turn_id": turn_id, + "content_revision": revision, + "presentation_version": "turn-present-ack-recover-v1", + "part_count": 2, + "source_ref": source["ref"], + } + ) + _put_final_part(api, plan_token=begun["plan_token"], ordinal=0, start=0, end=4) + _put_final_part(api, plan_token=begun["plan_token"], ordinal=1, start=4, end=8) + committed = api.prepare( + { + "schema_version": 1, + "action": "commit", + "name": "turn-final", + "plan_token": begun["plan_token"], + "source_ref": source["ref"], + } + ) + first_part = api.poll({"name": "turn-final", "limit": 10})["items"][0] + assert first_part["payload"]["part_ordinal"] == 0 + assert api.ack({"name": "turn-final", "ref": first_part["ref"]})[ + "status" + ] == "acknowledged" + + reclaimed = reclaim_expired_connector_leases( + db_path, + "host-a", + "turn-final", + now="9999-01-01T00:00:00+00:00", + ) + with sqlite3.connect(str(db_path)) as conn: + assert conn.execute( + "SELECT status FROM connector_outbox WHERE id = ?", + (source_id,), + ).fetchone() == ("queued",) + recovered = api.prepare( + { + "schema_version": 1, + "action": "recover", + "name": "turn-final", + "failed_plan_token": committed["plan_token"], + "request_id": "ack-deadline-recover-1", + } + ) + suffix = api.poll({"name": "turn-final", "limit": 10})["items"] + + assert reclaimed["reclaimed"] == 1 + assert recovered["status"] == "recovered" + assert recovered["acknowledged_prefix_count"] == 1 + assert recovered["executable_job_count"] == 1 + assert len(suffix) == 1 + assert suffix[0]["payload"]["part_ordinal"] == 1 + + +def test_awaiting_ack_without_plan_becomes_terminal_failed(tmp_path: Path) -> None: + db_path = tmp_path / "awaiting-ack-unrecoverable.db" + _enqueue_final_root( + db_path, + key_suffix="orphan-awaiting", + ordering_key="worker-a", + status="awaiting_ack", + ) + with sqlite3.connect(str(db_path)) as conn: + conn.execute( + """ + UPDATE connector_outbox + SET private_state_json = '{"ack_deadline_at":"2026-01-01T00:00:00+00:00"}' + """ + ) + reclaimed = reclaim_expired_connector_leases( + db_path, + "host-a", + "turn-final", + now="2026-01-01T00:00:01+00:00", + ) + with sqlite3.connect(str(db_path)) as conn: + status = conn.execute("SELECT status FROM connector_outbox").fetchone()[0] + assert reclaimed["reclaimed"] == 1 + assert status == "dead_letter" + + +def test_v16_migration_backfills_ordering_and_awaiting_ack_deadlines( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "connector-v16-migration.db" + stable_key = "wsk1_" + ("c" * 64) + enqueue_stable_key = "wsk1_" + ("d" * 64) + tombstoned_turn, _ = _canonical_turn( + db_path, + worker_id="worker-tombstoned", + source_turn_id="source-tombstoned", + stable_key=stable_key, + final_text="done", + ) + fallback_turn, _ = _canonical_turn( + db_path, + worker_id="worker-fallback", + source_turn_id="source-fallback", + stable_key="invalid", + final_text="done", + ) + deleted_turns = [ + _canonical_turn( + db_path, + worker_id=f"worker-deleted-{suffix}", + source_turn_id=f"source-deleted-{suffix}", + final_text="done", + )[0] + for suffix in ("a", "b") + ] + with sqlite3.connect(str(db_path)) as conn: + assert store_sqlite._tombstone_turn_conn( + conn, + "host-a", + tombstoned_turn, + superseded_by_turn_id=None, + superseded_at="2026-01-02T00:00:00+00:00", + ) + for turn_id, status in ( + (tombstoned_turn, "awaiting_ack"), + (fallback_turn, "dead_letter"), + (fallback_turn, "superseded"), + ): + payload = ( + { + "stable_key": enqueue_stable_key, + "stable_key_version": 1, + } + if status == "dead_letter" + else {} + ) + cursor = conn.execute( + """ + INSERT INTO connector_outbox ( + host_id, connector, delivery_key, delivery_kind, turn_id, + ordering_key, status, payload_json, private_state_json, + created_at, updated_at + ) VALUES (?, 'turn-final', ?, 'final_ready', ?, '', ?, ?, '{}', ?, ?) + """, + ( + "host-a", + f"migration-{turn_id}-{status}", + turn_id, + status, + json.dumps(payload), + "2026-01-01T00:00:00+00:00", + "2026-01-01T00:00:00+00:00", + ), + ) + if status == "awaiting_ack": + conn.execute( + """ + INSERT INTO connector_deliveries ( + outbox_id, host_id, connector, delivery_key, attempt, + status, response_json, private_state_json, created_at + ) VALUES (?, 'host-a', 'turn-final', 'migration-awaiting', 1, + 'awaiting_ack', '{}', '{}', ?) + """, + (cursor.lastrowid, "2026-01-01T00:00:00+00:00"), + ) + for deleted_turn in deleted_turns: + conn.execute( + "DELETE FROM turn_content_revisions WHERE host_id = ? AND turn_id = ?", + ("host-a", deleted_turn), + ) + conn.execute( + "DELETE FROM turns WHERE host_id = ? AND turn_id = ?", + ("host-a", deleted_turn), + ) + orphan_ids: list[int] = [] + for suffix, deleted_turn in zip(("a", "b"), deleted_turns, strict=True): + cursor = conn.execute( + """ + INSERT INTO connector_outbox ( + host_id, connector, delivery_key, delivery_kind, turn_id, + ordering_key, status, payload_json, private_state_json, + created_at, updated_at + ) VALUES ( + 'host-a', 'turn-final', ?, 'final_ready', ?, '', + 'dead_letter', '{}', '{}', ?, ? + ) + """, + ( + f"migration-deleted-{suffix}", + deleted_turn, + "2026-01-01T00:00:00+00:00", + "2026-01-01T00:00:00+00:00", + ), + ) + orphan_ids.append(int(cursor.lastrowid)) + conn.execute("DROP INDEX IF EXISTS idx_connector_outbox_final_ordering") + conn.execute("ALTER TABLE connector_outbox DROP COLUMN ordering_key") + conn.execute("PRAGMA user_version = 15") + migration_now = "2026-01-03T00:00:00+00:00" + monkeypatch.setenv("TENDWIRE_CONNECTOR_ACK_TTL_SECONDS", "999") + monkeypatch.setattr(store_sqlite, "utc_timestamp", lambda: migration_now) + init_store(db_path, connector_ack_ttl_seconds=123) + with sqlite3.connect(str(db_path)) as conn: + rows = conn.execute( + """ + SELECT delivery_key, turn_id, status, ordering_key, private_state_json + FROM connector_outbox + WHERE delivery_key LIKE 'migration-%' + ORDER BY id + """ + ).fetchall() + delivery_private = conn.execute( + "SELECT private_state_json FROM connector_deliveries WHERE status = 'awaiting_ack'" + ).fetchone()[0] + by_key = {row[0]: row[1:] for row in rows} + tombstoned_key = f"migration-{tombstoned_turn}-awaiting_ack" + dead_letter_key = f"migration-{fallback_turn}-dead_letter" + superseded_key = f"migration-{fallback_turn}-superseded" + assert by_key[tombstoned_key][0:3] == ( + tombstoned_turn, + "awaiting_ack", + stable_key, + ) + ack_deadline = json.loads(by_key[tombstoned_key][3])["ack_deadline_at"] + assert ( + datetime.fromisoformat(ack_deadline) + - datetime.fromisoformat(migration_now) + ).total_seconds() == 123 + assert by_key[dead_letter_key][0:3] == ( + fallback_turn, + "dead_letter", + enqueue_stable_key, + ) + assert by_key[superseded_key][0:3] == ( + fallback_turn, + "superseded", + "worker-fallback", + ) + assert by_key["migration-deleted-a"][2] == f"orphan:{orphan_ids[0]}" + assert by_key["migration-deleted-b"][2] == f"orphan:{orphan_ids[1]}" + assert by_key["migration-deleted-a"][2] != by_key["migration-deleted-b"][2] + assert json.loads(delivery_private)["ack_deadline_at"] == ack_deadline diff --git a/tests/test_daemon.py b/tests/test_daemon.py index 8c0b0a0..6f75b7b 100644 --- a/tests/test_daemon.py +++ b/tests/test_daemon.py @@ -932,6 +932,8 @@ def project( "limit": 17, "cursor": "twlist1.public", "since": None, + "turn_refresh_interval_seconds": 2.0, + "claim_hard_ttl_seconds": 86400, } for call in projection_calls ) @@ -1393,6 +1395,8 @@ def test_daemon_health_exposes_public_operational_status_without_private_values( max_workers=8, max_outbox_attempts=4, connector_claim_ttl_seconds=33, + connector_max_claim_ttl_seconds=222, + connector_ack_ttl_seconds=111, acknowledged_final_retention_days=40, acknowledged_final_retention_count=500, snapshot_retention_days=9, @@ -1523,6 +1527,8 @@ def operational_status(self) -> dict[str, Any]: "max_workers": 8, "max_outbox_attempts": 4, "outbox_claim_ttl_seconds": 33, + "outbox_max_claim_ttl_seconds": 222, + "outbox_ack_ttl_seconds": 111, "acknowledged_final_retention_days": 40, "acknowledged_final_retention_count": 500, "command_retry_horizon_seconds": 120, diff --git a/tests/test_delivery_retention.py b/tests/test_delivery_retention.py index 5ad8b66..9302a24 100644 --- a/tests/test_delivery_retention.py +++ b/tests/test_delivery_retention.py @@ -84,6 +84,8 @@ def _source_turns(db_path: Path, snapshot: Any, *, host_id: str = HOST_ID) -> li snapshot=snapshot, schema_version=2, limit=250, + turn_refresh_interval_seconds=1_000_000_000, + claim_hard_ttl_seconds=1_000_000_000, ) return [ turn @@ -319,7 +321,11 @@ def _aggressive_cleanup(db_path: Path, *, batch_size: int = 100) -> dict[str, An ) -def test_dead_letter_final_blocks_only_its_stable_owner(tmp_path: Path) -> None: +def test_dead_letter_final_blocks_no_later_finals(tmp_path: Path) -> None: + # dead_letter is terminal for FIFO ordering: it must not block ANY later + # final — not other owners (the original intent of this test) and not its + # own owner either (a poisoned reply must never wedge its worker; recovery + # goes through retry_final_ready_delivery/supersede instead). db_path = tmp_path / "dead-letter-owner-isolation.db" second_worker_id = "worker-2" second_stable_key = "wsk1_" + ("b" * 64) @@ -392,8 +398,9 @@ def test_dead_letter_final_blocks_only_its_stable_owner(tmp_path: Path) -> None: {"name": FINAL_NAME, "limit": 100, "lease_seconds": 60} ) assert unrelated["ok"] is True - assert len(unrelated["items"]) == 1 - assert unrelated["items"][0]["payload"]["stable_key"] == second_stable_key + assert len(unrelated["items"]) == 2 + polled_keys = {item["payload"]["stable_key"] for item in unrelated["items"]} + assert polled_keys == {STABLE_KEY, second_stable_key} with sqlite3.connect(str(db_path)) as conn: first_owner_states = conn.execute( @@ -407,7 +414,7 @@ def test_dead_letter_final_blocks_only_its_stable_owner(tmp_path: Path) -> None: """, (FINAL_NAME, STABLE_KEY), ).fetchall() - assert first_owner_states == [("dead_letter",), ("queued",)] + assert first_owner_states == [("dead_letter",), ("leased",)] def test_acknowledged_prefix_is_cleaned_but_failed_suffix_and_exact_final_survive( @@ -882,6 +889,8 @@ def test_completed_command_receipt_does_not_release_command_linked_pending_turn( snapshot=snapshot, schema_version=2, limit=250, + turn_refresh_interval_seconds=1_000_000_000, + claim_hard_ttl_seconds=1_000_000_000, )["turns"] protected = [ turn diff --git a/tests/test_delivery_retention_migration.py b/tests/test_delivery_retention_migration.py index 394b521..0e01fa9 100644 --- a/tests/test_delivery_retention_migration.py +++ b/tests/test_delivery_retention_migration.py @@ -875,7 +875,7 @@ def test_v10_to_v12_migration_retains_finals_without_reposting_or_leaking( finals = _seed_v10_finals(db_path) init_store(db_path) - assert store_sqlite.STORE_SCHEMA_VERSION == 14 + assert store_sqlite.STORE_SCHEMA_VERSION == 17 delivered_key = _final_key(*finals["delivered"]) hold_keys = {_final_key(*finals[label]) for label in ("hold-a", "hold-b")} diff --git a/tests/test_delivery_retention_projection.py b/tests/test_delivery_retention_projection.py index 3ce5841..0e8f150 100644 --- a/tests/test_delivery_retention_projection.py +++ b/tests/test_delivery_retention_projection.py @@ -165,7 +165,7 @@ def _turn_rows_snapshot(db_path: Path) -> tuple[tuple[Any, ...], ...]: def _assert_continuity_integrity(db_path: Path) -> None: with sqlite3.connect(str(db_path)) as conn: - assert conn.execute("PRAGMA user_version").fetchone() == (store_sqlite.STORE_SCHEMA_VERSION,) == (14,) + assert conn.execute("PRAGMA user_version").fetchone() == (store_sqlite.STORE_SCHEMA_VERSION,) == (17,) assert conn.execute("PRAGMA foreign_key_check").fetchall() == [] current_counts = conn.execute( """ diff --git a/tests/test_delivery_retention_recovery.py b/tests/test_delivery_retention_recovery.py index 81f0ea7..f91f1d0 100644 --- a/tests/test_delivery_retention_recovery.py +++ b/tests/test_delivery_retention_recovery.py @@ -973,7 +973,7 @@ def test_v12_migration_uses_effective_recovery_lineage_without_repost_or_hold( api = ConnectorOutboxAPI(db_path, HOST_ID) assert api.poll({"name": FINAL_NAME, "limit": 100})["items"] == [] with sqlite3.connect(str(db_path)) as conn: - assert conn.execute("PRAGMA user_version").fetchone()[0] == store_sqlite.STORE_SCHEMA_VERSION == 14 + assert conn.execute("PRAGMA user_version").fetchone()[0] == store_sqlite.STORE_SCHEMA_VERSION == 17 anchor = conn.execute( """ SELECT delivery_kind, status diff --git a/tests/test_store.py b/tests/test_store.py index bee9f48..7f1691b 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -12,7 +12,7 @@ import stat import threading import time -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any @@ -4953,6 +4953,66 @@ def test_command_pending_turn_terminal_effect_is_atomic_with_acceptance( assert turns[0]["user_text"] == "Continue safely." +def test_command_pending_turn_effect_is_atomic_with_send_start( + tmp_path: Path, +) -> None: + db_path = tmp_path / "pending-turn-send-start-effect.db" + init_store(db_path) + reservation = _reserve_test_request(db_path, request_id="turn-send-start") + invalid_effect = command_pending_turn_terminal_effect( + host_id="host-a", + worker={"id": "worker-public", "name": "Worker"}, + request_id="turn-send-start", + instruction_text="", + ) + with pytest.raises( + store_sqlite.StoreSchemaError, + match="command_pending_turn_terminal_effect_failed", + ): + mark_command_send_started( + db_path, + host_id="host-a", + request_id="turn-send-start", + canonical_fingerprint="send_instruction:turn-send-start", + owner_token=reservation["owner_token"], + binding_fingerprint="private-binding", + send_started_effect=invalid_effect, + now="2026-01-01T00:00:01+00:00", + ) + assert get_command_request( + db_path, + "host-a", + "turn-send-start", + )["state"] == "reserved" + assert turns_payload_from_store(db_path, "host-a")["turns"] == [] + + valid_effect = command_pending_turn_terminal_effect( + host_id="host-a", + worker={"id": "worker-public", "name": "Worker"}, + request_id="turn-send-start", + instruction_text="Continue safely.", + ) + started = mark_command_send_started( + db_path, + host_id="host-a", + request_id="turn-send-start", + canonical_fingerprint="send_instruction:turn-send-start", + owner_token=reservation["owner_token"], + binding_fingerprint="private-binding", + send_started_effect=valid_effect, + now="2026-01-01T00:00:01+00:00", + ) + assert started["status"] == "send_started" + assert started["effect_result"]["origin_command_id"] == "turn-send-start" + turns = turns_payload_from_store( + db_path, + "host-a", + claim_hard_ttl_seconds=1_000_000_000, + )["turns"] + assert len(turns) == 1 + assert turns[0]["id"] == started["effect_result"]["id"] + + def test_backend_pending_choice_terminal_effect_is_atomic_with_acceptance( tmp_path: Path, ) -> None: @@ -6196,146 +6256,1057 @@ def test_store_status_command_request_metrics_are_aggregate_only( request_id="send-started-private-id", now=old_at, ) - assert mark_command_send_started( - db_path, - host_id="host-a", - request_id="send-started-private-id", - canonical_fingerprint="send_instruction:send-started-private-id", - owner_token=stale_send["owner_token"], - binding_fingerprint="private-binding", - now=old_started_at, - )["status"] == "send_started" + assert mark_command_send_started( + db_path, + host_id="host-a", + request_id="send-started-private-id", + canonical_fingerprint="send_instruction:send-started-private-id", + owner_token=stale_send["owner_token"], + binding_fingerprint="private-binding", + now=old_started_at, + )["status"] == "send_started" + + status = store_status( + db_path, + "host-a", + command_retry_horizon_seconds=604_800, + command_receipt_retention_seconds=691_200, + command_receipt_retention_count=1, + ) + metrics = status["command_requests"] + assert metrics["total"] == 4 + assert metrics["states"] == { + "reserved": 2, + "send_started": 1, + "accepted": 0, + "rejected": 0, + "uncertain": 1, + } + assert metrics["stale_active"] == 1 + assert metrics["eligible"] == 1 + assert metrics["storage_pressure"] is True + assert metrics["retry_horizon_seconds"] == 604_800 + assert metrics["retention_seconds"] == 691_200 + assert metrics["retention_count"] == 1 + assert status["maintenance"]["backlog"] is True + public_json = json.dumps(metrics, sort_keys=True) + assert "private-request-id" not in public_json + assert "active-private-id" not in public_json + assert "expired-private-id" not in public_json + assert "send-started-private-id" not in public_json + assert "secret-evidence" not in public_json + + +def test_distinct_source_turns_mint_distinct_public_turn_ids(tmp_path: Path) -> None: + db_path = tmp_path / "source-turns.db" + config = Config(host_id="turn-host", db_path=db_path) + snapshot = project_from_raw( + config, + workers=[{"id": "worker-1", "name": "claude", "status": "active", "space_id": "space-1"}], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + + for index, (prompt, reply) in enumerate( + [("first question", "first answer"), ("second question", "second answer")], + start=1, + ): + merge_turn_content( + db_path, + "turn-host", + "worker-1", + { + "user_text": prompt, + "assistant_final_text": reply, + "complete": True, + "has_open_turn": False, + "source_turn_id": f"uuid-{index}", + }, + observed_at=f"2026-01-01T00:0{index}:00+00:00", + ) + + payload = turns_payload_from_store(db_path, "turn-host", snapshot=snapshot) + content_turns = [t for t in payload["turns"] if t.get("assistant_final_text")] + assert len(content_turns) == 2 + assert len({t["id"] for t in content_turns}) == 2 + # Newest first per worker; the worker's base row must not carry stale text. + assert content_turns[0]["assistant_final_text"] == "second answer" + base_rows = [t for t in payload["turns"] if not t.get("source_turn_id")] + assert all(not t.get("assistant_final_text") and not t.get("user_text") for t in base_rows) + + # Same source turn observed again updates its row, keeping the id stable. + merge_turn_content( + db_path, + "turn-host", + "worker-1", + {"assistant_final_text": "second answer, revised", "complete": True, "source_turn_id": "uuid-2"}, + ) + payload2 = turns_payload_from_store(db_path, "turn-host", snapshot=snapshot) + revised = [t for t in payload2["turns"] if t.get("assistant_final_text") == "second answer, revised"] + assert len(revised) == 1 + assert revised[0]["id"] in {t["id"] for t in content_turns} + + # Snapshot rewrites must not prune per-source-turn rows. + save_snapshot(db_path, snapshot) + payload3 = turns_payload_from_store(db_path, "turn-host", snapshot=snapshot) + assert len([t for t in payload3["turns"] if t.get("source_turn_id")]) == 2 + + +def test_final_identity_is_stable_neutral_and_argument_bound() -> None: + identity = turn_final_delivery_identity( + "host-private-value", + "turn-public-a", + "twrev1.revisionA", + ) + + assert identity == turn_final_delivery_identity( + "host-private-value", + "turn-public-a", + "twrev1.revisionA", + ) + assert identity.startswith("twfinal1.") + assert "host-private-value" not in identity + assert "turn-public-a" not in identity + assert identity != turn_final_delivery_identity( + "host-private-value", + "turn-public-a", + "twrev1.revisionB", + ) + assert identity != turn_final_delivery_identity( + "other-host", + "turn-public-a", + "twrev1.revisionA", + ) + with pytest.raises(ValueError, match="invalid_host_id"): + turn_final_delivery_identity("", "turn-public-a", "twrev1.revisionA") + + +def test_twenty_offline_source_finals_are_retained_as_unique_ready_anchors( + tmp_path: Path, +) -> None: + db_path = tmp_path / "source-turn-retention.db" + config = Config(host_id="turn-host", db_path=db_path) + snapshot = project_from_raw( + config, + workers=[ + { + "id": "worker-1", + "name": "claude", + "status": "active", + "space_id": "space-1", + "meta": { + "stable_key": "wsk1_" + ("2" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + for index in range(20): + assert merge_turn_content( + db_path, + "turn-host", + "worker-1", + { + "assistant_final_text": f"answer {index}", + "complete": True, + "source_turn_id": f"uuid-{index}", + }, + observed_at=f"2026-01-01T00:{index:02d}:00+00:00", + ) == 1 + + payload = turns_payload_from_store(db_path, "turn-host", snapshot=snapshot) + source_rows = [turn for turn in payload["turns"] if turn.get("source_turn_id")] + with sqlite3.connect(str(db_path)) as conn: + anchors = conn.execute( + """ + SELECT delivery_key, payload_json + FROM connector_outbox + WHERE host_id = ? + AND delivery_kind = 'final_ready' + AND status = 'queued' + ORDER BY id + """, + ("turn-host",), + ).fetchall() + + assert len(source_rows) == 20 + assert len(anchors) == len({row[0] for row in anchors}) == 20 + assert source_rows[0]["assistant_final_text"] == "answer 19" + assert all( + row[0].startswith("turn-final:revision:twfinal1.") + for row in anchors + ) + encoded_anchors = "\n".join(row[1] for row in anchors) + assert "answer 0" not in encoded_anchors + assert "answer 19" not in encoded_anchors + assert "source_turn_id" not in encoded_anchors + + +def test_turn_claim_sweeper_expires_never_observed_claim(tmp_path: Path) -> None: + db_path = tmp_path / "turn-claim-expiry.db" + host_id = "turn-claim-expiry-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("c" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + claim = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="never-observed-request", + instruction_text="never observed", + observed_at="2026-07-19T00:00:00+00:00", + ) + assert claim is not None + + assert store_sqlite.sweep_turn_claims( + db_path, + host_id, + grace_seconds=1, + hard_ttl_seconds=60, + now="2026-07-19T00:02:00+00:00", + ) == 1 + + with sqlite3.connect(str(db_path)) as conn: + raw = json.loads( + conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, claim["id"]), + ).fetchone()[0] + ) + foreign_keys = conn.execute("PRAGMA foreign_key_check").fetchall() + assert raw["complete"] is True + assert raw["has_open_turn"] is False + assert raw["status"] == "closed" + assert raw["superseded_at"] == "2026-07-19T00:02:00+00:00" + assert raw["superseded_by_turn_id"] is None + assert claim["id"] not in { + turn["id"] + for turn in turns_payload_from_store(db_path, host_id)["turns"] + } + assert foreign_keys == [] + + +def test_turn_claim_sweeper_resolves_only_one_identical_claim_per_done_turn( + tmp_path: Path, +) -> None: + db_path = tmp_path / "turn-claim-identical.db" + host_id = "turn-claim-identical-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[{"id": "worker-1", "name": "Worker", "status": "active"}], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "identical-observed-source", + "user_text": "same prompt", + "assistant_final_text": "one answer", + "complete": True, + "has_open_turn": False, + }, + observed_at="2099-07-19T00:00:00+00:00", + ) == 1 + observed = next( + turn + for turn in turns_payload_from_store(db_path, host_id)["turns"] + if turn.get("source_turn_id") + ) + first = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="identical-first", + instruction_text="same prompt", + observed_at="2099-07-19T00:00:01+00:00", + ) + second = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="identical-second", + instruction_text="same prompt", + observed_at="2099-07-19T00:00:02+00:00", + ) + assert first is not None and second is not None + + assert store_sqlite.sweep_turn_claims( + db_path, + host_id, + grace_seconds=1, + now="2099-07-19T00:01:00+00:00", + ) == 1 + + with sqlite3.connect(str(db_path)) as conn: + stored = { + turn_id: json.loads(payload_json) + for turn_id, payload_json in conn.execute( + """ + SELECT turn_id, payload_json + FROM turns + WHERE host_id = ? AND turn_id IN (?, ?) + """, + (host_id, first["id"], second["id"]), + ).fetchall() + } + assert stored[first["id"]]["superseded_by_turn_id"] == observed["id"] + assert stored[second["id"]].get("superseded_at") is None + assert stored[second["id"]]["has_open_turn"] is True + + +@pytest.mark.parametrize(("claim_count", "done_count"), [(1, 2), (2, 1)]) +def test_turn_claim_sweeper_fifo_requires_unambiguous_claim_and_done_sets( + tmp_path: Path, + claim_count: int, + done_count: int, +) -> None: + db_path = tmp_path / f"turn-fifo-{claim_count}-{done_count}.db" + host_id = "turn-fifo-unambiguous-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("7" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + claims = [ + store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id=f"fifo-claim-{index}", + instruction_text=f"claim prompt {index}", + observed_at=f"2026-07-19T00:00:0{index}+00:00", + ) + for index in range(claim_count) + ] + assert all(claim is not None for claim in claims) + for index in range(done_count): + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": f"fifo-done-{index}", + "user_text": f"unrelated done {index}", + "assistant_final_text": f"answer {index}", + "complete": True, + "has_open_turn": False, + }, + observed_at=f"2026-07-19T00:01:0{index}+00:00", + ) == 1 + + assert store_sqlite.sweep_turn_claims( + db_path, + host_id, + grace_seconds=1, + hard_ttl_seconds=1_000_000, + now="2026-07-19T00:02:00+00:00", + ) == 0 + with sqlite3.connect(str(db_path)) as conn: + stored = [ + json.loads( + conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, claim["id"]), + ).fetchone()[0] + ) + for claim in claims + if claim is not None + ] + assert all(payload.get("superseded_at") is None for payload in stored) + + +def test_turn_list_lazy_sweep_uses_the_callers_now(tmp_path: Path) -> None: + db_path = tmp_path / "turn-list-caller-now.db" + host_id = "turn-list-caller-now-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[{"id": "worker-1", "name": "Worker", "status": "active"}], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + claim = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="caller-now-claim", + instruction_text="caller clock", + observed_at="2000-01-01T00:00:00+00:00", + ) + assert claim is not None + + before_ttl = turns_payload_from_store( + db_path, + host_id, + now=datetime.fromisoformat("2000-01-01T00:00:30+00:00").timestamp(), + claim_hard_ttl_seconds=60, + ) + assert claim["id"] in {turn["id"] for turn in before_ttl["turns"]} + after_ttl = turns_payload_from_store( + db_path, + host_id, + now=datetime.fromisoformat("2000-01-01T00:02:00+00:00").timestamp(), + claim_hard_ttl_seconds=60, + ) + assert claim["id"] not in {turn["id"] for turn in after_ttl["turns"]} + + +def test_late_real_completion_never_adopts_tombstoned_claim(tmp_path: Path) -> None: + db_path = tmp_path / "turn-late-real-completion.db" + host_id = "turn-late-real-completion-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("8" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + claim = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="late-real-claim", + instruction_text="late real prompt", + observed_at="2026-07-19T00:00:00+00:00", + ) + assert claim is not None + assert store_sqlite.sweep_turn_claims( + db_path, + host_id, + grace_seconds=1, + hard_ttl_seconds=60, + now="2026-07-19T00:02:00+00:00", + ) == 1 + + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "late-real-source", + "user_text": "late real prompt", + "assistant_final_text": "late real answer", + "complete": True, + "has_open_turn": False, + }, + observed_at="2026-07-19T00:02:01+00:00", + ) == 1 + listed = turns_payload_from_store( + db_path, + host_id, + now=datetime.fromisoformat("2026-07-19T00:02:02+00:00").timestamp(), + )["turns"] + real = next(turn for turn in listed if turn.get("assistant_final_text") == "late real answer") + assert real["id"] != claim["id"] + with sqlite3.connect(str(db_path)) as conn: + claim_payload = json.loads( + conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, claim["id"]), + ).fetchone()[0] + ) + assert claim_payload.get("source_turn_id") is None + assert claim_payload["superseded_at"] == "2026-07-19T00:02:00+00:00" + + +def test_submission_adoption_rejects_old_completed_matching_text(tmp_path: Path) -> None: + db_path = tmp_path / "turn-old-observation-adoption.db" + host_id = "turn-old-observation-adoption-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("9" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "old-yes-source", + "user_text": "yes", + "assistant_final_text": "old answer", + "complete": True, + "has_open_turn": False, + }, + observed_at="2026-07-19T00:00:00+00:00", + ) == 1 + old = next( + turn + for turn in turns_payload_from_store( + db_path, + host_id, + now=datetime.fromisoformat("2026-07-19T00:00:01+00:00").timestamp(), + )["turns"] + if turn.get("assistant_final_text") == "old answer" + ) + + claim = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="new-yes-request", + instruction_text="yes", + observed_at="2026-07-19T00:05:00+00:00", + ) + assert claim is not None + assert claim["id"] != old["id"] + assert claim["complete"] is False + assert claim.get("assistant_final_text") in {None, ""} + assert old.get("origin_command_id") is None + + +def test_text_drift_tombstone_preserves_cursor_and_since_continuity( + tmp_path: Path, +) -> None: + db_path = tmp_path / "turn-claim-cursor.db" + host_id = "turn-claim-cursor-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("e" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + baseline = turns_payload_from_store(db_path, host_id, schema_version=2) + claim = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="drift-request", + instruction_text="final normalized prompt", + observed_at="2099-07-19T00:00:00+00:00", + ) + assert claim is not None + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "drift-source", + "user_text": "draft prompt", + "complete": False, + "has_open_turn": True, + }, + observed_at="2099-07-19T00:00:01+00:00", + ) == 1 + first_page = turns_payload_from_store( + db_path, + host_id, + schema_version=2, + limit=1, + ) + source_id = first_page["turns"][0]["id"] + assert source_id != claim["id"] + assert first_page["has_more"] is True + assert first_page["next_cursor"] is not None + + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "drift-source", + "user_text": "final normalized prompt", + "assistant_final_text": "done", + "complete": True, + "has_open_turn": False, + }, + observed_at="2099-07-19T00:00:02+00:00", + ) == 1 + + continuation = turns_payload_from_store( + db_path, + host_id, + schema_version=2, + limit=1, + cursor=first_page["next_cursor"], + ) + assert continuation.get("status") != "cursor_expired" + assert claim["id"] not in {turn["id"] for turn in continuation["turns"]} + since_poll = turns_payload_from_store( + db_path, + host_id, + schema_version=2, + since=baseline["since"], + ) + assert {turn["id"] for turn in since_poll["turns"]} == {source_id} + with sqlite3.connect(str(db_path)) as conn: + physical = conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, claim["id"]), + ).fetchone() + assert physical is not None + claim_payload = json.loads(physical[0]) + assert claim_payload["superseded_by_turn_id"] == source_id + assert claim_payload["superseded_at"] == "2099-07-19T00:00:02+00:00" + + +def test_v14_migration_tombstones_production_shape_duplicate_claim( + tmp_path: Path, +) -> None: + db_path = tmp_path / "turn-claim-v14.db" + host_id = "turn-claim-v14-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("d" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "migration-observed-source", + "user_text": "same migration prompt", + "assistant_final_text": "migration answer", + "complete": True, + "has_open_turn": False, + }, + observed_at="2026-07-19T00:00:00+00:00", + ) == 1 + observed = next( + turn + for turn in turns_payload_from_store(db_path, host_id, schema_version=2)["turns"] + if turn.get("assistant_final_text") == "migration answer" + ) + claim = store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id="migration-command-request", + instruction_text="temporary different prompt", + observed_at="2026-07-19T00:00:01+00:00", + ) + assert claim is not None + assert claim["id"] != observed["id"] + with sqlite3.connect(str(db_path)) as conn: + conn.execute( + """ + UPDATE turn_content_revisions + SET user_text = 'same migration prompt' + WHERE host_id = ? AND turn_id = ? AND is_current = 1 + """, + (host_id, claim["id"]), + ) + generation_before = conn.execute( + "SELECT traversal_generation FROM turn_list_hosts WHERE host_id = ?", + (host_id,), + ).fetchone()[0] + conn.execute("PRAGMA user_version = 14") + + init_store(db_path) + + with sqlite3.connect(str(db_path)) as conn: + claim_payload = json.loads( + conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, claim["id"]), + ).fetchone()[0] + ) + generation_after = conn.execute( + "SELECT traversal_generation FROM turn_list_hosts WHERE host_id = ?", + (host_id,), + ).fetchone()[0] + physical_rows = conn.execute( + "SELECT COUNT(*) FROM turns WHERE host_id = ? AND turn_id IN (?, ?)", + (host_id, observed["id"], claim["id"]), + ).fetchone()[0] + foreign_keys = conn.execute("PRAGMA foreign_key_check").fetchall() + public = turns_payload_from_store(db_path, host_id, schema_version=2)["turns"] + assert claim_payload["superseded_by_turn_id"] == observed["id"] + assert claim_payload["complete"] is True + assert generation_after == generation_before + 1 + assert physical_rows == 2 + assert public[0]["id"] == observed["id"] + assert claim["id"] not in {turn["id"] for turn in public} + assert foreign_keys == [] + + +def test_v14_migration_skips_ambiguous_claim_pair(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "turn-v14-ambiguous-pair.db" + host_id = "turn-v14-ambiguous-pair-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("a" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "ambiguous-migration-source", + "user_text": "ambiguous migration prompt", + "assistant_final_text": "one observed answer", + "complete": True, + "has_open_turn": False, + }, + observed_at="2026-07-19T00:00:00+00:00", + ) == 1 + claims = [ + store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id=f"ambiguous-migration-{index}", + instruction_text=f"temporary migration prompt {index}", + observed_at=f"2026-07-19T00:02:0{index}+00:00", + ) + for index in range(2) + ] + assert all(claim is not None for claim in claims) + with sqlite3.connect(str(db_path)) as conn: + for claim in claims: + assert claim is not None + conn.execute( + """ + UPDATE turn_content_revisions + SET user_text = 'ambiguous migration prompt' + WHERE host_id = ? AND turn_id = ? AND is_current = 1 + """, + (host_id, claim["id"]), + ) + conn.execute("PRAGMA user_version = 14") + monkeypatch.setenv("TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS", "999999999") + + init_store(db_path) + + with sqlite3.connect(str(db_path)) as conn: + payloads = [ + json.loads( + conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, claim["id"]), + ).fetchone()[0] + ) + for claim in claims + if claim is not None + ] + assert len(payloads) == 2 + assert all(payload.get("superseded_at") is None for payload in payloads) + + +def test_v14_migration_does_not_reuse_already_claimed_done_row( + tmp_path: Path, + monkeypatch, +) -> None: + db_path = tmp_path / "turn-v14-used-done.db" + host_id = "turn-v14-used-done-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[ + { + "id": "worker-1", + "name": "Worker", + "status": "active", + "meta": { + "stable_key": "wsk1_" + ("b" * 64), + "stable_key_version": 1, + }, + } + ], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + assert merge_turn_content( + db_path, + host_id, + "worker-1", + { + "source_turn_id": "used-done-source", + "user_text": "used done prompt", + "assistant_final_text": "used done answer", + "complete": True, + "has_open_turn": False, + }, + observed_at="2026-07-19T00:00:00+00:00", + ) == 1 + done = next( + turn + for turn in turns_payload_from_store( + db_path, + host_id, + now=datetime.fromisoformat("2026-07-19T00:00:01+00:00").timestamp(), + )["turns"] + if turn.get("assistant_final_text") == "used done answer" + ) + claims = [ + store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id=f"used-done-claim-{index}", + instruction_text=f"temporary used prompt {index}", + observed_at=f"2026-07-19T00:02:0{index}+00:00", + ) + for index in range(2) + ] + assert all(claim is not None for claim in claims) + with sqlite3.connect(str(db_path)) as conn: + for claim in claims: + assert claim is not None + conn.execute( + """ + UPDATE turn_content_revisions + SET user_text = 'used done prompt' + WHERE host_id = ? AND turn_id = ? AND is_current = 1 + """, + (host_id, claim["id"]), + ) + assert store_sqlite._tombstone_turn_conn( + conn, + host_id, + claims[0]["id"], + superseded_by_turn_id=done["id"], + superseded_at="2026-07-19T00:03:00+00:00", + ) + conn.execute("PRAGMA user_version = 14") + monkeypatch.setenv("TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS", "999999999") - status = store_status( + init_store(db_path) + + with sqlite3.connect(str(db_path)) as conn: + second_payload = json.loads( + conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, claims[1]["id"]), + ).fetchone()[0] + ) + assert second_payload.get("superseded_at") is None + assert second_payload["has_open_turn"] is True + + +def test_v14_migration_uses_configured_claim_hard_ttl( + tmp_path: Path, + monkeypatch, +) -> None: + db_path = tmp_path / "turn-v14-configured-ttl.db" + host_id = "turn-v14-configured-ttl-host" + snapshot = project_from_raw( + Config(host_id=host_id, db_path=db_path), + workers=[{"id": "worker-1", "name": "Worker", "status": "active"}], + ) + init_store(db_path) + save_snapshot(db_path, snapshot) + claim = store_sqlite.upsert_command_pending_turn( db_path, - "host-a", - command_retry_horizon_seconds=604_800, - command_receipt_retention_seconds=691_200, - command_receipt_retention_count=1, + host_id, + snapshot.workers[0], + request_id="configured-ttl-claim", + instruction_text="expire from configured ttl", + observed_at=(datetime.now(timezone.utc) - timedelta(seconds=120)).isoformat(), ) - metrics = status["command_requests"] - assert metrics["total"] == 4 - assert metrics["states"] == { - "reserved": 2, - "send_started": 1, - "accepted": 0, - "rejected": 0, - "uncertain": 1, - } - assert metrics["stale_active"] == 1 - assert metrics["eligible"] == 1 - assert metrics["storage_pressure"] is True - assert metrics["retry_horizon_seconds"] == 604_800 - assert metrics["retention_seconds"] == 691_200 - assert metrics["retention_count"] == 1 - assert status["maintenance"]["backlog"] is True - public_json = json.dumps(metrics, sort_keys=True) - assert "private-request-id" not in public_json - assert "active-private-id" not in public_json - assert "expired-private-id" not in public_json - assert "send-started-private-id" not in public_json - assert "secret-evidence" not in public_json + assert claim is not None + with sqlite3.connect(str(db_path)) as conn: + conn.execute("PRAGMA user_version = 14") + monkeypatch.setenv("TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS", "60") + init_store(db_path) -def test_distinct_source_turns_mint_distinct_public_turn_ids(tmp_path: Path) -> None: - db_path = tmp_path / "source-turns.db" - config = Config(host_id="turn-host", db_path=db_path) + with sqlite3.connect(str(db_path)) as conn: + payload = json.loads( + conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, claim["id"]), + ).fetchone()[0] + ) + assert payload.get("superseded_at") is not None + assert payload.get("superseded_by_turn_id") is None + + +def test_turn_cursor_continues_across_interior_tombstone(tmp_path: Path) -> None: + db_path = tmp_path / "turn-interior-tombstone-cursor.db" + host_id = "turn-interior-tombstone-cursor-host" snapshot = project_from_raw( - config, - workers=[{"id": "worker-1", "name": "claude", "status": "active", "space_id": "space-1"}], + Config(host_id=host_id, db_path=db_path), + workers=[{"id": "worker-1", "name": "Worker", "status": "active"}], ) init_store(db_path) save_snapshot(db_path, snapshot) - - for index, (prompt, reply) in enumerate( - [("first question", "first answer"), ("second question", "second answer")], - start=1, - ): - merge_turn_content( + for index in range(3): + assert merge_turn_content( db_path, - "turn-host", + host_id, "worker-1", { - "user_text": prompt, - "assistant_final_text": reply, + "source_turn_id": f"cursor-source-{index}", + "user_text": f"cursor prompt {index}", + "assistant_final_text": f"cursor answer {index}", "complete": True, "has_open_turn": False, - "source_turn_id": f"uuid-{index}", }, - observed_at=f"2026-01-01T00:0{index}:00+00:00", + observed_at=f"2026-07-19T00:0{index}:00+00:00", + ) == 1 + with sqlite3.connect(str(db_path)) as conn: + visible = conn.execute( + """ + SELECT turn_id + FROM turns + WHERE host_id = ? + AND COALESCE(json_extract(payload_json, '$.source_turn_id'), '') != '' + ORDER BY worker_id ASC, list_sequence DESC, turn_id ASC + """, + (host_id,), + ).fetchall() + ordered_ids = [str(row[0]) for row in visible] + assert len(ordered_ids) == 3 + first = turns_payload_from_store( + db_path, + host_id, + schema_version=2, + limit=1, + now=datetime.fromisoformat("2026-07-19T00:03:00+00:00").timestamp(), + ) + assert first["turns"][0]["id"] == ordered_ids[0] + assert first["next_cursor"] is not None + with sqlite3.connect(str(db_path)) as conn: + assert store_sqlite._tombstone_turn_conn( + conn, + host_id, + ordered_ids[1], + superseded_by_turn_id=ordered_ids[0], + superseded_at="2026-07-19T00:03:01+00:00", ) - payload = turns_payload_from_store(db_path, "turn-host", snapshot=snapshot) - content_turns = [t for t in payload["turns"] if t.get("assistant_final_text")] - assert len(content_turns) == 2 - assert len({t["id"] for t in content_turns}) == 2 - # Newest first per worker; the worker's base row must not carry stale text. - assert content_turns[0]["assistant_final_text"] == "second answer" - base_rows = [t for t in payload["turns"] if not t.get("source_turn_id")] - assert all(not t.get("assistant_final_text") and not t.get("user_text") for t in base_rows) - - # Same source turn observed again updates its row, keeping the id stable. - merge_turn_content( + continuation = turns_payload_from_store( db_path, - "turn-host", - "worker-1", - {"assistant_final_text": "second answer, revised", "complete": True, "source_turn_id": "uuid-2"}, + host_id, + schema_version=2, + limit=1, + cursor=first["next_cursor"], + now=datetime.fromisoformat("2026-07-19T00:03:02+00:00").timestamp(), ) - payload2 = turns_payload_from_store(db_path, "turn-host", snapshot=snapshot) - revised = [t for t in payload2["turns"] if t.get("assistant_final_text") == "second answer, revised"] - assert len(revised) == 1 - assert revised[0]["id"] in {t["id"] for t in content_turns} + assert continuation.get("status") != "cursor_expired" + assert [turn["id"] for turn in continuation["turns"]] == [ordered_ids[2]] - # Snapshot rewrites must not prune per-source-turn rows. - save_snapshot(db_path, snapshot) - payload3 = turns_payload_from_store(db_path, "turn-host", snapshot=snapshot) - assert len([t for t in payload3["turns"] if t.get("source_turn_id")]) == 2 +def test_turn_list_lazy_sweep_is_rate_limited( + tmp_path: Path, + monkeypatch, +) -> None: + db_path = tmp_path / "turn-list-sweep-rate-limit.db" + host_id = "turn-list-sweep-rate-limit-host" + init_store(db_path) + calls: list[tuple[float | None, str | None]] = [] -def test_final_identity_is_stable_neutral_and_argument_bound() -> None: - identity = turn_final_delivery_identity( - "host-private-value", - "turn-public-a", - "twrev1.revisionA", - ) + def capture_sweep(*_args, grace_seconds=None, now=None, **_kwargs): + calls.append((grace_seconds, now)) + return 0 - assert identity == turn_final_delivery_identity( - "host-private-value", - "turn-public-a", - "twrev1.revisionA", - ) - assert identity.startswith("twfinal1.") - assert "host-private-value" not in identity - assert "turn-public-a" not in identity - assert identity != turn_final_delivery_identity( - "host-private-value", - "turn-public-a", - "twrev1.revisionB", - ) - assert identity != turn_final_delivery_identity( - "other-host", - "turn-public-a", - "twrev1.revisionA", - ) - with pytest.raises(ValueError, match="invalid_host_id"): - turn_final_delivery_identity("", "turn-public-a", "twrev1.revisionA") + monkeypatch.setattr(store_sqlite, "sweep_turn_claims", capture_sweep) + for current in (1_000.0, 1_001.0, 1_002.0): + turns_payload_from_store( + db_path, + host_id, + now=current, + turn_refresh_interval_seconds=2.0, + ) + assert calls == [ + ( + 60.0, + datetime.fromtimestamp(1_000.0, tz=timezone.utc).isoformat(), + ), + ( + 60.0, + datetime.fromtimestamp(1_002.0, tz=timezone.utc).isoformat(), + ), + ] -def test_twenty_offline_source_finals_are_retained_as_unique_ready_anchors( +def test_ingestion_ambiguity_fallthrough_emits_structured_diagnostic( tmp_path: Path, + caplog, ) -> None: - db_path = tmp_path / "source-turn-retention.db" - config = Config(host_id="turn-host", db_path=db_path) + db_path = tmp_path / "turn-ingestion-ambiguity-diagnostic.db" + host_id = "turn-ingestion-ambiguity-diagnostic-host" snapshot = project_from_raw( - config, + Config(host_id=host_id, db_path=db_path), workers=[ { "id": "worker-1", - "name": "claude", + "name": "Worker", "status": "active", - "space_id": "space-1", "meta": { - "stable_key": "wsk1_" + ("2" * 64), + "stable_key": "wsk1_" + ("c" * 64), "stable_key_version": 1, }, } @@ -6343,45 +7314,44 @@ def test_twenty_offline_source_finals_are_retained_as_unique_ready_anchors( ) init_store(db_path) save_snapshot(db_path, snapshot) - for index in range(20): + for index in range(2): + assert store_sqlite.upsert_command_pending_turn( + db_path, + host_id, + snapshot.workers[0], + request_id=f"diagnostic-claim-{index}", + instruction_text="same ambiguous prompt", + observed_at=f"2026-07-19T00:00:0{index}+00:00", + ) is not None + + with caplog.at_level("WARNING", logger="tendwire.store.sqlite"): assert merge_turn_content( db_path, - "turn-host", + host_id, "worker-1", { - "assistant_final_text": f"answer {index}", + "source_turn_id": "diagnostic-source", + "user_text": "same ambiguous prompt", + "assistant_final_text": "independent answer", "complete": True, - "source_turn_id": f"uuid-{index}", + "has_open_turn": False, }, - observed_at=f"2026-01-01T00:{index:02d}:00+00:00", + observed_at="2026-07-19T00:01:00+00:00", ) == 1 - payload = turns_payload_from_store(db_path, "turn-host", snapshot=snapshot) - source_rows = [turn for turn in payload["turns"] if turn.get("source_turn_id")] - with sqlite3.connect(str(db_path)) as conn: - anchors = conn.execute( - """ - SELECT delivery_key, payload_json - FROM connector_outbox - WHERE host_id = ? - AND delivery_kind = 'final_ready' - AND status = 'queued' - ORDER BY id - """, - ("turn-host",), - ).fetchall() - - assert len(source_rows) == 20 - assert len(anchors) == len({row[0] for row in anchors}) == 20 - assert source_rows[0]["assistant_final_text"] == "answer 19" - assert all( - row[0].startswith("turn-final:revision:twfinal1.") - for row in anchors - ) - encoded_anchors = "\n".join(row[1] for row in anchors) - assert "answer 0" not in encoded_anchors - assert "answer 19" not in encoded_anchors - assert "source_turn_id" not in encoded_anchors + diagnostics = [ + getattr(record, "tendwire_diagnostic", None) + for record in caplog.records + if record.getMessage() == "turn_ingestion_ambiguity_fallthrough" + ] + assert diagnostics == [ + { + "code": "turn_ingestion_ambiguity_fallthrough", + "host_id": host_id, + "worker_id": "worker-1", + "candidate_count": 2, + } + ] def _reset_store_to_v5_with_legacy_turn( @@ -6785,7 +7755,7 @@ def test_v6_to_v7_repairs_mixed_turns_with_absent_content_descriptors( ) for row in first_rows ) - assert len(first_v2["turns"]) >= 3 + assert len(first_v2["turns"]) >= 2 for turn in first_v2["turns"]: assert turn["content"]["schema_version"] == 1 if turn["id"] not in missing_turn_ids: @@ -10221,7 +11191,7 @@ def test_v13_migration_repairs_nonpositive_turn_sequences_and_blocks_recurrence( init_store(db_path) with sqlite3.connect(str(db_path)) as conn: - assert _user_version(conn) == store_sqlite.STORE_SCHEMA_VERSION == 14 + assert _user_version(conn) == store_sqlite.STORE_SCHEMA_VERSION == 17 assert conn.execute( """ SELECT turn_id, list_sequence @@ -13289,7 +14259,12 @@ def test_same_source_completion_is_observation_monotonic_and_never_reopens( }, observed_at="2030-01-01T00:00:01+00:00", ) == 1 - payload = turns_payload_from_store(db_path, host_id, schema_version=2) + payload = turns_payload_from_store( + db_path, + host_id, + schema_version=2, + claim_hard_ttl_seconds=1_000_000_000, + ) source_turn = next(turn for turn in payload["turns"] if turn.get("source_turn_id")) assert source_turn["assistant_final_text"] == "revised final" assert source_turn["assistant_stream_text"] is None @@ -14199,7 +15174,7 @@ def test_stable_owner_snapshot_ambiguity_rolls_back_without_allocating_sequence( assert foreign_keys == [] -def test_stable_owner_source_turn_inherits_only_exact_matching_command_origin( +def test_stable_owner_exact_content_adoption_does_not_misattribute_unmatched_turn( tmp_path: Path, ) -> None: db_path = tmp_path / "stable-owner-command-provenance.db" @@ -14244,7 +15219,12 @@ def test_stable_owner_source_turn_inherits_only_exact_matching_command_origin( }, observed_at="2026-07-13T02:00:02+00:00", ) == 1 - payload = turns_payload_from_store(db_path, host_id, schema_version=2) + payload = turns_payload_from_store( + db_path, + host_id, + schema_version=2, + now=datetime.fromisoformat("2026-07-13T02:00:02+00:00").timestamp(), + ) exact_source = next( turn for turn in payload["turns"] if turn.get("assistant_final_text") == "exact final" @@ -14272,8 +15252,9 @@ def test_stable_owner_source_turn_inherits_only_exact_matching_command_origin( source_turn_id=raw_matching_source, meta=worker.meta, ) - assert exact_source["id"] == source_with_origin.id == source_without_origin.id - assert exact_source["id"] != command["id"] + assert exact_source["id"] == command["id"] + assert exact_source["id"] != source_with_origin.id + assert source_with_origin.id == source_without_origin.id assert exact_source["origin_command_id"] == "request-exact" assert exact_source["source_turn_id"] == source_with_origin.source_turn_id assert raw_matching_source not in json.dumps(payload, sort_keys=True) @@ -14301,18 +15282,29 @@ def test_stable_owner_source_turn_inherits_only_exact_matching_command_origin( }, observed_at="2026-07-13T02:00:04+00:00", ) == 1 - refreshed = turns_payload_from_store(db_path, host_id, schema_version=2) + refreshed = turns_payload_from_store( + db_path, + host_id, + schema_version=2, + claim_hard_ttl_seconds=1_000_000_000, + now=datetime.fromisoformat("2026-07-13T02:00:05+00:00").timestamp(), + ) unmatched_source = next( turn for turn in refreshed["turns"] if turn.get("assistant_final_text") == "independent final" ) assert unmatched_source.get("origin_command_id") is None assert unmatched_source["id"] != stale_command["id"] - assert any( - turn.get("id") == stale_command["id"] - and turn.get("origin_command_id") == "request-stale" - for turn in refreshed["turns"] - ) + assert stale_command["id"] in {turn["id"] for turn in refreshed["turns"]} + with sqlite3.connect(str(db_path)) as conn: + stale_payload = json.loads( + conn.execute( + "SELECT payload_json FROM turns WHERE host_id = ? AND turn_id = ?", + (host_id, stale_command["id"]), + ).fetchone()[0] + ) + assert stale_payload.get("superseded_at") is None + assert stale_payload.get("superseded_by_turn_id") is None encoded = json.dumps(refreshed, sort_keys=True) assert raw_matching_source not in encoded assert raw_unmatched_source not in encoded diff --git a/tests/test_turn_ingestion.py b/tests/test_turn_ingestion.py index 4cc85f4..dae09a2 100644 --- a/tests/test_turn_ingestion.py +++ b/tests/test_turn_ingestion.py @@ -394,6 +394,36 @@ def reader(_config, _binding, *, adapter_timeout_seconds): scheduler.stop() +def test_scheduler_scan_sweeps_turn_claims_with_configured_ttls( + tmp_path: Path, + monkeypatch, +) -> None: + config, _snapshot, _bindings = _scheduler_store(tmp_path, 1) + calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = [] + + monkeypatch.setattr( + herdr_turns, + "prune_backend_pending", + lambda *_args, **_kwargs: 0, + ) + + def observed_sweep(*args: Any, **kwargs: Any) -> int: + calls.append((args, kwargs)) + return 0 + + monkeypatch.setattr(herdr_turns, "sweep_turn_claims", observed_sweep) + scheduler = TurnIngestionScheduler(config, refresh_interval_seconds=7.5) + + scheduler._scan_bindings() + + assert len(calls) == 1 + args, kwargs = calls[0] + assert args == (config.db_path, config.host_id) + assert kwargs["grace_seconds"] == 75.0 + assert kwargs["hard_ttl_seconds"] == config.turn_claim_hard_ttl_seconds + assert isinstance(kwargs["now"], str) + + def test_transient_initial_binding_scan_retries_once_and_dispatches( tmp_path: Path, monkeypatch,