fix: race-proof command-turn lifecycle + non-wedging final outbox (production incident root causes)

.env.example

@@ -49,6 +49,8 @@ TENDWIRE_ACKNOWLEDGED_FINAL_RETENTION_COUNT=4096
 # malformed-owner, known-incomplete, and automation holds are inspection-only.
 TENDWIRE_MAX_OUTBOX_ATTEMPTS=10
 TENDWIRE_CONNECTOR_CLAIM_TTL_SECONDS=60
+TENDWIRE_CONNECTOR_MAX_CLAIM_TTL_SECONDS=300
+TENDWIRE_CONNECTOR_ACK_TTL_SECONDS=300
 
 # Automatic snapshot cleanup defaults to 14 days and 4096 rows per host.
 # Each host's latest row is always retained. At a 5-minute snapshot interval,
@@ -81,6 +83,7 @@ TENDWIRE_HERDR_TIMEOUT_SECONDS=1.0
 # TENDWIRE_MAX_WORKERS. The internal refresh queue is fixed at 64.
 TENDWIRE_TURN_REFRESH_INTERVAL_SECONDS=2.0
 TENDWIRE_TURN_REFRESH_WORKERS=4
+TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS=86400
 
 # Backend prompts remain visible as stale after a transient adapter failure for
 # one fixed, non-sliding grace window. Repeated failures do not extend it.

INSTALL.md

@@ -45,6 +45,7 @@ Environment=TENDWIRE_HERDR_BACKEND=socket
 Environment=TENDWIRE_DB_PATH=%h/.local/share/tendwire/tendwire.db
 Environment=TENDWIRE_TURN_REFRESH_INTERVAL_SECONDS=2.0
 Environment=TENDWIRE_TURN_REFRESH_WORKERS=4
+Environment=TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS=86400
 ExecStart=%h/.local/bin/tendwire daemon --db-path %h/.local/share/tendwire/tendwire.db
 Restart=always
 RestartSec=5s
@@ -71,6 +72,8 @@ reconciles. `TENDWIRE_TURN_REFRESH_WORKERS` defaults to `4`, must be from 1
 through 32, and cannot exceed `TENDWIRE_MAX_WORKERS`. Every adapter uses
 `TENDWIRE_HERDR_TIMEOUT_SECONDS`; the queue is fixed at 64. One private target
 is serialized with itself while distinct targets can use the worker pool.
+Unobserved command-turn claims become terminal tombstones after
+`TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS` (default `86400`).
 
 OMP JSONL cache/IPC state is coordinate-only: parse/EOF and replay offsets,
 observed file identity/size/timestamps, an open-turn flag, and validated project
@@ -628,10 +631,11 @@ following together:
 - `turn.list` schema v2 with descriptor schema v1, 1,000-character previews and
   insertion-stable paging; schema-v1 `turn.content.get` with a 49,152-byte
   UTF-8 page ceiling;
-- range-only schema-v1 `connector.prepare` begin/part/commit/recover, root-wide
-  leases, independent part ACKs, and immutable schema-v2 source-less route
-  lineage. Cleanup retains bounded delivered tombstones so repeated snapshots
-  cannot recreate/repost removed acknowledged roots; and
+- range-only schema-v1 `connector.prepare` begin/part/commit/recover, per-worker
+  ordered roots and parts, bounded renewable/releasable leases, deadline-based
+  awaiting-ACK recovery, independent part ACKs, and immutable schema-v2
+  source-less route lineage. Cleanup retains bounded delivered tombstones so
+  repeated snapshots cannot recreate/repost removed acknowledged roots; and
 - fair dead-letter inspection, exact root/failed-plan retry including
   source-less recovery, retained ACK prefix, cumulative attempts, request-ID
   idempotency, immutable recovery audit, and no provider-perfect exactly-once

README.md

@@ -534,6 +534,8 @@ variables:
 | `max_workers` | `TENDWIRE_MAX_WORKERS` | `512` | integer >= 1 |
 | `max_outbox_attempts` | `TENDWIRE_MAX_OUTBOX_ATTEMPTS` | `10` | integer >= 1 |
 | `connector_claim_ttl_seconds` | `TENDWIRE_CONNECTOR_CLAIM_TTL_SECONDS` | `60` | integer >= 1 |
+| `connector_max_claim_ttl_seconds` | `TENDWIRE_CONNECTOR_MAX_CLAIM_TTL_SECONDS` | `300` | integer >= 1; caps `turn-final` poll and renew requests |
+| `connector_ack_ttl_seconds` | `TENDWIRE_CONNECTOR_ACK_TTL_SECONDS` | `300` | integer >= 1; grace period for committed plans awaiting ACK completion |
 | `command_retry_horizon_seconds` | `TENDWIRE_COMMAND_RETRY_HORIZON_SECONDS` | `604800` | positive integer no greater than 604800 |
 | `command_receipt_retention_seconds` | `TENDWIRE_COMMAND_RECEIPT_RETENTION_SECONDS` | `2592000` | integer at least 691200 and strictly greater than the retry horizon |
 | `command_receipt_retention_count` | `TENDWIRE_COMMAND_RECEIPT_RETENTION_COUNT` | `4096` | positive integer; newest bounded inactive receipts per host |
@@ -545,6 +547,7 @@ variables:
 | `store_maintenance_cadence_seconds` | `TENDWIRE_STORE_MAINTENANCE_CADENCE_SECONDS` | `3600` | positive integer |
 | `turn_refresh_interval_seconds` | `TENDWIRE_TURN_REFRESH_INTERVAL_SECONDS` | `2.0` | finite positive float |
 | `turn_refresh_workers` | `TENDWIRE_TURN_REFRESH_WORKERS` | `4` | integer from 1 through 32 and no greater than `max_workers` |
+| `turn_claim_hard_ttl_seconds` | `TENDWIRE_TURN_CLAIM_HARD_TTL_SECONDS` | `86400` | positive integer; unobserved command claims become terminal after this interval |
 
 The socket/event backend uses `event_debounce_seconds` for event batching and
 `reconcile_interval_seconds` for bounded periodic full reconciles. Set
@@ -1134,15 +1137,17 @@ See `INSTALL.md` for the required maintenance and rollback order.
 
 Tendwire exposes a Tendwire-only connector delivery boundary above the SQLite
 store. The public daemon methods are `connector.prepare`, `connector.poll`,
-`connector.ack`, `connector.fail`, `connector.defer`, `connector.inspect`,
-`connector.retry`, and the operational helper `connector.reclaim`. Matching
-JSON-only CLI hooks include:
+`connector.ack`, `connector.fail`, `connector.defer`, `connector.renew`,
+`connector.release`, `connector.inspect`, `connector.retry`, and the operational
+helper `connector.reclaim`. Matching JSON-only CLI hooks include:
 
 ```bash
 tendwire connector poll --name attention --limit 10 --lease-seconds 60 --db-path /path/to/tendwire.db
 tendwire connector ack --name attention --ref '<opaque-ref>' --response-json '{"delivered":true}' --db-path /path/to/tendwire.db
 tendwire connector fail --name attention --ref '<opaque-ref>' --reason temporary --delay-seconds 60 --db-path /path/to/tendwire.db
 tendwire connector defer --name attention --ref '<opaque-ref>' --reason scheduled --available-at 2026-01-01T00:10:00+00:00 --db-path /path/to/tendwire.db
+tendwire connector renew --name turn-final --ref '<opaque-ref>' --lease-seconds 120 --db-path /path/to/tendwire.db
+tendwire connector release --name turn-final --ref '<opaque-ref>' --db-path /path/to/tendwire.db
 tendwire connector inspect --name turn-final --status dead_letter --limit 100 --db-path /path/to/tendwire.db
 tendwire connector retry --name turn-final --final-identity 'twfinal1.<opaque>' --db-path /path/to/tendwire.db
 ```
@@ -1247,20 +1252,44 @@ effect.
 `connector.poll` atomically leases due `connector_outbox` rows for one `name`
 and returns opaque per-attempt refs. A live lease prevents duplicate polling.
 Expired leases are reclaimed before polling and before ref-mutating operations;
-`connector.reclaim` can also be called directly. `connector.ack` validates the
-host, name, attempt, lease, and ref before marking the delivery and outbox item
-delivered. `connector.fail` records sanitized failure data and schedules retry
-availability. `connector.defer` records sanitized defer data and schedules future
-availability without treating the item as delivered. Stale, expired,
-wrong-host, wrong-name, and superseded refs fail closed with neutral errors.
+the daemon first performs a read-only due check on its periodic tick and takes
+the reclaim write transaction only when work is actually expired.
+`connector.reclaim` can be called directly. `connector.renew` extends a live
+lease without creating a delivery attempt. `connector.release` records the live
+attempt as released and makes the row immediately pollable again. `connector.ack`
+validates the host, name, attempt, lease, and ref before marking the delivery and
+outbox item delivered. `connector.fail` records sanitized failure data and
+schedules retry availability. `connector.defer` records sanitized defer data and
+schedules future availability without treating the item as delivered. Stale,
+expired, wrong-host, wrong-name, and superseded refs fail closed with neutral
+errors.
 When callers omit `lease_seconds`, the daemon and CLI use
 `connector_claim_ttl_seconds` from config, defaulting to 60 seconds; an explicit
-public `lease_seconds` value still wins. `max_outbox_attempts` prevents
+public `lease_seconds` value still wins. `turn-final` poll and renew requests are
+capped by `connector_max_claim_ttl_seconds`, defaulting to 300 seconds.
+`max_outbox_attempts` prevents
 unbounded retry loops. Once a failed job reaches the configured cap, Tendwire
 moves the outbox item to a neutral terminal `dead_letter` state and returns the
 public status `attempts_exhausted` without exposing private outbox or delivery
 state.
 
+Final-root FIFO is partitioned by the turn's immutable stable worker key, with
+the persisted worker ID as the enqueue-time fallback. Legacy backfill prefers
+the enqueue-era stable key carried in the outbox payload; rows that predate that
+metadata still fall back to the turn's current persisted worker ID. If neither
+identity can be resolved, each row receives its own `orphan:<id>` partition so
+unrelated degraded rows cannot block one another. A blocked worker therefore
+cannot starve another worker, while roots and plan parts for the same worker stay
+strictly ordered. `dead_letter`, `superseded`, and `delivered` roots are terminal
+for this gate. A committed source in `awaiting_ack` no longer blocks by itself;
+its plan jobs carry the ordering obligation, including source-less recovery
+plans. Commit stamps an ACK deadline, each acknowledged plan job extends it,
+and a valid part lease prevents deadline reclaim from interrupting in-flight
+delivery. ACK-deadline recovery does not consume the connector failure-attempt
+budget. An expired incomplete plan is otherwise failed and its source requeued.
+Missing or unrecoverable plan state terminates the source instead of leaving it
+pending.
+
 ### Delivery-aware final roots and acknowledged history
 
 Persisting an authoritative, owner-authenticated complete final atomically

scripts/turn_ingestion_benchmark.py

@@ -1241,7 +1241,16 @@ def submit_command(_config: Config, payload: str) -> Mapping[str, Any]:
             )
         ) - baseline_threads
         source_calls_final = len(adapter_records)
-        forbidden_values.extend(str(process_id) for process_id in process_ids)
+        forbidden_values.extend(
+            marker
+            for process_id in process_ids
+            for marker in (
+                f'"process_id":{process_id}',
+                f'"process_id":"{process_id}"',
+                f'"pid":{process_id}',
+                f'"pid":"{process_id}"',
+            )
+        )
         overlap_ns = _first_call_overlap_ns(adapter_records, args.blocked_workers)
         response_bytes_max = max(
             metric["response_bytes_max"] for metric in latency.values()

src/tendwire/backends/herdr_turns.py

@@ -36,9 +36,11 @@
     redact_private_prompt_text,
 )
 from ..store.sqlite import (
+    TURN_CLAIM_SWEEP_MIN_GRACE_SECONDS,
     apply_turn_refresh,
     list_worker_bindings,
     prune_backend_pending,
+    sweep_turn_claims,
 )
 
 
@@ -4409,6 +4411,20 @@ def _scan_bindings(self) -> None:
         except Exception:
             pass
 
+        try:
+            sweep_turn_claims(
+                self.config.db_path,
+                self.config.host_id,
+                grace_seconds=max(
+                    TURN_CLAIM_SWEEP_MIN_GRACE_SECONDS,
+                    10.0 * self.refresh_interval_seconds,
+                ),
+                hard_ttl_seconds=self.config.turn_claim_hard_ttl_seconds,
+                now=self._utc_clock(),
+            )
+        except Exception:
+            pass
+
     def _binding_for_item(self, item: _TurnRefreshItem) -> WorkerBinding | None:
         if self.config.db_path is None:
             return None

src/tendwire/cli.py

@@ -497,15 +497,18 @@ def add_common(action_parser: argparse.ArgumentParser) -> None:
     reclaim_parser = actions.add_parser("reclaim", help="Expire stale connector leases.")
     add_common(reclaim_parser)
 
-    for action in ("ack", "fail", "defer"):
+    for action in ("ack", "fail", "defer", "renew", "release"):
         action_parser = actions.add_parser(action, help=f"Apply connector.{action} to a live ref.")
         add_common(action_parser)
         action_parser.add_argument("--ref", required=True)
-        action_parser.add_argument("--response-json", dest="response_json", default=None)
+        if action in {"ack", "fail", "defer"}:
+            action_parser.add_argument("--response-json", dest="response_json", default=None)
         if action in {"fail", "defer"}:
             action_parser.add_argument("--reason", default="")
             action_parser.add_argument("--available-at", dest="available_at", default=None)
             action_parser.add_argument("--delay-seconds", dest="delay_seconds", type=int, default=None)
+        if action == "renew":
+            action_parser.add_argument("--lease-seconds", dest="lease_seconds", type=int, default=None)
 
 
 def _load_worker_bindings(config: Config) -> list[WorkerBinding]:
@@ -1035,6 +1038,8 @@ def cmd_turns(
                 limit=limit,
                 cursor=cursor,
                 since=since,
+                turn_refresh_interval_seconds=config.turn_refresh_interval_seconds,
+                claim_hard_ttl_seconds=config.turn_claim_hard_ttl_seconds,
             )
     elif daemon_attempt.error_kind == "timeout":
         payload = {
@@ -1103,7 +1108,10 @@ def cmd_turn_content_get(config: Config, args: argparse.Namespace) -> int:
     else:
         from .store.sqlite import get_turn_content, init_store
 
-        init_store(config.db_path)
+        init_store(
+            config.db_path,
+            connector_ack_ttl_seconds=config.connector_ack_ttl_seconds,
+        )
         payload = get_turn_content(
             config.db_path,
             config.host_id,
@@ -1425,9 +1433,9 @@ def _connector_params_from_args(args: argparse.Namespace) -> dict[str, Any]:
         params["limit"] = args.limit
         if args.lease_seconds is not None:
             params["lease_seconds"] = args.lease_seconds
-    if args.connector_action in {"ack", "fail", "defer"}:
+    if args.connector_action in {"ack", "fail", "defer", "renew", "release"}:
         params["ref"] = args.ref
-        if args.response_json:
+        if getattr(args, "response_json", None):
             try:
                 parsed = json.loads(args.response_json)
             except json.JSONDecodeError:
@@ -1440,6 +1448,8 @@ def _connector_params_from_args(args: argparse.Namespace) -> dict[str, Any]:
                 params["available_at"] = args.available_at
             if args.delay_seconds is not None:
                 params["delay_seconds"] = args.delay_seconds
+        if args.connector_action == "renew" and args.lease_seconds is not None:
+            params["lease_seconds"] = args.lease_seconds
     return params
 
 
@@ -1507,11 +1517,16 @@ def cmd_connector(config: Config, args: argparse.Namespace) -> int:
     from .connectors import ConnectorOutboxAPI
     from .store.sqlite import init_store
 
-    init_store(config.db_path)
+    init_store(
+        config.db_path,
+        connector_ack_ttl_seconds=config.connector_ack_ttl_seconds,
+    )
     payload = ConnectorOutboxAPI(
         config.db_path,
         config.host_id,
         default_lease_seconds=config.connector_claim_ttl_seconds,
+        max_lease_seconds=config.connector_max_claim_ttl_seconds,
+        ack_ttl_seconds=config.connector_ack_ttl_seconds,
         max_attempts=config.max_outbox_attempts,
     ).dispatch(method, params)
     print(_connector_payload_json(payload, indent=2))