diff --git a/planfile/api/server.py b/planfile/api/server.py index 156b5339..68edb1dc 100644 --- a/planfile/api/server.py +++ b/planfile/api/server.py @@ -2095,10 +2095,26 @@ async def _archive_history_daily(interval_seconds: float = 300.0) -> None: def _repair_ticket_index() -> dict: """Repair the newest state without requiring a quiet writer interval.""" store = get_planfile().store - # ensure_ticket_index coalesces rebuilders with its own cross-process lock - # and retries source-signature races. A disposable projection must never - # hold the YAML mutation lock while scanning the entire ticket history. - return store.ensure_ticket_index() + index = store._sqlite_ticket_index() + if not store.ticket_index_enabled(): + return index.status() + if time.monotonic() < store._ticket_index_rebuild_deferred_until: + raise TicketIndexContentionError( + "ticket_index_rebuild_deferred_after_source_contention" + ) + signature = store._ticket_index_signature() + if index.is_current(signature): + return index.status(signature) | {"rebuilt": False} + # Non-blocking lock probe coalesces multi-worker background loops without + # pinning an AnyIO worker thread for LOCK_TIMEOUT_SECONDS. + try: + with store.ticket_index_rebuild_lock(timeout_seconds=0.0): + signature = store._ticket_index_signature() + if index.is_current(signature): + return index.status(signature) | {"rebuilt": False} + return store._rebuild_ticket_index_unlocked(index, force=False) + except StoreLockTimeoutError: + return {"rebuilt": False, "coalesced": True} async def _maintain_ticket_index(interval_seconds: float = 3.0) -> None: @@ -2113,6 +2129,10 @@ async def _maintain_ticket_index(interval_seconds: float = 3.0) -> None: if await asyncio.to_thread(store._sqlite_ticket_index().is_current, signature): continue await asyncio.to_thread(_repair_ticket_index) + except (StoreLockTimeoutError, TicketIndexContentionError): + # Another worker or request already owns .rebuild.lock or deferred + # rebuilds after contention; retry quietly on the next tick. + continue except Exception as exc: # pragma: no cover - defensive runtime telemetry __import__("logging").getLogger("planfile.api").warning( "ticket index background repair failed: %s", exc @@ -4404,7 +4424,7 @@ async def websocket_dsl(websocket: WebSocket, project_path: str = "."): # ── Health ───────────────────────────────────────────────────────────────────── @app.get("/health", tags=["system"]) -def health(): +async def health() -> dict[str, Any]: import planfile return { "status": "ok", diff --git a/planfile/core/sqlite_index.py b/planfile/core/sqlite_index.py index bd135497..55b02aa4 100644 --- a/planfile/core/sqlite_index.py +++ b/planfile/core/sqlite_index.py @@ -17,6 +17,7 @@ _SCHEMA_VERSION = 1 _REBUILD_BATCH_SIZE = 128 _SOURCE_INDEXED_AT_NS_KEY = "source_indexed_at_ns" +_WAL_JOURNAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024 def _is_sqlite_contention(error: sqlite3.DatabaseError) -> bool: @@ -379,6 +380,14 @@ def rebuild( (_SOURCE_INDEXED_AT_NS_KEY, str(time.time_ns())), ) connection.commit() + try: + connection.execute("PRAGMA busy_timeout=0") + connection.execute( + f"PRAGMA journal_size_limit={_WAL_JOURNAL_SIZE_LIMIT_BYTES}" + ) + connection.execute("PRAGMA wal_checkpoint(TRUNCATE)") + except sqlite3.DatabaseError: + pass return count def get_ticket(self, ticket_id: str) -> dict[str, Any] | None: diff --git a/project/ticket-285/README.md b/project/ticket-285/README.md new file mode 100644 index 00000000..c0ce288f --- /dev/null +++ b/project/ticket-285/README.md @@ -0,0 +1,26 @@ +# Ticket 285: Prevent healthcheck threadpool starvation and coalesce background SQLite index rebuilds + +- **ID**: ticket-285 +- **Owner**: agent:antigravity +- **Status**: IN_PROGRESS +- **Workflow state**: PUBLICATION +- **Created**: 2026-11-16 + +SESSION_EXECUTION_AUTHORIZATION: Authorized autonomous diagnosis, implementation, validation, and independent Validator publication (`kontynuuj,scalaj,testuj`) inside canonical worktree `.worktrees/ticket-285--prevent-healthcheck-starvation`, preserving foreign untracked `project/ticket-280/` untouched on primary checkout. + +## Goal and scope + +1. Serve `GET /health` (`planfile/api/server.py`) directly on the asyncio event loop (`async def health`) so container liveness probes cannot time out when AnyIO's worker threadpool is saturated by concurrent API reads or SQLite index maintenance. +2. Acquire `.rebuild.lock` non-blockingly (`timeout_seconds=0.0`) inside `_repair_ticket_index()`, returning `{"rebuilt": False, "coalesced": True}` on contention, and coalesce `(StoreLockTimeoutError, TicketIndexContentionError)` in `_maintain_ticket_index()` so secondary `--workers` processes and overlapping ticks yield immediately without holding threadpool workers for 10 seconds or emitting false-positive `planfile_store_lock_timeout` warnings. +3. Bound disposable SQLite WAL file retention in `planfile/core/sqlite_index.py` by running non-blocking (`PRAGMA busy_timeout=0`) `PRAGMA journal_size_limit=67108864` and `PRAGMA wal_checkpoint(TRUNCATE)` immediately after `rebuild()` commits (without acquiring write locks inside `_connect()` on reader probes), so full archive rebuilds truncate multi-hundred-megabyte `-wal` files on disk without stalling concurrent readers or writers. + +## Acceptance criteria + +- [x] AC-01: `GET /health` is an `async def` coroutine handler that responds on the event loop even when the default AnyIO worker threadpool is exhausted. +- [x] AC-02: `_repair_ticket_index()` / `_maintain_ticket_index()` probe `.rebuild.lock` with `timeout_seconds=0.0` and coalesce active rebuilders without blocking for `LOCK_TIMEOUT_SECONDS` or logging spurious lock-timeout warnings. +- [x] AC-03: `SQLiteTicketIndex.rebuild()` enforces a 64 MiB `journal_size_limit` and runs a non-blocking `TRUNCATE` WAL checkpoint post-commit without reader lock contention, covered by regression tests, Ruff, and `./project/governance-check.sh`. + +## Tracking boundary + +This directory contains the minimal reviewed intent. Optional participant prose +and raw command logs are not required delivery output. diff --git a/project/ticket-285/intent.json b/project/ticket-285/intent.json new file mode 100644 index 00000000..62f6e4d8 --- /dev/null +++ b/project/ticket-285/intent.json @@ -0,0 +1,26 @@ +{ + "schema": "new-project.intent/v3", + "ticket": "ticket-285", + "summary": "Prevent healthcheck threadpool starvation and coalesce background SQLite index rebuilds", + "workstream": "application", + "classification": { + "kind": "BUG", + "priority": "P1", + "origin": "health" + }, + "allowedPaths": [ + "project/ticket-285/**", + "TODO.md", + "project/TICKETS.md", + "planfile/api/server.py", + "planfile/core/sqlite_index.py", + "tests/test_index_rebuild_contention.py" + ], + "forbiddenPaths": [ + "project/ticket-*/user-*.md" + ], + "stacks": [], + "dependsOn": [], + "conflictsWith": [], + "integrationTicket": null +} diff --git a/tests/test_index_rebuild_contention.py b/tests/test_index_rebuild_contention.py new file mode 100644 index 00000000..e19d3cdf --- /dev/null +++ b/tests/test_index_rebuild_contention.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import asyncio +import fcntl +import inspect +import logging +import threading +import time + +import anyio.to_thread +import httpx +import pytest + +from planfile import Planfile +from planfile.api import server +from planfile.core import sqlite_index + + +def _setup_store(tmp_path): + pf = Planfile(str(tmp_path)) + config = pf.store._read_config() + config["archive"]["enabled"] = False + pf.store._write_config(config) + return pf + + +def test_health_endpoint_is_async_and_responds_when_threadpool_is_exhausted(): + assert inspect.iscoroutinefunction(server.health) + + blocker_started = threading.Event() + release_blocker = threading.Event() + + def occupy_only_worker() -> None: + blocker_started.set() + release_blocker.wait(timeout=5.0) + + async def exercise() -> dict: + limiter = anyio.to_thread.current_default_thread_limiter() + original_tokens = limiter.total_tokens + limiter.total_tokens = 1 + try: + worker_task = asyncio.create_task( + anyio.to_thread.run_sync(occupy_only_worker) + ) + await asyncio.to_thread(blocker_started.wait, 2.0) + transport = httpx.ASGITransport(app=server.app) + async with httpx.AsyncClient( + transport=transport, base_url="http://testserver" + ) as client: + response = await asyncio.wait_for(client.get("/health"), timeout=1.0) + release_blocker.set() + await worker_task + assert response.status_code == 200 + return response.json() + finally: + release_blocker.set() + limiter.total_tokens = original_tokens + + payload = asyncio.run(exercise()) + assert payload["status"] == "ok" + assert "ticket.discovery.current_scoped_pages" in payload["capabilities"] + + +def test_background_index_maintenance_coalesces_active_rebuild_lock_without_blocking( + tmp_path, monkeypatch, caplog +): + pf = _setup_store(tmp_path) + pf.create_ticket(name="Stale index entry") + pf.store.configure_ticket_index(True) + # Invalidate SQLite projection so _maintain_ticket_index attempts repair. + pf.store._ticket_index_invalidation_path.write_text("stale-token", encoding="ascii") + + monkeypatch.setattr(server, "get_planfile", lambda: pf) + + lock_path = pf.store._ticket_index_rebuild_lock_path + lock_path.parent.mkdir(parents=True, exist_ok=True) + with lock_path.open("a+", encoding="utf-8") as held_lock: + fcntl.flock(held_lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + try: + started = time.monotonic() + coalesced = server._repair_ticket_index() + elapsed = time.monotonic() - started + assert coalesced == {"rebuilt": False, "coalesced": True} + assert elapsed < 0.5 + + ticks = 0 + + async def fast_tick(_interval: float) -> None: + nonlocal ticks + ticks += 1 + if ticks >= 2: + raise asyncio.CancelledError + + monkeypatch.setattr(asyncio, "sleep", fast_tick) + with caplog.at_level(logging.WARNING, logger="planfile.api"): + with pytest.raises(asyncio.CancelledError): + asyncio.run(server._maintain_ticket_index(interval_seconds=0.01)) + + assert ticks == 2 + assert not [ + rec + for rec in caplog.records + if "ticket index background repair failed" in rec.getMessage() + ] + finally: + fcntl.flock(held_lock.fileno(), fcntl.LOCK_UN) + + +def test_sqlite_index_checkpoints_and_truncates_wal_after_rebuild_without_blocking_readers( + tmp_path, monkeypatch +): + monkeypatch.setattr(sqlite_index, "_WAL_JOURNAL_SIZE_LIMIT_BYTES", 4096) + pf = _setup_store(tmp_path) + for idx in range(25): + pf.create_ticket(name=f"Ticket {idx}", description="x" * 4096) + pf.store.configure_ticket_index(True) + + index = pf.store._sqlite_ticket_index() + wal_path = index.path.with_name(index.path.name + "-wal") + report = pf.store.ensure_ticket_index(force=True) + assert report["rebuilt"] is True + if wal_path.exists(): + assert wal_path.stat().st_size <= 4096 + + # Holding an open read snapshot must not cause rebuild's post-commit + # checkpoint to block for SQLite's default 10s busy_timeout. + with index._connection() as reader: + reader.execute("BEGIN") + reader.execute("SELECT COUNT(*) FROM tickets").fetchone() + started = time.monotonic() + rebuilt_under_reader = pf.store.ensure_ticket_index(force=True) + elapsed = time.monotonic() - started + assert rebuilt_under_reader["rebuilt"] is True + assert elapsed < 2.0