Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 25 additions & 5 deletions planfile/api/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -2095,10 +2095,26 @@ async def _archive_history_daily(interval_seconds: float = 300.0) -> None:
def _repair_ticket_index() -> dict:
"""Repair the newest state without requiring a quiet writer interval."""
store = get_planfile().store
# ensure_ticket_index coalesces rebuilders with its own cross-process lock
# and retries source-signature races. A disposable projection must never
# hold the YAML mutation lock while scanning the entire ticket history.
return store.ensure_ticket_index()
index = store._sqlite_ticket_index()
if not store.ticket_index_enabled():
return index.status()
if time.monotonic() < store._ticket_index_rebuild_deferred_until:
raise TicketIndexContentionError(
"ticket_index_rebuild_deferred_after_source_contention"
)
signature = store._ticket_index_signature()
if index.is_current(signature):
return index.status(signature) | {"rebuilt": False}
# Non-blocking lock probe coalesces multi-worker background loops without
# pinning an AnyIO worker thread for LOCK_TIMEOUT_SECONDS.
try:
with store.ticket_index_rebuild_lock(timeout_seconds=0.0):
signature = store._ticket_index_signature()
if index.is_current(signature):
return index.status(signature) | {"rebuilt": False}
return store._rebuild_ticket_index_unlocked(index, force=False)
except StoreLockTimeoutError:
return {"rebuilt": False, "coalesced": True}


async def _maintain_ticket_index(interval_seconds: float = 3.0) -> None:
Expand All @@ -2113,6 +2129,10 @@ async def _maintain_ticket_index(interval_seconds: float = 3.0) -> None:
if await asyncio.to_thread(store._sqlite_ticket_index().is_current, signature):
continue
await asyncio.to_thread(_repair_ticket_index)
except (StoreLockTimeoutError, TicketIndexContentionError):
# Another worker or request already owns .rebuild.lock or deferred
# rebuilds after contention; retry quietly on the next tick.
continue
except Exception as exc: # pragma: no cover - defensive runtime telemetry
__import__("logging").getLogger("planfile.api").warning(
"ticket index background repair failed: %s", exc
Expand Down Expand Up @@ -4404,7 +4424,7 @@ async def websocket_dsl(websocket: WebSocket, project_path: str = "."):
# ── Health ─────────────────────────────────────────────────────────────────────

@app.get("/health", tags=["system"])
def health():
async def health() -> dict[str, Any]:
import planfile
return {
"status": "ok",
Expand Down
9 changes: 9 additions & 0 deletions planfile/core/sqlite_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
_SCHEMA_VERSION = 1
_REBUILD_BATCH_SIZE = 128
_SOURCE_INDEXED_AT_NS_KEY = "source_indexed_at_ns"
_WAL_JOURNAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024


def _is_sqlite_contention(error: sqlite3.DatabaseError) -> bool:
Expand Down Expand Up @@ -379,6 +380,14 @@ def rebuild(
(_SOURCE_INDEXED_AT_NS_KEY, str(time.time_ns())),
)
connection.commit()
try:
connection.execute("PRAGMA busy_timeout=0")
connection.execute(
f"PRAGMA journal_size_limit={_WAL_JOURNAL_SIZE_LIMIT_BYTES}"
)
connection.execute("PRAGMA wal_checkpoint(TRUNCATE)")
except sqlite3.DatabaseError:
pass
return count

def get_ticket(self, ticket_id: str) -> dict[str, Any] | None:
Expand Down
26 changes: 26 additions & 0 deletions project/ticket-285/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# Ticket 285: Prevent healthcheck threadpool starvation and coalesce background SQLite index rebuilds

- **ID**: ticket-285
- **Owner**: agent:antigravity
- **Status**: IN_PROGRESS
- **Workflow state**: PUBLICATION
- **Created**: 2026-11-16

SESSION_EXECUTION_AUTHORIZATION: Authorized autonomous diagnosis, implementation, validation, and independent Validator publication (`kontynuuj,scalaj,testuj`) inside canonical worktree `.worktrees/ticket-285--prevent-healthcheck-starvation`, preserving foreign untracked `project/ticket-280/` untouched on primary checkout.

## Goal and scope

1. Serve `GET /health` (`planfile/api/server.py`) directly on the asyncio event loop (`async def health`) so container liveness probes cannot time out when AnyIO's worker threadpool is saturated by concurrent API reads or SQLite index maintenance.
2. Acquire `.rebuild.lock` non-blockingly (`timeout_seconds=0.0`) inside `_repair_ticket_index()`, returning `{"rebuilt": False, "coalesced": True}` on contention, and coalesce `(StoreLockTimeoutError, TicketIndexContentionError)` in `_maintain_ticket_index()` so secondary `--workers` processes and overlapping ticks yield immediately without holding threadpool workers for 10 seconds or emitting false-positive `planfile_store_lock_timeout` warnings.
3. Bound disposable SQLite WAL file retention in `planfile/core/sqlite_index.py` by running non-blocking (`PRAGMA busy_timeout=0`) `PRAGMA journal_size_limit=67108864` and `PRAGMA wal_checkpoint(TRUNCATE)` immediately after `rebuild()` commits (without acquiring write locks inside `_connect()` on reader probes), so full archive rebuilds truncate multi-hundred-megabyte `-wal` files on disk without stalling concurrent readers or writers.

## Acceptance criteria

- [x] AC-01: `GET /health` is an `async def` coroutine handler that responds on the event loop even when the default AnyIO worker threadpool is exhausted.
- [x] AC-02: `_repair_ticket_index()` / `_maintain_ticket_index()` probe `.rebuild.lock` with `timeout_seconds=0.0` and coalesce active rebuilders without blocking for `LOCK_TIMEOUT_SECONDS` or logging spurious lock-timeout warnings.
- [x] AC-03: `SQLiteTicketIndex.rebuild()` enforces a 64 MiB `journal_size_limit` and runs a non-blocking `TRUNCATE` WAL checkpoint post-commit without reader lock contention, covered by regression tests, Ruff, and `./project/governance-check.sh`.

## Tracking boundary

This directory contains the minimal reviewed intent. Optional participant prose
and raw command logs are not required delivery output.
26 changes: 26 additions & 0 deletions project/ticket-285/intent.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
{
"schema": "new-project.intent/v3",
"ticket": "ticket-285",
"summary": "Prevent healthcheck threadpool starvation and coalesce background SQLite index rebuilds",
"workstream": "application",
"classification": {
"kind": "BUG",
"priority": "P1",
"origin": "health"
},
"allowedPaths": [
"project/ticket-285/**",
"TODO.md",
"project/TICKETS.md",
"planfile/api/server.py",
"planfile/core/sqlite_index.py",
"tests/test_index_rebuild_contention.py"
],
"forbiddenPaths": [
"project/ticket-*/user-*.md"
],
"stacks": [],
"dependsOn": [],
"conflictsWith": [],
"integrationTicket": null
}
134 changes: 134 additions & 0 deletions tests/test_index_rebuild_contention.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
from __future__ import annotations

import asyncio
import fcntl
import inspect
import logging
import threading
import time

import anyio.to_thread
import httpx
import pytest

from planfile import Planfile
from planfile.api import server
from planfile.core import sqlite_index


def _setup_store(tmp_path):
pf = Planfile(str(tmp_path))
config = pf.store._read_config()
config["archive"]["enabled"] = False
pf.store._write_config(config)
return pf


def test_health_endpoint_is_async_and_responds_when_threadpool_is_exhausted():
assert inspect.iscoroutinefunction(server.health)

blocker_started = threading.Event()
release_blocker = threading.Event()

def occupy_only_worker() -> None:
blocker_started.set()
release_blocker.wait(timeout=5.0)

async def exercise() -> dict:
limiter = anyio.to_thread.current_default_thread_limiter()
original_tokens = limiter.total_tokens
limiter.total_tokens = 1
try:
worker_task = asyncio.create_task(
anyio.to_thread.run_sync(occupy_only_worker)
)
await asyncio.to_thread(blocker_started.wait, 2.0)
transport = httpx.ASGITransport(app=server.app)
async with httpx.AsyncClient(
transport=transport, base_url="http://testserver"
) as client:
response = await asyncio.wait_for(client.get("/health"), timeout=1.0)
release_blocker.set()
await worker_task
assert response.status_code == 200
return response.json()
finally:
release_blocker.set()
limiter.total_tokens = original_tokens

payload = asyncio.run(exercise())
assert payload["status"] == "ok"
assert "ticket.discovery.current_scoped_pages" in payload["capabilities"]


def test_background_index_maintenance_coalesces_active_rebuild_lock_without_blocking(
tmp_path, monkeypatch, caplog
):
pf = _setup_store(tmp_path)
pf.create_ticket(name="Stale index entry")
pf.store.configure_ticket_index(True)
# Invalidate SQLite projection so _maintain_ticket_index attempts repair.
pf.store._ticket_index_invalidation_path.write_text("stale-token", encoding="ascii")

monkeypatch.setattr(server, "get_planfile", lambda: pf)

lock_path = pf.store._ticket_index_rebuild_lock_path
lock_path.parent.mkdir(parents=True, exist_ok=True)
with lock_path.open("a+", encoding="utf-8") as held_lock:
fcntl.flock(held_lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
try:
started = time.monotonic()
coalesced = server._repair_ticket_index()
elapsed = time.monotonic() - started
assert coalesced == {"rebuilt": False, "coalesced": True}
assert elapsed < 0.5

ticks = 0

async def fast_tick(_interval: float) -> None:
nonlocal ticks
ticks += 1
if ticks >= 2:
raise asyncio.CancelledError

monkeypatch.setattr(asyncio, "sleep", fast_tick)
with caplog.at_level(logging.WARNING, logger="planfile.api"):
with pytest.raises(asyncio.CancelledError):
asyncio.run(server._maintain_ticket_index(interval_seconds=0.01))

assert ticks == 2
assert not [
rec
for rec in caplog.records
if "ticket index background repair failed" in rec.getMessage()
]
finally:
fcntl.flock(held_lock.fileno(), fcntl.LOCK_UN)


def test_sqlite_index_checkpoints_and_truncates_wal_after_rebuild_without_blocking_readers(
tmp_path, monkeypatch
):
monkeypatch.setattr(sqlite_index, "_WAL_JOURNAL_SIZE_LIMIT_BYTES", 4096)
pf = _setup_store(tmp_path)
for idx in range(25):
pf.create_ticket(name=f"Ticket {idx}", description="x" * 4096)
pf.store.configure_ticket_index(True)

index = pf.store._sqlite_ticket_index()
wal_path = index.path.with_name(index.path.name + "-wal")
report = pf.store.ensure_ticket_index(force=True)
assert report["rebuilt"] is True
if wal_path.exists():
assert wal_path.stat().st_size <= 4096

# Holding an open read snapshot must not cause rebuild's post-commit
# checkpoint to block for SQLite's default 10s busy_timeout.
with index._connection() as reader:
reader.execute("BEGIN")
reader.execute("SELECT COUNT(*) FROM tickets").fetchone()
started = time.monotonic()
rebuilt_under_reader = pf.store.ensure_ticket_index(force=True)
elapsed = time.monotonic() - started
assert rebuilt_under_reader["rebuilt"] is True
assert elapsed < 2.0
Loading