Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 81 additions & 20 deletions planfile/core/allocator_sprints.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import re
import tempfile
from pathlib import Path
from typing import Any

from planfile.core.fastio import load_yaml_text

Expand Down Expand Up @@ -61,7 +62,85 @@ class SourceChangedError(RuntimeError):
"""Authoritative source changed during optimistic preparation."""


MAX_LOCKED_REBUILD_BYTES = 5_000_000
MAX_LOCKED_REBUILD_BYTES = 50_000_000


def save_sprint_highwater(source: Path, prefix: str, highest: int, sig: list[int] | None = None) -> None:
if sig is None:
try:
sig = signature(source)
except OSError:
return
key = hashlib.sha256((source.name + "\0" + prefix).encode()).hexdigest()
cache = source.parent / "index" / "allocator-sprints" / f"{key}.json"
value = {"version": 1, "signature": sig, "prefix": prefix, "source": source.name, "highest": highest}
value["checksum"] = _digest(value)
temporary = None
try:
cache.parent.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(mode="w", dir=cache.parent, delete=False) as stream:
temporary = Path(stream.name)
json.dump(value, stream)
stream.flush()
os.fsync(stream.fileno())
os.replace(temporary, cache)
except OSError:
pass # Derived cache failure must not prevent allocation.
finally:
if temporary is not None:
try:
temporary.unlink(missing_ok=True)
except OSError:
pass


def update_sprint_highwater_cache(source: Path, tickets: list[Any] | Any = ()) -> None:
"""Update on-disk allocator highwater cache after writing sprint file.

Preserves valid cached prefixes with the new file signature and updates
the highwater mark for prefixes contained in the added tickets.
"""
try:
sig = signature(source)
except OSError:
return
ticket_list = tickets if isinstance(tickets, list) else ([tickets] if tickets else [])
recs = []
for item in ticket_list:
if isinstance(item, dict):
recs.append(item)
elif hasattr(item, "model_dump"):
recs.append(item.model_dump(mode="json", exclude_none=True))
elif hasattr(item, "__dict__"):
recs.append(item.__dict__)

marker = re.compile(r"([A-Za-z0-9_-]+)-(\d+)")
all_prefixes: set[str] = set()
for r in recs:
for val in (r.get("id"), *(r.get("labels") or []), r.get("description")):
if val and isinstance(val, str):
for m in marker.finditer(val):
all_prefixes.add(m.group(1))

updated_prefixes: set[str] = set()
index_dir = source.parent / "index" / "allocator-sprints"
if index_dir.is_dir():
for cache_path in index_dir.glob("*.json"):
try:
content = json.loads(cache_path.read_text(encoding="utf-8"))
if content.get("source") == source.name and "prefix" in content and "highest" in content:
pfx = content["prefix"]
cur_highest = content["highest"]
added_high = record_highwater(recs, pfx) if recs else 0
new_highest = max(cur_highest, added_high)
save_sprint_highwater(source, pfx, new_highest, sig)
updated_prefixes.add(pfx)
except (OSError, ValueError, TypeError, KeyError):
continue

for pfx in all_prefixes - updated_prefixes:
high = record_highwater(recs, pfx)
save_sprint_highwater(source, pfx, high, sig)


def sprint_highwater(source: Path, prefix: str, *, max_rebuild_bytes: int | None = None) -> int:
Expand Down Expand Up @@ -101,23 +180,5 @@ def sprint_highwater(source: Path, prefix: str, *, max_rebuild_bytes: int | None
highest = record_highwater(records(data), prefix)
if signature(source) != before:
raise SourceChangedError("Sprint changed while rebuilding allocator identity evidence")
value = {"version": 1, "signature": before, "prefix": prefix, "source": source.name, "highest": highest}
value["checksum"] = _digest(value)
temporary = None
try:
cache.parent.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(mode="w", dir=cache.parent, delete=False) as stream:
temporary = Path(stream.name)
json.dump(value, stream)
stream.flush()
os.fsync(stream.fileno())
os.replace(temporary, cache)
except OSError:
pass # Derived cache failure must not prevent allocation.
finally:
if temporary is not None:
try:
temporary.unlink(missing_ok=True)
except OSError:
pass
save_sprint_highwater(source, prefix, highest, before)
return highest
6 changes: 6 additions & 0 deletions planfile/core/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -2654,6 +2654,8 @@ def _create_tickets_bulk_unlocked(self, tickets: list[Ticket]) -> list[Ticket]:
self._write_yaml_atomic(sprint_file, data, allow_unicode=True)
if hasattr(self, "_yaml_cache"):
self._yaml_cache.pop(str(sprint_file), None)
from planfile.core.allocator_sprints import update_sprint_highwater_cache
update_sprint_highwater_cache(sprint_file, sprint_tickets)

self._append_operational_lines(
[str(ticket.dsl) for ticket in tickets if ticket.dsl]
Expand Down Expand Up @@ -2795,6 +2797,8 @@ def _create_ticket_unlocked(self, ticket: Ticket) -> Ticket:
# Invalidate yaml cache
if hasattr(self, "_yaml_cache"):
self._yaml_cache.pop(str(sprint_file), None)
from planfile.core.allocator_sprints import update_sprint_highwater_cache
update_sprint_highwater_cache(sprint_file, ticket)

archive_report = (
self._archive_completed_unlocked()
Expand Down Expand Up @@ -3606,6 +3610,8 @@ def delete_tickets_bulk(self, ticket_ids: list[str]) -> tuple[list[str], list[st
self._write_yaml_atomic(sprint_file, data, allow_unicode=True)
if hasattr(self, "_yaml_cache"):
self._yaml_cache.pop(str(sprint_file), None)
from planfile.core.allocator_sprints import update_sprint_highwater_cache
update_sprint_highwater_cache(sprint_file)

from planfile.core.operational_dsl import line as operational_line
for ticket_id in deleted:
Expand Down
1 change: 1 addition & 0 deletions project/TICKETS.md
Original file line number Diff line number Diff line change
Expand Up @@ -173,4 +173,5 @@ This file indexes governance tickets without taking ownership of
| **ticket-272** | [`README.md`](./ticket-272/README.md) | - | - | - | - | - |
| **ticket-273** | [`README.md`](./ticket-273/README.md) | - | - | - | - | - |
| **ticket-274** | [`README.md`](./ticket-274/README.md) | - | - | - | - | - |
| **ticket-275** | [`README.md`](./ticket-275/README.md) | - | - | - | - | - |
<!-- AUTO:TICKET_INDEX:END -->
28 changes: 28 additions & 0 deletions project/ticket-277/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Ticket 277: update allocator highwater cache on ticket creation to prevent lock timeout on files > 5MB

- **ID**: ticket-277
- **Owner**: agent:antigravity
- **Status**: IN_PROGRESS
- **Workflow state**: VALIDATION
- **Created**: 2026-11-03

## Goal and scope

Prevent locked rebuild timeouts and `allocator_evidence_changed_retry` errors on large sprint files (> 5MB) by atomically updating the allocator highwater cache on ticket persistence and increasing `MAX_LOCKED_REBUILD_BYTES` to 50MB.

## Acceptance criteria

- [x] AC-01: Ticket creation atomically maintains allocator highwater cache or supports large sprint files under lock
- [x] AC-02: Sequential ticket creation on sprints > 5MB succeeds without crashing with allocator_evidence_changed_retry
- [x] AC-03: Tests in tests/test_allocator_preparation.py and tests/test_allocator_dedupe.py pass

## Implementation details

- `planfile/core/allocator_sprints.py`:
- Increased `MAX_LOCKED_REBUILD_BYTES` to 50,000,000 (50MB).
- Factored out `save_sprint_highwater` helper.
- Implemented `update_sprint_highwater_cache` to preserve existing prefix caches with the new file signature and update highwater marks for newly added tickets.
- `planfile/core/store.py`:
- Called `update_sprint_highwater_cache` after atomic YAML write in `_create_ticket_unlocked` and `_create_tickets_bulk_unlocked`.
- `tests/test_allocator_preparation.py`:
- Added `test_sequential_ticket_creation_on_large_sprint_succeeds_without_retry` verifying sequential ticket creations succeed with small `MAX_LOCKED_REBUILD_BYTES`.
26 changes: 26 additions & 0 deletions project/ticket-277/intent.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
{
"schema": "new-project.intent/v3",
"ticket": "ticket-277",
"summary": "update allocator highwater cache on ticket creation to prevent lock timeout on files > 5MB",
"workstream": "application",
"classification": {
"kind": "SERVICE",
"priority": "P2",
"origin": "health"
},
"allowedPaths": [
"project/ticket-277/**",
"TODO.md",
"project/TICKETS.md",
"planfile/core/allocator_sprints.py",
"planfile/core/store.py",
"tests/test_allocator_preparation.py"
],
"forbiddenPaths": [
"project/ticket-*/user-*.md"
],
"stacks": [],
"dependsOn": [],
"conflictsWith": [],
"integrationTicket": null
}
16 changes: 16 additions & 0 deletions tests/test_allocator_preparation.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,3 +185,19 @@ def invalid(text):
with pytest.raises(RuntimeError, match="invalid source"):
store.next_id()
assert store._read_config() == before


def test_sequential_ticket_creation_on_large_sprint_succeeds_without_retry(tmp_path, monkeypatch):
pf = Planfile(tmp_path)
pf.store.init()
t1 = pf.create_ticket("First ticket")
assert t1.id == "PLF-001"
sprint_file = pf.store._sprint_file("current")
sprint_file.write_text(sprint_file.read_text() + "\n# " + "x" * 200)
evidence.sprint_highwater(sprint_file, "PLF")
monkeypatch.setattr(evidence, "MAX_LOCKED_REBUILD_BYTES", 100)
t2 = pf.create_ticket("Second ticket")
assert t2.id == "PLF-002"
t3 = pf.create_ticket("Third ticket")
assert t3.id == "PLF-003"
assert pf.store.next_id() == "PLF-004"
Loading