diff --git a/planfile/core/allocator_sprints.py b/planfile/core/allocator_sprints.py index 720934de..7fb73930 100644 --- a/planfile/core/allocator_sprints.py +++ b/planfile/core/allocator_sprints.py @@ -7,6 +7,7 @@ import re import tempfile from pathlib import Path +from typing import Any from planfile.core.fastio import load_yaml_text @@ -61,7 +62,85 @@ class SourceChangedError(RuntimeError): """Authoritative source changed during optimistic preparation.""" -MAX_LOCKED_REBUILD_BYTES = 5_000_000 +MAX_LOCKED_REBUILD_BYTES = 50_000_000 + + +def save_sprint_highwater(source: Path, prefix: str, highest: int, sig: list[int] | None = None) -> None: + if sig is None: + try: + sig = signature(source) + except OSError: + return + key = hashlib.sha256((source.name + "\0" + prefix).encode()).hexdigest() + cache = source.parent / "index" / "allocator-sprints" / f"{key}.json" + value = {"version": 1, "signature": sig, "prefix": prefix, "source": source.name, "highest": highest} + value["checksum"] = _digest(value) + temporary = None + try: + cache.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(mode="w", dir=cache.parent, delete=False) as stream: + temporary = Path(stream.name) + json.dump(value, stream) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, cache) + except OSError: + pass # Derived cache failure must not prevent allocation. + finally: + if temporary is not None: + try: + temporary.unlink(missing_ok=True) + except OSError: + pass + + +def update_sprint_highwater_cache(source: Path, tickets: list[Any] | Any = ()) -> None: + """Update on-disk allocator highwater cache after writing sprint file. + + Preserves valid cached prefixes with the new file signature and updates + the highwater mark for prefixes contained in the added tickets. + """ + try: + sig = signature(source) + except OSError: + return + ticket_list = tickets if isinstance(tickets, list) else ([tickets] if tickets else []) + recs = [] + for item in ticket_list: + if isinstance(item, dict): + recs.append(item) + elif hasattr(item, "model_dump"): + recs.append(item.model_dump(mode="json", exclude_none=True)) + elif hasattr(item, "__dict__"): + recs.append(item.__dict__) + + marker = re.compile(r"([A-Za-z0-9_-]+)-(\d+)") + all_prefixes: set[str] = set() + for r in recs: + for val in (r.get("id"), *(r.get("labels") or []), r.get("description")): + if val and isinstance(val, str): + for m in marker.finditer(val): + all_prefixes.add(m.group(1)) + + updated_prefixes: set[str] = set() + index_dir = source.parent / "index" / "allocator-sprints" + if index_dir.is_dir(): + for cache_path in index_dir.glob("*.json"): + try: + content = json.loads(cache_path.read_text(encoding="utf-8")) + if content.get("source") == source.name and "prefix" in content and "highest" in content: + pfx = content["prefix"] + cur_highest = content["highest"] + added_high = record_highwater(recs, pfx) if recs else 0 + new_highest = max(cur_highest, added_high) + save_sprint_highwater(source, pfx, new_highest, sig) + updated_prefixes.add(pfx) + except (OSError, ValueError, TypeError, KeyError): + continue + + for pfx in all_prefixes - updated_prefixes: + high = record_highwater(recs, pfx) + save_sprint_highwater(source, pfx, high, sig) def sprint_highwater(source: Path, prefix: str, *, max_rebuild_bytes: int | None = None) -> int: @@ -101,23 +180,5 @@ def sprint_highwater(source: Path, prefix: str, *, max_rebuild_bytes: int | None highest = record_highwater(records(data), prefix) if signature(source) != before: raise SourceChangedError("Sprint changed while rebuilding allocator identity evidence") - value = {"version": 1, "signature": before, "prefix": prefix, "source": source.name, "highest": highest} - value["checksum"] = _digest(value) - temporary = None - try: - cache.parent.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile(mode="w", dir=cache.parent, delete=False) as stream: - temporary = Path(stream.name) - json.dump(value, stream) - stream.flush() - os.fsync(stream.fileno()) - os.replace(temporary, cache) - except OSError: - pass # Derived cache failure must not prevent allocation. - finally: - if temporary is not None: - try: - temporary.unlink(missing_ok=True) - except OSError: - pass + save_sprint_highwater(source, prefix, highest, before) return highest diff --git a/planfile/core/store.py b/planfile/core/store.py index 0d106dca..f9ae6942 100644 --- a/planfile/core/store.py +++ b/planfile/core/store.py @@ -2654,6 +2654,8 @@ def _create_tickets_bulk_unlocked(self, tickets: list[Ticket]) -> list[Ticket]: self._write_yaml_atomic(sprint_file, data, allow_unicode=True) if hasattr(self, "_yaml_cache"): self._yaml_cache.pop(str(sprint_file), None) + from planfile.core.allocator_sprints import update_sprint_highwater_cache + update_sprint_highwater_cache(sprint_file, sprint_tickets) self._append_operational_lines( [str(ticket.dsl) for ticket in tickets if ticket.dsl] @@ -2795,6 +2797,8 @@ def _create_ticket_unlocked(self, ticket: Ticket) -> Ticket: # Invalidate yaml cache if hasattr(self, "_yaml_cache"): self._yaml_cache.pop(str(sprint_file), None) + from planfile.core.allocator_sprints import update_sprint_highwater_cache + update_sprint_highwater_cache(sprint_file, ticket) archive_report = ( self._archive_completed_unlocked() @@ -3606,6 +3610,8 @@ def delete_tickets_bulk(self, ticket_ids: list[str]) -> tuple[list[str], list[st self._write_yaml_atomic(sprint_file, data, allow_unicode=True) if hasattr(self, "_yaml_cache"): self._yaml_cache.pop(str(sprint_file), None) + from planfile.core.allocator_sprints import update_sprint_highwater_cache + update_sprint_highwater_cache(sprint_file) from planfile.core.operational_dsl import line as operational_line for ticket_id in deleted: diff --git a/project/TICKETS.md b/project/TICKETS.md index 4306e4a8..e29fc205 100644 --- a/project/TICKETS.md +++ b/project/TICKETS.md @@ -173,4 +173,5 @@ This file indexes governance tickets without taking ownership of | **ticket-272** | [`README.md`](./ticket-272/README.md) | - | - | - | - | - | | **ticket-273** | [`README.md`](./ticket-273/README.md) | - | - | - | - | - | | **ticket-274** | [`README.md`](./ticket-274/README.md) | - | - | - | - | - | +| **ticket-275** | [`README.md`](./ticket-275/README.md) | - | - | - | - | - | diff --git a/project/ticket-277/README.md b/project/ticket-277/README.md new file mode 100644 index 00000000..9d41b593 --- /dev/null +++ b/project/ticket-277/README.md @@ -0,0 +1,28 @@ +# Ticket 277: update allocator highwater cache on ticket creation to prevent lock timeout on files > 5MB + +- **ID**: ticket-277 +- **Owner**: agent:antigravity +- **Status**: IN_PROGRESS +- **Workflow state**: VALIDATION +- **Created**: 2026-11-03 + +## Goal and scope + +Prevent locked rebuild timeouts and `allocator_evidence_changed_retry` errors on large sprint files (> 5MB) by atomically updating the allocator highwater cache on ticket persistence and increasing `MAX_LOCKED_REBUILD_BYTES` to 50MB. + +## Acceptance criteria + +- [x] AC-01: Ticket creation atomically maintains allocator highwater cache or supports large sprint files under lock +- [x] AC-02: Sequential ticket creation on sprints > 5MB succeeds without crashing with allocator_evidence_changed_retry +- [x] AC-03: Tests in tests/test_allocator_preparation.py and tests/test_allocator_dedupe.py pass + +## Implementation details + +- `planfile/core/allocator_sprints.py`: + - Increased `MAX_LOCKED_REBUILD_BYTES` to 50,000,000 (50MB). + - Factored out `save_sprint_highwater` helper. + - Implemented `update_sprint_highwater_cache` to preserve existing prefix caches with the new file signature and update highwater marks for newly added tickets. +- `planfile/core/store.py`: + - Called `update_sprint_highwater_cache` after atomic YAML write in `_create_ticket_unlocked` and `_create_tickets_bulk_unlocked`. +- `tests/test_allocator_preparation.py`: + - Added `test_sequential_ticket_creation_on_large_sprint_succeeds_without_retry` verifying sequential ticket creations succeed with small `MAX_LOCKED_REBUILD_BYTES`. diff --git a/project/ticket-277/intent.json b/project/ticket-277/intent.json new file mode 100644 index 00000000..bab36f76 --- /dev/null +++ b/project/ticket-277/intent.json @@ -0,0 +1,26 @@ +{ + "schema": "new-project.intent/v3", + "ticket": "ticket-277", + "summary": "update allocator highwater cache on ticket creation to prevent lock timeout on files > 5MB", + "workstream": "application", + "classification": { + "kind": "SERVICE", + "priority": "P2", + "origin": "health" + }, + "allowedPaths": [ + "project/ticket-277/**", + "TODO.md", + "project/TICKETS.md", + "planfile/core/allocator_sprints.py", + "planfile/core/store.py", + "tests/test_allocator_preparation.py" + ], + "forbiddenPaths": [ + "project/ticket-*/user-*.md" + ], + "stacks": [], + "dependsOn": [], + "conflictsWith": [], + "integrationTicket": null +} diff --git a/tests/test_allocator_preparation.py b/tests/test_allocator_preparation.py index 4b2b35dd..9819cc6d 100644 --- a/tests/test_allocator_preparation.py +++ b/tests/test_allocator_preparation.py @@ -185,3 +185,19 @@ def invalid(text): with pytest.raises(RuntimeError, match="invalid source"): store.next_id() assert store._read_config() == before + + +def test_sequential_ticket_creation_on_large_sprint_succeeds_without_retry(tmp_path, monkeypatch): + pf = Planfile(tmp_path) + pf.store.init() + t1 = pf.create_ticket("First ticket") + assert t1.id == "PLF-001" + sprint_file = pf.store._sprint_file("current") + sprint_file.write_text(sprint_file.read_text() + "\n# " + "x" * 200) + evidence.sprint_highwater(sprint_file, "PLF") + monkeypatch.setattr(evidence, "MAX_LOCKED_REBUILD_BYTES", 100) + t2 = pf.create_ticket("Second ticket") + assert t2.id == "PLF-002" + t3 = pf.create_ticket("Third ticket") + assert t3.id == "PLF-003" + assert pf.store.next_id() == "PLF-004"