From 54d1abfae8157aa701d5249069b3f2babbba2e49 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Tue, 3 Nov 2026 10:13:24 +0100 Subject: [PATCH 1/3] ticket-277: update allocator highwater cache on ticket creation to prevent lock timeout on files > 5MB --- planfile/core/allocator_sprints.py | 91 ++++++++++++++++++++++------- planfile/core/store.py | 6 ++ project/TICKETS.md | 1 + project/ticket-277/README.md | 28 +++++++++ project/ticket-277/intent.json | 26 +++++++++ tests/test_allocator_preparation.py | 16 +++++ 6 files changed, 148 insertions(+), 20 deletions(-) create mode 100644 project/ticket-277/README.md create mode 100644 project/ticket-277/intent.json diff --git a/planfile/core/allocator_sprints.py b/planfile/core/allocator_sprints.py index 720934de..517ffd31 100644 --- a/planfile/core/allocator_sprints.py +++ b/planfile/core/allocator_sprints.py @@ -8,6 +8,8 @@ import tempfile from pathlib import Path +from typing import Any + from planfile.core.fastio import load_yaml_text @@ -61,7 +63,74 @@ class SourceChangedError(RuntimeError): """Authoritative source changed during optimistic preparation.""" -MAX_LOCKED_REBUILD_BYTES = 5_000_000 +MAX_LOCKED_REBUILD_BYTES = 50_000_000 + + +def save_sprint_highwater(source: Path, prefix: str, highest: int, sig: list[int] | None = None) -> None: + if sig is None: + try: + sig = signature(source) + except OSError: + return + key = hashlib.sha256((source.name + "\0" + prefix).encode()).hexdigest() + cache = source.parent / "index" / "allocator-sprints" / f"{key}.json" + value = {"version": 1, "signature": sig, "prefix": prefix, "source": source.name, "highest": highest} + value["checksum"] = _digest(value) + temporary = None + try: + cache.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(mode="w", dir=cache.parent, delete=False) as stream: + temporary = Path(stream.name) + json.dump(value, stream) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, cache) + except OSError: + pass # Derived cache failure must not prevent allocation. + finally: + if temporary is not None: + try: + temporary.unlink(missing_ok=True) + except OSError: + pass + + +def update_sprint_highwater_cache(source: Path, tickets: list[Any] | Any) -> None: + """Update on-disk allocator highwater cache after writing sprint file. + + Preserves valid cached prefixes with the new file signature and updates + the highwater mark for prefixes contained in the added tickets. + """ + try: + sig = signature(source) + except OSError: + return + ticket_list = tickets if isinstance(tickets, list) else [tickets] + pattern = re.compile(r"^([A-Za-z0-9_-]+)-(\d+)$") + added_prefixes: dict[str, int] = {} + for item in ticket_list: + tid = getattr(item, "id", None) if hasattr(item, "id") else (item.get("id") if isinstance(item, dict) else str(item)) + if tid: + m = pattern.fullmatch(str(tid)) + if m: + pfx, num = m.group(1), int(m.group(2)) + added_prefixes[pfx] = max(added_prefixes.get(pfx, 0), num) + + index_dir = source.parent / "index" / "allocator-sprints" + if index_dir.is_dir(): + for cache_path in index_dir.glob("*.json"): + try: + content = json.loads(cache_path.read_text(encoding="utf-8")) + if content.get("source") == source.name and "prefix" in content and "highest" in content: + pfx = content["prefix"] + cur_highest = content["highest"] + new_highest = max(cur_highest, added_prefixes.pop(pfx, 0)) + save_sprint_highwater(source, pfx, new_highest, sig) + except (OSError, ValueError, TypeError, KeyError): + continue + + for pfx, highest in added_prefixes.items(): + save_sprint_highwater(source, pfx, highest, sig) def sprint_highwater(source: Path, prefix: str, *, max_rebuild_bytes: int | None = None) -> int: @@ -101,23 +170,5 @@ def sprint_highwater(source: Path, prefix: str, *, max_rebuild_bytes: int | None highest = record_highwater(records(data), prefix) if signature(source) != before: raise SourceChangedError("Sprint changed while rebuilding allocator identity evidence") - value = {"version": 1, "signature": before, "prefix": prefix, "source": source.name, "highest": highest} - value["checksum"] = _digest(value) - temporary = None - try: - cache.parent.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile(mode="w", dir=cache.parent, delete=False) as stream: - temporary = Path(stream.name) - json.dump(value, stream) - stream.flush() - os.fsync(stream.fileno()) - os.replace(temporary, cache) - except OSError: - pass # Derived cache failure must not prevent allocation. - finally: - if temporary is not None: - try: - temporary.unlink(missing_ok=True) - except OSError: - pass + save_sprint_highwater(source, prefix, highest, before) return highest diff --git a/planfile/core/store.py b/planfile/core/store.py index 0d106dca..8a18238c 100644 --- a/planfile/core/store.py +++ b/planfile/core/store.py @@ -2654,6 +2654,8 @@ def _create_tickets_bulk_unlocked(self, tickets: list[Ticket]) -> list[Ticket]: self._write_yaml_atomic(sprint_file, data, allow_unicode=True) if hasattr(self, "_yaml_cache"): self._yaml_cache.pop(str(sprint_file), None) + from planfile.core.allocator_sprints import update_sprint_highwater_cache + update_sprint_highwater_cache(sprint_file, sprint_tickets) self._append_operational_lines( [str(ticket.dsl) for ticket in tickets if ticket.dsl] @@ -2795,6 +2797,8 @@ def _create_ticket_unlocked(self, ticket: Ticket) -> Ticket: # Invalidate yaml cache if hasattr(self, "_yaml_cache"): self._yaml_cache.pop(str(sprint_file), None) + from planfile.core.allocator_sprints import update_sprint_highwater_cache + update_sprint_highwater_cache(sprint_file, ticket) archive_report = ( self._archive_completed_unlocked() @@ -3606,6 +3610,8 @@ def delete_tickets_bulk(self, ticket_ids: list[str]) -> tuple[list[str], list[st self._write_yaml_atomic(sprint_file, data, allow_unicode=True) if hasattr(self, "_yaml_cache"): self._yaml_cache.pop(str(sprint_file), None) + from planfile.core.allocator_sprints import update_sprint_highwater_cache + update_sprint_highwater_cache(sprint_file, sprint_tickets) from planfile.core.operational_dsl import line as operational_line for ticket_id in deleted: diff --git a/project/TICKETS.md b/project/TICKETS.md index 4306e4a8..e29fc205 100644 --- a/project/TICKETS.md +++ b/project/TICKETS.md @@ -173,4 +173,5 @@ This file indexes governance tickets without taking ownership of | **ticket-272** | [`README.md`](./ticket-272/README.md) | - | - | - | - | - | | **ticket-273** | [`README.md`](./ticket-273/README.md) | - | - | - | - | - | | **ticket-274** | [`README.md`](./ticket-274/README.md) | - | - | - | - | - | +| **ticket-275** | [`README.md`](./ticket-275/README.md) | - | - | - | - | - | diff --git a/project/ticket-277/README.md b/project/ticket-277/README.md new file mode 100644 index 00000000..9d41b593 --- /dev/null +++ b/project/ticket-277/README.md @@ -0,0 +1,28 @@ +# Ticket 277: update allocator highwater cache on ticket creation to prevent lock timeout on files > 5MB + +- **ID**: ticket-277 +- **Owner**: agent:antigravity +- **Status**: IN_PROGRESS +- **Workflow state**: VALIDATION +- **Created**: 2026-11-03 + +## Goal and scope + +Prevent locked rebuild timeouts and `allocator_evidence_changed_retry` errors on large sprint files (> 5MB) by atomically updating the allocator highwater cache on ticket persistence and increasing `MAX_LOCKED_REBUILD_BYTES` to 50MB. + +## Acceptance criteria + +- [x] AC-01: Ticket creation atomically maintains allocator highwater cache or supports large sprint files under lock +- [x] AC-02: Sequential ticket creation on sprints > 5MB succeeds without crashing with allocator_evidence_changed_retry +- [x] AC-03: Tests in tests/test_allocator_preparation.py and tests/test_allocator_dedupe.py pass + +## Implementation details + +- `planfile/core/allocator_sprints.py`: + - Increased `MAX_LOCKED_REBUILD_BYTES` to 50,000,000 (50MB). + - Factored out `save_sprint_highwater` helper. + - Implemented `update_sprint_highwater_cache` to preserve existing prefix caches with the new file signature and update highwater marks for newly added tickets. +- `planfile/core/store.py`: + - Called `update_sprint_highwater_cache` after atomic YAML write in `_create_ticket_unlocked` and `_create_tickets_bulk_unlocked`. +- `tests/test_allocator_preparation.py`: + - Added `test_sequential_ticket_creation_on_large_sprint_succeeds_without_retry` verifying sequential ticket creations succeed with small `MAX_LOCKED_REBUILD_BYTES`. diff --git a/project/ticket-277/intent.json b/project/ticket-277/intent.json new file mode 100644 index 00000000..bab36f76 --- /dev/null +++ b/project/ticket-277/intent.json @@ -0,0 +1,26 @@ +{ + "schema": "new-project.intent/v3", + "ticket": "ticket-277", + "summary": "update allocator highwater cache on ticket creation to prevent lock timeout on files > 5MB", + "workstream": "application", + "classification": { + "kind": "SERVICE", + "priority": "P2", + "origin": "health" + }, + "allowedPaths": [ + "project/ticket-277/**", + "TODO.md", + "project/TICKETS.md", + "planfile/core/allocator_sprints.py", + "planfile/core/store.py", + "tests/test_allocator_preparation.py" + ], + "forbiddenPaths": [ + "project/ticket-*/user-*.md" + ], + "stacks": [], + "dependsOn": [], + "conflictsWith": [], + "integrationTicket": null +} diff --git a/tests/test_allocator_preparation.py b/tests/test_allocator_preparation.py index 4b2b35dd..9819cc6d 100644 --- a/tests/test_allocator_preparation.py +++ b/tests/test_allocator_preparation.py @@ -185,3 +185,19 @@ def invalid(text): with pytest.raises(RuntimeError, match="invalid source"): store.next_id() assert store._read_config() == before + + +def test_sequential_ticket_creation_on_large_sprint_succeeds_without_retry(tmp_path, monkeypatch): + pf = Planfile(tmp_path) + pf.store.init() + t1 = pf.create_ticket("First ticket") + assert t1.id == "PLF-001" + sprint_file = pf.store._sprint_file("current") + sprint_file.write_text(sprint_file.read_text() + "\n# " + "x" * 200) + evidence.sprint_highwater(sprint_file, "PLF") + monkeypatch.setattr(evidence, "MAX_LOCKED_REBUILD_BYTES", 100) + t2 = pf.create_ticket("Second ticket") + assert t2.id == "PLF-002" + t3 = pf.create_ticket("Third ticket") + assert t3.id == "PLF-003" + assert pf.store.next_id() == "PLF-004" From 012d6bf74157cdf44a850f3c01783fc232ff3fb6 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Tue, 3 Nov 2026 10:19:28 +0100 Subject: [PATCH 2/3] fix(allocator): scan ticket descriptions for markers and fix bulk delete call (ticket-277) --- planfile/core/allocator_sprints.py | 39 +++++++++++++++++++----------- planfile/core/store.py | 2 +- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/planfile/core/allocator_sprints.py b/planfile/core/allocator_sprints.py index 517ffd31..21373d43 100644 --- a/planfile/core/allocator_sprints.py +++ b/planfile/core/allocator_sprints.py @@ -95,7 +95,7 @@ def save_sprint_highwater(source: Path, prefix: str, highest: int, sig: list[int pass -def update_sprint_highwater_cache(source: Path, tickets: list[Any] | Any) -> None: +def update_sprint_highwater_cache(source: Path, tickets: list[Any] | Any = ()) -> None: """Update on-disk allocator highwater cache after writing sprint file. Preserves valid cached prefixes with the new file signature and updates @@ -105,17 +105,25 @@ def update_sprint_highwater_cache(source: Path, tickets: list[Any] | Any) -> Non sig = signature(source) except OSError: return - ticket_list = tickets if isinstance(tickets, list) else [tickets] - pattern = re.compile(r"^([A-Za-z0-9_-]+)-(\d+)$") - added_prefixes: dict[str, int] = {} + ticket_list = tickets if isinstance(tickets, list) else ([tickets] if tickets else []) + recs = [] for item in ticket_list: - tid = getattr(item, "id", None) if hasattr(item, "id") else (item.get("id") if isinstance(item, dict) else str(item)) - if tid: - m = pattern.fullmatch(str(tid)) - if m: - pfx, num = m.group(1), int(m.group(2)) - added_prefixes[pfx] = max(added_prefixes.get(pfx, 0), num) - + if isinstance(item, dict): + recs.append(item) + elif hasattr(item, "model_dump"): + recs.append(item.model_dump(mode="json", exclude_none=True)) + elif hasattr(item, "__dict__"): + recs.append(item.__dict__) + + marker = re.compile(r"([A-Za-z0-9_-]+)-(\d+)") + all_prefixes: set[str] = set() + for r in recs: + for val in (r.get("id"), *(r.get("labels") or []), r.get("description")): + if val and isinstance(val, str): + for m in marker.finditer(val): + all_prefixes.add(m.group(1)) + + updated_prefixes: set[str] = set() index_dir = source.parent / "index" / "allocator-sprints" if index_dir.is_dir(): for cache_path in index_dir.glob("*.json"): @@ -124,13 +132,16 @@ def update_sprint_highwater_cache(source: Path, tickets: list[Any] | Any) -> Non if content.get("source") == source.name and "prefix" in content and "highest" in content: pfx = content["prefix"] cur_highest = content["highest"] - new_highest = max(cur_highest, added_prefixes.pop(pfx, 0)) + added_high = record_highwater(recs, pfx) if recs else 0 + new_highest = max(cur_highest, added_high) save_sprint_highwater(source, pfx, new_highest, sig) + updated_prefixes.add(pfx) except (OSError, ValueError, TypeError, KeyError): continue - for pfx, highest in added_prefixes.items(): - save_sprint_highwater(source, pfx, highest, sig) + for pfx in all_prefixes - updated_prefixes: + high = record_highwater(recs, pfx) + save_sprint_highwater(source, pfx, high, sig) def sprint_highwater(source: Path, prefix: str, *, max_rebuild_bytes: int | None = None) -> int: diff --git a/planfile/core/store.py b/planfile/core/store.py index 8a18238c..f9ae6942 100644 --- a/planfile/core/store.py +++ b/planfile/core/store.py @@ -3611,7 +3611,7 @@ def delete_tickets_bulk(self, ticket_ids: list[str]) -> tuple[list[str], list[st if hasattr(self, "_yaml_cache"): self._yaml_cache.pop(str(sprint_file), None) from planfile.core.allocator_sprints import update_sprint_highwater_cache - update_sprint_highwater_cache(sprint_file, sprint_tickets) + update_sprint_highwater_cache(sprint_file) from planfile.core.operational_dsl import line as operational_line for ticket_id in deleted: From ea70473a87d8ff4af130042d7904cbd2898dac3e Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Tue, 3 Nov 2026 10:43:43 +0100 Subject: [PATCH 3/3] style(allocator): format imports in allocator_sprints to satisfy ruff (ticket-277) --- planfile/core/allocator_sprints.py | 1 - 1 file changed, 1 deletion(-) diff --git a/planfile/core/allocator_sprints.py b/planfile/core/allocator_sprints.py index 21373d43..7fb73930 100644 --- a/planfile/core/allocator_sprints.py +++ b/planfile/core/allocator_sprints.py @@ -7,7 +7,6 @@ import re import tempfile from pathlib import Path - from typing import Any from planfile.core.fastio import load_yaml_text