From d771492b745957641a196441496a7c16dbb5cf79 Mon Sep 17 00:00:00 2001 From: sethigoldy <57476662+sethigoldy@users.noreply.github.com> Date: Sun, 27 Sep 2026 13:20:13 +0000 Subject: [PATCH] chore: sync hooks from memvara/memvara@6cd83bf6abb64a4f4a6140df4baf24d82102e76d --- hooks.lock | 2 +- hooks/daemon.py | 39 +++++++++++--- hooks/lib/deadline.py | 45 ++++++++++++++++ hooks/lib/fast.py | 31 +++++++++-- hooks/lib/hosted.py | 18 +++++++ hooks/lib/open.py | 15 +++++- hooks/lib/standing.py | 72 +++++++++++++++++++++++++ hooks/lib/write.py | 10 ++-- hooks/recall.py | 92 +++++++++++++++++++------------- hooks/session_start.py | 117 ++++++++++++++++++++++++++++++++--------- 10 files changed, 361 insertions(+), 80 deletions(-) create mode 100644 hooks/lib/deadline.py diff --git a/hooks.lock b/hooks.lock index e70b424..c5eaeaf 100644 --- a/hooks.lock +++ b/hooks.lock @@ -6,6 +6,6 @@ # no hooks.json here and nothing generates one; tools/generate.py refuses this host by # name for exactly that reason. repo=memvara/memvara -sha=026be5cfd815419fa4c7eda2cb0ada109ea22cab +sha=6cd83bf6abb64a4f4a6140df4baf24d82102e76d path=plugin/hooks host=opencode diff --git a/hooks/daemon.py b/hooks/daemon.py index 65631b6..44da262 100644 --- a/hooks/daemon.py +++ b/hooks/daemon.py @@ -253,15 +253,31 @@ def run(self) -> int: server.bind(self.path) except OSError: return 0 + # The file's identity, read straight after `bind()`, so that `_release` removes + # only this daemon's socket. It is read from the path because `fstat` on a socket + # does not describe the socket's file. A second daemon needs a failed `bind()`, an + # import and a refused probe before it can replace the file, which is far longer + # than the one call between `bind()` and this one. A file that is already gone + # belongs to that other daemon now, so this one leaves. try: - os.chmod(self.path, 0o600) + ours = os.stat(self.path) except OSError: - pass - self._sweep_stale() - + server.close() + return 0 + # Listen at once. A second daemon whose `bind()` fails probes this path, and takes + # a refused probe to mean the owner is dead: a socket that is bound and not yet + # listening refuses too, so listening only after the sweep below let a loser + # unlink this path and bind its own, leaving this daemon on a socket nothing + # could reach (#344). The socket is private before `chmod` as well, because its + # directory is 0700 (`lib.ipc.runtime_dir`). server.listen(16) - server.settimeout(30.0) try: + try: + os.chmod(self.path, 0o600) + except OSError: + pass + self._sweep_stale() + server.settimeout(30.0) while True: try: conn, _ = server.accept() @@ -280,10 +296,17 @@ def run(self) -> int: threading.Thread(target=self._serve, args=(conn,), daemon=True).start() finally: server.close() - try: + self._release(ours) + + def _release(self, ours: os.stat_result) -> None: + """Remove the socket file, if the path still names this daemon's socket. Another + daemon may have replaced it, and removing that daemon's socket would leave it + serving an address no client can reach.""" + try: + if os.path.samestat(os.stat(self.path), ours): os.unlink(self.path) - except OSError: - pass + except OSError: + pass def main() -> int: diff --git a/hooks/lib/deadline.py b/hooks/lib/deadline.py new file mode 100644 index 0000000..f2b3de9 --- /dev/null +++ b/hooks/lib/deadline.py @@ -0,0 +1,45 @@ +"""One deadline for the whole hook process, which every hosted call respects. + +The host stops a hook at its time limit: 10 seconds for recall and 20 for session start on +every host. The hosted client waits up to `hosted.TIMEOUT_SEC` for each request and retries +a request that got no answer once, and one hook makes several calls, so an endpoint that +accepted connections and never answered kept session start for about 60 seconds and recall +for about 36 (#345). The host killed the hook first, so the turn got no memories and not +even the status line that would have said why. + +A reading hook sets the deadline once: `MARGIN_SEC` seconds before its host's limit, which +leaves that long for writing its reply. Every hosted call then waits at most the time left, and none is started or retried +once it is spent. A process that sets no deadline, such as the daemon or the capture hook, +waits as it always has. + +Only `time` is imported, because recall imports this on every prompt. +""" + +from __future__ import annotations + +import time + +#: What a hook keeps back from its host's limit, for the work after its last hosted call: +#: writing the reply and its log lines. The interpreter's start is inside the host's limit +#: too, before the hook can set anything. +MARGIN_SEC = 1.5 + +_at: "float | None" = None + + +def set_from_limit(limit: float) -> None: + """Set the deadline to `MARGIN_SEC` seconds before a limit of `limit` seconds from + now. With a 10-second limit, the deadline is 8.5 seconds from now.""" + global _at + _at = time.monotonic() + max(0.0, limit - MARGIN_SEC) + + +def left() -> "float | None": + """Seconds until the deadline, which may be 0 or less, or None when none is set.""" + return None if _at is None else _at - time.monotonic() + + +def clear() -> None: + """Remove the deadline.""" + global _at + _at = None diff --git a/hooks/lib/fast.py b/hooks/lib/fast.py index f82d337..d9a99bc 100644 --- a/hooks/lib/fast.py +++ b/hooks/lib/fast.py @@ -51,6 +51,7 @@ import sys import time +from . import deadline from .ipc import CLIENT_TIMEOUT_SEC, log_line, send, socket_path, store_key #: Set in a spawned daemon's environment so a daemon can never spawn a daemon. @@ -197,6 +198,11 @@ def _spawn(root: str) -> None: DAILY = "daily" +#: The reason token for a local store that is configured and could not open, followed by +#: the exception's class after a colon, such as `open:DatabaseError`. +OPEN = "open" + + def _reason(exc: "BaseException") -> str: """The short token for a failure, or `""` when there is nothing useful to add. @@ -254,7 +260,9 @@ def recall(query: str, *, k: int = 6, budget: int = 700, header: str | None = No The third slot is `reason`: `""` when there is nothing to add, else a short token the caller can turn into words: `"quota"` for a spent monthly allowance and `"daily"` for a - spent daily one, each with its reset after a colon when the refusal said. It exists + spent daily one, each with its reset after a colon when the refusal said, and + `"open:"` for a local store that is configured and could not open, + which comes with `ok=False` where "nothing configured" comes with `ok=None`. It exists because `False` alone sent a user to read a log about a store that was answering perfectly and telling him, in the body of a 402, exactly which allowance was spent and when it resets. @@ -276,6 +284,16 @@ def recall(query: str, *, k: int = 6, budget: int = 700, header: str | None = No except Exception: path = None + # The hook's deadline bounds the daemon's wait as it bounds a hosted call's (see + # `lib.deadline`): a hook that used most of its time on earlier calls must not spend + # this wait on top, and one that used all of it goes straight to the fallbacks below. + left = deadline.left() + if left is not None and left <= 0: + path = None + if left is not None: + # The same for the in-process rewrite below, which waits `rewrite_wait` for a model. + rewrite_wait = min(rewrite_wait, max(left, 0.0)) + if path is not None: request = {"q": query, "k": k, "budget": budget} if min_score: @@ -294,6 +312,8 @@ def recall(query: str, *, k: int = 6, budget: int = 700, header: str | None = No # plain read. request["query_rewrite"] = True wait = rewrite_wait if query_rewrite else CLIENT_TIMEOUT_SEC + if left is not None: + wait = min(wait, left) began = _clock() answer = send(path, request, timeout=wait) served = _served(answer) @@ -318,8 +338,13 @@ def recall(query: str, *, k: int = 6, budget: int = 700, header: str | None = No client = open_hosted() if client is None: - # Nothing is configured at all -- no local database, no library to read one - # with, and no credentials file. Distinct from a store that would not answer. + # No hosted store either. That is "nothing configured" -- no local database, no + # library to read one with, and no credentials file -- only when the local + # store was not configured, rather than configured and unable to open (#337). + from . import open as opener # noqa: PLC0415 - already imported by _local_store + + if opener.failure is not None: + return "", False, f"{OPEN}:{type(opener.failure).__name__}" return "", None, "" try: # No `query_rewrite` here, whatever the caller asked: the hosted client always diff --git a/hooks/lib/hosted.py b/hooks/lib/hosted.py index 07be09b..a3b47b6 100644 --- a/hooks/lib/hosted.py +++ b/hooks/lib/hosted.py @@ -38,6 +38,7 @@ import os.path import ssl +from . import deadline from .ipc import log_line from .project import ENV as PROJECT_ENV @@ -243,9 +244,26 @@ def _rpc(self, method: str, params: "dict | None" = None, if project: headers[PROJECT_HEADER] = project + # The hook's deadline, when one is set (see `lib.deadline`): no call starts once it + # has passed, and none waits longer than the time left. A call not made is the + # same as a call not answered, so the caller reports the store as not answering. + left = deadline.left() + if left is not None and left <= 0: + return None + try: if self._conn is None: self._conn = self._connect() + if left is not None: + # A kept-alive connection has the timeout it was made with, and the time + # left is shorter now. Only with a deadline: otherwise the timeout is the + # one the connection already has, and setting it again costs a system call + # on every request of a daemon that lives for half an hour. + wait = min(TIMEOUT_SEC, left) + self._conn.timeout = wait + sock = getattr(self._conn, "sock", None) + if sock is not None: + sock.settimeout(wait) self._conn.request("POST", MCP_PATH, json.dumps(body), headers) response = self._conn.getresponse() raw = response.read() diff --git a/hooks/lib/open.py b/hooks/lib/open.py index 6cfb7db..d3d6e0c 100644 --- a/hooks/lib/open.py +++ b/hooks/lib/open.py @@ -27,6 +27,13 @@ #: Written by `memvara-mcp login`, read when there is no local store to open. _CREDENTIALS = Path.home() / ".memvara" / "credentials.json" +#: Why the last `open_store` call could not open the local store it was configured to +#: open, or None. `open_store` still answers None then, as it does when nothing is +#: configured, so the hooks read this to tell the two apart: a store that is configured +#: and cannot open is a failure, and reporting it as "not configured" sent a person +#: looking for configuration that was there (#337). +failure: "BaseException | None" = None + def _import_memvara(env: Mapping[str, str]) -> Any: """Import the library, honouring a PYTHONPATH that only the server block knows. @@ -57,6 +64,8 @@ def open_store() -> Any | None: for a second client to be better at. It briefly took a `recalls` flag, when the MCP surface could not carry `sources=` and the library's client could. """ + global failure + failure = None # The client's block loses to a real environment variable; `client_env` is where that # rule is written, for this function, the daemon's address and the rewrite decision. env = client_env() @@ -112,8 +121,10 @@ def open_store() -> Any | None: # degrades to the route that works instead of to a silent outage. return None return build_memvara(config) - except Exception: + except Exception as exc: # Deliberately bare. ConfigError, ImportError, EmbedderMismatchError, a corrupt # SQLite file and a revoked API key are all the same event from here: no memory - # this turn. + # this turn. Kept in `failure`, because it is not the same event as having no + # store configured, and the hooks say which it was. + failure = exc return None diff --git a/hooks/lib/standing.py b/hooks/lib/standing.py index 98e9bfe..e846df4 100644 --- a/hooks/lib/standing.py +++ b/hooks/lib/standing.py @@ -38,12 +38,16 @@ from __future__ import annotations +import hashlib +import os import re import unicodedata from typing import Any, Callable, NamedTuple +from . import state_file from .mark import marked from .mark import on as mark_on +from .mark import unmark_block #: A row of the "Believed now, not believed then" half of a `memory_since` reply. The other #: half is claims the store STOPPED believing, and parsing it into the standing set would @@ -374,3 +378,71 @@ def standing_block(store: Any, *, hosted: bool, budget: int, header: str, return str(fallback() or "") except Exception: return "" + + +# -- what the session has already seen ---------------------------------------------------- + +def seen_dir() -> str: + """Where the recall hook keeps each session's state, `recall.SEEN_DIR`. + + Worked out when called rather than at import, so that session start writes under the + home directory it runs with. + """ + return os.path.join(os.path.expanduser("~"), ".memvara", ".hooks", "recalled") + + +def state_path(directory: str, session: str) -> "str | None": + """The state file of `session` under `directory`, or None for an id that cannot name + one. A NUL byte makes every `os` call raise `ValueError`, not the `OSError` the state + functions are written to absorb, so such an id gets no state file at all.""" + if not session or "/" in session or "\0" in session or session in (".", ".."): + return None + return os.path.join(directory, f"{session}.json") + + +def fingerprint(text: str) -> str: + """A short hash of `text` with its whitespace collapsed. Recall's dedup hashes each + memory line with it, and `digest` hashes the standing block with it.""" + return hashlib.sha256(" ".join(text.split()).encode("utf-8")).hexdigest()[:16] + + +def digest(block: str) -> str: + """The digest the recall hook compares to tell whether the standing block changed. + + Taken over the block without the recall mark. The mark is presentation, and hashing it + made every running session report "standing preferences updated" once after an upgrade + and again each time the `recall_mark` switch changed. + """ + return fingerprint(unmark_block(block)) + + +def normalised(data: object) -> dict: + """A session's state file contents as a dict. A bare list is the older format of the + file, which held only the seen hashes.""" + if isinstance(data, list): + return {"seen": [h for h in data if isinstance(h, str)]} + return data if isinstance(data, dict) else {} + + +def record_injected(session: str, block: str, now: float, + directory: "str | None" = None) -> None: + """Record in `session`'s recall state that `block` was injected at `now`. + + Session start calls this after it injects the standing block. The recall hook checks + the block again once `recall.STANDING_REFRESH_SECONDS` have passed since the time + recorded here, and injects it only when its digest differs. With nothing recorded, the + first prompt of every session found the check due and the digest different, and + injected the whole block a second time under "standing preferences updated" (#343). + Every other key in the state file is kept. Nothing here raises: a failure costs one + repeated block, which is less than a failed session start. + """ + directory = directory or seen_dir() + path = state_path(directory, session) + if path is None: + return + + def change(raw: object) -> dict: + return {**normalised(raw), "standing": digest(block), "standing_at": now} + + state_file.update_json(path, change, lock_path=os.path.join(directory, ".lock"), + prefix=".recalled-") diff --git a/hooks/lib/write.py b/hooks/lib/write.py index 2693c1b..4098006 100644 --- a/hooks/lib/write.py +++ b/hooks/lib/write.py @@ -198,10 +198,12 @@ def remember_kwargs(memory_type: "str | None", turn: str, hosted: bool, """ kwargs: dict = {"confidence": 0.7} if memory_type: - # The hosted tool takes the type's name. The local library takes its enum, and a - # plain string fails there with `AttributeError: 'str' object has no attribute - # 'value'` when the claim is stored. Every fact this hook wrote to a local store - # failed that way, and capture.log recorded each one under `failed=`. + # The hosted tool takes the type's name. The local library takes its enum, and + # before #270 a plain string failed there with `AttributeError: 'str' object has + # no attribute 'value'` when the claim was stored, so every fact this hook wrote + # to a local store failed and capture.log recorded each one under `failed=`. The + # library converts a name itself now; converting here as well keeps the hook + # working with an installed library older than that fix. kwargs["memory_type"] = memory_type if hosted else _memory_type(memory_type) # The label the host writes under. It is stored on every claim and rendered back # by `memory_why`, so it is a fact about recorded history rather than a string to diff --git a/hooks/recall.py b/hooks/recall.py index 5df9941..128b71b 100644 --- a/hooks/recall.py +++ b/hooks/recall.py @@ -60,7 +60,6 @@ from __future__ import annotations -import hashlib import json import os.path import time @@ -72,7 +71,7 @@ from core.envelope import read_event, write # noqa: E402 from core.host import Reply, active # noqa: E402 -from lib import counts, state_file # noqa: E402 +from lib import counts, deadline, state_file # noqa: E402 from lib.fast import REWRITE_WAIT_SEC # noqa: E402 from lib.fast import recall as fast_recall # noqa: E402 from lib.ipc import ( # noqa: E402 @@ -82,9 +81,10 @@ from lib.mark import count as count_memories # noqa: E402 from lib.mark import marked # noqa: E402 from lib.mark import on as mark_on # noqa: E402 -from lib.mark import unmark_block # noqa: E402 from lib.project import bind as bind_project # noqa: E402 from lib.read_model import allowed as rewrite_allowed # noqa: E402 +from lib.standing import digest as standing_digest # noqa: E402 +from lib.standing import fingerprint, normalised, seen_dir, state_path # noqa: E402 #: The client this process is answering, resolved once. `run.py` binds it before importing #: this module; a bare `python3 recall.py` gets Claude Code, which is what that invocation @@ -204,17 +204,12 @@ #: this hook exists to make and runs regardless of elapsed time; skipping it to stay inside #: a budget would be answering the timeout by not doing the hook's own job. #: -#: What this does NOT close: it is checked before starting a call, not while one is -#: already running, so it stops a SECOND slow call from compounding a first one but cannot -#: shorten a call already in flight. On a fresh process the connection cache above is -#: empty, so whichever hosted call happens to run first -- the standing refresh, if its own -#: 15-minute interval is due, or otherwise the primary call itself -- gets no benefit from -#: it and can still cost the full worst case on its own. If that first call is the standing -#: refresh, in the worst case it alone can outlast this hook's entire 10s allowance before -#: the primary call the budget was written to protect ever starts. Closing that fully would -#: mean bounding the DURATION of an in-flight call -- a deadline enforced inside -#: `lib.hosted` itself, shared by every caller of it, not a clock kept in this one file -- -#: which is a deeper change than a wall-clock gate on whether to start a second one. +#: It is checked before starting a call, not while one is running, so on its own it could +#: not shorten a call already in flight: the standing refresh alone could outlast the whole +#: 10s allowance before the primary call ever started. `lib.deadline` closes that. `main` +#: sets it from the host's limit, and every hosted call and the daemon's wait stop at it +#: (#345). This budget still decides whether to START optional work, and is kept below the +#: deadline so that the primary call has time left when the optional work is done. OVERALL_BUDGET_SEC = 7.5 #: Prompts that are not questions to the model: a slash command, a bash escape, a comment. @@ -258,7 +253,7 @@ #: Where the per-session record of what has already been injected lives. Beside the store, #: not in the plugin, which is replaced wholesale on update. -SEEN_DIR = os.path.join(os.path.expanduser("~"), ".memvara", ".hooks", "recalled") +SEEN_DIR = seen_dir() #: Enough to cover a long session without the file becoming something that needs managing. MAX_SEEN = 500 @@ -303,6 +298,21 @@ #: not crowd out the words the user actually typed this turn. MAX_CARRY_CHARS = 300 +#: The longest prompt recall reads. A longer one keeps its first and last half of this +#: many characters. The store's time grows with the query, about 2 to 3 seconds a +#: megabyte on a laptop, so a pasted log file of a few megabytes ran past the host's +#: 10-second limit and the turn got no memories at all (#348). A question is at the start +#: or the end of what someone pastes, and retrieval gains nothing from the middle. +MAX_PROMPT_CHARS = 8000 + + +def _bounded(prompt: str) -> str: + """`prompt`, or its first and last `MAX_PROMPT_CHARS // 2` characters when longer.""" + if len(prompt) <= MAX_PROMPT_CHARS: + return prompt + half = MAX_PROMPT_CHARS // 2 + return f"{prompt[:half]}\n{prompt[-half:]}" + #: The leading clause is load-bearing beyond its wording: `transcript.RECALL_MARKERS` #: matches on it to keep an injected block out of the text that gets mined. Change the #: clause and the block starts being read back as conversation -- see @@ -320,7 +330,7 @@ def _digest(line: str) -> str: Hashing the marked line instead would make every memory a session had already seen look new on the first prompt after the upgrade, and inject all of them again. """ - return hashlib.sha256(" ".join(line.split()).encode("utf-8")).hexdigest()[:16] + return fingerprint(line) def _count_recalled(session: str, n: int) -> None: @@ -330,11 +340,7 @@ def _count_recalled(session: str, n: int) -> None: def _seen_path(session: str) -> "str | None": - # A NUL byte makes every `os` call raise `ValueError`, not the `OSError` the state - # functions below are written to absorb, so such an id gets no state file at all. - if not session or "/" in session or "\0" in session or session in (".", ".."): - return None - return os.path.join(SEEN_DIR, f"{session}.json") + return state_path(SEEN_DIR, session) def _state_json(session: str) -> dict: @@ -351,14 +357,9 @@ def _state_json(session: str) -> dict: data = json.load(fh) except (OSError, ValueError): return {} - return _normalised(data) + return normalised(data) -def _normalised(data: object) -> dict: - """A state file's contents as a dict, reading the old bare-list format too.""" - if isinstance(data, list): - return {"seen": [h for h in data if isinstance(h, str)]} - return data if isinstance(data, dict) else {} def _read_state(session: str) -> "tuple[list[str], str]": @@ -419,7 +420,7 @@ def _write_state(session: str, hashes: "list[str]", query: str, return def change(raw: object) -> dict: - was = _normalised(raw) + was = normalised(raw) kept = set(hashes) earlier = [h for h in was.get("seen") or [] if isinstance(h, str) and h not in kept] was_digest = was.get("standing") @@ -629,10 +630,9 @@ def _standing_refresh(session: str, now: float, cwd: str = "") -> "tuple[str, tu # the next one either. return "", (digest, now) - # Hashed without the recall mark, as `_digest` promises: the mark is presentation, and - # hashing it made every running session report "standing preferences updated" once - # after the upgrade and again each time the `recall_mark` switch changed. - fresh = _digest(unmark_block(block)) + # The same digest session start records when it injects the block, so the first + # prompt of a session finds it unchanged. + fresh = standing_digest(block) if not block.strip() or fresh == digest: return "", (digest or fresh, now) return block.rstrip(), (fresh, now) @@ -753,7 +753,7 @@ def _belongs_here(bullet: str, cwd: str) -> bool: node = parent -def main() -> int: +def _main() -> int: # The clock the optional hosted work below is measured against -- see # OVERALL_BUDGET_SEC. `monotonic`, not `time.time()`: this is an ELAPSED-time budget, # and `daemon.py` already uses `time.monotonic()` for its own idle-timeout for the same @@ -764,6 +764,9 @@ def main() -> int: # them so it covers the whole invocation, even though nothing before the first hosted # call is expensive enough to matter in practice. start = time.monotonic() + # Before any hosted call: every one of them stops at this, so the hook answers inside + # its host's limit however the endpoint behaves (#345). + deadline.set_from_limit(HOST.timeouts.get("recall", 10)) if under_extraction(): # The prompt in front of us is `capture.py`'s own extraction request, not a @@ -779,7 +782,12 @@ def main() -> int: # matters because the miss is silent: the dedup file is keyed on session, so a renamed # key re-injects every memory on every turn while every banner still reads healthy. event = read_event(HOST, "recall", payload()) - prompt = event.prompt.strip() + # Half of a surrogate pair is dropped. A client written in JavaScript can send one, an + # emoji cut in two, and JSON.stringify escapes it as \ud83d, which Python decodes into + # a string that cannot be encoded. The store hashes each query with `text.encode()`, + # so the whole recall failed on it although the store was healthy (#347). Half a + # character carries nothing a search can use. + prompt = _bounded(event.prompt.encode("utf-8", "ignore").decode("utf-8").strip()) session = event.session if not prompt or prompt.startswith(SKIP_PREFIXES): @@ -898,11 +906,14 @@ def _emit(reply: Reply) -> None: # retrying without it (`lib.hosted.HostedRecall.recall`). if time.monotonic() - start < OVERALL_BUDGET_SEC: try: - # Plain: a rewrite here would be a second model call on one prompt. + # Plain: a rewrite here would be a second model call on one prompt. And no + # daemon: the first read already started one when none was serving, and it + # is usually not listening yet, so a second start would be a second daemon + # for the same store (#344). wider, wider_ok, _ = fast_recall(query, k=EPISODE_K, budget=EPISODE_BUDGET, header=HEADER, include_episodes=True, min_score=_min_score(), - query_rewrite=False) + query_rewrite=False, spawn=False) except Exception: wider, wider_ok = "", False if wider_ok and wider: @@ -968,5 +979,14 @@ def _emit(reply: Reply) -> None: return 0 +def main() -> int: + """Run the hook, then clear the deadline it set, for a caller that runs it in this + process and goes on to make hosted calls of its own.""" + try: + return _main() + finally: + deadline.clear() + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/hooks/session_start.py b/hooks/session_start.py index 49161df..fb7d91b 100644 --- a/hooks/session_start.py +++ b/hooks/session_start.py @@ -30,6 +30,7 @@ from __future__ import annotations import sys +import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -37,16 +38,16 @@ from core.envelope import read_event, write # noqa: E402 from core.host import Reply, active # noqa: E402 from lib.ipc import ( # noqa: E402 - due_capture_alert, payload, plural, status, under_extraction, with_alert, + due_capture_alert, log_line, payload, plural, status, under_extraction, with_alert, ) -from lib import counts, project # noqa: E402 +from lib import counts, deadline, project # noqa: E402 from lib.agentic import sweep_configs as sweep_capture_configs # noqa: E402 -from lib.fast import read_kinds # noqa: E402 +from lib.fast import OPEN, read_kinds # noqa: E402 from lib.mark import count as count_memories # noqa: E402 from lib.mark import mark_block # noqa: E402 from lib.mark import on as mark_on # noqa: E402 from lib.project import bind as bind_project # noqa: E402 -from lib.standing import standing_block # noqa: E402 +from lib.standing import record_injected, standing_block # noqa: E402 from lib.write import open_writer # noqa: E402 #: Wider than the per-prompt hook: this runs once per session, not once per turn. @@ -104,8 +105,13 @@ def _why(exc: "BaseException") -> str: return "notes unavailable (quota)" if getattr(exc, "code", "") == "quota_exhausted" else "" +#: What the status line calls each section when it did not arrive. +_SECTIONS = {"binding": "scope", "standing": "standing preferences", "notes": "notes"} + + def _local_binding(store: object) -> str: - """The binding line from a library handle, or '' if it cannot be read. + """The binding line from a library handle. A store that cannot be read raises, so that + `main` can say the store did not answer rather than that it is empty. `scope` means two different things on the two classes this can be handed. On a `ScopedMemvara` it is the bound `Scope`; on a bare `Memvara` it is the *method* that @@ -113,13 +119,10 @@ def _local_binding(store: object) -> str: this wrong is silent — the attribute exists either way — which is why it is resolved explicitly rather than by a `try` that would swallow the difference. """ - try: - scope_attr = getattr(store, "scope") - scoped = store if not callable(scope_attr) else store.scope() # type: ignore[operator] - scope = scoped.scope.key() - visible = scoped.count() - except Exception: - return "" + scope_attr = getattr(store, "scope") + scoped = store if not callable(scope_attr) else store.scope() # type: ignore[operator] + scope = scoped.scope.key() + visible = scoped.count() return _binding_line(scope, f"{visible} claim(s)") @@ -127,12 +130,10 @@ def _hosted_binding(store: object) -> str: """The binding line from the hosted endpoint's own `memory_stats` report. The server already formats the scope and the count, so this reads them back rather than - deriving a second version that could disagree with the first. + deriving a second version that could disagree with the first. A failed call raises, so + that `main` can say the store did not answer rather than that it is empty. """ - try: - report = str(store.stats() or "") # type: ignore[attr-defined] - except Exception: - return "" + report = str(store.stats() or "") # type: ignore[attr-defined] scope, visible = "", "" for line in report.splitlines(): line = line.strip() @@ -160,7 +161,7 @@ def _binding_line(scope: str, visible: str) -> str: return line -def main() -> int: +def _main() -> int: if under_extraction(): # `claude -p` opens a session like any other, so this hook fired inside every # extraction and built the whole standing block for a child that was about to be @@ -184,6 +185,9 @@ def main() -> int: # valid banner and fail nothing. alert = due_capture_alert() host = active() + # Before any hosted call: every one of them stops at this, so the hook answers inside + # its host's limit however the endpoint behaves (#345). + deadline.set_from_limit(host.timeouts.get("session_start", 20)) def _emit(reply: Reply) -> None: if reply.status: @@ -194,7 +198,8 @@ def _emit(reply: Reply) -> None: # checkout, and `cwd` is how the second half is known. An unreadable payload gives "", # which `_mine` treats as "user notes only" -- the safe direction, since the failure it # avoids is carrying another project's instructions into this one. - cwd = read_event(host, "session_start", payload()).cwd + event = read_event(host, "session_start", payload()) + cwd = event.cwd # Before the store is opened: the hosted client sends this project with every call. bind_project(cwd) # Once per session rather than on every write: the per-session counters and the @@ -207,6 +212,16 @@ def _emit(reply: Reply) -> None: sweep_capture_configs() store, close = open_writer() if store is None: + # `open_writer` answers None both when nothing is configured and when a local + # store is configured and cannot open. The second is a failure, and saying "not + # configured" of it sent a person looking for configuration that was there (#337). + from lib import open as opener # noqa: PLC0415 + + if opener.failure is not None: + log_line("session_start", + f"failed reason={OPEN}:{type(opener.failure).__name__}") + _emit(Reply("session_start", status=status("recall failed"))) + return 0 _emit(Reply("session_start", status=status("not configured"))) return 0 @@ -223,25 +238,47 @@ def _emit(reply: Reply) -> None: #: What a section could not be fetched for, in words. Set before the `try` so that #: every path to the banner below has it, including the ones that leave early. missing = "" + #: `(section, exception)` for each section the store did not answer for. A store that + #: did not answer is not an empty one, and before this list the hook said "nothing + #: stored yet" of an endpoint it could not reach, and logged nothing (#339). + failed: "list[tuple[str, BaseException]]" = [] mark = mark_on() try: parts = [] - binding = _hosted_binding(store) if hosted else _local_binding(store) + try: + binding = _hosted_binding(store) if hosted else _local_binding(store) + except Exception as exc: + binding = "" + failed.append(("binding", exc)) if binding: parts.append(binding) + #: Whether the standing block came from the ranked read below. Recall's refresh + #: never uses that read, so its digest would differ from the full block the next + #: refresh builds, and that refresh would inject the block again as "updated". + legacy = False + def _legacy_standing() -> str: - return str(store.recall(QUERY, k=STANDING_K, - budget=STANDING_FALLBACK_TOKENS, - header=STANDING_HEADER, - memory_types=STANDING, **plain_read) or "") + nonlocal legacy + legacy = True + # `standing_block` catches every failure of its own routes and of this one, and + # answers "" for all of them, so a failure here is recorded before it is caught. + try: + return str(store.recall(QUERY, k=STANDING_K, + budget=STANDING_FALLBACK_TOKENS, + header=STANDING_HEADER, + memory_types=STANDING, **plain_read) or "") + except Exception as exc: + failed.append(("standing", exc)) + raise try: standing = standing_block(store, hosted=hosted, budget=STANDING_BUDGET, header=STANDING_HEADER, fallback=_legacy_standing, cwd=cwd) - except Exception: + except Exception as exc: standing = "" + failed.append(("standing", exc)) if standing.strip(): # Marked here as well as in `render`, because the legacy fallback returns the # server's own block, whose bullets carry no mark. Marking twice is harmless. @@ -258,18 +295,31 @@ def _legacy_standing() -> str: # memory. Measured on a spent quota: three sections and 15,324 characters # became two and 13,541, with the banner unchanged. notes, missing = "", _why(exc) + failed.append(("notes", exc)) if notes.strip(): parts.append(mark_block(notes.rstrip(), mark)) finally: if close is not None: close() + for section, exc in failed: + # On every host, as recall logs its failures, because most hosts show no status + # line and the log is the only account there is. The exception's class, which + # names the kind of failure, and nothing the store's reply said. + log_line("session_start", f"failed section={section} reason={type(exc).__name__}") + if failed and not missing: + # A session that opened without a section says which one, so a partial session is + # not mistaken for a whole one. A spent quota has already said it in its own words. + missing = " and ".join(dict.fromkeys(_SECTIONS[name] for name, _ in failed)) + missing += " unavailable" + if not parts: # "Nothing stored yet" is a claim about the store's contents. Only make it when # every section came back empty rather than unavailable -- otherwise a store that # is merely unreachable is reported as one that is empty, and nobody investigates # an empty store. - _emit(Reply("session_start", status=status(missing or "nothing stored yet"))) + empty = "recall failed" if failed else "nothing stored yet" + _emit(Reply("session_start", status=status(missing or empty))) return 0 count = count_memories("\n\n".join(parts)) @@ -279,8 +329,23 @@ def _legacy_standing() -> str: _emit(Reply("session_start", status=status(f"{opened} · {missing}" if missing else opened), context="\n\n".join(parts))) + if standing.strip() and not legacy and "recall" in host.events: + # After the reply is written, so only a block that was delivered is recorded. The + # recall hook reads this, and without it the first prompt injected the same block + # again (#343). Not on a host that runs no recall, such as Cursor: nothing there + # would read the record, and recall is what prunes these files. + record_injected(event.session, standing, time.time()) return 0 +def main() -> int: + """Run the hook, then clear the deadline it set, for a caller that runs it in this + process and goes on to make hosted calls of its own.""" + try: + return _main() + finally: + deadline.clear() + + if __name__ == "__main__": raise SystemExit(main())