diff --git a/memory/README.md b/memory/README.md index dbfcbd9..dd4c84c 100644 --- a/memory/README.md +++ b/memory/README.md @@ -2,113 +2,53 @@ Agent-agnostic session memory and private data sync for dotagents. -This directory makes AI session context and private skill data durable across -tools and machines without tying the layout to any one agent or memory provider. -Hook entrypoints capture compact session digests, sync selected memory facts into -the knowledge vault, and keep the derived search index current. +This directory is the **canonical upstream** of the memory layer: hook +entrypoints capture compact session digests, `rem` captures explicit facts, +consolidation is report-first, and `knowledge-sync` keeps the knowledge vault +git-synced. User repos deployed via `dotagents setup` carry their own copy of +`memory/lib`, `memory/hooks`, and `memory/tools`; `dotagents sync` builds the +tools and this repo carries the tested reference implementations. -`memory/` is the stable abstraction. The dependency-free basic tier writes -bounded Markdown digests directly; `memsearch` remains an optional indexed -provider behind the full hook pipeline. - -## Setup - -Choose a tier through the CLI: +## The rem workflow ```bash -dotagents setup --memory basic # default; Python 3 only -dotagents setup --memory off # no managed memory hooks -dotagents setup --memory memsearch # requires memsearch on PATH +rem add -src claude "prefers pnpm for Node work" # capture a candidate fact anywhere +rem dream # consolidation report (report-only) +rem dream --apply # collapse exact-duplicate records +rem search "quota preferences" # semantic search via memsearch +rem sync # flush the vault via knowledge-sync ``` -Basic memory appends compact digests under -`$KNOWLEDGE_DIR/sessions/YYYY-MM-DD.md` and injects bounded recent context at -session start. It does not invoke memsearch or persist raw transcripts. - -The memsearch tier registers the indexed SessionStart, Stop, and SessionEnd -pipeline. Configure its vault through `~/.agents/memsearch.conf`. +Candidates are inert until promoted into durable instructions — consolidation +is report-first by design. See [tools/rem/README.md](tools/rem/README.md). -## Dream review +## Setup -Run the dream pass manually to find exact repeated preferences or corrections, -duplicate session records, and conflicting copies: +Choose a tier through the CLI: ```bash -python3 ~/.agents/memory/lib/basic_memory.py dream \ - --output "$KNOWLEDGE_DIR/reviews/memory-dream-2026-08-19.json" -``` - -The output is a deterministic, review-only JSON artifact. The command reads -complete marker-delimited records under `sessions/` plus exact duplicate facts -in the legacy `sessions/knowledge.md` export. It does not scan -`profile/USER.md`, edit canonical memory, archive records, or delete anything. -The output path must be new and live under `$KNOWLEDGE_DIR/reviews/`. - -Candidates are deliberately conservative: - -- Repeated preferences and corrections require the same explicit statement in - at least two distinct session IDs. -- A stale duplicate requires an identical session block (ignoring trailing - whitespace) outside its unique UTC-dated canonical file. -- Legacy cleanup reports only exact facts repeated across distinct `## Sync` - records; it does not infer staleness from age. -- Reused session IDs with differing content are reported as conflicts, without - choosing a survivor. -- Assistant output, truncated statements, incomplete records, fuzzy semantic - matches, and age alone never produce candidates. - -Review the source coordinates and candidate IDs before making a separate, -explicit edit to canonical memory. There is intentionally no apply or delete -mode. - -## Canonical paths - -The knowledge vault path is set in `~/.agents/memsearch.conf` as `KNOWLEDGE_DIR`. -All skills and hooks resolve data paths relative to this variable: - -``` -$KNOWLEDGE_DIR/ # vault root (default: ~/Workspace/knowledge) - sessions/ # dated session digests (YYYY-MM-DD.md) - notes/ # handwritten notes - profile/ # USER.md, WORK.md - skills// # private data for shared skills +dotagents setup --memory basic # default; Python 3 only +dotagents setup --memory off # no managed memory hooks +dotagents setup --memory memsearch # indexed search; requires memsearch on PATH ``` -Do not hardcode `~/Workspace/knowledge` in code. Use `$KNOWLEDGE_DIR`. - -Skills with private data store it under `$KNOWLEDGE_DIR/skills//`. -Example: `$KNOWLEDGE_DIR/skills/jobs/opportunities.yaml`. - -## Design principles - -- No symlinks. One canonical location per data file. -- Reuse existing sync pipelines when generalizing. Do not add ad-hoc rsync, - cron, or new sync tooling for individual skills. -- The knowledge vault is the canonical store for all private data that needs - to be available across agents and machines. -- The search index (memsearch) is derived state. Markdown in the vault is - canonical. -- Keep lifecycle concepts generic: `session-start`, `stop`, `session-end`, sync. -- Avoid per-agent directories unless an integration boundary forces them. -- Prefer payload dispatch and small parser modules over duplicated hooks. -- Index durable summaries and curated vault content, not raw full transcripts. - -## Security - -The knowledge vault contains private data. It must not be pushed to any public -remote. Distribution is limited to local git and private sync between trusted -machines (Mac and VPS via the knowledge-sync tool). +| Tier | Behavior | Dependency | +|---|---|---| +| `off` | no managed memory hooks | none | +| `basic` | bounded session digests into the knowledge vault | Python 3 | +| `memsearch` | adds a derived search index over the vault | `memsearch` | ## Layout -- `hooks/`: executable lifecycle hook entrypoints. -- `lib/`: transcript digest, vault sync, and indexing helpers. -- `tools/`: small repo-owned operational helpers for the memory subsystem. -- `AGENTS.md`: local rules for keeping this area agent-agnostic. +- `hooks/` — lifecycle entrypoints (session start/end/stop) registered per harness +- `lib/` — Python implementation: `basic_memory.py` (digests, dream-pass parsing), + `sync.py` (Hermes memory ↔ vault), `safety.py` +- `tools/` — Go binaries built by `dotagents sync`: `rem`, `knowledge-sync` +- `tests/` — reference test suite -## Hook registration +## Relationship to user repos -Memory-tier selection and native hook registration live in the dotagents CLI, -not in copy-pasted setup snippets. Re-run `dotagents setup --memory ` to -change the managed hooks. Keep manual troubleshooting details in -`skills/dotagents/references/memory-sync.md`. +`dotagents` (this repo) is upstream: changes land here first, with tests. Your +`~/.agents` repository (created by `setup`) carries the deployed copy; `sync` +rebuilds the binaries whenever the sources change. Keep the two in sync by +porting changes here, then pulling in the user repo. diff --git a/memory/lib/basic_memory.py b/memory/lib/basic_memory.py index 0d54db8..4a922bc 100755 --- a/memory/lib/basic_memory.py +++ b/memory/lib/basic_memory.py @@ -186,7 +186,7 @@ def inline_messages(payload: dict[str, Any]) -> list[dict[str, Any]]: def read_transcript(path: Path) -> tuple[list[dict[str, Any]], datetime | None, str | None]: if not path.exists(): - raise HookError(f"transcript_path does not exist: {path}") + return [], None, None if not path.is_file(): raise HookError(f"transcript_path is not a file: {path}") diff --git a/memory/lib/sync.py b/memory/lib/sync.py index 739b13a..48915e0 100755 --- a/memory/lib/sync.py +++ b/memory/lib/sync.py @@ -17,6 +17,7 @@ python sync.py both # bidirectional """ +import hashlib import os import re import sys @@ -62,6 +63,25 @@ def normalize(entry: str) -> str: return re.sub(r"\s+", " ", text).strip() +def exported_entry_fingerprints(text: str, memory_entries: list[str]) -> set[bytes]: + """Find exact serialized entries, preferring longer overlapping matches.""" + occupied: list[tuple[int, int]] = [] + fingerprints: set[bytes] = set() + for entry in sorted(memory_entries, key=len, reverse=True): + pattern = re.compile( + rf"(?m)^- {re.escape(entry)}\n(?=- |\n## Sync |\Z)" + ) + for match in pattern.finditer(text): + start, end = match.span() + if any(start < used_end and used_start < end for used_start, used_end in occupied): + continue + occupied.append((start, end)) + fingerprint = hashlib.sha256(normalize(entry).encode("utf-8")).digest() + fingerprints.add(fingerprint) + break + return fingerprints + + # -- Direction 1: Memory -> Vault ------------------------------------------ def memory_to_vault(paths: dict): """Export Hermes memory facts to the knowledge vault.""" @@ -71,17 +91,17 @@ def memory_to_vault(paths: dict): memory_entries = parse_hermes_memory(paths["hermes_memory"]) knowledge_path = paths["vault_knowledge"] - existing_knowledge = set() + existing_knowledge: set[bytes] = set() if knowledge_path.exists(): - for line in knowledge_path.read_text(encoding="utf-8").splitlines(): - line = line.strip().lstrip("- ").strip() - if line: - existing_knowledge.add(normalize(line)) + knowledge_text = knowledge_path.read_text(encoding="utf-8") + existing_knowledge = exported_entry_fingerprints(knowledge_text, memory_entries) new_entries = [] for entry in memory_entries: - if normalize(entry) not in existing_knowledge: + fingerprint = hashlib.sha256(normalize(entry).encode("utf-8")).digest() + if fingerprint not in existing_knowledge: new_entries.append(entry) + existing_knowledge.add(fingerprint) if new_entries: knowledge_path.parent.mkdir(parents=True, exist_ok=True)