diff --git a/benchmarks/src/basic_memory_benchmarks/agent_tasks/driver.py b/benchmarks/src/basic_memory_benchmarks/agent_tasks/driver.py index d098e6f2f..d349c700b 100644 --- a/benchmarks/src/basic_memory_benchmarks/agent_tasks/driver.py +++ b/benchmarks/src/basic_memory_benchmarks/agent_tasks/driver.py @@ -390,18 +390,22 @@ def verify_seeded_recency(*, prefix: list[str], env: dict[str, str], project: Ta result = run_command( prefix + [ - "tail", + "tool", + "project", + "recent", "--project", project.name, - "--timeframe", + "--since", SEEDED_RECENCY_WINDOW, - "-n", + "--items", + "note", + "--page-size", "100", "--json", ], env=env, ) - recent = {row["file_path"] for row in json.loads(result.stdout)} + recent = {row["file_path"] for row in json.loads(result.stdout)["results"]} mismatch = recency_mismatch(recent, set(TIMESTAMPS)) if mismatch is not None: raise SeedingError( diff --git a/benchmarks/tests/agent_tasks/test_driver.py b/benchmarks/tests/agent_tasks/test_driver.py index 314bc1881..4baf16f8e 100644 --- a/benchmarks/tests/agent_tasks/test_driver.py +++ b/benchmarks/tests/agent_tasks/test_driver.py @@ -101,13 +101,13 @@ def _seeded_status(*, indexed: int, files: int, phase: str = "idle") -> str: def _bm_stdout(command: list[str]) -> str: """What the stubbed bm prints for the seed guards. - `status --json` reports a fully indexed project and `tail --json` the aged - gold set, so the guards pass unless a test overrides one of them. + `status --json` reports a fully indexed project and `tool project recent + --json` the aged gold set, so the guards pass unless a test overrides one of them. """ if "status" in command: return _seeded_status(indexed=25, files=25) - if "tail" in command: - return json.dumps([{"file_path": relpath} for relpath in TIMESTAMPS]) + if "recent" in command: + return json.dumps({"results": [{"file_path": relpath} for relpath in TIMESTAMPS]}) return "" @@ -314,15 +314,19 @@ def test_state_graded_task_reindexes_before_grading( # seeded project (settle_index is stubbed, so these are the only status calls). statuses = [cmd[cmd.index("status") :] for cmd in commands if "status" in cmd] assert statuses == [["status", "--project", project, "--json", "--local"]] - tails = [cmd[cmd.index("tail") :] for cmd in commands if "tail" in cmd] - assert tails == [ + recents = [cmd[cmd.index("tool") :] for cmd in commands if "recent" in cmd] + assert recents == [ [ - "tail", + "tool", + "project", + "recent", "--project", project, - "--timeframe", + "--since", driver.SEEDED_RECENCY_WINDOW, - "-n", + "--items", + "note", + "--page-size", "100", "--json", ] @@ -704,7 +708,7 @@ def test_grouped_manifest_run_shares_projects_and_reports_groups( # Grouped corpora carry no aged-mtime contract, so the recency guard is # specific to the shipped corpus and must not run here; the index-count # guard runs for every seeded project, once per persona. - assert not [cmd for cmd in commands if "tail" in cmd] + assert not [cmd for cmd in commands if "recent" in cmd] # Slice from the verb: the uv prefix carries its own `--project `. statuses = sorted(cmd[cmd.index("status") + 2] for cmd in commands if "status" in cmd) assert statuses == added_projects @@ -1018,12 +1022,14 @@ def test_seeding_records_a_project_that_lost_its_aged_mtimes( the files outside the gold set, and the run keeps the other projects' spend. """ _stub_bm(monkeypatch) - whole_corpus = json.dumps([{"file_path": relpath} for relpath in corpus_files(CORPUS_DIR)]) + whole_corpus = json.dumps( + {"results": [{"file_path": relpath} for relpath in corpus_files(CORPUS_DIR)]} + ) monkeypatch.setattr( driver, "run_command", lambda command, **kwargs: SimpleNamespace( - stdout=whole_corpus if "tail" in command else _bm_stdout(command), stderr="" + stdout=whole_corpus if "recent" in command else _bm_stdout(command), stderr="" ), ) monkeypatch.chdir(tmp_path) diff --git a/docs/1.0.md b/docs/1.0.md index 1e1533d91..d01b0c359 100644 --- a/docs/1.0.md +++ b/docs/1.0.md @@ -172,7 +172,29 @@ pass `--json`. Without `--project`, commands use the configured default project. | `--plain` | removed; the default output is the tool's text result | | (new) | `move`, `list`, `find` | -The bundled manual page `tools-migration(7)` (`bm man tools-migration`) carries both tables. +The shell read verbs now run the same tools the same way: `bm cat` and `bm head` call +`bm_read`, `bm grep` calls `bm_search` in text mode, `bm ls` calls `bm_list`, and +`bm find` and `bm tree` call `bm_find`. Their output is the tool's text result, or its +JSON result with `--json`. + +| Previous shell verb | 1.0 shell verb | +| --- | --- | +| `bm tail` | removed; use `bm tool project recent` | +| `--plain`, Rich tables, JSON when piped | removed; text result by default, `--json` for JSON | +| `--project-id UUID` | `--project UUID` | +| no `--project` with several projects: refused | the configured default project | +| `/path` routes to that project by itself | pass `--project `; a prefix naming another project is refused | +| `bm ls /` lists the projects | lists the default project's root; `bm project list` lists projects | +| `bm cat --lines` with `--no-frontmatter` counts body lines | `--lines` always counts the whole file, frontmatter included; `--frontmatter` applies to whole-note reads | +| `bm head --frontmatter/--no-frontmatter` | removed; `head` reads lines 1-N of the file | +| `bm grep -C/--context-lines N` | `bm grep -C/--lines N` | +| `bm grep ""` refused; `-C` with `--semantic` refused | follow `bm_search` | +| (new) `bm grep --path DIR` | search one folder | +| (new) `bm ls --glob`, `--sort` | `bm_list`'s file-name glob and order | +| `ls`/`find`/`tree` `--page-size` up to 200 | 1-100; `tree` defaults to 100 | +| `bm find --fields` hit rows rendered as a table | `bm_find`'s text result | + +The bundled manual page `tools-migration(7)` (`bm man tools-migration`) carries these tables. | Previous MCP tool | 1.0 replacement | | --- | --- | @@ -184,7 +206,7 @@ The bundled manual page `tools-migration(7)` (`bm man tools-migration`) carries | `list_directory`, `ls` | `bm_list` for one level; `bm_find` for recursive inventory | | `find` | `bm_find` | | `recent_activity` | `bm_project(command="recent")` | -| `tail` | `bm_project(command="recent")`; map `timeframe` to `since` and `lines` to `page_size`. Recent JSON includes `created_at`, not `updated_at`; shell users needing the last-change timestamp can use `bm tail --json`. | +| `tail` | `bm_project(command="recent")`; map `timeframe` to `since` and `lines` to `page_size`. Recent JSON includes `created_at`, not `updated_at`. The `bm tail` shell command is removed; use `bm tool project recent`. | | `list_memory_projects` | `bm_project(command="list")` | | `list_workspaces` | `bm_project(command="list")` groups projects by workspace, so it omits a workspace with no projects; `bm cloud workspace list` in a shell lists every workspace | | `create_memory_project` | `bm_project(command="create", name=..., path=...)` | diff --git a/plugins/claude-code/skills/bm-setup/SKILL.md b/plugins/claude-code/skills/bm-setup/SKILL.md index 717378176..b9b234678 100644 --- a/plugins/claude-code/skills/bm-setup/SKILL.md +++ b/plugins/claude-code/skills/bm-setup/SKILL.md @@ -232,16 +232,21 @@ nudge — the config's presence is the signal that setup has run. ### 4. Smoke-test the wiring Before you close, prove recall actually resolves — this catches a misnamed project, missing cloud credentials, or a ref that doesn't route, while the user is still here -to fix it. Run the same structured query the SessionStart hook runs, via the CLI it -uses (`basic-memory` / `bm` / `uvx --prerelease=allow basic-memory`): - -- **Primary:** `… tool search --type schema --page-size 10` against - `primaryProject` — use `--project-id ` for a UUID, `--project ` - otherwise. It should return the schemas you just seeded. For a coding setup, - confirm the `Coding Session` schema is present. -- **One shared project** (only if `secondaryProjects` is non-empty): a - `--type decision --status open` query against the first ref. It just needs to - return *cleanly* — `0 results` is fine; an **error** means the ref doesn't route. +to fix it. Use the CLI the hook uses (`basic-memory` / `bm` / +`uvx --prerelease=allow basic-memory`). `--project` takes a project name, +`workspace/project`, or a project id. + +- **Primary:** run the open-decision query the SessionStart hook runs against + `primaryProject`: + `… tool search --note-types decision --filters '{"status": "open"}' --page-size 5 --project `. + It just needs to return *cleanly* — `0 results` is fine; an **error** means the + ref doesn't route. +- **Seeded schemas:** `… tool search --note-types schema --page-size 10 --project ` + should return the schemas you just seeded. For a coding setup, confirm the + `Coding Session` schema is present. +- **One shared project** (only if `secondaryProjects` is non-empty): the same + open-decision query with `--project `. Again `0 results` is fine; an + **error** means the ref doesn't route. - **Hook health:** run `basic-memory hook status --harness claude --project-dir ` with the first available `basic-memory`, `bm`, or `uvx --prerelease=allow basic-memory` launcher. It diff --git a/scripts/update_man_pages.py b/scripts/update_man_pages.py index 64327aa98..f50920850 100644 --- a/scripts/update_man_pages.py +++ b/scripts/update_man_pages.py @@ -60,7 +60,7 @@ # fast path in cli/main.py skips these imports, so a script that walks the tree # must import them explicitly or every subcommand comes back empty. from basic_memory.cli.app import app -import basic_memory.cli.commands.posix # noqa: F401 (registers cat/grep/ls/find/tail/head/tree) +import basic_memory.cli.commands.posix # noqa: F401 (registers cat/grep/ls/find/head/tree) import basic_memory.cli.commands.okf # noqa: F401 (registers OKF commands) import basic_memory.cli.commands.man # noqa: F401 (registers `bm man apropos`) diff --git a/skills/memory-literary-analysis/SKILL.md b/skills/memory-literary-analysis/SKILL.md index 553d354fe..fb10e9ca9 100644 --- a/skills/memory-literary-analysis/SKILL.md +++ b/skills/memory-literary-analysis/SKILL.md @@ -65,7 +65,7 @@ answer, exit 0, no warning — so learn it here rather than from a graph you tho audited: - **`find` pages, and the default page is 10.** Any query whose answer is "all N chapters" - needs `--page-size 200` (the maximum) — see [Coverage Checks](#coverage-checks). + needs `--page-size 100` (the maximum) and paging — see [Coverage Checks](#coverage-checks). - **`--name` cannot combine with `--meta`.** The metadata search has no filename glob. Scope a `--meta` query with the positional path instead: `find /characters --meta 'note_type=character'`. That path is matched on a directory boundary against the *file path* @@ -361,14 +361,14 @@ re-reading the whole book into context: ```bash grep -n '^CHAPTER ' ~/basic-memory/moby-dick/moby-dick.txt # heading -> line number -bm cat moby-dick/moby-dick.txt --lines 4200-4890 --plain # one chapter, not the whole text +bm cat moby-dick.txt --lines 4200-4890 --project moby-dick # one chapter, not the whole text ``` `grep -n` here is the shell's grep on a filesystem path (this is the map-building step, and -it needs the file). `bm cat` then takes the *note identifier* and returns exactly that slice plus a -`lines 4200-4890 of N` footer. Spell the identifier project-qualified -(`/.txt`) so it routes to the right project. `bm head moby-dick/moby-dick.txt -n 40` is the cheap -way to eyeball the heading format before writing the grep pattern. +it needs the file). `bm cat` then takes the *note identifier* and returns exactly that slice, +numbered, under a `Lines 4200-4890 of N` header. Pass `--project ` so it reads from the +right project. `bm head moby-dick.txt -n 40 --project moby-dick` is the cheap way to eyeball +the heading format before writing the grep pattern. Store the map in the project (a note or a small JSON file) so later batches — and a resumed run after context compaction — do not have to rediscover it. On a long work this is the @@ -493,17 +493,17 @@ After all chapters are processed: Do not re-read every note to decide what is thin. Query for it: ```bash -bm find --meta 'note_type=chapter' --fields chapter_number,pov,setting --page-size 200 -bm find --meta 'note_type=character' --fields role,status --page-size 200 # who is still a stub -bm find --meta 'chapter_number>100' --fields pov --page-size 200 # late-book POV drift +bm find --meta 'note_type=chapter' --fields chapter_number,pov,setting --page-size 100 +bm find --meta 'note_type=character' --fields role,status --page-size 100 # who is still a stub +bm find --meta 'chapter_number>100' --fields pov --page-size 100 # late-book POV drift ``` -A field a note never set comes back as a blank cell (`null` under `--json`), so rows with +A field a note never set comes back as `None` (`null` under `--json`), so rows with blanks are the work queue. This turns "audit the graph" from a read of every note into one call per question. `bm_write` snake-cases `note_type` (`Chapter` is stored as `chapter`), and `--meta` -type predicates are normalized the same way, so either spelling matches. `--page-size 200` +type predicates are normalized the same way, so either spelling matches. `--page-size 100` is not decoration: without it these return the first 10 rows and the work queue looks ten items long. @@ -582,14 +582,14 @@ Fix issues found — common fixes: Schema validation proves notes match their shape. These prove the graph is *complete*: ```bash -bm find --meta 'note_type=chapter' --fields chapter_number --page-size 200 # every chapter present? -bm find --meta 'note_type=chapter' --fields pov,setting --page-size 200 # missing context? -bm find /characters --meta 'note_type=character' --fields role --page-size 200 # inventory vs. seed list +bm find --meta 'note_type=chapter' --fields chapter_number --page-size 100 # every chapter present? +bm find --meta 'note_type=chapter' --fields pov,setting --page-size 100 # missing context? +bm find /characters --meta 'note_type=character' --fields role --page-size 100 # inventory vs. seed list ``` **A coverage check that pages is not a coverage check.** `bm find` defaults to `--page-size 10`, so the un-sized form of the first query "proves" a 138-chapter work has 10 -chapters. 200 is the maximum page size; past that, iterate with `--page 2`, `--page 3`, … . +chapters. 100 is the maximum page size; past that, iterate with `--page 2`, `--page 3`, … . Scope a `--meta` query with the positional path (`/characters`), never `--name` — the two options are mutually exclusive, because the metadata search has no filename glob. The positional path scopes by the *file path* a note is indexed under, matched on a directory @@ -597,12 +597,10 @@ boundary: `/characters` admits `characters/major/ahab.md` but never `characters- not a permalink match, so a note that pins its own `permalink:` is still found where its file lives. -Read the count off the footer, not off the rows you can see. In a terminal every `find` -result reports `page 1 • total 138`, and appends `• more available (--page)` when the page -truncated the answer — that suffix appearing is the check *failing*, whatever the visible -rows say. **The footer is a TTY feature.** Piped output without `--plain` is JSON, which -carries `total` and `has_more`; with `--plain` the rows go to stdout and a has-more notice -goes to stderr. For scripts, read `has_more` from `--json`. +Read the count off the result, not off the rows you can see. The text result opens with the +total (`138 notes under '/' match:`) and ends with `More results available.` when the page +truncated the answer — that line appearing is the check *failing*, whatever the visible rows +say. For scripts, read `total` and `has_more` from `--json`. For the sequence gap — the failure that a count alone cannot catch — take the numbers from `--json`, which carries `total`, `total_is_exact`, and `has_more`: @@ -611,7 +609,7 @@ For the sequence gap — the failure that a count alone cannot catch — take th expected=138; page=1; rows='[]' while :; do resp=$(bm find --meta 'note_type=chapter' --fields chapter_number \ - --page-size 200 --page $page --project --json) + --page-size 100 --page $page --project --json) rows=$(jq -n --argjson acc "$rows" --argjson r "$resp" \ '$acc + [$r.results[] | {title, n: .fields.chapter_number}]') [ "$(jq -r '.has_more' <<<"$resp")" = "true" ] || break @@ -627,9 +625,9 @@ jq -n --argjson rows "$rows" --argjson expected "$expected" ' out_of_range: ($n | map(select(. < 1 or . > $expected)) | unique) }' ``` -The loop is not ceremony. `--page-size` caps at 200, so a single call cannot inventory a work -with more than 200 chapters — and rerunning it with `--page 2` *replaces* the numbers rather -than accumulating them, which reports chapters 1-200 as missing on a corpus that is complete. +The loop is not ceremony. `--page-size` caps at 100, so a single call cannot inventory a work +with more than 100 chapters — and rerunning it with `--page 2` *replaces* the numbers rather +than accumulating them, which reports chapters 1-100 as missing on a corpus that is complete. Walk until `has_more` is false and check the union. `--fields` returns YAML-native types, so `chapter_number: 63` comes back as `63`. The @@ -679,13 +677,27 @@ orphans behind has made the graph worse. With the graph complete, traverse it to find what the chapter-by-chapter pass could not see: ```bash -bm grep "features [[" --page-size 200 --project --json # every chapter's cast, one call -bm find --meta 'note_type=theme' --fields prevalence --page-size 200 # thematic weight +bm find --meta 'note_type=theme' --fields prevalence --page-size 100 # thematic weight bm grep "doubloon" --page-size 100 --project # every mention of a symbol ``` -"Which characters share the most chapters" is the first line plus a local parse of each -row's `content` for `features [[...]]` — one call instead of a read per chapter. Do not +"Which characters share the most chapters" is a search for `features [[` plus a local parse +of each row's `content` for `features [[...]]` — one call per 100 chapters instead of a read +per chapter. Page until `has_more` is false, as the coverage check does; a 138-chapter work +needs two pages: + +```bash +page=1; rows='[]' +while :; do + resp=$(bm grep "features [[" --page-size 100 --page $page --project --json) + rows=$(jq -n --argjson acc "$rows" --argjson r "$resp" '$acc + $r.results') + [ "$(jq -r '.has_more' <<<"$resp")" = "true" ] || break + page=$((page + 1)) +done +jq 'length' <<<"$rows" # one row per chapter that lists its cast +``` + +Do not reach for `bm tool context 'memory://characters/major/*'` here: a wildcard context is capped at 100 related rows across all primaries and returns one primary row per indexed observation, so it neither enumerates the cast nor walks the web. @@ -699,14 +711,15 @@ the default page would have dropped. `--semantic` ranks by meaning instead, whic "what is this about?" rather than "where does this word appear?". Two more facts about `grep` rows. Matching is case-insensitive and note-level: a hit is a -note, not a line, and there is no `-n` or context. And each row's `content` is capped at +note, not a line, unless you pass `-C N` for matching lines with context. And each row's +`content` is capped at 4000 characters; `content_truncated: true` marks a cut row, so a long note's tail (such as a final chapter's `follows`/`precedes` relations) may be missing from a grep-driven scan. When a parse depends on the end of a note, `cat` that note. `--page-size` raises the ceiling, it does not remove it. A symbol in a long work can exceed -even 100, so check whether the last page was full and walk `--page 2`, `--page 3` until it is -not. A truncated symbol search fails the same silent way as an unpaginated `find`: a plausible +even 100, so walk `--page 2`, `--page 3` until `has_more` is false, as in the cast loop +above. A truncated symbol search fails the same silent way as an unpaginated `find`: a plausible answer, exit 0, and no sign that the tail is missing. And `grep` searches **your notes, not the source**. The `.txt` is indexed as an entity, diff --git a/src/basic_memory/cli/app.py b/src/basic_memory/cli/app.py index 3aacedc59..a1cdd8f11 100644 --- a/src/basic_memory/cli/app.py +++ b/src/basic_memory/cli/app.py @@ -54,7 +54,6 @@ def version_callback(value: bool) -> None: "grep", "ls", "find", - "tail", "head", "tree", } diff --git a/src/basic_memory/cli/commands/config.py b/src/basic_memory/cli/commands/config.py index b55526aa9..3903dec8b 100644 --- a/src/basic_memory/cli/commands/config.py +++ b/src/basic_memory/cli/commands/config.py @@ -98,7 +98,7 @@ def _env_var_name(key: str) -> str: def _redact_for_display(key: str, raw: str) -> str: - """Mask secrets and URL credentials the same way `basic_memory_diagnostics` does (#963).""" + """Mask secrets and URL credentials with the shared redaction rules (#963).""" if key in SECRET_FIELDS: return SECRET_MASK if key in URL_FIELDS: diff --git a/src/basic_memory/cli/commands/find_predicates.py b/src/basic_memory/cli/commands/find_predicates.py new file mode 100644 index 000000000..9138ca73e --- /dev/null +++ b/src/basic_memory/cli/commands/find_predicates.py @@ -0,0 +1,256 @@ +"""`bm find --meta` predicates: the shell spelling of bm_find's `filters`. + +A predicate string such as 'status=active' or 'confidence>0.6' becomes one entry +of the metadata_filters dict bm_find takes, in a subset of the grammar the search +API's parse_metadata_filters supports. Mistakes are refused here, in find's own +words, before any request is built. +""" + +import json +import math +import re +from typing import Any + +from basic_memory.repository.metadata_filters import parse_metadata_path + +# find's `--meta` strings translate onto the search API's metadata_filters dict — +# a subset of the grammar parse_metadata_filters supports (eq, $gt/$gte/$lt/$lte, +# $in, array-contains-all, $between), nothing more. Word ops need whitespace around +# them and symbol ops are characters the key cannot contain, so a predicate has +# exactly one reading: the word form, or a split at its first operator character. +# The key admits '.' anywhere on purpose — it is looser than a dot path. +# parse_metadata_path, which owns the frontmatter path grammar, is what decides +# a well-formed one, applied in parse_meta_predicates once the predicate has +# split. Tightening the key instead would make '.owner=null' split no way at all +# and be reported as a missing operator rather than as the bad key it is. +_PREDICATE_KEY_RE = re.compile(r"[A-Za-z0-9_.-]+") +_PREDICATE_WORD_RE = re.compile(r"^([A-Za-z0-9_.-]+)\s+(in|has|between)\s+(.+)$") +_SYMBOL_OPERATOR_CHARS = frozenset("<>=") +_TWO_CHARACTER_OPERATORS = (">=", "<=") +_SYMBOL_OPERATORS = {">": "$gt", ">=": "$gte", "<": "$lt", "<=": "$lte"} +_SUPPORTED_PREDICATE_OPS = "= > >= < <= in has between" +# The symbol split consumes the first operator it recognizes, so an operator +# spelling outside the supported set ("==", "=>", ">>", ">=>") leaves its +# second character at the head of the value. These are the characters that can +# be left behind that way. A set, not a string: "" is a substring of any string +# but is not a member here, so an empty token never reads as operator-prefixed. +_OPERATOR_VALUE_PREFIXES = frozenset("=<>") +# Mirrors bm_find's alias: "note_type" (the entity model column) means the +# frontmatter "type" key, so a repeated key is caught under either spelling. +_METADATA_KEY_ALIASES = {"note_type": "type"} + + +def _opens_an_unterminated_quote(text: str) -> bool: + """True when a double quote opens in `text` and nothing closes it. + + One scanner decides this for every value token, scalar or list element, so + the two paths cannot disagree about what "quoted" means. + """ + in_quotes = False + escaped = False + for char in text: + if escaped: + escaped = False + elif in_quotes and char == "\\": + escaped = True + elif char == '"': + in_quotes = not in_quotes + return in_quotes + + +def _predicate_scalar(token: str, predicate: str) -> Any: + """Read one predicate value token, refusing everything a search cannot answer. + + "true"/"false"/"null"/numbers become bool/None/int/float so the produced + filters dict is byte-equal to what a bm_find caller passes as JSON; a JSON-quoted token ('"true"') forces a literal string; anything that + is not a JSON scalar stays the raw string. + """ + text = token.strip() + # Trigger: an unquoted value opens with one of the operator characters. + # Why: only the operators in _SUPPORTED_PREDICATE_OPS are real, but the + # split takes the longest supported one and hands the rest to the + # value — 'status==active' would filter for the string "=active" and + # 'count>>3' for ">3", so a typo'd operator answered as an empty (or + # worse, a non-empty but wrong) result set instead of the refusal the + # grammar documents. + # Outcome: refuse, naming the supported set and the quoting escape hatch a + # value that genuinely starts with '=', '<' or '>' needs. + if text[:1] in _OPERATOR_VALUE_PREFIXES: + raise ValueError( + f"find: unsupported predicate operator in '{predicate}'; " + f"supported: {_SUPPORTED_PREDICATE_OPS}; quote the value as " + f'"{text}" to match text that starts with that character' + ) + # Trigger: a quote opens in the token and never closes. + # Why: json.loads rejects it, and the raw-text fallback would then keep the + # dangling quote as part of a literal value — 'status="active' would + # search for the seven-character text '"active', report no matches, + # and hide the typo behind an ordinary empty result. + # Outcome: refuse for every value token, so the scalar operators and the + # list operators (where a severed quote would also mis-split the + # list) answer a dangling quote the same way. + if _opens_an_unterminated_quote(text): + raise ValueError( + f"find: predicate '{predicate}' has an unterminated quoted value; " + "close the quote — 'status=\"active\"' forces a literal string, and " + "'label in \"a,b\",c' protects a comma inside a list element" + ) + try: + value = json.loads(text) + except json.JSONDecodeError: + return text + # Trigger: the token parsed to a non-finite float. Python's JSON reader + # accepts NaN/Infinity/-Infinity as an extension, and overflows a + # large exponent ("1e999") to infinity. + # Why: none of those are JSON the request encoder will emit, so the filter + # died at transport with "Out of range float values are not JSON + # compliant" — a network-shaped error for what is a predicate typo. + # Outcome: refuse here, in the same shape as the grammar's other refusals. + if isinstance(value, float) and not math.isfinite(value): + raise ValueError( + f"find: predicate '{predicate}' has a non-finite number '{text}'; " + f'predicate values must be finite numbers; quote the value as "{text}" ' + "to match that literal text" + ) + if value is None or isinstance(value, (bool, int, float, str)): + return value + return text + + +def _split_predicate_items(raw_value: str, predicate: str) -> list[str]: + """Split a list-op value on its top-level commas, refusing empty elements. + + A comma inside a JSON-quoted token belongs to the value, not to the list, so + the quoting escape hatch the scalar operators document works for `in`, `has` + and `between` too: 'label in "a,b",c' yields ['"a,b"', 'c'], which + _predicate_scalar then reads as the literal strings "a,b" and "c". Splitting + the raw string first would sever the quoted token into '"a' and 'b"' and + filter for values nothing carries — wrong, and silent. + + A split only happens outside quotes, so an unterminated quote here always + ends up inside one element and _predicate_scalar refuses it; this function + does not repeat that check. + """ + items: list[str] = [] + current: list[str] = [] + in_quotes = False + escaped = False + for char in raw_value: + current.append(char) + if escaped: + escaped = False + elif in_quotes and char == "\\": + escaped = True + elif char == '"': + in_quotes = not in_quotes + elif char == "," and not in_quotes: + current.pop() + items.append("".join(current)) + current = [] + items.append("".join(current)) + stripped = [item.strip() for item in items] + if any(not item for item in stripped): + raise ValueError(f"find: predicate '{predicate}' has an empty list element") + return stripped + + +def _refuse_null_outside_equality(values: list[Any], op: str, predicate: str) -> None: + """Refuse a null bound, list element, or comparison value. + + Trigger: `null` reached an operator other than '='. + Why: '=' compiles to IS NULL server-side, which is the question null asks — + does this note carry a value here at all. Every other operator compares + against the value, and a SQL comparison with NULL is never true, so + 'score>null' or 'priority in null,high' would answer zero rows for + every note in the project rather than name the query it cannot run. + Outcome: refuse, pointing at the equality spelling that does work. + """ + if any(value is None for value in values): + raise ValueError( + f"find: predicate '{predicate}' uses null with '{op}'; null matches only " + "as equality ('owner=null' finds notes carrying no owner); quote the " + 'value as "null" to match that literal text' + ) + + +def _split_symbol_predicate(predicate: str) -> tuple[str, str, str] | None: + """Split 'key OP value' at its first operator character, or None. + + A two-character operator (>=, <=) wins over its first character, so ">=" + never reads as ">" plus a value starting with "=". Whitespace around the + operator is not part of the key or the value. A plain scan rather than a + regex: the operator characters are '<', '>' and '=', and a regex over them + reads to static analysis like an HTML comment filter, which this is not. + """ + index = next( + (position for position, char in enumerate(predicate) if char in _SYMBOL_OPERATOR_CHARS), + None, + ) + if index is None: + return None + key = predicate[:index].rstrip() + if _PREDICATE_KEY_RE.fullmatch(key) is None: + return None + pair = predicate[index : index + 2] + op = pair if pair in _TWO_CHARACTER_OPERATORS else predicate[index] + return key, op, predicate[index + len(op) :].lstrip() + + +def parse_meta_predicates(predicates: list[str]) -> dict[str, Any]: + """Translate POSIX-style predicate strings into the search API metadata_filters dict. + + One predicate per string; predicates AND together. Exactly one predicate + per key — the API admits one operator per key, so a repeated key fails fast + instead of last-wins. Raises ValueError on any operator outside the + supported set, and on any key outside the search API's dot-path grammar. + """ + filters: dict[str, Any] = {} + for predicate in predicates: + word = _PREDICATE_WORD_RE.match(predicate.strip()) + parts = word.groups() if word else _split_symbol_predicate(predicate.strip()) + if parts is None: + raise ValueError( + f"find: unsupported predicate operator in '{predicate}'; " + f"supported: {_SUPPORTED_PREDICATE_OPS}" + ) + raw_key, op, raw_value = parts + key = _METADATA_KEY_ALIASES.get(raw_key, raw_key) + # Trigger: the key capture accepted something that is not a dot path — + # a doubled, leading or trailing dot ('review..approved', + # '.owner', 'owner.'). + # Why: the search API refuses these keys, so the query was never going + # to run. Letting it travel spends a request to come back with + # "Unsupported metadata filter key", which names neither find nor + # the shape a key must have — every other predicate mistake is + # refused here, before transport, in find's own words. + # Outcome: refuse locally, naming the offending key and the grammar. + if parse_metadata_path(key) is None: + raise ValueError( + f"find: malformed predicate key '{key}' in '{predicate}'; keys are " + "dot-separated names of letters, digits, '_' or '-' " + "(e.g. 'status' or 'review.approved')" + ) + if key in filters: + raise ValueError( + f"find: duplicate predicate key '{key}' in '{predicate}'; " + "use 'between' for ranges (e.g. 'score between 0.3,0.8')" + ) + if op in ("in", "has", "between"): + items = [ + _predicate_scalar(item, predicate) + for item in _split_predicate_items(raw_value, predicate) + ] + _refuse_null_outside_equality(items, op, predicate) + if op == "between" and len(items) != 2: + raise ValueError(f"find: 'between' needs exactly min,max in '{predicate}'") + filters[key] = ( + {"$in": items} if op == "in" else items if op == "has" else {"$between": items} + ) + else: + if not raw_value.strip(): + raise ValueError(f"find: predicate '{predicate}' has no value") + value = _predicate_scalar(raw_value, predicate) + if op != "=": + _refuse_null_outside_equality([value], op, predicate) + filters[key] = value if op == "=" else {_SYMBOL_OPERATORS[op]: value} + return filters diff --git a/src/basic_memory/cli/commands/hook.py b/src/basic_memory/cli/commands/hook.py index 985abf001..3f28dd6e3 100644 --- a/src/basic_memory/cli/commands/hook.py +++ b/src/basic_memory/cli/commands/hook.py @@ -1083,7 +1083,7 @@ def _checkpoint_note( ).hexdigest() title = f"Pi session {identity}" - # Frontmatter as a dict (write_note serializes + quotes it); `type` rides the + # Frontmatter as a dict (bm_write serializes + quotes it); `type` rides the # note_type arg. Order preserved for stable, readable output. metadata: dict[str, Any] = { "status": "open", diff --git a/src/basic_memory/cli/commands/mcp.py b/src/basic_memory/cli/commands/mcp.py index 5b94b25aa..2f7cef7bd 100644 --- a/src/basic_memory/cli/commands/mcp.py +++ b/src/basic_memory/cli/commands/mcp.py @@ -138,7 +138,7 @@ def _run_background_auto_update() -> None: elif transport == "streamable-http" or transport == "sse": # Trigger: HTTP/SSE bind host is not a loopback address. # Why: these transports have no inbound authentication, so every MCP tool - # (write_note, delete_note, create_memory_project, ...) is reachable by + # (bm_write, bm_delete, bm_project create, ...) is reachable by # anyone who can reach the port. # Outcome: warn on stderr and in the MCP log; startup proceeds because # containers legitimately bind 0.0.0.0 behind their own network. diff --git a/src/basic_memory/cli/commands/output.py b/src/basic_memory/cli/commands/output.py index e4c6ce45a..ebecf57b2 100644 --- a/src/basic_memory/cli/commands/output.py +++ b/src/basic_memory/cli/commands/output.py @@ -1,6 +1,6 @@ -"""Output modes and renderers for the shell-native read commands. +"""Output modes for `bm inspect`, and the shared Rich console. -`bm cat`, `bm grep` and `bm inspect` print in one of three modes: +`bm inspect` prints in one of three modes: - **JSON**: raw machine-readable JSON, with ``--json`` or whenever stdout is not a terminal, so scripts stay parseable. @@ -12,22 +12,15 @@ error. """ -import json import sys -from typing import Any, Literal +from typing import Literal import typer from rich.console import Console -from rich.markdown import Markdown -from rich.markup import escape as markup_escape -from rich.panel import Panel -from rich.table import Table -from rich.text import Text from basic_memory.config import ConfigManager -from basic_memory.file_utils import has_frontmatter, remove_frontmatter -# Shared Rich console (stderr=False so output goes to stdout, matching print_json). +# Shared Rich console (stderr=False so output goes to stdout). console = Console() @@ -84,202 +77,3 @@ def resolve_output_mode(json_output: bool, plain: bool) -> OutputMode: # Why: let users choose their default terminal experience without a flag. # Outcome: honor cli_output_style (rich out of the box, plain if configured). return ConfigManager().config.cli_output_style - - -def print_json(result: Any) -> None: - """Print a result as formatted JSON.""" - print(json.dumps(result, indent=2, ensure_ascii=True, default=str)) - - -def search_result_summary(result: dict[str, Any]) -> str: - """Describe search count and pagination without inventing a final page.""" - results = result.get("results", []) - raw_total = result.get("total") - raw_total_is_exact = result.get("total_is_exact") - if isinstance(raw_total_is_exact, bool): - total_is_known = raw_total_is_exact and isinstance(raw_total, int) - else: - # Older payloads used a positive total for exact counts and zero as the - # unknown-total sentinel, so retain that fallback during compatibility. - total_is_known = isinstance(raw_total, int) and raw_total > 0 - total = raw_total if total_is_known else len(results) - page = result.get("current_page") or result.get("page", 1) - page_size = result.get("page_size", len(results)) or 1 - - summary = f"{total} result(s) • page {page}" - if total_is_known: - summary += f" of {max(1, -(-total // page_size))}" - if result.get("has_more") is True: - summary += " • more results available" - return summary - - -# --- Rich formatters --- - - -def display_search_results(result: dict[str, Any], query: str = "") -> None: - """Render search-notes results as a Rich table. - - Real SearchResponse.model_dump() shape: - results: list of SearchResult dicts (title, type, permalink, score, matched_chunk, content) - current_page: int (NOT "page") - page_size: int - total: int - total_is_exact: bool - has_more: bool - """ - results = result.get("results", []) - # Trigger: query is user-supplied text that may contain Rich markup characters. - # Why: interpolating it directly into a markup string causes brackets to be - # parsed as style tags, swallowing or restyling bracketed content. - # Outcome: escape the query so its literal characters are always displayed. - escaped_query = markup_escape(query) if query else query - title = f"Search: [bold cyan]{escaped_query}[/bold cyan]" if query else "Search results" - subtitle = search_result_summary(result) - - if not results: - console.print(Panel(Text("No results found.", style="dim"), title=title, expand=False)) - return - - table = Table(show_header=True, header_style="bold", expand=False) - table.add_column("Type", style="dim", width=12) - table.add_column("Title", style="bold cyan") - table.add_column("Score", style="yellow", width=7) - table.add_column("Permalink", style="green") - table.add_column("Snippet", style="dim", max_width=60) - - for item in results: - item_type = item.get("type", "") - # Trigger: user-sourced title/permalink may contain bracketed text. - # Why: Rich table cells with a style column interpret markup in cell values, - # swallowing brackets (e.g. "Spec [draft] v2" → "Spec v2"). - # Outcome: escape every user-sourced cell value before adding to the table. - item_title = markup_escape(item.get("title") or item.get("permalink", "")) - permalink = markup_escape(item.get("permalink", "")) - score = item.get("score") - score_str = f"{score:.2f}" if score is not None else "" - # Prefer matched_chunk as the most relevant snippet; fall back to content. - raw_snippet = item.get("matched_chunk") or item.get("content") or "" - # Truncate to ~200 chars so the table stays readable. - snippet = markup_escape(raw_snippet[:200].replace("\n", " ")) if raw_snippet else "" - table.add_row(item_type, item_title, score_str, permalink, snippet) - - console.print(Panel(table, title=title, subtitle=subtitle, expand=False)) - - -def display_read_note(result: dict[str, Any], *, include_frontmatter: bool = False) -> None: - """Render read-note result: header panel + optional frontmatter + rendered Markdown content.""" - title = str(result.get("title") or "") - permalink = str(result.get("permalink") or "") - raw_content = result.get("content") - content = raw_content if isinstance(raw_content, str) else "" - raw_frontmatter = result.get("frontmatter") - frontmatter: dict[str, Any] = raw_frontmatter if isinstance(raw_frontmatter, dict) else {} - - # Trigger: read_note returns an explicit all-None payload when no note resolves. - # Why: treating those values as strings crashes Rich and hides any related-note - # suggestions that the MCP fallback already found. - # Outcome: render a stable not-found panel, including safe, copyable suggestions. - if raw_content is None and not title and not permalink: - raw_related = result.get("related_results") - related_results: list[dict[str, Any]] = ( - [item for item in raw_related if isinstance(item, dict)] - if isinstance(raw_related, list) - else [] - ) - if related_results: - table = Table(show_header=True, header_style="bold", expand=False) - table.add_column("Title", style="bold cyan") - table.add_column("Permalink", style="green") - table.add_column("File", style="dim") - for item in related_results: - table.add_row( - markup_escape(str(item.get("title") or "")), - markup_escape(str(item.get("permalink") or "")), - markup_escape(str(item.get("file_path") or "")), - ) - console.print( - Panel( - table, - title="Note not found", - subtitle=f"{len(related_results)} related result(s)", - expand=False, - ) - ) - else: - message = str(result.get("error") or "No note or related content found.") - console.print(Panel(Text(message, style="dim"), title="Note not found", expand=False)) - return - - # The header already uses Text.append so title is never markup-interpreted. - header = Text() - header.append(title, style="bold cyan") - if permalink: - header.append(f" [{permalink}]", style="dim green") - - console.print(Panel(header, expand=False)) - - # Trigger: --frontmatter was passed; the MCP tool populates "frontmatter". - # Why: the JSON payload always carries a "frontmatter" key regardless of the flag, - # so checking non-empty alone would render it even without the flag. The flag - # must be threaded in to gate the panel. - # Outcome: print a dim key/value block above the content only when the flag is set. - if include_frontmatter and frontmatter: - fm_table = Table(show_header=False, box=None, padding=(0, 1), expand=False) - fm_table.add_column("key", style="dim") - fm_table.add_column("value", style="dim") - for key, value in frontmatter.items(): - # Trigger: frontmatter keys/values are user-sourced and may contain markup. - # Why: Rich table cells with a style column parse markup, so bracketed - # keys or values would be silently consumed or restyled. - # Outcome: escape both key and value before adding them to the table. - fm_table.add_row(markup_escape(str(key)), markup_escape(str(value))) - console.print(Panel(fm_table, title="[dim]frontmatter[/dim]", expand=False)) - - # Trigger: --frontmatter makes the API return the literal file, so - # content starts with the frontmatter block the panel above already shows. - # Why: rendering it again through Markdown duplicates the frontmatter (and - # Markdown mangles the --- fences into rules/headings). - # Outcome: strip the block from the body; the panel is the frontmatter view. - body = content - if include_frontmatter and content and has_frontmatter(content): - body = remove_frontmatter(content) - if body and body.strip(): - console.print(Markdown(body)) - else: - console.print(Text("(no content)", style="dim")) - - -# --- Plain formatters --- -# -# Plain output is NOT Rich markup: it is undecorated, greppable text printed via -# the builtin print(). Literal brackets ([draft], [fact]) must survive verbatim, -# so we deliberately do NOT call rich.markup.escape here -- escaping is only for -# the Rich path and would corrupt literal brackets in plain text. - - -def plain_search_results(result: dict[str, Any], query: str = "") -> None: - """Render search-notes results as numbered plain-text entries. - - Mirrors the Rich table content: a header line, then one numbered block per - result (title / score / permalink) with an indented snippet line. - """ - results = result.get("results", []) - header = f"Search: {query}" if query else "Search results" - print(header) - print(search_result_summary(result).replace(" • ", " | ")) - - if not results: - print("No results found.") - return - - for index, item in enumerate(results, start=1): - item_title = item.get("title") or item.get("permalink", "") - permalink = item.get("permalink", "") - score = item.get("score") - score_str = f"{score:.2f}" if score is not None else "" - print(f"{index}. {item_title} (score: {score_str}) {permalink}") - raw_snippet = item.get("matched_chunk") or item.get("content") or "" - if raw_snippet: - snippet = raw_snippet[:200].replace("\n", " ") - print(f" {snippet}") diff --git a/src/basic_memory/cli/commands/posix.py b/src/basic_memory/cli/commands/posix.py index ff5ebce1d..45eff77e4 100644 --- a/src/basic_memory/cli/commands/posix.py +++ b/src/basic_memory/cli/commands/posix.py @@ -1,94 +1,70 @@ """POSIX-style read verbs for the Basic Memory CLI (SPEC-47, #1404). -Seven top-level verbs — ``bm cat``, ``bm grep``, ``bm ls``, ``bm find``, -``bm tail``, ``bm head``, ``bm tree`` — provide shell access through shared -Python implementations in ``basic_memory.mcp.tools.posix_tools``. These -functions no longer register MCP tools. ``head`` and ``tree`` combine ``cat`` -and ``find`` behavior on the client. - -The retired ``enable_posix_tools`` configuration is a no-op. These shell -commands remain available independently of MCP registration. - -Honesty rules (SPEC-47): these are subcommands of ``bm`` only — nothing -shadows real cat/grep on PATH, and there is no shell-redirection magic. -Output follows the ``bm tool`` contract: Rich rendering on a TTY, stable JSON -on ``--json`` or when piped, undecorated text with ``--plain``. +`bm cat`, `bm head`, `bm grep`, `bm ls`, `bm find` and `bm tree` are shell +spellings of the published bm_* tools, run the same way `bm tool` runs them +(see ``basic_memory.cli.commands.tool``): + +- cat and head call bm_read; +- grep calls bm_search in text mode (meaning search with --semantic); +- ls calls bm_list; +- find calls bm_find, with --meta predicates translated to its `filters`; +- tree calls bm_find and draws the listing as a hierarchy. + +Output is the tool's text result, or its JSON result with --json. A refusal or an +invalid argument prints to stderr and exits 1. Without --project the configured +default project is used, as with every other `bm` command. + +Honesty rules (SPEC-47): these are subcommands of ``bm`` only; nothing shadows +real cat/grep on PATH. """ import json -import sys from dataclasses import dataclass, field from typing import Annotated, Any, Optional import typer from rich.markup import escape as markup_escape -from rich.panel import Panel -from rich.table import Table -from rich.text import Text from rich.tree import Tree from basic_memory.cli.app import app -from basic_memory.cli.commands.command_utils import run_with_cleanup -from basic_memory.cli.commands.routing import force_routing, validate_routing_flags -from basic_memory.cli.commands.output import ( - display_read_note, - display_search_results, - plain_search_results, - print_json, - resolve_output_mode, - validate_output_flags, - console, -) - -from basic_memory.schemas.directory import DEFAULT_DIRECTORY_PAGE_SIZE +from basic_memory.cli.commands.output import console +from basic_memory.cli.commands.tool import call_tool, run_tool +from basic_memory.config import ConfigManager # MCP tool functions are imported inside each command: importing # basic_memory.mcp.tools loads the entire tool stack (fastmcp, mcp SDK, # SQLAlchemy), which would slow every CLI invocation, including --help (#886). -# --- Shared option types --- -# One definition per flag so the seven verbs cannot drift apart; the help text -# matches the `bm tool` commands verbatim. +# --- Shared options --- ProjectOption = Annotated[ Optional[str], - typer.Option(help="The project to use. If not provided, the default project will be used."), -] -ProjectIdOption = Annotated[ - Optional[str], - typer.Option( - "--project-id", - help=( - "Project external_id (UUID). Takes precedence over --project; use to " - "disambiguate same-named projects across cloud workspaces." - ), - ), -] -JsonOption = Annotated[ - bool, typer.Option("--json", help="Output raw JSON instead of formatted display") -] -PlainOption = Annotated[ - bool, typer.Option( - "--plain", help="Output undecorated plain text (no colors/markup), even when piped" + help="Project name, 'workspace/project', or project id. Defaults to the default project." ), ] +JsonOption = Annotated[bool, typer.Option("--json", help="Print the tool's JSON result")] LocalOption = Annotated[ bool, typer.Option("--local", help="Force local API routing (ignore cloud mode)") ] CloudOption = Annotated[bool, typer.Option("--cloud", help="Force cloud API routing")] +PageOption = Annotated[Optional[int], typer.Option("--page", help="Page number (1-indexed)")] +PageSizeOption = Annotated[ + Optional[int], typer.Option("--page-size", help="Results per page (1-100)") +] -# --- Shared helpers --- +def _refuse(message: str) -> typer.Exit: + typer.echo(f"Error: {message}", err=True) + return typer.Exit(1) def _parse_lines(lines: str) -> tuple[int, Optional[int]]: """Parse cat's --lines forms into a (start_line, end_line) pair. - Accepted forms, 1-indexed inclusive: "N-M" (range), "N-" (to end), - "N" (one line). Bounds validation (start >= 1, end >= start) stays in the - cat tool itself — the CLI adds only the string parse. + Accepted forms, 1-indexed inclusive: "N-M" (range), "N-" (to end), "N" (one + line). Bounds are bm_read's to check; the CLI adds only the string parse. """ start_text, separator, end_text = lines.strip().partition("-") try: @@ -100,407 +76,26 @@ def _parse_lines(lines: str) -> tuple[int, Optional[int]]: else: end = int(end_text) except ValueError: - raise ValueError( + raise _refuse( f'--lines must be "N-M", "N-" (to end), or "N" (one line), got {lines!r}' ) from None return start, end -def _directory_page_summary(result: dict[str, Any]) -> str: - """Describe a directory listing page without inventing a final page.""" - summary = f"page {result.get('page', 1)} • total {result.get('total', 0)}" - if result.get("has_more") is True: - summary += " • more available (--page)" - return summary - - -def _write_plain_page_footer(result: dict[str, Any], *, search: bool = False) -> None: - """Report omitted rows on stderr without contaminating pipe-friendly stdout.""" - if result.get("has_more") is not True: - return - summary = _search_page_summary(result) if search else _directory_page_summary(result) - sys.stdout.flush() - print(summary, file=sys.stderr) - - -# --- cat / head rendering --- - - -def _write_slice_footer( - result: dict[str, Any], *, plain: bool, content_terminated: bool = True -) -> None: - """Describe an applied slice under the content. - - Rich mode prints a dim footer on stdout; plain mode sends it to stderr so - stdout stays pure note content (real-cat behavior, redirection-safe). - """ - notes: list[str] = [] - start_line = result.get("start_line") - end_line = result.get("end_line") - if start_line is not None and end_line is not None: - note = f"lines {start_line}-{end_line}" - if result.get("total_lines") is not None: - note += f" of {result['total_lines']}" - notes.append(note) - if result.get("section"): - notes.append(f"section {result['section']}") - if result.get("truncated"): - continue_line = result.get("continue_line") - if continue_line is not None: - notes.append(f"truncated — continue with --lines {continue_line}-") - else: - notes.append("truncated") - if not notes: - return - text = " • ".join(notes) - if plain: - # Content was written with sys.stdout.write and may not end in a - # newline, so its last line can sit in stdout's line buffer on a TTY; - # flush before the unbuffered stderr write or the footer prints first. - sys.stdout.flush() - # An unterminated slice would visually concatenate the footer onto the - # last content line in a terminal or merged capture. Lead with the - # newline on STDERR so stdout stays byte-exact for pipes. - if not content_terminated: - text = f"\n{text}" - print(text, file=sys.stderr) - else: - console.print(Text(text, style="dim")) - - -def _render_cat( - result: dict[str, Any], *, json_output: bool, plain: bool, include_frontmatter: bool -) -> None: - """Render a cat/head payload in the resolved output mode.""" - mode = resolve_output_mode(json_output, plain) - if mode == "json": - print_json(result) - return - if mode == "plain": - # Plain stdout is the content and nothing else — exact bytes, no added - # newline — so `bm cat x --plain` pipes and redirects like cat(1) and - # round-trips the file; slice info goes to stderr. - content = result.get("content") - text = content if isinstance(content, str) else "" - sys.stdout.write(text) - _write_slice_footer(result, plain=True, content_terminated=text.endswith("\n") or not text) - return - # cat's payload is the read_note JSON shape, so the read-note renderer applies. - display_read_note(result, include_frontmatter=include_frontmatter) - _write_slice_footer(result, plain=False) - - -# --- ls rendering --- - - -def _display_ls(result: dict[str, Any], path: str) -> None: - """Render one directory level as a Rich table.""" - nodes: list[dict[str, Any]] = list(result.get("nodes", [])) - title = f"ls [bold cyan]{markup_escape(path)}[/bold cyan]" - subtitle = _directory_page_summary(result) - - if not nodes: - console.print(Panel(Text("Empty directory.", style="dim"), title=title, subtitle=subtitle)) - return - - table = Table(show_header=True, header_style="bold", expand=False) - table.add_column("Name", style="bold cyan") - table.add_column("Type", style="dim", width=9) - table.add_column("Title") - table.add_column("Permalink", style="green") - table.add_column("Updated", style="dim") - for node in nodes: - # User-sourced cell text must be escaped so bracketed names ("[draft]") - # survive Rich's markup parsing. - name = markup_escape(str(node.get("name") or "")) - if node.get("type") == "directory": - name += "/" - table.add_row( - name, - str(node.get("type") or ""), - markup_escape(str(node.get("title") or "")), - markup_escape(str(node.get("permalink") or "")), - str(node.get("updated_at") or ""), - ) - console.print(Panel(table, title=title, subtitle=subtitle, expand=False)) - - -def _plain_ls(result: dict[str, Any]) -> None: - """Render one directory level as ls -1 style lines (one path per line).""" - for node in result.get("nodes", []): - suffix = "/" if node.get("type") == "directory" else "" - print(f"{node.get('directory_path', '')}{suffix}") - - -# --- find rendering --- - - -def _display_find(result: dict[str, Any], path: str, name: Optional[str]) -> None: - """Render recursive find results as a Rich table.""" - nodes: list[dict[str, Any]] = list(result.get("nodes", [])) - title = f"find [bold cyan]{markup_escape(path)}[/bold cyan]" - if name: - title += f" [dim]--name {markup_escape(name)}[/dim]" - subtitle = _directory_page_summary(result) - - if not nodes: - console.print(Panel(Text("No matches.", style="dim"), title=title, subtitle=subtitle)) - return - - table = Table(show_header=True, header_style="bold", expand=False) - table.add_column("Path", style="bold cyan") - table.add_column("Title") - table.add_column("Permalink", style="green") - for node in nodes: - node_path = str(node.get("file_path") or node.get("directory_path") or "") - table.add_row( - markup_escape(node_path), - markup_escape(str(node.get("title") or "")), - markup_escape(str(node.get("permalink") or "")), - ) - console.print(Panel(table, title=title, subtitle=subtitle, expand=False)) - - -def _plain_find(result: dict[str, Any]) -> None: - """Render find results as find(1) style lines (one path per line).""" - for node in result.get("nodes", []): - print(str(node.get("file_path") or node.get("directory_path") or "")) - - -# --- find --meta --fields rendering --- -# Metadata predicates flip find's payload to the search response shape; without -# --fields the shared search renderers (grep's) apply, with --fields these two -# small renderers add the projected columns so the shared ones stay untouched. - - -def _search_page_summary(result: dict[str, Any]) -> str: - """Describe a search results page without inventing a final page.""" - summary = f"page {result.get('current_page', 1)} • total {result.get('total', 0)}" - if result.get("has_more") is True: - summary += " • more available (--page)" - return summary - - -def _field_cell(value: Any) -> str: - """Render one projected field value: strings bare, null empty, the rest compact JSON.""" - if value is None: - return "" - if isinstance(value, str): - return value - return json.dumps(value, separators=(",", ":")) - - -def _display_find_fields( - result: dict[str, Any], path: str, meta: list[str], fields: list[str] -) -> None: - """Render metadata hits with their projected fields as a Rich table.""" - rows: list[dict[str, Any]] = list(result.get("results", [])) - title = ( - f"find [bold cyan]{markup_escape(path)}[/bold cyan]" - f" [dim]--meta {markup_escape(' AND '.join(meta))}[/dim]" - ) - subtitle = _search_page_summary(result) - - if not rows: - console.print(Panel(Text("No matches.", style="dim"), title=title, subtitle=subtitle)) - return - - table = Table(show_header=True, header_style="bold", expand=False) - table.add_column("Path", style="bold cyan") - for field_name in fields: - table.add_column(markup_escape(field_name)) - for row in rows: - projected = row.get("fields") or {} - cells = [markup_escape(str(row.get("file_path") or ""))] - cells.extend(markup_escape(_field_cell(projected.get(field_name))) for field_name in fields) - table.add_row(*cells) - console.print(Panel(table, title=title, subtitle=subtitle, expand=False)) - - -def _plain_find_fields(result: dict[str, Any]) -> None: - """Render metadata hits as file-pathcompact-JSON-fields lines.""" - for row in result.get("results", []): - fields_json = json.dumps(row.get("fields") or {}, separators=(",", ":")) - print(f"{row.get('file_path', '')}\t{fields_json}") - - -def _display_grep_lines(result: dict[str, Any], *, plain: bool) -> None: - """Print bounded windows without reintroducing the search response's full body.""" - output: list[str] = [f"Search candidates: page {result['current_page']}"] - for row in result["results"]: - path = row["file_path"] - output.append(f"{path}: {row['match_count']} matching line(s)") - for window in row["windows"]: - for number, line in zip( - range(window["start_line"], window["end_line"] + 1), - window["content"].split("\n"), - strict=True, - ): - marker = ":" if number in window["match_lines"] else "-" - output.append(f"{path}{marker}{number}{marker}{line}") - if row["next_match_line"] is not None: - output.append(f"… more matches; read {path} at line {row['next_match_line']}") - if result["has_more"]: - output.append(f"… more candidates; use --page {result['current_page'] + 1}") - text = "\n".join(output) - if plain: - print(text) - else: - console.print(Text(text)) - - -# --- tail rendering --- -# tail's row shape ({type, title, permalink, file_path, created_at}) differs -# from recent-activity's payload, so it gets its own small renderers rather -# than reusing _display_recent_activity. - - -def _tail_changed_at(row: dict[str, Any]) -> str: - """When a tail row last changed; tail orders by it.""" - # COMPAT(v0.24): servers before v0.24 send only created_at. - return str(row.get("updated_at") or row.get("created_at") or "") - - -def _display_tail(rows: list[dict[str, Any]]) -> None: - """Render recently changed notes as a Rich table, newest first.""" - if not rows: - console.print(Panel(Text("No recent changes.", style="dim"), title="tail", expand=False)) - return - - table = Table(show_header=True, header_style="bold", expand=False) - table.add_column("Updated", style="dim") - table.add_column("Type", style="dim", width=12) - table.add_column("Title", style="bold cyan") - table.add_column("Permalink", style="green") - table.add_column("File", style="dim") - for row in rows: - table.add_row( - _tail_changed_at(row), - str(row.get("type") or ""), - markup_escape(str(row.get("title") or "")), - markup_escape(str(row.get("permalink") or "")), - markup_escape(str(row.get("file_path") or "")), - ) - console.print(Panel(table, title="tail", expand=False)) - - -def _plain_tail(rows: list[dict[str, Any]]) -> None: - """Render recently changed notes as tab-separated lines.""" - for row in rows: - columns = ( - _tail_changed_at(row), - str(row.get("type") or ""), - str(row.get("title") or ""), - str(row.get("permalink") or ""), - str(row.get("file_path") or ""), - ) - print("\t".join(columns)) - - -# --- tree rendering --- - - -@dataclass -class _TreeEntry: - """One rebuilt tree node: whether it is a directory, and its children.""" - - is_dir: bool - children: dict[str, "_TreeEntry"] = field(default_factory=dict) - - -def _build_tree(nodes: list[dict[str, Any]], root: str) -> dict[str, _TreeEntry]: - """Rebuild a hierarchy from the API's flat page of directory nodes. - - The directory API strips children from paginated listings, so the page is - flat; nesting is recovered from each node's directory_path segments. A glob - filter can return files without their parent directories, so intermediate - segments are synthesized as directories. - """ - root_prefix = root.strip("/") - top: dict[str, _TreeEntry] = {} - for node in nodes: - raw_path = str(node.get("directory_path") or node.get("file_path") or "").strip("/") - if root_prefix: - if raw_path == root_prefix: - continue - raw_path = raw_path.removeprefix(f"{root_prefix}/") - if not raw_path: - continue - segments = raw_path.split("/") - level = top - for depth, segment in enumerate(segments): - is_leaf = depth == len(segments) - 1 - entry = level.get(segment) - if entry is None: - entry = _TreeEntry(is_dir=not is_leaf or node.get("type") == "directory") - level[segment] = entry - elif is_leaf and node.get("type") == "directory": - # A synthesized intermediate may already exist; the node's own - # listing is authoritative about it being a directory. - entry.is_dir = True - level = entry.children - return top - - -def _add_tree_branches(branch: Tree, entries: dict[str, _TreeEntry]) -> None: - for segment, entry in entries.items(): - label = markup_escape(segment + ("/" if entry.is_dir else "")) - child = branch.add(f"[bold]{label}[/bold]" if entry.is_dir else label) - _add_tree_branches(child, entry.children) - - -def _display_tree(result: dict[str, Any], label: str, root: str) -> None: - """Render find results as a Rich tree rooted at the search path. - - ``label`` is the caller's spelling of the root; ``root`` is the routed - project-relative path the node paths actually start with — a qualified - '/dir' input strips its project prefix in the shared tool layer, - so the two differ exactly when the input carried a project prefix (#1415). - """ - entries = _build_tree(list(result.get("nodes", [])), root) - tree = Tree(f"[bold cyan]{markup_escape(label)}[/bold cyan]") - if not entries: - tree.add("[dim]empty[/dim]") - _add_tree_branches(tree, entries) - console.print(tree) - if result.get("has_more") is True: - console.print( - Text(f"… more entries (page {result.get('page', 1)}; use --page)", style="dim") - ) - - -def _print_plain_tree_level(entries: dict[str, _TreeEntry], depth: int) -> None: - for segment, entry in entries.items(): - suffix = "/" if entry.is_dir else "" - print(f"{' ' * depth}{segment}{suffix}") - _print_plain_tree_level(entry.children, depth + 1) - - -def _plain_tree(result: dict[str, Any], label: str, root: str) -> None: - """Render the tree as two-space-indented lines; pagination note on stderr. - - ``label``/``root`` split as in ``_display_tree``: print the caller's - spelling, strip the routed project-relative root from node paths. - """ - print(label) - _print_plain_tree_level(_build_tree(list(result.get("nodes", [])), root), depth=1) - if result.get("has_more") is True: - print(f"… more entries (page {result.get('page', 1)}; use --page)", file=sys.stderr) - - -# --- Commands --- +# --- cat / head --- @app.command() def cat( identifier: Annotated[ - str, typer.Argument(help="Note title, permalink, or memory:// URL (resolved exactly)") + str, typer.Argument(help="Note path, permalink, memory:// URL, or title") ], lines: Annotated[ Optional[str], typer.Option( "--lines", - help='Line range "N-M", "N-" (to end), or "N" (one line); 1-indexed inclusive', + help='Line range "N-M", "N-" (to end), or "N" (one line); 1-indexed, ' + "frontmatter included", ), ] = None, section: Annotated[ @@ -513,127 +108,83 @@ def cat( Optional[int], typer.Option( "--max-tokens", - help="Approximate token budget; truncates at a section/paragraph boundary", + help="Approximate token budget; cuts at a section or paragraph boundary", ), ] = None, include_frontmatter: Annotated[ bool, typer.Option( "--frontmatter/--no-frontmatter", - help="Include the YAML frontmatter block (ignored for section/token slices)", + help="Include the YAML frontmatter in a whole-note read", ), ] = True, - json_output: JsonOption = False, - plain: PlainOption = False, project: ProjectOption = None, - project_id: ProjectIdOption = None, + json_output: JsonOption = False, local: LocalOption = False, cloud: CloudOption = False, ) -> None: - """Print a note's content, optionally sliced by lines, section, or token budget. + """Print a note, optionally sliced by lines, section, or token budget (bm_read). Examples: bm cat specs/search bm cat specs/search --lines 20-40 bm cat specs/search --section Decisions - bm cat specs/search --max-tokens 500 --plain + bm cat specs/search --max-tokens 500 """ - # Deferred: loading the MCP tool stack at module import slows CLI startup (#886). - from basic_memory.mcp.tools import cat as mcp_cat - from fastmcp.exceptions import ToolError - - try: - validate_routing_flags(local, cloud) - validate_output_flags(json_output, plain) - start_line, end_line = _parse_lines(lines) if lines is not None else (None, None) - - with force_routing(local=local, cloud=cloud): - result = run_with_cleanup( - mcp_cat( - identifier, - start_line=start_line, - end_line=end_line, - section=section, - max_tokens=max_tokens, - include_frontmatter=include_frontmatter, - project=project, - project_id=project_id, - ) - ) - _render_cat( - result, json_output=json_output, plain=plain, include_frontmatter=include_frontmatter - ) - # ToolError is cat's strict-resolve miss ("Entity not found"); ValueError is - # the tool's own argument validation. Both are user-facing failures. - except (ValueError, ToolError) as e: - typer.echo(f"Error: {e}", err=True) - raise typer.Exit(1) - except Exception as e: # pragma: no cover - if not isinstance(e, typer.Exit): - typer.echo(f"Error during cat: {e}", err=True) - raise typer.Exit(1) - raise + from basic_memory.mcp.tools import bm_read + + start_line, end_line = _parse_lines(lines) if lines is not None else (None, None) + run_tool( + bm_read, + identifier=identifier, + start_line=start_line, + end_line=end_line, + section=section, + max_tokens=max_tokens, + include_frontmatter=include_frontmatter, + project=project, + json_output=json_output, + local=local, + cloud=cloud, + ) @app.command() def head( identifier: Annotated[ - str, typer.Argument(help="Note title, permalink, or memory:// URL (resolved exactly)") + str, typer.Argument(help="Note path, permalink, memory:// URL, or title") ], n: Annotated[ int, typer.Option("-n", "--lines", min=1, help="Number of lines to print (from line 1)") ] = 10, - include_frontmatter: Annotated[ - bool, - typer.Option("--frontmatter/--no-frontmatter", help="Include the YAML frontmatter block"), - ] = True, - json_output: JsonOption = False, - plain: PlainOption = False, project: ProjectOption = None, - project_id: ProjectIdOption = None, + json_output: JsonOption = False, local: LocalOption = False, cloud: CloudOption = False, ) -> None: - """Print the first lines of a note (head(1) over `bm cat`). + """Print the first lines of a note (bm_read with a line range). Examples: bm head specs/search - bm head specs/search -n 3 --plain + bm head specs/search -n 3 """ - # Deferred: loading the MCP tool stack at module import slows CLI startup (#886). - # head is a client-side recombination of cat: same tool, fixed line range, - # so its JSON contract is exactly cat's payload. - from basic_memory.mcp.tools import cat as mcp_cat - from fastmcp.exceptions import ToolError + from basic_memory.mcp.tools import bm_read + + run_tool( + bm_read, + identifier=identifier, + start_line=1, + end_line=n, + project=project, + json_output=json_output, + local=local, + cloud=cloud, + ) - try: - validate_routing_flags(local, cloud) - validate_output_flags(json_output, plain) - - with force_routing(local=local, cloud=cloud): - result = run_with_cleanup( - mcp_cat( - identifier, - start_line=1, - end_line=n, - include_frontmatter=include_frontmatter, - project=project, - project_id=project_id, - ) - ) - _render_cat( - result, json_output=json_output, plain=plain, include_frontmatter=include_frontmatter - ) - except (ValueError, ToolError) as e: - typer.echo(f"Error: {e}", err=True) - raise typer.Exit(1) - except Exception as e: # pragma: no cover - if not isinstance(e, typer.Exit): - typer.echo(f"Error during head: {e}", err=True) - raise typer.Exit(1) - raise + +# --- grep --- @app.command() @@ -647,159 +198,118 @@ def grep( Optional[int], typer.Option( "-C", - "--context-lines", - min=0, - max=10, - help="Compact line matches with surrounding context (not with --semantic)", + "--lines", + help="Return matching lines with this many lines of context (0-10)", ), ] = None, max_matches: Annotated[ - int, - typer.Option( - "--max-matches", min=1, max=100, help="Matching lines per candidate in context mode" - ), - ] = 10, - page: Annotated[int, typer.Option("--page", help="Page number (1-indexed)")] = 1, - page_size: Annotated[int, typer.Option("--page-size", help="Results per page")] = 10, - json_output: JsonOption = False, - plain: PlainOption = False, + Optional[int], + typer.Option("--max-matches", help="Matching lines per note with -C (1-100)"), + ] = None, + path: Annotated[Optional[str], typer.Option("--path", help="Search only this folder")] = None, + page: PageOption = None, + page_size: PageSizeOption = None, project: ProjectOption = None, - project_id: ProjectIdOption = None, + json_output: JsonOption = False, local: LocalOption = False, cloud: CloudOption = False, ) -> None: - """Search note content by keyword (--semantic to rank by meaning). + """Search note content by keyword, or by meaning with --semantic (bm_search). + + Keyword matching is the default: nearest-neighbour ranking always returns + something, so a grep for a term that appears nowhere would list unrelated + notes instead of nothing (#1685). Examples: - bm grep "retry" -C 3 --plain + bm grep "retry" -C 3 bm grep --semantic "auth token rotation" - bm grep "BASIC_MEMORY_FORCE_LOCAL" - bm grep "deploy checklist" --page-size 20 --json + bm grep "deploy checklist" --path specs --json """ - # Deferred: loading the MCP tool stack at module import slows CLI startup (#886). - from basic_memory.mcp.tools import grep as mcp_grep - from fastmcp.exceptions import ToolError + from basic_memory.mcp.tools import bm_search + + run_tool( + bm_search, + query=pattern, + mode="hybrid" if semantic else "text", + # grep answers which notes contain the pattern, one row per note, so it + # names that rather than relying on the search's default item kinds. + items=["note"], + lines=context_lines, + max_matches=max_matches, + path=path, + page=page, + page_size=page_size, + project=project, + json_output=json_output, + local=local, + cloud=cloud, + ) + + +# --- ls --- + - try: - validate_routing_flags(local, cloud) - validate_output_flags(json_output, plain) - - with force_routing(local=local, cloud=cloud): - result = run_with_cleanup( - mcp_grep( - pattern, - semantic=semantic, - context_lines=context_lines, - max_matches=max_matches, - page=page, - page_size=page_size, - project=project, - project_id=project_id, - ) - ) - # grep returns the search-notes response shape, so those renderers apply. - mode = resolve_output_mode(json_output, plain) - if mode == "json": - print_json(result) - elif context_lines is not None: - _display_grep_lines(result, plain=mode == "plain") - elif mode == "plain": - plain_search_results(result, query=pattern) - else: - display_search_results(result, query=pattern) - except (ValueError, ToolError) as e: - typer.echo(f"Error: {e}", err=True) - raise typer.Exit(1) - except Exception as e: # pragma: no cover - if not isinstance(e, typer.Exit): - typer.echo(f"Error during grep: {e}", err=True) - raise typer.Exit(1) - raise - - -# Naming note: `bm ls` lists one directory inside one project; the unrelated -# `bm project ls` lists projects at the workspace level. @app.command() def ls( - path: Annotated[str, typer.Argument(help="Directory path to list")] = "/", - page: Annotated[int, typer.Option("--page", help="Page number (1-indexed)")] = 1, - page_size: Annotated[ - int, typer.Option("--page-size", help="Nodes per page") - ] = DEFAULT_DIRECTORY_PAGE_SIZE, - json_output: JsonOption = False, - plain: PlainOption = False, + path: Annotated[str, typer.Argument(help="Folder to list")] = "/", + glob: Annotated[ + Optional[str], typer.Option("--glob", help="File-name glob, e.g. '*.md'") + ] = None, + sort: Annotated[Optional[str], typer.Option("--sort", help="File order")] = None, + page: PageOption = None, + page_size: PageSizeOption = None, project: ProjectOption = None, - project_id: ProjectIdOption = None, + json_output: JsonOption = False, local: LocalOption = False, cloud: CloudOption = False, ) -> None: - """List the immediate contents of one directory. + """List the immediate contents of one folder (bm_list). + + `bm project list` lists the projects themselves. Examples: bm ls - bm ls /specs - bm ls /notes --page 2 --plain + bm ls specs + bm ls notes --page 2 """ - # Deferred: loading the MCP tool stack at module import slows CLI startup (#886). - from basic_memory.mcp.tools import ls as mcp_ls - from fastmcp.exceptions import ToolError + from basic_memory.mcp.tools import bm_list + + run_tool( + bm_list, + path=path, + glob=glob, + sort=sort, + page=page, + page_size=page_size, + project=project, + json_output=json_output, + local=local, + cloud=cloud, + ) - try: - validate_routing_flags(local, cloud) - validate_output_flags(json_output, plain) - - with force_routing(local=local, cloud=cloud): - result = run_with_cleanup( - mcp_ls( - path, - page=page, - page_size=page_size, - project=project, - project_id=project_id, - ) - ) - mode = resolve_output_mode(json_output, plain) - if mode == "json": - print_json(result) - elif mode == "plain": - _plain_ls(result) - _write_plain_page_footer(result) - else: - _display_ls(result, path) - except (ValueError, ToolError) as e: - typer.echo(f"Error: {e}", err=True) - raise typer.Exit(1) - except Exception as e: # pragma: no cover - if not isinstance(e, typer.Exit): - typer.echo(f"Error during ls: {e}", err=True) - raise typer.Exit(1) - raise + +# --- find --- @app.command() def find( - path: Annotated[str, typer.Argument(help="Directory to start from")] = "/", + path: Annotated[str, typer.Argument(help="Folder to start from")] = "/", name: Annotated[ Optional[str], typer.Option("--name", help='File-name glob, e.g. "*.md"') ] = None, depth: Annotated[ - int, typer.Option("--depth", min=1, max=10, help="Recursion depth (API bound 1-10)") - ] = 10, - page: Annotated[int, typer.Option("--page", help="Page number (1-indexed)")] = 1, - page_size: Annotated[ - int, typer.Option("--page-size", help="Nodes per page") - ] = DEFAULT_DIRECTORY_PAGE_SIZE, + Optional[int], typer.Option("--depth", help="Folder levels to search (1-10)") + ] = None, meta: Annotated[ Optional[list[str]], typer.Option( "--meta", help=( - "Metadata predicate, repeatable: 'status=active', 'confidence>0.6', " + "Frontmatter predicate, repeatable: 'status=active', 'confidence>0.6', " "'priority in high,critical', 'tags has security', 'score between 0.3,0.8', " - "'owner=null' (key missing or null). " - "PATH still scopes the query, by file path" + "'owner=null' (key missing or null). PATH still scopes the query" ), ), ] = None, @@ -807,210 +317,169 @@ def find( Optional[str], typer.Option( "--fields", - help=( - 'Comma-separated frontmatter fields to show per hit, e.g. "title,priority" ' - "(requires --meta)" - ), + help='Comma-separated frontmatter fields to show per hit, e.g. "title,priority" ' + "(needs --meta)", ), ] = None, - json_output: JsonOption = False, - plain: PlainOption = False, + page: PageOption = None, + page_size: PageSizeOption = None, project: ProjectOption = None, - project_id: ProjectIdOption = None, + json_output: JsonOption = False, local: LocalOption = False, cloud: CloudOption = False, ) -> None: - """Recursively list files by name glob, or query notes by frontmatter metadata. + """Find files by name glob, or notes by frontmatter, recursively (bm_find). Examples: bm find --name "*.md" - bm find /specs --depth 3 - bm find /notes --name "auth*" --plain - bm find /specs --meta "status=active" --meta "confidence>0.6" --fields title,priority + bm find specs --depth 3 + bm find specs --meta "status=active" --meta "confidence>0.6" --fields title,priority """ - # Deferred: loading the MCP tool stack at module import slows CLI startup (#886). - from basic_memory.mcp.tools import find as mcp_find - from fastmcp.exceptions import ToolError + from basic_memory.cli.commands.find_predicates import parse_meta_predicates + from basic_memory.mcp.tools import bm_find try: - validate_routing_flags(local, cloud) - validate_output_flags(json_output, plain) - # The CLI only splits the comma form; validation (non-empty names, - # requires --meta) lives in the shared tool layer. - field_list = [item.strip() for item in fields.split(",")] if fields is not None else None - - with force_routing(local=local, cloud=cloud): - result = run_with_cleanup( - mcp_find( - path, - name=name, - depth=depth, - page=page, - page_size=page_size, - meta=meta, - fields=field_list, - project=project, - project_id=project_id, - ) - ) - mode = resolve_output_mode(json_output, plain) - if mode == "json": - print_json(result) - elif "results" in result: - # --meta flips the payload to the search response shape: projected - # fields get the dedicated renderers, otherwise grep's search - # renderers apply. - if field_list: - if mode == "plain": - _plain_find_fields(result) - _write_plain_page_footer(result, search=True) - else: - _display_find_fields(result, path, meta or [], field_list) - elif mode == "plain": - plain_search_results(result, query=" AND ".join(meta or [])) - _write_plain_page_footer(result, search=True) - else: - display_search_results(result, query=" AND ".join(meta or [])) - elif mode == "plain": - _plain_find(result) - _write_plain_page_footer(result) - else: - _display_find(result, path, name) - except (ValueError, ToolError) as e: - typer.echo(f"Error: {e}", err=True) - raise typer.Exit(1) - except Exception as e: # pragma: no cover - if not isinstance(e, typer.Exit): - typer.echo(f"Error during find: {e}", err=True) - raise typer.Exit(1) - raise + filters = parse_meta_predicates(meta) if meta is not None else None + except ValueError as error: + raise _refuse(str(error)) from None + run_tool( + bm_find, + path=path, + name=name, + depth=depth, + filters=filters, + # The comma form is the CLI's; bm_find validates the names themselves. + fields=[item.strip() for item in fields.split(",")] if fields is not None else None, + page=page, + page_size=page_size, + project=project, + json_output=json_output, + local=local, + cloud=cloud, + ) -@app.command() -def tail( - timeframe: Annotated[ - str, typer.Option("--timeframe", help='Time window, e.g. "7d", "yesterday"') - ] = "7d", - n: Annotated[ - int, typer.Option("-n", "--lines", min=1, max=100, help="Rows to show (1-100)") - ] = 10, - json_output: JsonOption = False, - plain: PlainOption = False, - project: ProjectOption = None, - project_id: ProjectIdOption = None, - local: LocalOption = False, - cloud: CloudOption = False, -) -> None: - """Show the most recently changed notes, newest first. +# --- tree --- - Examples: - bm tail - bm tail -n 20 --timeframe 1d - bm tail --plain +@dataclass +class _TreeEntry: + """One rebuilt tree node: whether it is a directory, and its children.""" + + is_dir: bool + children: dict[str, "_TreeEntry"] = field(default_factory=dict) + + +def _build_tree(nodes: list[dict[str, Any]], root: str) -> dict[str, _TreeEntry]: + """Rebuild a hierarchy from bm_find's flat page of directory nodes. + + The listing is flat; nesting is recovered from each node's directory_path + segments. A glob can return files without their parent directories, so + intermediate segments are synthesized as directories. + """ + root_prefix = root.strip("/") + top: dict[str, _TreeEntry] = {} + for node in nodes: + raw_path = str(node["directory_path"]).strip("/") + if root_prefix: + if raw_path == root_prefix: + continue + raw_path = raw_path.removeprefix(f"{root_prefix}/") + segments = raw_path.split("/") + level = top + for depth, segment in enumerate(segments): + is_leaf = depth == len(segments) - 1 + entry = level.get(segment) + if entry is None: + entry = _TreeEntry(is_dir=not is_leaf or node["type"] == "directory") + level[segment] = entry + elif is_leaf and node["type"] == "directory": + # A synthesized intermediate may already exist; the node's own + # listing is authoritative about it being a directory. + entry.is_dir = True + level = entry.children + return top + + +def _listing_root(path: str, project: str | None) -> str: + """The folder bm_find's node paths are relative to, in their own spelling. + + A path that starts with the project's name ('My Research/specs') comes back + qualified with the project's permalink ('/my-research/specs/...'), so the root + stripped to rebuild the hierarchy must carry that spelling too, not the + caller's. Any other path comes back as typed. """ - # Deferred: loading the MCP tool stack at module import slows CLI startup (#886). - from basic_memory.mcp.tools import tail as mcp_tail - from fastmcp.exceptions import ToolError + # Deferred: the identifier helpers load the MCP routing stack (#886). + from basic_memory.mcp.project_context_identifiers import split_project_permalink_prefix + from basic_memory.utils import generate_permalink - try: - validate_routing_flags(local, cloud) - validate_output_flags(json_output, plain) - - with force_routing(local=local, cloud=cloud): - result = run_with_cleanup( - mcp_tail( - timeframe=timeframe, - lines=n, - project=project, - project_id=project_id, - ) - ) - mode = resolve_output_mode(json_output, plain) - if mode == "json": - print_json(result) - elif mode == "plain": - _plain_tail(result) - else: - _display_tail(result) - except (ValueError, ToolError) as e: - typer.echo(f"Error: {e}", err=True) - raise typer.Exit(1) - except Exception as e: # pragma: no cover - if not isinstance(e, typer.Exit): - typer.echo(f"Error during tail: {e}", err=True) - raise typer.Exit(1) - raise + if project is None: + return path + claimed = split_project_permalink_prefix(path.strip("/"), [generate_permalink(project)]) + if claimed is None: + return path + permalink, remainder = claimed + return f"{permalink}/{remainder}" if remainder else permalink + + +def _add_tree_branches(branch: Tree, entries: dict[str, _TreeEntry]) -> None: + for segment, entry in entries.items(): + label = markup_escape(segment + ("/" if entry.is_dir else "")) + child = branch.add(f"[bold]{label}[/bold]" if entry.is_dir else label) + _add_tree_branches(child, entry.children) @app.command() def tree( - path: Annotated[str, typer.Argument(help="Directory to start from")] = "/", + path: Annotated[str, typer.Argument(help="Folder to start from")] = "/", name: Annotated[ Optional[str], typer.Option("--name", help='File-name glob, e.g. "*.md"') ] = None, depth: Annotated[ - int, typer.Option("--depth", min=1, max=10, help="Recursion depth (API bound 1-10)") - ] = 10, - page: Annotated[int, typer.Option("--page", help="Page number (1-indexed)")] = 1, - page_size: Annotated[ - int, typer.Option("--page-size", help="Nodes per page") - ] = DEFAULT_DIRECTORY_PAGE_SIZE, - json_output: JsonOption = False, - plain: PlainOption = False, + Optional[int], typer.Option("--depth", help="Folder levels to show (1-10)") + ] = None, + page: PageOption = None, + page_size: Annotated[int, typer.Option("--page-size", help="Entries per page (1-100)")] = 100, project: ProjectOption = None, - project_id: ProjectIdOption = None, + json_output: JsonOption = False, local: LocalOption = False, cloud: CloudOption = False, ) -> None: - """Show a directory hierarchy (tree(1) over `bm find`). + """Show a folder hierarchy (bm_find drawn as a tree). Examples: bm tree - bm tree /specs --depth 2 - bm tree --name "*.md" --plain + bm tree specs --depth 2 + bm tree --name "*.md" """ - # Deferred: loading the MCP tool stack at module import slows CLI startup (#886). - # tree is a client-side recombination of find: the same flat listing, with - # the hierarchy rebuilt from directory paths for display, so its JSON - # contract is exactly find's payload. - from basic_memory.mcp.tools.posix_tools import find_listing - from fastmcp.exceptions import ToolError + from basic_memory.mcp.tools import bm_find + + listing = call_tool( + bm_find, + path=path, + name=name, + depth=depth, + page=page, + page_size=page_size, + project=project, + output_format="json", + local=local, + cloud=cloud, + ) + if json_output: + print(json.dumps(listing, indent=2, ensure_ascii=True, default=str)) + return - try: - validate_routing_flags(local, cloud) - validate_output_flags(json_output, plain) - - # find returns the listing and the root its node paths are relative to - # from one resolution. Resolving here as well cost a second project-list - # round trip on every cloud call, because a CLI invocation has no - # FastMCP context for the per-request cache to live in (#1421). - with force_routing(local=local, cloud=cloud): - result, root = run_with_cleanup( - find_listing( - path, - name=name, - depth=depth, - page=page, - page_size=page_size, - project=project, - project_id=project_id, - ) - ) - mode = resolve_output_mode(json_output, plain) - if mode == "json": - print_json(result) - elif mode == "plain": - _plain_tree(result, path, root) - else: - _display_tree(result, path, root) - except (ValueError, ToolError) as e: - typer.echo(f"Error: {e}", err=True) - raise typer.Exit(1) - except Exception as e: # pragma: no cover - if not isinstance(e, typer.Exit): - typer.echo(f"Error during tree: {e}", err=True) - raise typer.Exit(1) - raise + drawn = Tree(f"[bold cyan]{markup_escape(path)}[/bold cyan]") + entries = _build_tree( + listing["nodes"], + _listing_root(path, project or ConfigManager().config.default_project), + ) + if not entries: + drawn.add("[dim]empty[/dim]") + _add_tree_branches(drawn, entries) + console.print(drawn) + if listing["has_more"]: + console.print(f"[dim]… more entries (page {listing['page']}; use --page)[/dim]") diff --git a/src/basic_memory/cli/commands/project.py b/src/basic_memory/cli/commands/project.py index ad079cb54..9d55af267 100644 --- a/src/basic_memory/cli/commands/project.py +++ b/src/basic_memory/cli/commands/project.py @@ -997,7 +997,7 @@ async def _add_project(): # Why: registration alone leaves every read surface reporting an empty # project while the notes sit unindexed on disk, and there is no # daemon-reachability probe to delegate to -- so index inline, the - # same way MCP's create_memory_project already does (#1414). + # same way MCP's bm_project(command="create") already does (#1414). # Outcome: the project's existing notes are queryable when this returns. try: with force_routing(local=local, cloud=cloud): diff --git a/src/basic_memory/cli/commands/tool.py b/src/basic_memory/cli/commands/tool.py index 431d0dc94..60ebc99f8 100644 --- a/src/basic_memory/cli/commands/tool.py +++ b/src/basic_memory/cli/commands/tool.py @@ -64,6 +64,39 @@ def run_tool( **arguments: Any, ) -> None: """Call a bm_* tool with MCP argument validation and print its result.""" + result = call_tool( + tool, + project=project, + output_format="json" if json_output else "text", + local=local, + cloud=cloud, + default_to_config_project=default_to_config_project, + **arguments, + ) + if json_output: + # bm_read returns image and document content as MCP content models; dump them + # with their wire names (mimeType), as an MCP client sees them. + if isinstance(result, BaseModel): + result = result.model_dump(mode="json", by_alias=True) + print(json.dumps(result, indent=2, ensure_ascii=True, default=str)) + else: + print(result) + + +def call_tool( + tool: Callable[..., Coroutine[Any, Any, Any]], + *, + project: str | None, + output_format: str, + local: bool, + cloud: bool, + default_to_config_project: bool = True, + **arguments: Any, +) -> Any: + """Call a bm_* tool with MCP argument validation and return its result. + + A refusal or an invalid argument prints to stderr and exits 1. + """ # Deferred with the tool stack (#886). from fastmcp.exceptions import ToolError from pydantic import ValidationError, validate_call @@ -90,7 +123,7 @@ def run_tool( result = run_with_cleanup( validated_tool( **supplied, - output_format="json" if json_output else "text", + output_format=output_format, # Outside an MCP request there is no FastMCP context to inject. context=None, ) @@ -103,15 +136,7 @@ def run_tool( except ToolError as error: typer.echo(f"Error: {error}", err=True) raise typer.Exit(1) - - if json_output: - # bm_read returns image and document content as MCP content models; dump them - # with their wire names (mimeType), as an MCP client sees them. - if isinstance(result, BaseModel): - result = result.model_dump(mode="json", by_alias=True) - print(json.dumps(result, indent=2, ensure_ascii=True, default=str)) - else: - print(result) + return result # --- Notes --- diff --git a/src/basic_memory/man/__init__.py b/src/basic_memory/man/__init__.py index 016864023..cb2e19a40 100644 --- a/src/basic_memory/man/__init__.py +++ b/src/basic_memory/man/__init__.py @@ -302,8 +302,8 @@ def render_parameters(tool_name: str, parameters: Mapping[str, Any]) -> str: (the order clients see) — mirroring render_synopsis. Each bullet names the parameter, its type when the schema gives one, whether it is required or optional (with the default for optionals that carry one), and its description. - Returns "" when the schema has no properties, so tools like - basic_memory_diagnostics get no section. + Returns "" when the schema has no properties, so a tool that takes no + arguments gets no section. """ required: list[str] = parameters.get("required") or [] properties: Mapping[str, Any] = parameters.get("properties") or {} @@ -377,8 +377,8 @@ def params(self) -> Sequence[ClickParam]: ... # Option pairs the CLI rejects in combination — ``--json``/``--plain`` guarded by -# _validate_output_flags and ``--local``/``--cloud`` by validate_routing_flags in -# cli/commands/posix.py; cat's --lines/--section conflict in mcp/tools/posix_tools.py. +# validate_output_flags (cli/commands/output.py), ``--local``/``--cloud`` by +# validate_routing_flags, and cat's --lines/--section by bm_read's refusal. # Click carries no cross-parameter constraint, so the # generator must name them here: the SYNOPSIS shows a fully-present pair as one # ``[--json | --plain]`` alternative rather than two freely-combinable tokens, the diff --git a/src/basic_memory/man/bm.1 b/src/basic_memory/man/bm.1 index 553e6a19c..33dc785f5 100644 --- a/src/basic_memory/man/bm.1 +++ b/src/basic_memory/man/bm.1 @@ -48,9 +48,6 @@ metadata (\-\-meta). .B bm tree Show a directory hierarchy. .TP -.B bm tail -Show recently changed notes, newest first. -.TP .B bm status Show sync status between files and the database. .TP diff --git a/src/basic_memory/man/man1/cat(1).md b/src/basic_memory/man/man1/cat(1).md index cc0c2450b..85457ec0f 100644 --- a/src/basic_memory/man/man1/cat(1).md +++ b/src/basic_memory/man/man1/cat(1).md @@ -18,43 +18,43 @@ generated: cli ``` bm cat IDENTIFIER [--lines LINES | --section SECTION] [--max-tokens MAX_TOKENS] [--frontmatter | --no-frontmatter] - [--json | --plain] [--project PROJECT] [--project-id PROJECT_ID] - [--local | --cloud] + [--project PROJECT] [--json] [--local | --cloud] ``` ## DESCRIPTION -Prints one note, resolved exactly by title, permalink, or memory:// URL. -The content can be sliced by a 1-indexed inclusive line range (`--lines +Prints one note through `bm_read`, resolved by path, permalink, memory:// +URL, or exact title. The content can be sliced by a 1-indexed inclusive line range (`--lines "20-40"`, `"20-"` to the end, `"20"` for one line), by a heading (`--section Decisions`, path form `Auth/Decisions`, or `Heading[1]` for a duplicate), or truncated to an approximate token budget (`--max-tokens`). -`--lines` and `--section` cannot be combined. A section response includes -start_line and end_line for follow-up `--lines` reads. +A text file that is not a note (an indexed `.txt` source, say) takes +`--lines` too; `--section` and `--max-tokens` are for notes. Line numbers count +the whole file, frontmatter included; `--frontmatter` and +`--no-frontmatter` shape only a whole-note read. `--lines` and `--section` +cannot be combined. A section response includes start_line and end_line for +follow-up `--lines` reads. -Projects in another cloud workspace are not among the session's own -projects. Address them as `//`, or pass -`--project-id` (pass the UUID as `project` in MCP calls). A bare `/` never -reaches another workspace; when its first segment names a project there, the -refusal names the workspace and both spellings that work. -`bm_project(command="list")` (`bm project list`) lists every workspace's projects. +Without `--project` the configured default project is used. A path may start +with the project's own name (`/`) when it agrees with `--project`; +a prefix naming a different project is refused. Projects in another cloud +workspace are addressed as `--project /` or +`--project `; `bm project list` lists every workspace's projects. -On a TTY the note renders as formatted Markdown; `--plain` writes the raw -content to stdout (slice details go to stderr); `--json`, or piped output, -emits the structured payload with slice metadata (start_line, end_line, -total_lines, truncated, continue_line). +A whole-note read prints the note itself. A slice prints numbered lines under +a `Lines N-M of T` header with the next range to read. `--json` prints +`bm_read`'s payload with slice metadata (start_line, end_line, total_lines, +has_more, next_start_line, truncated, continue_line). ## OPTIONS -- **--lines** — Line range "N-M", "N-" (to end), or "N" (one line); 1-indexed inclusive +- **--lines** — Line range "N-M", "N-" (to end), or "N" (one line); 1-indexed, frontmatter included - **--section** — Heading slice: "Decisions", "Auth/Decisions", or "Heading[1]" -- **--max-tokens** — Approximate token budget; truncates at a section/paragraph boundary -- **--frontmatter / --no-frontmatter** (default: --frontmatter) — Include the YAML frontmatter block (ignored for section/token slices) -- **--json** — Output raw JSON instead of formatted display -- **--plain** — Output undecorated plain text (no colors/markup), even when piped -- **--project** — The project to use. If not provided, the default project will be used. -- **--project-id** — Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces. +- **--max-tokens** — Approximate token budget; cuts at a section or paragraph boundary +- **--frontmatter / --no-frontmatter** (default: --frontmatter) — Include the YAML frontmatter in a whole-note read +- **--project** — Project name, 'workspace/project', or project id. Defaults to the default project. +- **--json** — Print the tool's JSON result - **--local** — Force local API routing (ignore cloud mode) - **--cloud** — Force cloud API routing @@ -62,7 +62,7 @@ total_lines, truncated, continue_line). ``` bm cat specs/search -bm cat specs/search --lines 20-40 --plain +bm cat specs/search --lines 20-40 bm cat specs/search --section Decisions --max-tokens 500 ``` diff --git a/src/basic_memory/man/man1/find(1).md b/src/basic_memory/man/man1/find(1).md index 9367f9ecf..cd8285a2a 100644 --- a/src/basic_memory/man/man1/find(1).md +++ b/src/basic_memory/man/man1/find(1).md @@ -17,26 +17,25 @@ generated: cli ``` bm find [PATH] [--name NAME] [--depth DEPTH] [--page PAGE] - [--page-size PAGE_SIZE] [--json | --plain] [--project PROJECT] - [--project-id PROJECT_ID] [--local | --cloud] + [--page-size PAGE_SIZE] [--project PROJECT] [--json] + [--local | --cloud] -bm find [PATH] [--page PAGE] [--page-size PAGE_SIZE] - --meta META [--meta META ...] [--fields FIELDS] [--json | --plain] - [--project PROJECT] [--project-id PROJECT_ID] [--local | --cloud] +bm find [PATH] --meta META [--meta META ...] [--fields FIELDS] [--page PAGE] + [--page-size PAGE_SIZE] [--project PROJECT] [--json] + [--local | --cloud] ``` ## DESCRIPTION Two modes, chosen by `--meta`. -Without `--meta`, find recursively lists files under a directory (default: -the project root), optionally filtered by a file-name glob. Depth is bounded -1-10 by the directory API. On a TTY results render as a table; `--plain` -prints one path per line, find(1) style; `--json` (or piped output) emits the -listing with pagination and totals. +find calls `bm_find`. Without `--meta`, it recursively lists files under a +directory (default: the project root), optionally filtered by a file-name +glob. Depth is bounded 1-10 by the directory API. `--json` emits the listing +with pagination and totals. -With `--meta`, find queries notes by their frontmatter instead: every -predicate must hold, and `PATH` still scopes the results — server-side, by +With `--meta`, find queries notes by their frontmatter instead: the +predicates become `bm_find`'s `filters`, every predicate must hold, and `PATH` still scopes the results — server-side, by file-path prefix, so the totals are exact and every page is reachable. The payload becomes the search response shape (the same one `bm grep` returns), not the directory listing. Non-markdown files carry no frontmatter and are @@ -54,16 +53,15 @@ Only the surrounding separators and a leading `./` are notation — the plain listing reads them the same way, so one `PATH` names one subtree with or without `--meta`. Everything else belongs to the path, including whitespace: a directory named `" specs "` is addressed by that exact spelling. -`PATH` may also name a project (`bm find myproject --meta ...`) — that is a -routing prefix, a mount point rather than a subtree, and scopes to that -project's root. +`PATH` may also start with the project's own name +(`bm find myproject --meta ... --project myproject`) — that prefix is a mount +point rather than a subtree, and scopes to that project's root. -Projects in another cloud workspace are not among the session's own -projects. Address them as `//`, or pass -`--project-id` (pass the UUID as `project` in MCP calls). A bare `/` never -reaches another workspace; when its first segment names a project there, the -refusal names the workspace and both spellings that work. -`bm_project(command="list")` (`bm project list`) lists every workspace's projects. +Without `--project` the configured default project is used. A path may start +with the project's own name (`/`) when it agrees with `--project`; +a prefix naming a different project is refused. Projects in another cloud +workspace are addressed as `--project /` or +`--project `; `bm project list` lists every workspace's projects. `--fields` is the SELECT to the predicates' WHERE: each hit comes back as the note's identity — title, permalink, file path, external id, last-updated — plus @@ -139,15 +137,13 @@ predicates keep their existing frontmatter comparison behavior. ## OPTIONS - **--name** — File-name glob, e.g. "*.md" -- **--depth** (default: 10) — Recursion depth (API bound 1-10) -- **--page** (default: 1) — Page number (1-indexed) -- **--page-size** (default: 10) — Nodes per page -- **--meta** — Metadata predicate, repeatable: 'status=active', 'confidence>0.6', 'priority in high,critical', 'tags has security', 'score between 0.3,0.8', 'owner=null' (key missing or null). PATH still scopes the query, by file path -- **--fields** — Comma-separated frontmatter fields to show per hit, e.g. "title,priority" (requires --meta) -- **--json** — Output raw JSON instead of formatted display -- **--plain** — Output undecorated plain text (no colors/markup), even when piped -- **--project** — The project to use. If not provided, the default project will be used. -- **--project-id** — Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces. +- **--depth** — Folder levels to search (1-10) +- **--meta** — Frontmatter predicate, repeatable: 'status=active', 'confidence>0.6', 'priority in high,critical', 'tags has security', 'score between 0.3,0.8', 'owner=null' (key missing or null). PATH still scopes the query +- **--fields** — Comma-separated frontmatter fields to show per hit, e.g. "title,priority" (needs --meta) +- **--page** — Page number (1-indexed) +- **--page-size** — Results per page (1-100) +- **--project** — Project name, 'workspace/project', or project id. Defaults to the default project. +- **--json** — Print the tool's JSON result - **--local** — Force local API routing (ignore cloud mode) - **--cloud** — Force cloud API routing @@ -156,13 +152,13 @@ predicates keep their existing frontmatter comparison behavior. ``` bm find --name "*.md" bm find /specs --depth 3 -bm find /notes --name "auth*" --plain +bm find /notes --name "auth*" bm find --meta "status=active" bm find /specs --meta "status=active" --meta "confidence>0.6" -bm find myproject --meta "status=active" +bm find myproject --meta "status=active" --project myproject bm find --meta "owner=null" --fields title bm find --meta "tags has security,oauth" --fields title,priority -bm find --meta "status=active" --fields title --plain +bm find --meta "status=active" --fields title --json ``` ## SEE ALSO diff --git a/src/basic_memory/man/man1/grep(1).md b/src/basic_memory/man/man1/grep(1).md index 0f2ecf200..e08b43f47 100644 --- a/src/basic_memory/man/man1/grep(1).md +++ b/src/basic_memory/man/man1/grep(1).md @@ -16,55 +16,53 @@ generated: cli ## SYNOPSIS ``` -bm grep PATTERN [--semantic] [--context-lines CONTEXT_LINES] - [--max-matches MAX_MATCHES] [--page PAGE] [--page-size PAGE_SIZE] - [--json | --plain] [--project PROJECT] [--project-id PROJECT_ID] +bm grep PATTERN [--semantic] [--lines CONTEXT_LINES] + [--max-matches MAX_MATCHES] [--path PATH] [--page PAGE] + [--page-size PAGE_SIZE] [--project PROJECT] [--json] [--local | --cloud] ``` ## DESCRIPTION -Searches note content by full-text keyword matching, so a pattern that appears in -no note returns no results. It is not a regex: the pattern is tokenized like any +Searches note content through `bm_search` in text mode: full-text keyword +matching, so a pattern that appears in no note returns no results. It is not a regex: the pattern is tokenized like any full-text query. `-s` (`--semantic`) ranks by meaning instead, with hybrid semantic plus full-text search when semantic search is enabled for the project. Semantic results are nearest neighbours, so they can include notes that do not -contain the pattern. Results carry title, score, permalink, and the matched -snippet; on a TTY they render as a table, and `--json` (or piped output) emits -the search response with pagination. +contain the pattern. The text result lists each hit with its path and a match +excerpt; `--json` emits the search response with pagination. -With `-C N`, return compact literal match windows instead. The full-text index -selects a page of candidate notes, then their current content is checked for -case-insensitive literal substrings. Matching line numbers include frontmatter -and can be passed directly to `bm_read(start_line=..., end_line=...)` or -`bm cat ... --lines N-M`. Overlapping context windows merge to avoid repeated text. -The full body and search excerpt are omitted in this mode, including JSON output. +With `-C N`, each hit returns its matching lines with N lines of context +instead of an excerpt. The full-text index selects a page of ranked notes, then +their current content is scanned for lines containing the query's words or +quoted phrases. Line numbers include frontmatter and can be passed directly to +`bm cat ... --lines N-M`. Overlapping context windows merge to avoid repeated +text, and `--max-matches` bounds the lines shown per note. This is not an exhaustive filesystem grep: index tokenization, stemming, or edits -can produce a candidate with zero literal matches, or omit a substring-only match. -Pagination and totals refer to search candidates, not exact matching notes or lines. -Each candidate reports `match_count`, `total_lines`, `windows`, and `next_match_line` -(the first omitted matching line, or null). Use the latter for a targeted note read. -Line positions can change if the note is edited between calls. +can rank a note with no matching line, and such notes are dropped from the page. +`total` therefore counts ranked candidates and `total_is_exact` is false; page with +`has_more`. Each hit reports `match_count`, `windows`, and `next_match_line` (the +first omitted matching line, or null). Line positions can change if the note is +edited between calls. ## OPTIONS - **-s, --semantic** — Rank by meaning instead of matching keywords -- **-C, --context-lines** — Compact line matches with surrounding context (not with --semantic) -- **--max-matches** (default: 10) — Matching lines per candidate in context mode -- **--page** (default: 1) — Page number (1-indexed) -- **--page-size** (default: 10) — Results per page -- **--json** — Output raw JSON instead of formatted display -- **--plain** — Output undecorated plain text (no colors/markup), even when piped -- **--project** — The project to use. If not provided, the default project will be used. -- **--project-id** — Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces. +- **-C, --lines** — Return matching lines with this many lines of context (0-10) +- **--max-matches** — Matching lines per note with -C (1-100) +- **--path** — Search only this folder +- **--page** — Page number (1-indexed) +- **--page-size** — Results per page (1-100) +- **--project** — Project name, 'workspace/project', or project id. Defaults to the default project. +- **--json** — Print the tool's JSON result - **--local** — Force local API routing (ignore cloud mode) - **--cloud** — Force cloud API routing ## EXAMPLES ``` -bm grep "retry" -C 3 --max-matches 5 --plain +bm grep "retry" -C 3 --max-matches 5 bm grep --semantic "auth token rotation" bm grep "BASIC_MEMORY_FORCE_LOCAL" bm grep "deploy checklist" --json | jq '.results[].permalink' diff --git a/src/basic_memory/man/man1/head(1).md b/src/basic_memory/man/man1/head(1).md index db7581c51..05c3dfbd9 100644 --- a/src/basic_memory/man/man1/head(1).md +++ b/src/basic_memory/man/man1/head(1).md @@ -16,26 +16,23 @@ generated: cli ## SYNOPSIS ``` -bm head IDENTIFIER [--lines N] [--frontmatter | --no-frontmatter] - [--json | --plain] [--project PROJECT] [--project-id PROJECT_ID] +bm head IDENTIFIER [--lines N] [--project PROJECT] [--json] [--local | --cloud] ``` ## DESCRIPTION -Prints the first `-n` lines (default 10) of one note, resolved exactly by -title, permalink, or memory:// URL. head is `bm cat` with a fixed line -range, so its JSON payload is exactly cat's: content plus start_line, -end_line, and total_lines. +Prints the first `-n` lines (default 10) of one note through `bm_read`, +resolved by path, permalink, memory:// URL, or exact title. Lines count the +whole file, frontmatter included. The text result numbers the lines; `--json` +is `bm_read`'s slice payload: content plus start_line, end_line, total_lines, +and the next range to read. ## OPTIONS - **-n, --lines** (default: 10) — Number of lines to print (from line 1) -- **--frontmatter / --no-frontmatter** (default: --frontmatter) — Include the YAML frontmatter block -- **--json** — Output raw JSON instead of formatted display -- **--plain** — Output undecorated plain text (no colors/markup), even when piped -- **--project** — The project to use. If not provided, the default project will be used. -- **--project-id** — Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces. +- **--project** — Project name, 'workspace/project', or project id. Defaults to the default project. +- **--json** — Print the tool's JSON result - **--local** — Force local API routing (ignore cloud mode) - **--cloud** — Force cloud API routing @@ -43,11 +40,10 @@ end_line, and total_lines. ``` bm head specs/search -bm head specs/search -n 3 --plain +bm head specs/search -n 3 ``` ## SEE ALSO - see_also [[cat(1)]] -- see_also [[tail(1)]] - see_also [[bm-read(3)]] diff --git a/src/basic_memory/man/man1/ls(1).md b/src/basic_memory/man/man1/ls(1).md index 02306607f..402ae2f4a 100644 --- a/src/basic_memory/man/man1/ls(1).md +++ b/src/basic_memory/man/man1/ls(1).md @@ -16,40 +16,35 @@ generated: cli ## SYNOPSIS ``` -bm ls [PATH] [--page PAGE] [--page-size PAGE_SIZE] [--json | --plain] - [--project PROJECT] [--project-id PROJECT_ID] [--local | --cloud] +bm ls [PATH] [--glob GLOB] [--sort SORT] [--page PAGE] + [--page-size PAGE_SIZE] [--project PROJECT] [--json] + [--local | --cloud] ``` ## DESCRIPTION -Lists the immediate contents of one directory (default: the project root). -Directories print with a trailing slash. On a TTY the listing renders as a -table with title, permalink, and update time; `--plain` prints one path per -line, ls -1 style; `--json` (or piped output) emits the listing with -pagination and totals. +Lists the immediate contents of one directory (default: the project root) +through `bm_list`. `--glob` filters file names and `--sort` orders files; +folders always come first. `--json` emits the listing with pagination and +totals. `bm ls` lists files inside one project; the unrelated `bm project ls` lists projects. -The MCP `bm_list` tool lists the session's projects as mount points at `/`. A cloud -session that can reach several workspaces also gets their slugs and a note on -reaching their projects. - -Projects in another cloud workspace are not among the session's own -projects. Address them as `//`, or pass -`--project-id` (pass the UUID as `project` in MCP calls). A bare `/` never -reaches another workspace; when its first segment names a project there, the -refusal names the workspace and both spellings that work. -`bm_project(command="list")` (`bm project list`) lists every workspace's projects. +Without `--project` the configured default project is used. A path may start +with the project's own name (`/`) when it agrees with `--project`; +a prefix naming a different project is refused. Projects in another cloud +workspace are addressed as `--project /` or +`--project `; `bm project list` lists every workspace's projects. ## OPTIONS -- **--page** (default: 1) — Page number (1-indexed) -- **--page-size** (default: 10) — Nodes per page -- **--json** — Output raw JSON instead of formatted display -- **--plain** — Output undecorated plain text (no colors/markup), even when piped -- **--project** — The project to use. If not provided, the default project will be used. -- **--project-id** — Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces. +- **--glob** — File-name glob, e.g. '*.md' +- **--sort** — File order +- **--page** — Page number (1-indexed) +- **--page-size** — Results per page (1-100) +- **--project** — Project name, 'workspace/project', or project id. Defaults to the default project. +- **--json** — Print the tool's JSON result - **--local** — Force local API routing (ignore cloud mode) - **--cloud** — Force cloud API routing @@ -58,7 +53,7 @@ refusal names the workspace and both spellings that work. ``` bm ls bm ls /specs -bm ls /notes --plain +bm ls /notes --glob '*.md' --sort updated_desc ``` ## SEE ALSO diff --git a/src/basic_memory/man/man1/tail(1).md b/src/basic_memory/man/man1/tail(1).md deleted file mode 100644 index 3e5f887c8..000000000 --- a/src/basic_memory/man/man1/tail(1).md +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: tail(1) -type: manpage -section: 1 -name: tail -summary: show recently changed notes -generated: cli ---- - -# tail(1) - -## NAME - -**tail** — show recently changed notes - -## SYNOPSIS - -``` -bm tail [--timeframe TIMEFRAME] [--lines N] [--json | --plain] - [--project PROJECT] [--project-id PROJECT_ID] [--local | --cloud] -``` - -## DESCRIPTION - -Shows the most recently changed notes in a project, newest first — tail as -in "the tail of the change log", not of one file. Each row carries the -last-change time, type, title, permalink, and file path; `--json` rows also -keep `created_at`. On a TTY rows render as a table; `--plain` prints -tab-separated lines; `--json` (or piped output) emits the rows as a JSON array. - -## OPTIONS - -- **--timeframe** (default: "7d") — Time window, e.g. "7d", "yesterday" -- **-n, --lines** (default: 10) — Rows to show (1-100) -- **--json** — Output raw JSON instead of formatted display -- **--plain** — Output undecorated plain text (no colors/markup), even when piped -- **--project** — The project to use. If not provided, the default project will be used. -- **--project-id** — Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces. -- **--local** — Force local API routing (ignore cloud mode) -- **--cloud** — Force cloud API routing - -## EXAMPLES - -``` -bm tail -bm tail -n 20 --timeframe 1d -bm tail --plain | cut -f3 -``` - -## SEE ALSO - -- see_also [[head(1)]] -- see_also [[bm-project(3)]] diff --git a/src/basic_memory/man/man1/tree(1).md b/src/basic_memory/man/man1/tree(1).md index 73c2f0605..01bc5badd 100644 --- a/src/basic_memory/man/man1/tree(1).md +++ b/src/basic_memory/man/man1/tree(1).md @@ -17,28 +17,25 @@ generated: cli ``` bm tree [PATH] [--name NAME] [--depth DEPTH] [--page PAGE] - [--page-size PAGE_SIZE] [--json | --plain] [--project PROJECT] - [--project-id PROJECT_ID] [--local | --cloud] + [--page-size PAGE_SIZE] [--project PROJECT] [--json] + [--local | --cloud] ``` ## DESCRIPTION Shows the hierarchy under a directory (default: the project root), rebuilt -from the same recursive listing `bm find` uses. Directories print with a -trailing slash. On a TTY the hierarchy renders as a tree; `--plain` prints -two-space-indented lines; `--json` (or piped output) emits find's flat -listing payload — the hierarchy is a display concern. +from the recursive `bm_find` listing `bm find` prints. Directories print with +a trailing slash. `--json` emits `bm_find`'s flat listing payload — the +hierarchy is a display concern. ## OPTIONS - **--name** — File-name glob, e.g. "*.md" -- **--depth** (default: 10) — Recursion depth (API bound 1-10) -- **--page** (default: 1) — Page number (1-indexed) -- **--page-size** (default: 10) — Nodes per page -- **--json** — Output raw JSON instead of formatted display -- **--plain** — Output undecorated plain text (no colors/markup), even when piped -- **--project** — The project to use. If not provided, the default project will be used. -- **--project-id** — Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces. +- **--depth** — Folder levels to show (1-10) +- **--page** — Page number (1-indexed) +- **--page-size** (default: 100) — Entries per page (1-100) +- **--project** — Project name, 'workspace/project', or project id. Defaults to the default project. +- **--json** — Print the tool's JSON result - **--local** — Force local API routing (ignore cloud mode) - **--cloud** — Force cloud API routing @@ -47,7 +44,7 @@ listing payload — the hierarchy is a display concern. ``` bm tree bm tree /specs --depth 2 -bm tree --name "*.md" --plain +bm tree --name "*.md" ``` ## SEE ALSO diff --git a/src/basic_memory/man/man3/bm-project(3).md b/src/basic_memory/man/man3/bm-project(3).md index 7dd97ac81..9c96885e0 100644 --- a/src/basic_memory/man/man3/bm-project(3).md +++ b/src/basic_memory/man/man3/bm-project(3).md @@ -41,7 +41,7 @@ While the local server is running, its watcher picks up a new project's folder at the next project reload. Once the watch is attached, a one-time index pass includes files copied in before attachment; later changes use normal file events. -The shell counterpart for `recent` is tail(1). +The shell spelling of `recent` is `bm tool project recent`. ## PARAMETERS @@ -78,6 +78,5 @@ bm_project(command="recent", since="1d", project="demo") ## SEE ALSO -- see_also [[tail(1)]] - see_also [[bm-project-delete(3)]] - see_also [[bm-list(3)]] diff --git a/src/basic_memory/man/man3/bm-read(3).md b/src/basic_memory/man/man3/bm-read(3).md index dc87ef377..42b23ea16 100644 --- a/src/basic_memory/man/man3/bm-read(3).md +++ b/src/basic_memory/man/man3/bm-read(3).md @@ -48,8 +48,11 @@ continue. `updated_at`. The checksum is the `expected_checksum` that bm_write, bm_edit, bm_move and bm_delete take to guard a change. -Images come back as images and other binary files as documents. A missing note -is a `NOT_FOUND` error; find notes with bm_search. +A text file that is not a note (`.txt`, `.json`, ...) takes `start_line` and +`end_line` too, with the same coordinates and continuation; `section` and +`max_tokens` apply to Markdown notes only. Images come back as images and other +binary files as documents, always whole. A missing note is a `NOT_FOUND` error; +find notes with bm_search. The shell counterpart is cat(1). diff --git a/src/basic_memory/man/man7/tools-migration(7).md b/src/basic_memory/man/man7/tools-migration(7).md index 47da64144..0cd1d96f8 100644 --- a/src/basic_memory/man/man7/tools-migration(7).md +++ b/src/basic_memory/man/man7/tools-migration(7).md @@ -40,7 +40,7 @@ The guide's "Tool-by-tool change list" gives each tool's renamed arguments and r | `list_directory`, `ls` | `bm_list` for one level; `bm_find` for recursive inventory | | `find` | `bm_find` | | `recent_activity` | `bm_project(command="recent")` | -| `tail` | `bm_project(command="recent")`; map `timeframe` to `since` and `lines` to `page_size`. Recent JSON includes `created_at`, not `updated_at`; shell users needing the last-change timestamp can use `bm tail --json`. | +| `tail` | `bm_project(command="recent")`; map `timeframe` to `since` and `lines` to `page_size`. Recent JSON includes `created_at`, not `updated_at`. The `bm tail` shell command is removed; use `bm tool project recent`. | | `list_memory_projects` | `bm_project(command="list")` | | `list_workspaces` | `bm_project(command="list")` groups projects by workspace, so it omits a workspace with no projects; `bm cloud workspace list` in a shell lists every workspace | | `create_memory_project` | `bm_project(command="create", name=..., path=...)` | @@ -80,6 +80,30 @@ local replacement, and `basic_memory_diagnostics` is `bm doctor` and `bm config `bm tool` prints the tool's text result, or its JSON result with `--json`. It no longer switches to JSON when piped, so scripts pass `--json`. +## SHELL READ VERBS + +The shell read verbs now run the same tools the same way: `bm cat` and `bm head` call +`bm_read`, `bm grep` calls `bm_search` in text mode, `bm ls` calls `bm_list`, and +`bm find` and `bm tree` call `bm_find`. Their output is the tool's text result, or its +JSON result with `--json`. + +| Previous shell verb | 1.0 shell verb | +| --- | --- | +| `bm tail` | removed; use `bm tool project recent` | +| `--plain`, Rich tables, JSON when piped | removed; text result by default, `--json` for JSON | +| `--project-id UUID` | `--project UUID` | +| no `--project` with several projects: refused | the configured default project | +| `/path` routes to that project by itself | pass `--project `; a prefix naming another project is refused | +| `bm ls /` lists the projects | lists the default project's root; `bm project list` lists projects | +| `bm cat --lines` with `--no-frontmatter` counts body lines | `--lines` always counts the whole file, frontmatter included; `--frontmatter` applies to whole-note reads | +| `bm head --frontmatter/--no-frontmatter` | removed; `head` reads lines 1-N of the file | +| `bm grep -C/--context-lines N` | `bm grep -C/--lines N` | +| `bm grep ""` refused; `-C` with `--semantic` refused | follow `bm_search` | +| (new) `bm grep --path DIR` | search one folder | +| (new) `bm ls --glob`, `--sort` | `bm_list`'s file-name glob and order | +| `ls`/`find`/`tree` `--page-size` up to 200 | 1-100; `tree` defaults to 100 | +| `bm find --fields` hit rows rendered as a table | `bm_find`'s text result | + ## SEE ALSO - see_also [[bm-man(3)]] diff --git a/src/basic_memory/markdown/line_scanning.py b/src/basic_memory/markdown/line_scanning.py index 6c62cc08f..c17912bb7 100644 --- a/src/basic_memory/markdown/line_scanning.py +++ b/src/basic_memory/markdown/line_scanning.py @@ -86,7 +86,8 @@ def format_line_read( range(start_line, end_line + 1), content.split("\n"), strict=True ) ) - if end_line >= start_line + # An empty document is the range 0-0 of 0 and has no lines to number. + if total_lines and end_line >= start_line else "" ) if end_line < total_lines: diff --git a/src/basic_memory/markdown/sections.py b/src/basic_memory/markdown/sections.py index 729faffc9..71307856a 100644 --- a/src/basic_memory/markdown/sections.py +++ b/src/basic_memory/markdown/sections.py @@ -324,7 +324,8 @@ class NoteSlice: ``content`` never carries a frontmatter block unless an explicit line range covered it; line numbers always count over the full stored document - (frontmatter included), matching line-range follow-up reads. + (frontmatter included), matching line-range follow-up reads. An empty + document has no line 1, so any slice of it is the empty range 0-0 of 0. """ content: str @@ -586,7 +587,14 @@ def slice_note_content( start_line = frontmatter_lines + 1 end_line = total_lines - selected = doc_lines[start_line - 1 : end_line] + # Trigger: the document has no lines at all (an empty text file). + # Why: a range from line 1 to the last line came back as 1-0, a range that + # ends before it starts, which no follow-up read can use. + # Outcome: the empty range 0-0 of 0, with empty content. + if total_lines == 0: + start_line = end_line = 0 + + selected = doc_lines[max(start_line - 1, 0) : end_line] content = "\n".join(selected) truncated = False continue_line: int | None = None diff --git a/src/basic_memory/markdown/temporal_qualifier.py b/src/basic_memory/markdown/temporal_qualifier.py index 336f866f3..88230c3c1 100644 --- a/src/basic_memory/markdown/temporal_qualifier.py +++ b/src/basic_memory/markdown/temporal_qualifier.py @@ -23,7 +23,7 @@ Quoting settles where the token ends; it does not make a date mean something fixed, so `@occurred:"2 days ago"` is delimited and still refused. The closing quote is the token boundary, so whatever follows it is ordinary content, and a `\\"` inside the value does -not end the token. The scan mirrors `_split_predicate_items` in `mcp/tools/posix_tools.py`, down to +not end the token. The scan mirrors `_split_predicate_items` in `cli/commands/find_predicates.py`, down to its rule that an unterminated quote is a typo to report rather than a boundary to guess at -- scanning on to end of line would hand the author's prose to dateparser. Only the double quote opens the form: an apostrophe is ordinary punctuation, and a scan looking diff --git a/src/basic_memory/mcp/directory_listing.py b/src/basic_memory/mcp/directory_listing.py index 7bb616785..e2621c690 100644 --- a/src/basic_memory/mcp/directory_listing.py +++ b/src/basic_memory/mcp/directory_listing.py @@ -1,63 +1,12 @@ -"""Directory listings: the project mount view and the text rendering of a page. +"""The text rendering of a directory listing page. -The legacy mount view lists addressable projects as directories. The bm_* surface -discovers projects with bm_project and requires scope on bm_list. The text -formatter renders any listing page with the next-page call when more pages exist. +The formatter renders any listing page with the next-page call when more pages +exist. """ from datetime import datetime -from typing import Any -from fastmcp import Context - -from basic_memory.mcp.project_context import ( - OTHER_WORKSPACES_HINT, - addressable_projects, - other_workspace_slugs, -) -from basic_memory.schemas.directory import DirectoryListResponse, DirectoryNode - - -async def project_mount_listing( - *, page: int, page_size: int, context: Context | None -) -> dict[str, Any]: - """Render the addressable projects as directory entries (the mount-point view). - - Sources ``addressable_projects`` — the same set the path resolver routes by - — so every mount advertised here is reachable as '/path' (#1421). - Each row's ``directory_path`` is the copyable '/' prefix form, and - the set already arrives sorted by project name. - - A cloud session that can see several workspaces also gets their slugs and a - note on reaching them (#1778). They are not listed as rows: a bare slug is not - a routable path, and every row here must be one. - """ - rows = [ - DirectoryNode( - name=item.name, - directory_path=f"/{item.permalink}", - permalink=item.permalink, - type="directory", - ) - for item in await addressable_projects(context=context) - ] - start = (page - 1) * page_size - listing = DirectoryListResponse( - nodes=rows[start : start + page_size], - page=page, - page_size=page_size, - total=len(rows), - has_more=start + page_size < len(rows), - ).model_dump(mode="json") - workspaces = await other_workspace_slugs(context=context) - if workspaces: - listing["workspaces"] = list(workspaces) - listing["note"] = ( - "the projects above are this session's workspace; " - + OTHER_WORKSPACES_HINT - + f" (workspace slugs: {', '.join(workspaces)})" - ) - return listing +from basic_memory.schemas.directory import DirectoryListResponse def format_listing_text( diff --git a/src/basic_memory/mcp/note_targets.py b/src/basic_memory/mcp/note_targets.py index 27d7e3021..9bdec4b50 100644 --- a/src/basic_memory/mcp/note_targets.py +++ b/src/basic_memory/mcp/note_targets.py @@ -233,8 +233,9 @@ class PathQualifier: def for_route(cls, route: ProjectPathRoute) -> "PathQualifier": if not route.stripped or route.project is None: return cls(prefix=None) - # The permalink form is the spelling `ls "/"` advertises and the resolver - # normalizes to, so it round-trips for display names too ('My Research'). + # The permalink form is the spelling addressable_projects carries and the + # resolver normalizes to, so it round-trips for display names too + # ('My Research'). return cls(prefix=generate_permalink(route.project)) def path(self, value: str) -> str: diff --git a/src/basic_memory/mcp/project_context.py b/src/basic_memory/mcp/project_context.py index 8a08f8aaf..5952bd269 100644 --- a/src/basic_memory/mcp/project_context.py +++ b/src/basic_memory/mcp/project_context.py @@ -272,7 +272,7 @@ async def resolve_workspace_qualified_identifier( 'memory://main/notes' readable as project 'main' with note 'notes': a workspace route has to name something *inside* the project, or the same string would have two readings and the project-prefix resolver would lose - the ones it has always owned. The posix resolver takes the pathless form + the ones it has always owned. The path resolver takes the pathless form (a project root is a legitimate thing to list) and says so at its own call site. """ @@ -981,7 +981,7 @@ class DetectedProjectRoute: ``project_id`` is the external_id of the exact project that claimed the segments, or None for a locally routed session, whose config holds no UUIDs and which has no second workspace to be confused with. Callers pass both, - the way the posix verbs pass ``ProjectPathRoute``'s pair: a project name is + the way path routing passes ``ProjectPathRoute``'s pair: a project name is unique only inside one workspace, so handing on the name alone lets it be re-resolved against a different accessible workspace that happens to hold the same permalink (#1432). @@ -1025,7 +1025,7 @@ async def detect_project_from_identifier_prefix( normal case on this surface — 'specs/search-spec' is an ordinary permalink, and a search query is arbitrary text — so the answer for it is None and the caller's active/default project applies. That is why these tools do not - adopt the posix verbs' refusal rule (#1421): there a first segment naming + adopt path routing's refusal rule (#1421): there a first segment naming nothing is an error, here it is the common input. What they do adopt is the *set*, and its precedence. A path-shaped @@ -1035,19 +1035,20 @@ async def detect_project_from_identifier_prefix( stays a fact about the known projects rather than a guess about the string: 1. the local config (``detect_project_from_url_prefix``), free to read; - 2. ``addressable_projects`` — the session's own mount table, the same set - ``ls /`` advertises and ``resolve_project_path_route`` routes by. In a + 2. ``addressable_projects`` — the session's own projects, the same set + ``resolve_project_path_route`` routes by. In a hosted session that listing comes from the session's own tenant route, so it is exactly the projects of the workspace this session is bound to; 3. then, and only then, an explicitly workspace-qualified '//' route, for projects in workspaces this session's own route does not list. - Mounts before workspace routes is ``resolve_project_path_route``'s order, - for its reason: only one reading of '/...' can win, and a name ``ls - /`` advertises must address that mount or we advertise an address that - resolves somewhere else. Reading it the other way here meant `ls team/docs` - and `bm_read("team/docs/x")` disagreed about what 'team' named. It is also + Addressable projects before workspace routes is + ``resolve_project_path_route``'s order, for its reason: only one reading of + '/...' can win, and a project prefix that tool output emits must + address that project, or a returned path resolves somewhere else. Reading it + the other way here meant `bm_list("team/docs")` and + `bm_read("team/docs/x")` disagreed about what 'team' named. It is also the cheaper order — a mount hit never builds the account-wide workspace index at all. @@ -1124,24 +1125,24 @@ async def detect_project_from_identifier_prefix( ) -# --- Project-qualified path routing (POSIX tools, #1415) --- +# --- Project-qualified path routing (#1415) --- # Projects are mount points: '/path' inputs route to that project so # tool inputs accept exactly the prefixed identifiers tool outputs and stored -# permalinks produce. One resolver serves the MCP posix tools and, through -# them, the CLI verbs. +# permalinks produce. One resolver serves the bm_* tools (through +# open_project_target) and, through them, the CLI verbs. # -# The mount table and the routing table are one set (addressable_projects), so -# every mount `ls /` advertises is reachable by name and an unqualified -# reference in a many-project workspace refuses instead of picking a default -# (#1421). +# The set a prefix is matched against is one set (addressable_projects), so +# every project a tool names in its output is reachable by name, and an +# unqualified reference in a many-project workspace refuses instead of picking a +# default (#1421). # -# Precedence: the mount table wins the leading path segments, ahead of +# Precedence: addressable projects win the leading path segments, ahead of # workspace-qualified '//' parsing. A project permalink # can also be an accessible workspace's slug, and only one reading of '/...' -# can win. Mount-wins is what the advertised list promises — a name `ls /` shows -# must address that mount, or we advertise a name that resolves somewhere else, -# which is worse than not advertising it at all. "Segments", plural: a project -# name may contain '/', so its advertised permalink can span more than one. +# can win. Mount-wins keeps the paths tools emit honest — a project prefix in +# tool output must address that project, or a returned path resolves somewhere +# else, which is worse than not routing it at all. "Segments", plural: a project +# name may contain '/', so its permalink can span more than one. # # Route versus path is inherently ambiguous, so the answer is a precedence # order, not a parser. 'acme/docs/foo' is a well-formed workspace route AND a @@ -1169,12 +1170,12 @@ async def detect_project_from_identifier_prefix( # collides with a real workspace and project has an explicit, documented fix in # rule 1. Coincidence loses to the address we published. # -# Rule 3 (mounts) sits above all of this: a name `ls /` advertises always -# addresses that mount. +# Rule 3 (mounts) sits above all of this: an addressable project's permalink +# always addresses that project. # # The cost of that choice, stated plainly: when a project's permalink equals an # accessible workspace's slug, that workspace's OTHER projects lose their -# qualified path spelling. With '/team' advertised as a mount, 'team/docs/x' is +# qualified path spelling. With 'team' an addressable project, 'team/docs/x' is # project 'team', path 'docs/x' — never workspace 'team', project 'docs'. Those # projects stay addressable through the project param (project='team/docs' with # the project-relative path 'x'), which is the escape hatch the prefix-conflict @@ -1184,7 +1185,7 @@ async def detect_project_from_identifier_prefix( @dataclass(frozen=True) class ProjectPathRoute: - """Effective routing for one posix call: project params + project-relative path. + """Effective routing for one tool call: project params + project-relative path. ``stripped=False`` means the input carried no recognized project prefix: ``path`` is the caller's input byte-for-byte and ``project`` is the @@ -1196,9 +1197,10 @@ class ProjectPathRoute: the remainder with no leading slash, "" meaning the project root. ``project_id`` is the effective external_id for the call — the caller's own - when they passed one, otherwise the id of the advertised mount that claimed - the prefix. Callers pass both fields to ``get_project_client`` verbatim; the - id is what keeps a mount bound to the workspace that advertised it, since a + when they passed one, otherwise the id of the addressable project that + claimed the prefix. Callers pass both fields to ``get_project_client`` + verbatim; the id is what keeps a mount bound to the workspace that listed it, + since a bare project name can name a different project in another accessible workspace (#1421). """ @@ -1223,13 +1225,13 @@ class AmbiguousMountError(ValueError): @dataclass(frozen=True) class AddressableProject: - """One project this session can both advertise and route to. + """One project this session can route to by name. ``name`` is the routing identifier handed to ``get_project_client`` and ``permalink`` is the path prefix agents copy out of tool output. A cloud session also carries ``external_id``: project names are unique only inside one workspace, so the UUID is the only identifier that pins a mount to the - workspace whose listing advertised it. A locally routed session reads its + workspace whose listing returned it. A locally routed session reads its mounts from config, which holds no UUIDs, and has no second workspace to be confused with, so ``external_id`` is None there. """ @@ -1239,8 +1241,8 @@ class AddressableProject: external_id: Optional[str] = None -# The session's own project listing. Routing and the mount view ask the same -# question, so it is memoized — but in *session* state, which fastmcp keys by +# The session's own project listing. Path routing and identifier detection ask +# the same question, so it is memoized — but in *session* state, which fastmcp keys by # session id and persists across tool calls, not for one request as this comment # used to claim. That is the whole finding: a project created out of band — by a # teammate, or by the CLI — stayed invisible for the life of the session, @@ -1271,10 +1273,8 @@ class AddressableProject: # account-wide workspace index — a control-plane call plus one listing per # accessible workspace — on *every* miss. # -# Applied in the loader rather than at a call site, so identifier detection, the -# posix path resolver, and the `ls "/"` mount view get the same freshness from -# one place. A miss-triggered refresh could not have covered `ls "/"` at all: -# listing the mounts has no miss to trigger on. +# Applied in the loader rather than at a call site, so identifier detection and +# the path resolver get the same freshness from one place. _SESSION_PROJECT_LIST_MAX_AGE_SECONDS = 60.0 @@ -1360,17 +1360,16 @@ async def addressable_projects( ) -> tuple[AddressableProject, ...]: """Return every project this session can address, sorted by name. - One source answers two questions that must never disagree: which mounts - ``ls /`` advertises, and which first path segment names a project. When - they came from different sources, a cloud session could advertise - ``/research`` at the root and then fail to recognize ``research/notes/x``, - silently routing it to a default project instead (#1421). + One source answers which first path segment names a project, for both the + path resolver and identifier detection, so the two never disagree. When a + cloud session's sources differed, it could list ``research`` and then fail + to recognize ``research/notes/x``, silently routing it to a default project + instead (#1421). - A locally routed session's config IS its mount table, so it answers with no + A locally routed session's config IS its project set, so it answers with no network call. A cloud session's projects live in the tenant database and its local config holds only the placeholder 'main' entry, so the session's - own project listing answers instead — the same call the mount view has - always made to render the root. + own project listing answers instead. """ if _session_routes_to_cloud(): project_list = await _session_project_list(context=context) @@ -1479,7 +1478,7 @@ async def _detect_workspace_qualified_route( session can actually address. Workspace-qualified memory URLs require three segments so that - 'memory://main/notes' stays readable as project 'main'. A posix path only + 'memory://main/notes' stays readable as project 'main'. A routed path only reaches here after the mount table declined its leading segments, so nothing addressable can be meant by it and the two-segment form unambiguously names that project's root — without which 'ls acme/docs/notes' resolved while @@ -1590,35 +1589,6 @@ def _agreed_route_project( ) -async def other_workspace_slugs(context: Optional[Context] = None) -> tuple[str, ...]: - """Slugs of the accessible workspaces, for a cloud session that can see several. - - The ``ls /`` mount view lists this session's own projects only, so without this - an agent reading the root takes it for the whole world (#1778). Every slug - returned is a working '//' prefix, the session's own - included. Empty for a session that does not route to the cloud, so a local - root listing never calls the control plane, and for a single-workspace account. - """ - if not _cloud_workspace_discovery_available(ConfigManager().config): - return () - # Deferred: ToolError lives in FastMCP's runtime, which must not load at CLI startup (#886). - from fastmcp.exceptions import ToolError - - try: - workspaces = await get_available_workspaces(context=context) - except ToolError as exc: - # Trigger: the control plane timed out or failed while the session's own - # project listing already succeeded. - # Why: the slugs only annotate the root listing; losing them must not turn a - # working `ls /` into an error. - # Outcome: the root lists its mounts without the workspace note. - logger.warning(f"Workspace discovery failed; listing mounts without it: {exc}") - return () - if len(workspaces) < 2: - return () - return tuple(sorted(workspace.slug for workspace in workspaces)) - - def _hint_for_other_workspace_project( entry: WorkspaceProjectEntry, path: str, @@ -1723,7 +1693,7 @@ async def resolve_project_path_route( project_id: Optional[str], context: Optional[Context] = None, ) -> ProjectPathRoute: - """Resolve a posix tool's path/identifier into an effective project route. + """Resolve a tool's path/identifier into an effective project route. First-segment project resolution (#1415), in order: @@ -1746,9 +1716,9 @@ async def resolve_project_path_route( UnqualifiedPathRefusedError instead of silently defaulting; a session that addresses at most one project keeps today's default resolution. - Rules 3 and 5 read the set from ``addressable_projects`` — the same set - ``ls /`` advertises — so a project can never be listed at the root and then - go unrecognized as a path prefix (#1421). Rule 3 deliberately precedes rule + Rules 3 and 5 read the set from ``addressable_projects``, the session's own + projects, so a project the session lists can never go unrecognized as a path + prefix (#1421). Rule 3 deliberately precedes rule 4; see the mount-precedence note above this section for the collision that ordering resolves and the spelling it costs. """ @@ -1766,15 +1736,15 @@ async def resolve_project_path_route( mount_project_id: Optional[str] = None addressable: tuple[AddressableProject, ...] | None = None - # --- Rule 3: the advertised mount table claims the leading segments --- - # Trigger: the input carries a leading segment at all — a bare mount name - # ('ls research') or a path under one. - # Why: this set is what `ls /` advertises, and an advertised name that - # resolves somewhere else is worse than one never advertised. The workspace + # --- Rule 3: addressable projects claim the leading segments --- + # Trigger: the input carries a leading segment at all — a bare project name + # ('research') or a path under one. + # Why: tool output emits these permalinks as path prefixes, and an emitted + # name that resolves somewhere else is worse than one never emitted. The workspace # parse below would take '//...' first, so a project whose # permalink is also an accessible workspace slug would hand 'team/docs/x' # to project 'docs' in workspace 'team' instead of the mount named 'team' - # — reading another project's data under an advertised name. + # — reading another project's data under a name tool output emitted. # Outcome: a leading segment matching an addressable project routes there # with the remainder as the project-relative path, and workspace discovery # is never consulted for it. A local session pays nothing (its config is @@ -1818,7 +1788,7 @@ async def resolve_project_path_route( detected, remainder = explicit, explicit_claim[1] # --- Rule 4: explicitly workspace-qualified spellings for everything else --- - # Trigger: no advertised mount claimed the leading segments, the input has + # Trigger: no addressable project claimed the leading segments, the input has # more than one segment, and no project was named. # Why: '/[/]' addresses projects in workspaces this # session's own route does not list, so they are absent from the mount diff --git a/src/basic_memory/mcp/tools/__init__.py b/src/basic_memory/mcp/tools/__init__.py index 64d740d52..eee0083a8 100644 --- a/src/basic_memory/mcp/tools/__init__.py +++ b/src/basic_memory/mcp/tools/__init__.py @@ -5,16 +5,6 @@ all tools with the MCP server. """ -# Shared POSIX implementations used by the shell commands. -from basic_memory.mcp.tools.posix_tools import ( - cat as cat, - find as find, - grep as grep, - ls as ls, - man as man, - tail as tail, -) - # 1.0 tool surface (SPEC-35.2, #1827). from basic_memory.mcp.tools.context import bm_context from basic_memory.mcp.tools.delete import bm_delete diff --git a/src/basic_memory/mcp/tools/posix_tools.py b/src/basic_memory/mcp/tools/posix_tools.py deleted file mode 100644 index 3fa4db1e6..000000000 --- a/src/basic_memory/mcp/tools/posix_tools.py +++ /dev/null @@ -1,962 +0,0 @@ -"""Shared POSIX read implementations for the shell CLI and bm_* MCP tools. - -These functions do not register MCP tools. - -Projects are mount points (#1415): when no ``project``/``project_id`` param is -given, a path or identifier whose first segment names an addressable project -routes there, with the remainder as the project-relative path — inputs accept -exactly the '/path' identifiers tool outputs produce. An explicit -project param plus an agreeing prefix strips the prefix; a disagreeing one -refuses naming both. Where more than one project is addressable — several local -projects, or a cloud workspace holding several — an unrecognized first segment -refuses with the project list rather than silently defaulting (#1421); the -mount view and the resolver read one list, so anything ``ls "/"`` advertises is -addressable by name. The resolver answers with both routing fields — the project -and its external_id — and every tool hands that pair to ``get_project_client`` -verbatim, which is what keeps a cloud mount bound to the workspace whose listing -advertised it. - -Collision rule: the project always wins over a same-named top-level folder in -the default project, so that folder is only reachable unqualified when a single -project is addressable (where there is no ambiguity); the qualified -'/folder/...' form always reaches it. ``man`` is excluded — its -``project`` param names the manual project, not a data project. -""" - -import json -import math -import os -import re -from dataclasses import asdict -from typing import Annotated, Any, Optional - -from fastmcp import Context -from fastmcp.exceptions import ToolError -from pydantic import BeforeValidator - -from basic_memory.man import ( - bundled_pages, - find_page, - parse_page_ref, - render_index, - search_pages, -) -from basic_memory.markdown.line_scanning import scan_literal_lines -from basic_memory.markdown.sections import document_lines, line_range_past_end -from basic_memory.mcp.directory_listing import project_mount_listing -from basic_memory.mcp.note_reads import read_note_json_by_external_id -from basic_memory.mcp.metadata_find import find_by_metadata, parse_field_paths -from basic_memory.mcp.note_targets import ( - ProjectTarget, - PathQualifier, -) -from basic_memory.mcp.server import mcp -from basic_memory.mcp.project_context import ( - ProjectPathRoute, - get_project_client, - resolve_project_path_route, -) -from basic_memory.repository.metadata_filters import MetadataPath, parse_metadata_path -from basic_memory.schemas.directory import ( - DEFAULT_DIRECTORY_PAGE_SIZE, - MAX_DIRECTORY_PAGE_SIZE, -) -from basic_memory.schemas.memory import EntitySummary -from basic_memory.schemas.search import SearchItemType, SearchQuery, SearchRetrievalMode -from basic_memory.utils import coerce_list, generate_permalink - -# API bound on directory recursion (directory_router depth query: ge=1, le=10). -_MAX_FIND_DEPTH = 10 - -# recent_activity's page-size cap; tail's `lines` maps onto it. -_MAX_TAIL_LINES = 100 - - -async def cat( - identifier: str, - start_line: Optional[int] = None, - end_line: Optional[int] = None, - section: Optional[str] = None, - max_tokens: Optional[int] = None, - include_frontmatter: bool = True, - project: Optional[str] = None, - project_id: Optional[str] = None, - context: Context | None = None, -) -> dict[str, Any]: - """Print a note's content, optionally sliced by line range, section, or token budget. - - Args: - identifier: Note title, permalink, memory:// URL, or '/path' - identifier (resolved exactly). - start_line: First line to include (1-indexed, inclusive). - end_line: Last line to include (inclusive). Defaults to the last line. - section: Heading to slice to: "Decisions", path form "Auth/Decisions" - to disambiguate by parent, or bracket form "Heading[1]" for the - second duplicate heading. Cannot combine with start_line/end_line; - the response's start_line/end_line support follow-up range reads. - max_tokens: Approximate token budget. Longer content is truncated at a - section or paragraph boundary with an explicit ellipsis marker; the - response carries truncated/continue_line for resuming. - include_frontmatter: Include the YAML frontmatter block in `content`. - Ignored for section/max_tokens reads — those slices never carry a - frontmatter block. A start_line/end_line range combined with - max_tokens addresses the full document (frontmatter included) and - therefore requires include_frontmatter=True. - project: Project name. Optional - the server resolves the default. - project_id: Project external_id (UUID); takes precedence over `project`. - context: Optional FastMCP context. - - Returns: - The read_note JSON payload (title, permalink, file_path, content, - frontmatter), plus start_line/end_line/total_lines when a slice applied, - `section` for section reads, and truncated/continue_line when max_tokens - cut the content. - """ - if section is not None and (start_line is not None or end_line is not None): - raise ValueError( - "cat: 'section' cannot be combined with start_line/end_line; use the " - "returned start_line/end_line for follow-up range reads" - ) - if max_tokens is not None and max_tokens < 1: - raise ValueError(f"max_tokens must be >= 1, got {max_tokens}") - if start_line is not None and start_line < 1: - raise ValueError(f"start_line must be >= 1, got {start_line}") - if end_line is not None and end_line < (start_line or 1): - raise ValueError(f"end_line must be >= start_line, got {end_line}") - # Trigger: a line range rides along with max_tokens while frontmatter is opted out. - # Why: server-side line ranges are document-absolute (frontmatter included), but - # include_frontmatter=False range reads slice the frontmatter-stripped body — - # the same numbers would address different lines, and the served range could - # carry frontmatter text despite the explicit opt-out. - # Outcome: the combination is rejected so every read keeps one coordinate system. - if ( - max_tokens is not None - and not include_frontmatter - and (start_line is not None or end_line is not None) - ): - raise ValueError( - "cat: max_tokens with start_line/end_line requires include_frontmatter=True — " - "those ranges address the full document (frontmatter included); drop max_tokens " - "for a body-relative range, or keep include_frontmatter=True" - ) - - # Trigger: section or max_tokens is set. - # Why: those slices need the server-side section scan and token budgeting; - # plain line ranges keep their original client-side slicing untouched. - # Outcome: the read carries the slice params (line bounds ride along as a - # lines= range) and the server-supplied payload returns as-is. - server_side_slice = section is not None or max_tokens is not None - lines_param: Optional[str] = None - if server_side_slice and (start_line is not None or end_line is not None): - lines_param = f"{start_line or 1}-{'' if end_line is None else end_line}" - - # '/path' identifiers route to their project; route.path is the - # identifier unchanged when no prefix was recognized. - route = await resolve_project_path_route( - identifier, project=project, project_id=project_id, context=context - ) - if route.stripped and not route.path: - raise ValueError(f"cat: '{identifier}' names a project, not a note") - - async with get_project_client(route.project, context=context, project_id=route.project_id) as ( - client, - active_project, - ): - # Import here to avoid circular import - from basic_memory.mcp.clients import KnowledgeClient, ResourceClient - - knowledge_client = KnowledgeClient(client, active_project.external_id) - entity_id = await knowledge_client.resolve_entity(route.path, strict=True) - payload: dict[str, Any] = dict( - await read_note_json_by_external_id( - knowledge_client=knowledge_client, - resource_client=ResourceClient(client, active_project.external_id), - entity_external_id=entity_id, - include_frontmatter=include_frontmatter, - section=section, - lines=lines_param, - max_tokens=max_tokens, - ) - ) - - if server_side_slice or (start_line is None and end_line is None): - return PathQualifier.for_route(route).note(payload) - - lines = document_lines(payload["content"]) - total_lines = len(lines) - first = start_line or 1 - past_end = line_range_past_end(first, total_lines) - if past_end is not None: - raise ValueError(f"cat: {past_end}") - last = min(end_line, total_lines) if end_line is not None else total_lines - payload["content"] = "\n".join(lines[first - 1 : last]) - payload["start_line"] = first - payload["end_line"] = last - payload["total_lines"] = total_lines - return PathQualifier.for_route(route).note(payload) - - -def _grep_retrieval_mode(semantic: bool) -> SearchRetrievalMode: - """Pick grep's retrieval mode: keyword full-text by default, hybrid on request. - - Keyword matching is the default because nearest-neighbour ranking always returns - something, so a grep for a term that appears nowhere would list unrelated notes - instead of nothing (#1685). - """ - return SearchRetrievalMode.HYBRID if semantic else SearchRetrievalMode.FTS - - -async def grep( - pattern: str, - semantic: bool = False, - page: int = 1, - page_size: int = 10, - project: Optional[str] = None, - project_id: Optional[str] = None, - context: Context | None = None, - context_lines: int | None = None, - max_matches: int = 10, -) -> dict[str, Any]: - """Search note content by full-text keywords, or by meaning with semantic=True. - - Args: - pattern: Text to search for. - semantic: Rank by meaning with hybrid semantic plus full-text retrieval instead - of the default keyword matching. Cannot be combined with context_lines. - page: Page number (1-indexed). - page_size: Results per page (maximum 100 in line-scanning mode). - context_lines: Return compact literal match windows with 0-10 surrounding lines - (not with semantic=True). Case-insensitive substrings, coordinates including - frontmatter, overlapping windows merged. Scans current content of this - indexed candidate page; pagination/totals count candidates, not exact matches. - max_matches: Maximum matching lines to show per candidate in line mode (1-100, - default 10). Omitted matches carry next_match_line for a targeted read_note - or cat read. Edits between calls can shift line positions. - project: Project name. Required when more than one project is addressable. - project_id: Project external_id (UUID); takes precedence over `project`. - context: Optional FastMCP context. - - Returns: - Normally the search response as JSON. With context_lines, only note identity, - match windows/counts, and candidate pagination; no full body or duplicate excerpt. - """ - if not pattern or not pattern.strip(): - raise ValueError("pattern must not be empty") - if page < 1: - raise ValueError(f"page must be >= 1, got {page}") - if page_size < 1: - raise ValueError(f"page_size must be >= 1, got {page_size}") - if context_lines is not None: - if semantic: - raise ValueError("grep: context_lines cannot be combined with semantic=True") - if not 0 <= context_lines <= 10: - raise ValueError("grep: context_lines must be between 0 and 10") - if "\n" in pattern or "\r" in pattern: - raise ValueError("grep: line scanning requires a single-line pattern") - if page_size > 100: - raise ValueError("grep: line scanning page_size must be <= 100") - elif max_matches != 10: - raise ValueError("grep: max_matches requires context_lines") - if not 1 <= max_matches <= 100: - raise ValueError("grep: max_matches must be between 1 and 100") - - # grep's pattern is never parsed as a path — search text like "error/timeout" - # must not be mistaken for a mount. Routing participates for the refusal rule - # only: unqualified multi-project calls fail loudly instead of defaulting. - route = await resolve_project_path_route( - "", project=project, project_id=project_id, context=context - ) - - query = SearchQuery( - text=pattern, - retrieval_mode=_grep_retrieval_mode(semantic), - entity_types=[SearchItemType.ENTITY], - ) - async with get_project_client(route.project, context=context, project_id=route.project_id) as ( - client, - active_project, - ): - # Import here to avoid circular import - from basic_memory.mcp.clients import KnowledgeClient, ResourceClient, SearchClient - - search_client = SearchClient(client, active_project.external_id) - response = await search_client.search(query.model_dump(), page=page, page_size=page_size) - payload = response.model_dump(mode="json", exclude_none=True) - if context_lines is None: - return payload - - # Search excerpts are truncated/indexed: derive coordinates from the same - # accepted content read_note serves. Hydrate only this bounded candidate page. - knowledge_client = KnowledgeClient(client, active_project.external_id) - resource_client = ResourceClient(client, active_project.external_id) - rows: list[dict[str, Any]] = [] - for candidate in response.results: - if candidate.external_id is None: - raise ToolError("grep: line scanning requires search results with external_id") - note = await read_note_json_by_external_id( - knowledge_client=knowledge_client, - resource_client=resource_client, - entity_external_id=candidate.external_id, - include_frontmatter=True, - ) - scan = scan_literal_lines( - note["content"], pattern, context_lines=context_lines, max_matches=max_matches - ) - rows.append( - { - "title": note["title"], - "permalink": note["permalink"], - "file_path": note["file_path"], - "external_id": candidate.external_id, - **asdict(scan), - } - ) - payload["results"] = rows - payload["pagination_scope"] = "search_candidates" - return payload - - -async def ls( - path: str = "/", - page: int = 1, - page_size: int = DEFAULT_DIRECTORY_PAGE_SIZE, - project: Optional[str] = None, - project_id: Optional[str] = None, - context: Context | None = None, -) -> dict[str, Any]: - """List the immediate contents of one directory. - - Args: - path: Directory path to list. '/' (the default) with no project param - lists the active projects as mount points; '/path' routes - into that project. - page: Page number (1-indexed). - page_size: Nodes per page. - project: Project name. Optional - '/' lists projects; qualified paths - route themselves; other unqualified paths refuse when several - projects are addressable. - project_id: Project external_id (UUID); takes precedence over `project`. - context: Optional FastMCP context. - - Returns: - The directory listing as JSON: nodes, pagination, and totals. - """ - if page < 1: - raise ValueError(f"page must be >= 1, got {page}") - if page_size < 1: - raise ValueError(f"page_size must be >= 1, got {page_size}") - if page_size > MAX_DIRECTORY_PAGE_SIZE: - raise ValueError(f"page_size must be <= {MAX_DIRECTORY_PAGE_SIZE}, got {page_size}") - - # Trigger: bare root with no project addressed (param, UUID, or env constraint). - # Why: the mount-point view puts project discovery in-band — ls "/" shows the - # mount table, ls "" shows that project's root (#1415). - # Outcome: list the active projects as directory entries; no project client. - if ( - project is None - and project_id is None - and not os.environ.get("BASIC_MEMORY_MCP_PROJECT") - and not path.strip().strip("/") - ): - return await project_mount_listing(page=page, page_size=page_size, context=context) - - route = await resolve_project_path_route( - path, project=project, project_id=project_id, context=context - ) - list_path = f"/{route.path}" if route.stripped else path - - async with get_project_client(route.project, context=context, project_id=route.project_id) as ( - client, - active_project, - ): - # Import here to avoid circular import - from basic_memory.mcp.clients import DirectoryClient - - directory_client = DirectoryClient(client, active_project.external_id) - listing = await directory_client.list(list_path, depth=1, page=page, page_size=page_size) - return PathQualifier.for_route(route).listing(listing.model_dump(mode="json")) - - -def routed_listing_root(path: str, route: ProjectPathRoute) -> str: - """The directory the returned paths are relative to, in *their* address space. - - Returned paths carry the project prefix whenever the call put it in the - path, so the root a caller strips to rebuild a hierarchy must carry it too — - and in the permalink spelling the payload uses, not the caller's ('My - Research' vs 'my-research'). - """ - if not route.stripped or route.project is None: - return path - prefix = generate_permalink(route.project) - return f"{prefix}/{route.path}" if route.path else prefix - - -async def find_listing( - path: str = "/", - *, - name: Optional[str] = None, - depth: int = _MAX_FIND_DEPTH, - page: int = 1, - page_size: int = DEFAULT_DIRECTORY_PAGE_SIZE, - project: Optional[str] = None, - project_id: Optional[str] = None, - context: Context | None = None, -) -> tuple[dict[str, Any], str]: - """find's body: the listing, plus the root its paths are relative to. - - `bm tree` needs both halves — the listing to render and the root to strip — - and resolving the path twice to get them cost a second project-list round - trip on every cloud CLI call, since CLI calls carry no FastMCP context for - the per-request cache to live in. One resolve now answers both. - """ - if depth < 1 or depth > _MAX_FIND_DEPTH: - raise ValueError(f"depth must be between 1 and {_MAX_FIND_DEPTH}, got {depth}") - if page < 1: - raise ValueError(f"page must be >= 1, got {page}") - if page_size < 1: - raise ValueError(f"page_size must be >= 1, got {page_size}") - if page_size > MAX_DIRECTORY_PAGE_SIZE: - raise ValueError(f"page_size must be <= {MAX_DIRECTORY_PAGE_SIZE}, got {page_size}") - - # The directory API is project-scoped, so cross-project find does not exist: - # find "/" with no project in a multi-project config refuses, teaching the - # per-project '/path' form instead. - route = await resolve_project_path_route( - path, project=project, project_id=project_id, context=context - ) - list_path = f"/{route.path}" if route.stripped else path - - async with get_project_client(route.project, context=context, project_id=route.project_id) as ( - client, - active_project, - ): - # Import here to avoid circular import - from basic_memory.mcp.clients import DirectoryClient - - directory_client = DirectoryClient(client, active_project.external_id) - listing = await directory_client.list( - list_path, - depth=depth, - file_name_glob=name, - page=page, - page_size=page_size, - ) - payload = PathQualifier.for_route(route).listing(listing.model_dump(mode="json")) - - return payload, routed_listing_root(path, route) - - -# --- find metadata predicates --- -# find's `meta` strings translate onto the search API's metadata_filters dict — -# a subset of the grammar parse_metadata_filters supports (eq, $gt/$gte/$lt/$lte, -# $in, array-contains-all, $between), nothing more. Word ops need whitespace around -# them and symbol ops exclude the key character class, so exactly one regex can -# match any given predicate. Two-char symbols sit first in the alternation so -# ">=" never parses as ">" plus a value starting with "=". -# The key capture admits '.' anywhere on purpose — it is looser than a dot path. -# parse_metadata_path, which owns the frontmatter path grammar, is what decides -# a well-formed one, applied in _parse_meta_predicates once the predicate has -# split. Tightening the capture instead would make '.owner=null' match no regex -# at all and be reported as a missing operator rather than as the bad key it is. -_PREDICATE_WORD_RE = re.compile(r"^([A-Za-z0-9_.-]+)\s+(in|has|between)\s+(.+)$") -_PREDICATE_SYMBOL_RE = re.compile(r"^([A-Za-z0-9_.-]+)\s*(>=|<=|=|>|<)\s*(.*)$") -_SYMBOL_OPERATORS = {">": "$gt", ">=": "$gte", "<": "$lt", "<=": "$lte"} -_SUPPORTED_PREDICATE_OPS = "= > >= < <= in has between" -# The symbol regex consumes the first operator it recognizes, so an operator -# spelling outside the supported set ("==", "=>", ">>", ">=>") leaves its -# second character at the head of the value. These are the characters that can -# be left behind that way. A set, not a string: "" is a substring of any string -# but is not a member here, so an empty token never reads as operator-prefixed. -_OPERATOR_VALUE_PREFIXES = frozenset("=<>") -# Mirrors search_notes' alias: "note_type" (the entity model column) means the -# frontmatter "type" key, so the two surfaces accept the same spelling. -_METADATA_KEY_ALIASES = {"note_type": "type"} - - -def _opens_an_unterminated_quote(text: str) -> bool: - """True when a double quote opens in `text` and nothing closes it. - - One scanner decides this for every value token, scalar or list element, so - the two paths cannot disagree about what "quoted" means. - """ - in_quotes = False - escaped = False - for char in text: - if escaped: - escaped = False - elif in_quotes and char == "\\": - escaped = True - elif char == '"': - in_quotes = not in_quotes - return in_quotes - - -def _predicate_scalar(token: str, predicate: str) -> Any: - """Read one predicate value token, refusing everything a search cannot answer. - - "true"/"false"/"null"/numbers become bool/None/int/float so the produced - filters dict is byte-equal to what a rich search_notes caller passes as - JSON; a JSON-quoted token ('"true"') forces a literal string; anything that - is not a JSON scalar stays the raw string. - """ - text = token.strip() - # Trigger: an unquoted value opens with one of the operator characters. - # Why: only the operators in _SUPPORTED_PREDICATE_OPS are real, but the - # regexes match the longest supported one and hand the rest to the - # value — 'status==active' would filter for the string "=active" and - # 'count>>3' for ">3", so a typo'd operator answered as an empty (or - # worse, a non-empty but wrong) result set instead of the refusal the - # grammar documents. - # Outcome: refuse, naming the supported set and the quoting escape hatch a - # value that genuinely starts with '=', '<' or '>' needs. - if text[:1] in _OPERATOR_VALUE_PREFIXES: - raise ValueError( - f"find: unsupported predicate operator in '{predicate}'; " - f"supported: {_SUPPORTED_PREDICATE_OPS}; quote the value as " - f'"{text}" to match text that starts with that character' - ) - # Trigger: a quote opens in the token and never closes. - # Why: json.loads rejects it, and the raw-text fallback would then keep the - # dangling quote as part of a literal value — 'status="active' would - # search for the seven-character text '"active', report no matches, - # and hide the typo behind an ordinary empty result. - # Outcome: refuse for every value token, so the scalar operators and the - # list operators (where a severed quote would also mis-split the - # list) answer a dangling quote the same way. - if _opens_an_unterminated_quote(text): - raise ValueError( - f"find: predicate '{predicate}' has an unterminated quoted value; " - "close the quote — 'status=\"active\"' forces a literal string, and " - "'label in \"a,b\",c' protects a comma inside a list element" - ) - try: - value = json.loads(text) - except json.JSONDecodeError: - return text - # Trigger: the token parsed to a non-finite float. Python's JSON reader - # accepts NaN/Infinity/-Infinity as an extension, and overflows a - # large exponent ("1e999") to infinity. - # Why: none of those are JSON the request encoder will emit, so the filter - # died at transport with "Out of range float values are not JSON - # compliant" — a network-shaped error for what is a predicate typo. - # Outcome: refuse here, in the same shape as the grammar's other refusals. - if isinstance(value, float) and not math.isfinite(value): - raise ValueError( - f"find: predicate '{predicate}' has a non-finite number '{text}'; " - f'predicate values must be finite numbers; quote the value as "{text}" ' - "to match that literal text" - ) - if value is None or isinstance(value, (bool, int, float, str)): - return value - return text - - -def _split_predicate_items(raw_value: str, predicate: str) -> list[str]: - """Split a list-op value on its top-level commas, refusing empty elements. - - A comma inside a JSON-quoted token belongs to the value, not to the list, so - the quoting escape hatch the scalar operators document works for `in`, `has` - and `between` too: 'label in "a,b",c' yields ['"a,b"', 'c'], which - _predicate_scalar then reads as the literal strings "a,b" and "c". Splitting - the raw string first would sever the quoted token into '"a' and 'b"' and - filter for values nothing carries — wrong, and silent. - - A split only happens outside quotes, so an unterminated quote here always - ends up inside one element and _predicate_scalar refuses it; this function - does not repeat that check. - """ - items: list[str] = [] - current: list[str] = [] - in_quotes = False - escaped = False - for char in raw_value: - current.append(char) - if escaped: - escaped = False - elif in_quotes and char == "\\": - escaped = True - elif char == '"': - in_quotes = not in_quotes - elif char == "," and not in_quotes: - current.pop() - items.append("".join(current)) - current = [] - items.append("".join(current)) - stripped = [item.strip() for item in items] - if any(not item for item in stripped): - raise ValueError(f"find: predicate '{predicate}' has an empty list element") - return stripped - - -def _refuse_null_outside_equality(values: list[Any], op: str, predicate: str) -> None: - """Refuse a null bound, list element, or comparison value. - - Trigger: `null` reached an operator other than '='. - Why: '=' compiles to IS NULL server-side, which is the question null asks — - does this note carry a value here at all. Every other operator compares - against the value, and a SQL comparison with NULL is never true, so - 'score>null' or 'priority in null,high' would answer zero rows for - every note in the project rather than name the query it cannot run. - Outcome: refuse, pointing at the equality spelling that does work. - """ - if any(value is None for value in values): - raise ValueError( - f"find: predicate '{predicate}' uses null with '{op}'; null matches only " - "as equality ('owner=null' finds notes carrying no owner); quote the " - 'value as "null" to match that literal text' - ) - - -def _parse_meta_predicates(predicates: list[str]) -> dict[str, Any]: - """Translate POSIX-style predicate strings into the search API metadata_filters dict. - - One predicate per string; predicates AND together. Exactly one predicate - per key — the API admits one operator per key, so a repeated key fails fast - instead of last-wins. Raises ValueError (surfaced to MCP callers as - ToolError) on any operator outside the supported set, and on any key outside - the search API's dot-path grammar. - """ - filters: dict[str, Any] = {} - for predicate in predicates: - match = _PREDICATE_WORD_RE.match(predicate.strip()) or _PREDICATE_SYMBOL_RE.match( - predicate.strip() - ) - if match is None: - raise ValueError( - f"find: unsupported predicate operator in '{predicate}'; " - f"supported: {_SUPPORTED_PREDICATE_OPS}" - ) - raw_key, op, raw_value = match.groups() - key = _METADATA_KEY_ALIASES.get(raw_key, raw_key) - # Trigger: the key capture accepted something that is not a dot path — - # a doubled, leading or trailing dot ('review..approved', - # '.owner', 'owner.'). - # Why: the search API refuses these keys, so the query was never going - # to run. Letting it travel spends a request to come back with - # "Unsupported metadata filter key", which names neither find nor - # the shape a key must have — every other predicate mistake is - # refused here, before transport, in find's own words. - # Outcome: refuse locally, naming the offending key and the grammar. - if parse_metadata_path(key) is None: - raise ValueError( - f"find: malformed predicate key '{key}' in '{predicate}'; keys are " - "dot-separated names of letters, digits, '_' or '-' " - "(e.g. 'status' or 'review.approved')" - ) - if key in filters: - raise ValueError( - f"find: duplicate predicate key '{key}' in '{predicate}'; " - "use 'between' for ranges (e.g. 'score between 0.3,0.8')" - ) - if op in ("in", "has", "between"): - items = [ - _predicate_scalar(item, predicate) - for item in _split_predicate_items(raw_value, predicate) - ] - _refuse_null_outside_equality(items, op, predicate) - if op == "between" and len(items) != 2: - raise ValueError(f"find: 'between' needs exactly min,max in '{predicate}'") - filters[key] = ( - {"$in": items} if op == "in" else items if op == "has" else {"$between": items} - ) - else: - if not raw_value.strip(): - raise ValueError(f"find: predicate '{predicate}' has no value") - value = _predicate_scalar(raw_value, predicate) - if op != "=": - _refuse_null_outside_equality([value], op, predicate) - filters[key] = value if op == "=" else {_SYMBOL_OPERATORS[op]: value} - return filters - - -async def find( - path: str = "/", - name: Optional[str] = None, - depth: int = _MAX_FIND_DEPTH, - page: int = 1, - page_size: int = DEFAULT_DIRECTORY_PAGE_SIZE, - meta: Annotated[Optional[list[str]], BeforeValidator(coerce_list)] = None, - fields: Annotated[Optional[list[str]], BeforeValidator(coerce_list)] = None, - project: Optional[str] = None, - project_id: Optional[str] = None, - context: Context | None = None, -) -> dict[str, Any]: - """Recursively list files by name glob, or query notes by frontmatter metadata. - - Without `meta`, this is a recursive directory listing. With `meta`, find - routes through the metadata search instead: predicates AND together, `path` - still scopes the results — server-side, by the indexed file path, so a note - is scoped by where it actually lives rather than by a permalink that may no - longer say — and non-markdown files (which carry no frontmatter) are never - hits. `name` and `depth` are refused alongside `meta`: the search API has no - filename-glob or depth-bound facility, and silently ignoring either would - misreport the match set. - - Args: - path: Directory to start from (default: project root). '/path' - routes into that project. With `meta`, scopes the search to this - subtree (matched on a directory boundary, so "specs" never admits - "specs-archive/"). - name: File-name glob to match, e.g. "*.md". None matches everything. - Cannot combine with `meta` — scope with `path` instead. - depth: How many levels to recurse (1-10, default: 10). A non-default - depth cannot combine with `meta` (the subtree scope is - all-or-nothing). - page: Page number (1-indexed). - page_size: Nodes per page. - meta: Frontmatter metadata predicates, repeatable; every predicate must - hold. One predicate per string, one predicate per key, at least one - predicate; omit `meta` entirely for the directory listing. Forms: - "status=active" equality - "confidence>0.6" comparison: > >= < <= - "priority in high,critical" any of the listed values - "tags has security,oauth" array contains ALL listed values - "score between 0.3,0.8" inclusive range - "owner=null" key missing or explicitly null - Values are JSON-scalar inferred ("true"/"false"/"null"/numbers - become booleans/None/numbers); quote a token to force a literal - string (e.g. 'status="true"'), including inside a list, where the - quotes also protect a comma ('label in "a,b",c' matches "a,b" or - "c") and a value that itself starts with an operator character - ('range=">=5"'). Numbers must be finite, and null is only meaningful - with "=" — the other operators compare against the value, and a - comparison with null is never true. Keys accept dot-paths - ("review.approved"); "note_type" aliases the frontmatter "type" key. - String equality and "in" on either type spelling use normalized - note-type matching ("LiteraryDevice" matches "literary_device"). - Other metadata keys and operators retain their usual comparisons. - Any other operator fails fast naming the supported set. - fields: Frontmatter fields to return per hit (dot-paths allowed), e.g. - ["title", "priority"]. Requires `meta`. A field missing on a hit - renders as null — rows are never dropped. Requesting fields also - narrows each row to the note's identity plus those values: the - projection replaces the note body rather than riding alongside it. - project: Project name. Optional - qualified paths route themselves; - unqualified paths refuse when several projects are addressable. - project_id: Project external_id (UUID); takes precedence over `project`. - context: Optional FastMCP context. - - Returns: - Without `meta`: the directory listing as JSON (nodes, pagination, - totals). With `meta`: the search response as JSON (results, pagination, - totals). Adding `fields` projects each result down to the note's - identity — title, permalink, file_path, external_id, updated_at — plus - the requested `fields` object; no note content comes back. - """ - # Combination rules, before any I/O. The metadata search takes no filename - # glob and no depth bound, so `name` and `depth` are refused rather than - # silently ignored; `path` survives, because the search API does express a - # file-path subtree. `fields` is the SELECT to the predicates' WHERE; - # without predicates the directory listing stays byte-identical to today. - if meta is not None: - # Trigger: 'meta' present but carrying no predicates. - # Why: an empty list parses to an empty filters dict, which is not None - # and would route into the metadata search with no predicate at - # all — an unfiltered project-wide match where the caller asked for - # a filtered set, and not the directory listing either. - # Outcome: refuse, exactly as 'fields' refuses an empty list. - if not meta: - raise ValueError( - "find: 'meta' must carry at least one predicate — omit 'meta' entirely " - "for the plain directory listing" - ) - if name is not None: - raise ValueError( - "find: 'name' cannot combine with 'meta' — the metadata search has no " - "filename glob; scope with 'path' instead" - ) - if depth != _MAX_FIND_DEPTH: - raise ValueError( - "find: 'depth' cannot combine with 'meta' — the metadata search scopes " - "by whole subtree; scope with 'path' instead" - ) - projected_fields: list[MetadataPath] | None = None - if fields is not None: - if meta is None: - raise ValueError( - "find: 'fields' requires 'meta' predicates — without predicates find " - "returns the plain directory listing" - ) - try: - projected_fields = parse_field_paths(fields) - except ValueError as error: - raise ValueError(f"find: {error}") from error - metadata_filters = _parse_meta_predicates(meta) if meta is not None else None - - # Trigger: predicates are present, so this call queries metadata rather than - # walking directories. - # Why: the two arms call different project-scoped APIs, and each resolves the - # route exactly once — find_listing already owns validation and - # resolution for the listing arm, so branching before resolving keeps a - # routed call to one project-list round trip (which is why find_listing - # exists at all). - # Outcome: the metadata arm answers with search results; otherwise the - # directory listing comes back unchanged. - if metadata_filters is not None: - # Pagination is an argument check, so it refuses here with the rest of - # them, before any I/O. The listing arm inherits these same bounds from - # find_listing, which the metadata arm never reaches — stated once per - # arm rather than once for both, so neither can drift onto the other's - # error message. - if page < 1: - raise ValueError(f"page must be >= 1, got {page}") - if page_size < 1: - raise ValueError(f"page_size must be >= 1, got {page_size}") - if page_size > MAX_DIRECTORY_PAGE_SIZE: - raise ValueError(f"page_size must be <= {MAX_DIRECTORY_PAGE_SIZE}, got {page_size}") - - # The search API is project-scoped too, so cross-project find does not - # exist here either: an unqualified path in a multi-project config - # refuses, teaching the per-project '/path' form instead. - route = await resolve_project_path_route( - path, project=project, project_id=project_id, context=context - ) - # The whole route travels, not three fields pulled off it: the arm needs - # the project and its id to open the client, route.path to scope the - # search, and — the part #1435 missed — the same route again to - # re-qualify the paths it answers with. - async with get_project_client( - route.project, context=context, project_id=route.project_id - ) as (client, active_project): - return await find_by_metadata( - ProjectTarget(client=client, project=active_project, route=route), - metadata_filters=metadata_filters, - fields=projected_fields, - page=page, - page_size=page_size, - ) - - listing, _ = await find_listing( - path, - name=name, - depth=depth, - page=page, - page_size=page_size, - project=project, - project_id=project_id, - context=context, - ) - return listing - - -async def tail( - timeframe: str = "7d", - lines: int = 10, - project: Optional[str] = None, - project_id: Optional[str] = None, - context: Context | None = None, -) -> list[dict[str, Any]]: - """Show the most recently changed notes in a project. - - Args: - timeframe: Time window, e.g. "7d", "yesterday", "2 days ago". - lines: Maximum number of rows to return (1-100). - project: Project name. Required when more than one project is addressable. - project_id: Project external_id (UUID); takes precedence over `project`. - context: Optional FastMCP context. - - Returns: - Rows of {type, title, permalink, file_path, created_at, updated_at}, newest - change first. - """ - if lines < 1: - raise ValueError(f"lines must be >= 1, got {lines}") - if lines > _MAX_TAIL_LINES: - raise ValueError(f"lines must be <= {_MAX_TAIL_LINES}, got {lines}") - - # tail has no path to carry a project prefix, so routing participates for - # the refusal rule only: unqualified multi-project calls fail loudly. - route = await resolve_project_path_route( - "", project=project, project_id=project_id, context=context - ) - - async with get_project_client(route.project, context=context, project_id=route.project_id) as ( - client, - active_project, - ): - # Import here to avoid circular import - from basic_memory.mcp.clients import MemoryClient - - memory_client = MemoryClient(client, active_project.external_id) - activity = await memory_client.recent( - timeframe=timeframe, - depth=1, - types=[SearchItemType.ENTITY.value], - page=1, - page_size=lines, - ) - - rows: list[dict[str, Any]] = [] - for result in activity.results: - primary = result.primary_result - # The recent query above asks for entity rows only. - if not isinstance(primary, EntitySummary): # pragma: no cover - raise TypeError(f"tail expected entity rows, got {primary.type}") - rows.append( - { - "type": primary.type, - "title": primary.title, - "permalink": primary.permalink, - "file_path": primary.file_path, - "created_at": primary.created_at.isoformat() if primary.created_at else None, - # The order is by last change, so the row says when that was. - "updated_at": primary.updated_at.isoformat() if primary.updated_at else None, - } - ) - return rows - - -async def man( - page: Optional[str] = None, - query: Optional[str] = None, - project: Optional[str] = None, - project_id: Optional[str] = None, - context: Context | None = None, -) -> str | dict[str, Any]: - """Look up one manual page, search the manual, or render the index. - - Modes: - - No arguments: the manual index (apropos view), as markdown. - - page: one bundled page by reference (e.g. "bm-search(3)"). - - query: the bundled pages that mention every word of the query. - - Args: - page: Page reference, e.g. "bm-search(3)". Any common spelling works. - query: Apropos search text. Mutually exclusive with `page`. - project: Ignored; the manual is bundled with the package. - project_id: Ignored; the manual is bundled with the package. - context: Optional FastMCP context. - - Returns: - Markdown (index or one page), or the matching pages as JSON. - """ - if page is not None and query is not None: - raise ValueError("man: pass either 'page' or 'query', not both") - - if page is None and query is None: - # Mirror the memory://man resource: mark pages whose tool this server - # does not register so an agent does not call a tool that is not there. - tools = await mcp.list_tools(run_middleware=False) - return render_index(bundled_pages(), frozenset(tool.name for tool in tools)) - - # The manual is the bundled pages. A fallback to a "manual" project failed for - # nearly every hosted caller, whose workspaces have no such project (about 90% of - # man's production errors, 2026-10); `project` and `project_id` are accepted and - # ignored so existing callers keep working. - if page is not None: - try: - bundled = find_page(parse_page_ref(page)) - except ValueError: - bundled = None - if bundled is None: - raise ToolError(f"No manual entry for {page}. Call man() for the index.") - return bundled.read() - - assert query is not None - matches = search_pages(query) - return { - "results": [ - {"title": match.title, "summary": match.summary, "uri": match.uri} for match in matches - ], - "total": len(matches), - } diff --git a/src/basic_memory/mcp/tools/read.py b/src/basic_memory/mcp/tools/read.py index 00455ce44..97b470796 100644 --- a/src/basic_memory/mcp/tools/read.py +++ b/src/basic_memory/mcp/tools/read.py @@ -14,6 +14,7 @@ from pydantic import Field from basic_memory.markdown.line_scanning import format_line_read +from basic_memory.markdown.sections import LineRange, NoteSliceError, slice_note_content from basic_memory.mcp.clients import KnowledgeClient, ResourceClient from basic_memory.mcp.file_content import ( DocumentFile, @@ -25,8 +26,8 @@ classify_file_response, ) from basic_memory.mcp.note_reads import format_note_metadata, read_note_json_by_external_id -from basic_memory.mcp.note_targets import open_project_target, resolve_note -from basic_memory.mcp.refusals import refusal +from basic_memory.mcp.note_targets import PathQualifier, open_project_target, resolve_note +from basic_memory.mcp.refusals import api_refusals, refusal from basic_memory.mcp.server import mcp from basic_memory.mcp.tool_params import NoteIdentifier, OutputFormat, ProjectParam from basic_memory.runtime.storage import runtime_file_path_is_markdown_note @@ -40,7 +41,8 @@ description=( "Read a Basic Memory note or file by path, permalink, memory:// URL or exact title. " "Slice a long note with `section` (a heading), `start_line`/`end_line` or " - "`max_tokens`; line numbers count the whole file, frontmatter included. " + "`max_tokens`; line numbers count the whole file, frontmatter included. A text " + "file such as .txt takes start_line/end_line too. " "`include_content=false` returns only the metadata and the checksum that " "bm_write and bm_edit take as expected_checksum. Images come back as images and " "other binary files as documents. A missing note is an error: find notes with " @@ -137,22 +139,46 @@ async def bm_read( resource_client = ResourceClient(target.client, target.project.external_id) # --- Files that are not Markdown notes --- - # Trigger: the resolved file is an image, PDF, JSON or other non-note file. - # Why: slicing is defined over Markdown lines and headings only. - # Outcome: the file is returned whole, in the form the client can display. + # Trigger: the resolved file is an image, PDF, JSON, plain text or other + # non-note file. + # Why: sections and token budgets are defined over Markdown headings and + # paragraphs, but lines are not: a long .txt source (a book being + # analyzed) is read a range at a time, like a note. + # Outcome: a text file honors start_line/end_line; any other slice, or + # a line range on a binary file, is refused; otherwise the file comes + # back whole, in the form the client can display. if not runtime_file_path_is_markdown_note(note.file_path): - if sliced: + if section is not None or max_tokens is not None: raise refusal( "UNSUPPORTED", - f"'{note.file_path}' is not a Markdown note; section, lines and " - "max_tokens apply only to notes.", - try_instead=f"bm_read(identifier={identifier!r}, project={project!r})", + f"'{note.file_path}' is not a Markdown note; section and max_tokens " + "apply only to notes.", + try_instead="read a range with start_line/end_line, or the whole file", output_format=output_format, ) - response = await resource_client.read(note.external_id) - presented = _present_file( - classify_file_response(response), note.file_path, note.title, output_format - ) + # An indexed file whose bytes are gone is the API's 404; it reaches a + # JSON caller as a NOT_FOUND refusal rather than as plain text. + with api_refusals(output_format): + response = await resource_client.read(note.external_id) + content = classify_file_response(response) + if line_range: + if not isinstance(content, TextFile): + raise refusal( + "UNSUPPORTED", + f"'{note.file_path}' is not a text file; start_line and end_line " + "apply only to notes and text files.", + try_instead=f"bm_read(identifier={identifier!r}, project={project!r})", + output_format=output_format, + ) + return _text_file_lines( + content, + note.file_path, + note.title, + LineRange(start=start_line or 1, end=end_line), + output_format, + target.paths, + ) + presented = _present_file(content, note.file_path, note.title, output_format) if isinstance(presented, dict): return target.paths.note(presented) return presented @@ -176,11 +202,7 @@ async def bm_read( first, last, total = payload["start_line"], payload["end_line"], payload["total_lines"] if output_format == "json": - width = last - first + 1 - payload["has_more"] = last < total - payload["next_start_line"] = last + 1 if last < total else None - payload["next_end_line"] = min(total, last + width) if last < total else None - return target.paths.note(payload) + return target.paths.note({**payload, **_continuation(first, last, total)}) # Trigger: max_tokens cut the content. # Why: the cut content ends with an ellipsis marker, so its lines no longer match # the coordinates one for one and cannot be numbered. @@ -194,6 +216,47 @@ async def bm_read( return format_line_read(payload["content"], start_line=first, end_line=last, total_lines=total) +def _continuation(first: int, last: int, total: int) -> dict[str, Any]: + """Where a slice ends relative to the document, and the next range of its width.""" + if last >= total: + return {"has_more": False, "next_start_line": None, "next_end_line": None} + width = last - first + 1 + return { + "has_more": True, + "next_start_line": last + 1, + "next_end_line": min(total, last + width), + } + + +def _text_file_lines( + text_file: TextFile, + file_path: str, + title: str, + lines: LineRange, + output_format: OutputFormat, + paths: PathQualifier, +) -> str | dict[str, Any]: + """One line range of a text file, with the coordinates a note slice carries.""" + sliced = slice_note_content(text_file.text, lines=lines) + if isinstance(sliced, NoteSliceError): + raise refusal("INVALID_ARGUMENTS", sliced.message, output_format=output_format) + first, last, total = sliced.start_line, sliced.end_line, sliced.total_lines + if output_format == "json": + return paths.note( + { + "title": title, + "file_path": file_path, + "content_type": text_file.content_type, + "content": sliced.content, + "start_line": first, + "end_line": last, + "total_lines": total, + **_continuation(first, last, total), + } + ) + return format_line_read(sliced.content, start_line=first, end_line=last, total_lines=total) + + def _present_file( content: FileContent, file_path: str, diff --git a/src/basic_memory/redaction.py b/src/basic_memory/redaction.py index bc081c2d8..cc3befc85 100644 --- a/src/basic_memory/redaction.py +++ b/src/basic_memory/redaction.py @@ -1,8 +1,8 @@ """Shared secret-redaction rules for surfacing ``BasicMemoryConfig`` values. -Used by the ``basic_memory_diagnostics`` MCP tool (#963) and the ``bm config`` -CLI command group (#991) so both surfaces apply identical scrubbing rules -instead of drifting apart over time. +Used by the ``bm config`` CLI command group (#991), which shows config values +the way the retired ``basic_memory_diagnostics`` tool did (#963), so secrets are +scrubbed by one set of rules. """ from typing import Any diff --git a/src/basic_memory/services/project_service.py b/src/basic_memory/services/project_service.py index 27c80fad6..1ecb5fd57 100644 --- a/src/basic_memory/services/project_service.py +++ b/src/basic_memory/services/project_service.py @@ -88,9 +88,9 @@ def project_permalink(name: str, *, top_level: bool) -> str: # Why: the permalink is the project's address, and the resolver matches # it segment by segment against a path whose leading slashes are # already stripped. An empty segment means no path can ever match it: - # '' advertises at the root as '/', indistinguishable from every other - # such project, and '/foo' advertises '//foo' and cannot be entered. - # Either way the mount view lists something unaddressable (#1421). A '.' + # '' is indistinguishable from every other such project, and '/foo' + # becomes the prefix '//foo', which no path can enter. Either way the + # project is listed but unaddressable (#1421). A '.' # or '..' segment is worse: as a directory it names the parent or the # root itself, so deleting that project's files would delete the root. # Outcome: refused at the boundary that creates projects, so an @@ -326,8 +326,8 @@ async def add_project(self, name: str, path: str, set_default: bool = False) -> # Trigger: a different name that normalizes to an existing project's # permalink ('My Docs' beside 'my-docs'). # Why: the permalink is the address, so two of them are one address - # for two projects. The mount view advertises both at the same - # path and the resolver can only pick one, leaving the other + # for two projects. Both carry the same path prefix and the + # resolver can only pick one, leaving the other # unreachable and its paths reading the wrong project's content. # Outcome: refused here, where the second one would be created. for existing in existing_projects: diff --git a/test-int/cli/test_find_note_type_integration.py b/test-int/cli/test_find_note_type_integration.py index 88ae67940..057f9bdf4 100644 --- a/test-int/cli/test_find_note_type_integration.py +++ b/test-int/cli/test_find_note_type_integration.py @@ -132,11 +132,10 @@ def test_find_type_membership_preserves_scope_and_other_predicates( "status=Active", "--fields", "chapter_number", - "--plain", ], ) assert result.exit_code == 0, result.output - assert 'book/Chapter One.md\t{"chapter_number":1}' in result.stdout + assert "- book/Chapter One.md | Chapter One | chapter_number=1" in result.stdout assert "Metaphor" not in result.stdout assert "Other Chapter" not in result.stdout diff --git a/test-int/cli/test_posix_commands_integration.py b/test-int/cli/test_posix_commands_integration.py new file mode 100644 index 000000000..23f7fa58e --- /dev/null +++ b/test-int/cli/test_posix_commands_integration.py @@ -0,0 +1,653 @@ +"""`bm cat/head/grep/ls/find/tree` against a real project, end to end. + +Each verb is a thin command over a bm_* tool, so these tests check what the shell +adds (the --lines and --meta spellings, the tree drawing, the default project) and +that each verb reaches its tool's behavior. Notes are seeded with `bm tool write`. +Every CLI call costs a few hundred milliseconds, so each test seeds once and makes +several checks against that state: on Postgres each test also pays a few seconds +of schema reset, so related checks share a test. Argument refusals that land before any I/O live +in tests/cli/test_cli_posix_argument_errors.py. +""" + +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import pytest +import pytest_asyncio +import yaml +from typer.testing import CliRunner, Result + +from basic_memory import db +from basic_memory.cli.main import app as cli_app +from basic_memory.index.local_project import LocalProjectIndexRunner +from basic_memory.mcp.clients import SearchClient +from basic_memory.models import Entity +from basic_memory.repository.entity_repository import EntityRepository +from basic_memory.repository.project_repository import ProjectRepository +from basic_memory.schemas.search import SearchItemType, SearchResponse, SearchResult + +pytestmark = pytest.mark.usefixtures("app") + +runner = CliRunner() + +SECOND = ["--project", "second-project"] + + +# --- CLI helpers --- + + +def bm(*args: str) -> Result: + return runner.invoke(cli_app, list(args)) + + +def bm_text(*args: str) -> str: + """Run a command and return its text output.""" + result = bm(*args) + assert result.exit_code == 0, result.output + return result.stdout + + +def bm_json(*args: str) -> Any: + """Run a command with --json and return its parsed payload.""" + return json.loads(bm_text(*args, "--json")) + + +def refusal(*args: str) -> str: + """Run a command that must fail and return what it printed on stderr.""" + result = bm(*args) + assert result.exit_code == 1, result.output + return result.stderr + + +def write_note( + title: str, + directory: str, + content: str, + *, + project: str = "test-project", + metadata: dict[str, Any] | None = None, +) -> None: + args = ["tool", "write", "--title", title, "--directory", directory, "--content", content] + if metadata is not None: + args += ["--metadata", json.dumps(metadata)] + bm_json(*args, "--project", project) + + +def titles(payload: dict[str, Any]) -> set[str]: + return {row["title"] for row in payload["results"]} + + +def meta_args(*predicates: str) -> list[str]: + return [arg for predicate in predicates for arg in ("--meta", predicate)] + + +# --- Fixtures --- + + +def seed_graph() -> None: + """Five notes under test/.""" + for title in ("Deeper Entity", "Deep Entity", "Connected Entity 2", "Connected Entity 1"): + write_note(title, "test", f"# {title}") + write_note("Root", "test", "# Root\n- connects_to [[Connected Entity 1]]") + + +def seed_meta_notes() -> None: + """Metadata-bearing notes across two directories (one name has a space).""" + write_note( + "Alpha Spec", + "specs", + "# Alpha Spec\n\nalpha body", + metadata={ + "status": "active", + "priority": "high", + "confidence": 0.9, + "chapter_number": 63, + "tags": ["security", "oauth"], + "review": {"approved": True}, + }, + ) + write_note( + "Beta Spec", + "specs", + "# Beta Spec\n\nbeta body", + metadata={"status": "draft", "priority": "low", "confidence": 0.2, "tags": ["security"]}, + ) + write_note( + "Gamma Note", + "My Notes", + "# Gamma Note\n\ngamma body", + metadata={"status": "active", "priority": "critical", "confidence": 0.5, "tags": ["oauth"]}, + ) + + +@pytest.fixture +def second_project(app, tmp_path_factory) -> Path: + """A second project, outside the first one's home (nested paths are refused).""" + path = tmp_path_factory.mktemp("second-project-home") + result = bm("project", "add", "second-project", str(path)) + assert result.exit_code == 0, result.output + return path + + +# --- cat / head --- + + +def check_cat_whole_notes_and_slices() -> None: + write_note("Cat Note", "test", "# Cat Note\n\nline one\nline two") + note = ["cat", "Cat Note"] + + whole = bm_json(*note) + assert whole["title"] == "Cat Note" + assert whole["file_path"] == "test/Cat Note.md" + assert whole["frontmatter"]["title"] == "Cat Note" + assert whole["content"].startswith("---") + assert "start_line" not in whole + # The text result of a whole-note read is the note itself. + assert bm_text(*note) == whole["content"] + "\n" + # --no-frontmatter drops the YAML block from a whole-note read. + body = bm_json(*note, "--no-frontmatter")["content"] + assert not body.startswith("---") + assert "line one" in body + + # --lines counts the whole document, frontmatter included, as bm_read does. + lines = whole["content"].splitlines() + ranged = bm_json(*note, "--lines", "2-3") + assert ranged["content"] == "\n".join(lines[1:3]) + assert (ranged["start_line"], ranged["end_line"], ranged["total_lines"]) == (2, 3, len(lines)) + assert ranged["has_more"] is True + assert bm_json(*note, "--lines", "2-")["content"] == "\n".join(lines[1:]) + assert bm_json(*note, "--lines", "3")["content"] == lines[2] + clamped = bm_json(*note, "--lines", "1-999") + assert clamped["content"] == "\n".join(lines) + assert clamped["end_line"] == len(lines) + # A range that starts after the last line is an error (#1634). + assert f"is past the end of the document ({len(lines)} lines)" in refusal( + *note, "--lines", "500-" + ) + # The text result of a slice numbers its lines. + assert f"3: {lines[2]}" in bm_text(*note, "--lines", "3") + + assert "Error:" in refusal("cat", "no-such-note-anywhere") + + +def check_cat_sections_and_token_budget() -> None: + write_note("Sections", "test", "# Guide\nintro\n## First\nalpha\n## Second\nbeta") + write_note("Paths", "test", "# Auth\n## Decisions\na\n# Ops\n## Decisions\nb") + write_note("Duplicates", "test", "# Spec\n## Auth\nfirst\n## Auth\nsecond") + write_note("Token Budget", "test", "# One\n" + "a" * 40 + "\n# Two\n" + "b" * 40) + + full_lines = bm_json("cat", "Sections")["content"].splitlines() + section = bm_json("cat", "Sections", "--section", "First") + assert section["section"] == "Guide/First" + assert section["content"].splitlines() == ["## First", "alpha"] + # Coordinates are document-absolute, so they address the same lines again. + assert section["content"] == "\n".join( + full_lines[section["start_line"] - 1 : section["end_line"]] + ) + assert section["frontmatter"] is None + + by_parent = bm_json("cat", "Paths", "--section", "Ops/Decisions") + assert by_parent["content"].splitlines() == ["## Decisions", "b"] + duplicate = bm_json("cat", "Duplicates", "--section", "Auth[1]") + assert duplicate["section"] == "Spec/Auth[1]" + assert duplicate["content"].splitlines() == ["## Auth", "second"] + missing = refusal("cat", "Sections", "--section", "Nope") + assert "Available sections" in missing + assert "Guide/First" in missing + + # A cut read says where to continue; reading from there rebuilds the rest. + budget_lines = bm_json("cat", "Token Budget")["content"].splitlines() + truncated = bm_json("cat", "Token Budget", "--max-tokens", "20") + assert truncated["truncated"] is True + kept = truncated["content"].splitlines()[:-1] + assert "truncated at max_tokens=20" in truncated["content"].splitlines()[-1] + rest = bm_json("cat", "Token Budget", "--lines", f"{truncated['continue_line']}-") + assert kept + rest["content"].splitlines() == budget_lines[truncated["start_line"] - 1 :] + + +def check_head_prints_the_first_lines() -> None: + write_note("Head Note", "test", "alpha\nbravo\ncharlie\ndelta") + lines = bm_json("cat", "Head Note")["content"].splitlines() + + first_three = bm_json("head", "Head Note", "-n", "3") + assert first_three["content"] == "\n".join(lines[:3]) + assert (first_three["start_line"], first_three["end_line"]) == (1, 3) + assert f"1: {lines[0]}" in bm_text("head", "Head Note", "-n", "3") + + +@pytest_asyncio.fixture +async def text_source(engine_factory, test_project) -> None: + """A .txt source kept verbatim in the project and indexed, as the + literary-analysis skill sets up its book.""" + books = Path(test_project.path) / "books" + books.mkdir() + (books / "source.txt").write_text( + "CHAPTER 1\nLoomings.\nCall me Ishmael.\nCHAPTER 2\nThe Carpet-Bag.\n" + ) + _, session_maker = engine_factory + runner = LocalProjectIndexRunner( + project_repository=ProjectRepository(), session_maker=session_maker + ) + await runner.index_project(test_project.id, force_full=True) + + +def check_text_source_reads_by_line() -> None: + """An indexed .txt source reads a range at a time through cat and head.""" + chapter = bm_json("cat", "books/source.txt", "--lines", "2-3") + assert chapter["content"] == "Loomings.\nCall me Ishmael." + assert (chapter["start_line"], chapter["end_line"], chapter["total_lines"]) == (2, 3, 5) + assert "3: Call me Ishmael." in bm_text("cat", "books/source.txt", "--lines", "2-3") + assert bm_text("head", "books/source.txt", "-n", "1").startswith("Lines 1-1 of 5") + # Sections are headings, which a text file does not have. + assert "section and max_tokens apply only to notes" in refusal( + "cat", "books/source.txt", "--section", "CHAPTER 1" + ) + + +def test_cat_and_head(text_source): + check_cat_whole_notes_and_slices() + check_cat_sections_and_token_budget() + check_head_prints_the_first_lines() + check_text_source_reads_by_line() + + +# --- grep --- + + +def test_grep_matches_keywords_and_lines(): + write_note("Grep Target", "test", "# Grep Target\n\nThe posixgrepneedle hides here.") + write_note("Other Target", "other", "# Other Target\n\nposixgrepneedle again.") + + found = bm_json("grep", "posixgrepneedle") + assert titles(found) == {"Grep Target", "Other Target"} + # Keyword matching is the default (#1685): an absent term lists nothing. + assert bm_json("grep", "zzqqabsentterm")["results"] == [] + # --path limits the search to one folder. + assert titles(bm_json("grep", "posixgrepneedle", "--path", "test")) == {"Grep Target"} + + # -C returns the matching lines with their document line numbers. + with_lines = bm_json("grep", "posixgrepneedle", "-C", "0", "--path", "test") + [row] = with_lines["results"] + assert row["file_path"] == "test/Grep Target.md" + assert row["match_count"] == 1 + [window] = row["windows"] + assert "posixgrepneedle" in window["content"] + note_lines = bm_json("cat", "Grep Target")["content"].splitlines() + assert note_lines[window["match_lines"][0] - 1] == window["content"] + + # REGRESSION: a note whose observation and relation match too is one row, + # not one per index row, with or without -C (#1960 review). + write_note( + "Facts Target", + "facts", + "# Facts Target\n\n- [fact] posixgrepneedle observed\n" + "- mentions [[Posixgrepneedle Source]]", + ) + facts = bm_json("grep", "posixgrepneedle", "--path", "facts") + assert [(row["type"], row["title"]) for row in facts["results"]] == [("entity", "Facts Target")] + assert facts["total"] == 1 + facts_lines = bm_json("grep", "posixgrepneedle", "--path", "facts", "-C", "0") + assert [row["file_path"] for row in facts_lines["results"]] == ["facts/Facts Target.md"] + + +# --- ls --- + + +def check_ls_lists_one_folder() -> None: + root = bm_json("ls") + assert [node["name"] for node in root["nodes"]] == ["test"] + assert root["nodes"][0]["type"] == "directory" + + folder = bm_json("ls", "test") + assert folder["total"] == 5 + assert {node["name"] for node in folder["nodes"]} == { + "Connected Entity 1.md", + "Connected Entity 2.md", + "Deep Entity.md", + "Deeper Entity.md", + "Root.md", + } + + first_page = bm_json("ls", "test", "--page-size", "2") + last_page = bm_json("ls", "test", "--page", "3", "--page-size", "2") + assert len(first_page["nodes"]) == 2 + assert first_page["has_more"] is True + assert len(last_page["nodes"]) == 1 + assert last_page["has_more"] is False + + assert {node["name"] for node in bm_json("ls", "test", "--glob", "Deep*")["nodes"]} == { + "Deep Entity.md", + "Deeper Entity.md", + } + + +def check_ls_and_tree_on_an_empty_project() -> None: + result = bm_json("ls") + + assert result["total"] == 0 + assert result["nodes"] == [] + assert "empty" in bm_text("tree") + + +# --- find / tree (listing) --- + + +def check_find_and_tree_list_recursively() -> None: + globbed = bm_json("find", "--name", "*.md") + assert globbed["total"] == 5 + assert all(node["type"] == "file" for node in globbed["nodes"]) + assert set(globbed) == {"nodes", "page", "page_size", "total", "has_more"} + + # The default depth recurses: the test folder plus its five files. + everything = bm_json("find") + assert everything["total"] == 6 + + # tree draws the same listing as a hierarchy; --json is bm_find's payload. + drawn = bm_text("tree", "--name", "*.md") + assert drawn.splitlines()[0] == "/" + assert "test/" in drawn + assert "Root.md" in drawn + assert bm_json("tree")["nodes"] == bm_json("find", "--page-size", "100")["nodes"] + + # A truncated page says so under the tree. + assert "more entries (page 1; use --page)" in bm_text("tree", "--page-size", "2") + + # A rooted tree does not draw its root as its own child. + rooted = bm_text("tree", "test") + assert rooted.splitlines()[0] == "test" + assert "test/" not in rooted + + +def test_ls_find_and_tree(): + check_ls_and_tree_on_an_empty_project() + seed_graph() + check_ls_lists_one_folder() + check_find_and_tree_list_recursively() + + +# --- find --meta --- + + +FIND_META_CASES = [ + (["status=active"], {"Alpha Spec", "Gamma Note"}), + (["confidence>0.5"], {"Alpha Spec"}), + (["confidence>=0.5"], {"Alpha Spec", "Gamma Note"}), + (["confidence<0.5"], {"Beta Spec"}), + (["confidence<=0.5"], {"Beta Spec", "Gamma Note"}), + (["priority in high,critical"], {"Alpha Spec", "Gamma Note"}), + # `has` is contains-ALL, so a two-element list narrows to one note. + (["tags has security"], {"Alpha Spec", "Beta Spec"}), + (["tags has security,oauth"], {"Alpha Spec"}), + (["confidence between 0.1,0.6"], {"Beta Spec", "Gamma Note"}), + (["review.approved=true"], {"Alpha Spec"}), + # 'key=null' finds the notes carrying no value there: the complement. + (["review.approved=null"], {"Beta Spec", "Gamma Note"}), + (["note_type=note"], {"Alpha Spec", "Beta Spec", "Gamma Note"}), + # Repeated predicates AND together. + (["status=active", "priority=high"], {"Alpha Spec"}), + (["status=active", "priority=nonexistent"], set()), +] + + +def check_find_meta_predicates() -> None: + shape = bm_json("find", "--meta", "status=active") + assert set(shape) == { + "results", + "current_page", + "page_size", + "total", + "total_is_exact", + "has_more", + } + assert shape["total"] == 2 + assert shape["total_is_exact"] is True + for row in shape["results"]: + assert "content" in row + assert "fields" not in row + + for predicates, expected_titles in FIND_META_CASES: + assert titles(bm_json("find", *meta_args(*predicates))) == expected_titles, predicates + + # PARITY: the predicate spelling and bm_find's JSON filters are one query. + searched = bm_json("tool", "find", "--filters", '{"tags": ["security", "oauth"]}') + assert titles(searched) == {"Alpha Spec"} + + # A 400-digit bound is a filter refusal, not a server error. + assert "not a finite number" in refusal("find", "--meta", f"score>{'9' * 400}") + + +def check_find_meta_scopes_paginates_and_projects() -> None: + write_note("Archived Spec", "specs-archive", "# Archived Spec", metadata={"status": "active"}) + + scopes = [ + ("/", {"Alpha Spec", "Beta Spec", "Gamma Note", "Archived Spec"}), + ("specs", {"Alpha Spec", "Beta Spec"}), + ("/specs", {"Alpha Spec", "Beta Spec"}), + ("./specs/", {"Alpha Spec", "Beta Spec"}), + # A spaced directory name is a file path, not a slug. + ("My Notes", {"Gamma Note"}), + ("nonexistent", set()), + ] + for path, expected_titles in scopes: + scoped = bm_json("find", path, "--meta", "note_type=note") + assert titles(scoped) == expected_titles, path + assert scoped["total"] == len(expected_titles), path + + first = bm_json("find", "specs", "--meta", "note_type=note", "--page-size", "1") + second = bm_json("find", "specs", "--meta", "note_type=note", "--page", "2", "--page-size", "1") + assert first["has_more"] is True + assert second["has_more"] is False + assert first["results"][0]["permalink"] != second["results"][0]["permalink"] + + projected = bm_json( + "find", + "--meta", + "priority in high,critical", + "--fields", + "title,priority,chapter_number,review.approved,tags,missing_field", + ) + by_title = {row["title"]: row for row in projected["results"]} + assert by_title["Alpha Spec"]["fields"] == { + "title": "Alpha Spec", + "priority": "high", + "chapter_number": 63, + "review.approved": True, + "tags": ["security", "oauth"], + "missing_field": None, + } + assert by_title["Gamma Note"]["fields"]["review.approved"] is None + # A projected row is the note's identity plus the fields, and no body. + for row in projected["results"]: + assert set(row) == { + "title", + "permalink", + "file_path", + "external_id", + "updated_at", + "fields", + } + assert "alpha body" not in json.dumps(row) + + +def check_find_meta_quoting_escapes() -> None: + write_note("Comma Note", "notes", "# Comma Note", metadata={"label": "a,b"}) + write_note("Range Note", "notes", "# Range Note", metadata={"range": ">=5"}) + + # A quoted list element keeps its comma. + assert titles(bm_json("find", "--meta", 'label in "a,b",c')) == {"Comma Note"} + # Unquoted, '>=5' reads as a mis-spelled operator; quoted, it matches literally. + assert "unsupported predicate operator" in refusal("find", "--meta", "range=>=5") + assert titles(bm_json("find", "--meta", 'range=">=5"')) == {"Range Note"} + + +def test_find_meta_scope_follows_the_file_path_not_the_permalink(): + """REGRESSION: a scope built from permalinks answers this pair backwards. + + An explicit frontmatter permalink is honored verbatim (#93), so these notes + live under one folder and permalink under the other. + """ + write_note( + "Housed In Specs", + "specs", + "---\npermalink: archive/housed-in-specs\nstatus: active\n---\n\n# Housed In Specs", + ) + write_note( + "Housed In Archive", + "archive", + "---\npermalink: specs/housed-in-archive\nstatus: active\n---\n\n# Housed In Archive", + ) + + specs = bm_json("find", "specs", "--meta", "status=active") + archive = bm_json("find", "archive", "--meta", "status=active") + + assert [row["file_path"] for row in specs["results"]] == ["specs/Housed In Specs.md"] + assert [row["file_path"] for row in archive["results"]] == ["archive/Housed In Archive.md"] + + +@pytest_asyncio.fixture +async def scanned_pdf(engine_factory, test_project, search_service) -> None: + """An indexed regular file: an ENTITY row with no frontmatter keys at all.""" + _, session_maker = engine_factory + now = datetime.now(timezone.utc) + async with db.scoped_session(session_maker) as session: + scan = await EntityRepository(project_id=test_project.id).add( + session, + Entity( + project_id=test_project.id, + title="Scanned Contract", + note_type="file", + content_type="application/pdf", + file_path="specs/Scanned Contract.pdf", + permalink="specs/scanned-contract", + created_at=now, + updated_at=now, + ), + ) + await search_service.index_entity_data(scan) + + +def test_find_meta(scanned_pdf): + """Predicates, scopes, projection and quoting over one seeded set. + + The set includes an indexed PDF. REGRESSION: a regular file carries no + frontmatter keys, which is what `key=null` asks for, so it used to be counted + as a hit; the null cases in FIND_META_CASES pin that it no longer is. + """ + seed_meta_notes() + check_find_meta_predicates() + # Each later check adds notes, so the order matters. + check_find_meta_scopes_paginates_and_projects() + check_find_meta_quoting_escapes() + + +def test_find_meta_fields_requires_hit_external_ids(monkeypatch): + """A server old enough to omit external_id fails fast instead of returning + unprojected rows. The old server is simulated at the search client: no real + server in this process can omit the field.""" + + async def legacy_search(self, query, page=1, page_size=10): + return SearchResponse( + results=[ + SearchResult( + title="Alpha Spec", + type=SearchItemType.ENTITY, + score=1.0, + permalink="test-project/specs/alpha-spec", + file_path="specs/Alpha Spec.md", + ) + ], + current_page=page, + page_size=page_size, + total=1, + ) + + monkeypatch.setattr(SearchClient, "search", legacy_search) + + stderr = refusal("find", "--meta", "status=active", "--fields", "title") + + assert "too old for `fields`" in stderr + + +# --- projects --- + + +def check_verbs_address_another_project(test_project) -> None: + write_note( + "Second Note", + "notes", + "# Second Note\n\nsecond body", + project="second-project", + metadata={"status": "active"}, + ) + + # --project selects the project; without it the default project answers. + assert bm_json("cat", "notes/second-note", *SECOND)["title"] == "Second Note" + assert "Error:" in refusal("cat", "notes/second-note") + assert titles(bm_json("grep", "second", *SECOND)) == {"Second Note"} + assert bm_json("grep", "second")["results"] == [] + + # A path qualified with the named project strips the prefix and answers in + # the qualified spelling, so a returned path is accepted again. + listing = bm_json("ls", "second-project/notes", *SECOND) + [node] = listing["nodes"] + assert node["file_path"] == "second-project/notes/Second Note.md" + assert bm_json("cat", node["file_path"], *SECOND)["title"] == "Second Note" + found = bm_json("find", "second-project", "--meta", "status=active", *SECOND) + assert [row["file_path"] for row in found["results"]] == [node["file_path"]] + # An unqualified call keeps project-relative paths. + assert bm_json("ls", "notes", *SECOND)["nodes"][0]["file_path"] == "notes/Second Note.md" + + # A prefix naming a different project than --project is refused. + assert "second-project" in refusal("cat", "second-project/notes/second-note") + + # --project also takes a project's external id. + write_note("Home Note", "notes", "# Home Note") + by_id = bm_json("cat", "notes/home-note", "--project", test_project.external_id) + assert by_id["title"] == "Home Note" + + +def check_routed_cat_never_rewrites_note_frontmatter(second_project: Path) -> None: + """Re-qualifying the transport path never touches the note's own YAML, + even when it carries keys spelled like transport fields.""" + write_note( + "Imported Note", + "notes", + "# Imported Note\n\nbody", + project="second-project", + metadata={"file_path": "imports/source.md", "directory_path": "/imports"}, + ) + + payload = bm_json("cat", "second-project/notes/imported-note", *SECOND) + + stored = (second_project / "notes" / "Imported Note.md").read_text() + _, _, rest = stored.partition("---\n") + block, _, _ = rest.partition("\n---") + assert yaml.safe_load(block) == payload["frontmatter"] + assert payload["frontmatter"]["file_path"] == "imports/source.md" + assert payload["file_path"] == "second-project/notes/Imported Note.md" + + +def check_tree_under_a_display_name_project(tmp_path_factory) -> None: + """A project named with a space and capitals answers in its permalink + spelling ('my-research/specs/...'); tree must strip that, not the caller's + 'My Research/specs', or the project segment nests under the root.""" + home = tmp_path_factory.mktemp("my-research-home") + added = bm("project", "add", "My Research", str(home)) + assert added.exit_code == 0, added.output + write_note("Plan", "specs/auth", "# Plan", project="My Research") + research = ["--project", "My Research"] + + qualified = bm_text("tree", "My Research/specs", *research) + relative = bm_text("tree", "specs", *research) + + assert qualified.splitlines() == ["My Research/specs", "└── auth/", " └── Plan.md"] + # Only the printed root differs between the two spellings of one folder. + assert qualified.splitlines()[1:] == relative.splitlines()[1:] + + +def test_verbs_with_two_projects(second_project, test_project, tmp_path_factory): + check_verbs_address_another_project(test_project) + check_routed_cat_never_rewrites_note_frontmatter(second_project) + check_tree_under_a_display_name_project(tmp_path_factory) diff --git a/test-int/mcp/test_adapter_project_routing.py b/test-int/mcp/test_adapter_project_routing.py index 81c22ae3a..921609320 100644 --- a/test-int/mcp/test_adapter_project_routing.py +++ b/test-int/mcp/test_adapter_project_routing.py @@ -6,11 +6,6 @@ from mcp import types as mt -import implementation_mcp - -implementation_server = implementation_mcp.implementation_server - - async def seed_projects( client: Client[FastMCPTransport], default_project: str, other_path: str ) -> None: @@ -31,9 +26,9 @@ async def seed_projects( @pytest.mark.parametrize("prompt_name", ["search_knowledge_base", "continue_conversation"]) @pytest.mark.parametrize("explicit_project", [False, True]) async def test_prompts_select_project_with_multiple_projects( - implementation_server, app, test_project, tmp_path, prompt_name, explicit_project + mcp_server, app, test_project, tmp_path, prompt_name, explicit_project ): - async with Client(implementation_server) as client: + async with Client(mcp_server) as client: await seed_projects( client, test_project.name, str(tmp_path.parent / (tmp_path.name + "-other")) ) diff --git a/test-int/mcp/test_read_note_missing_identifier_integration.py b/test-int/mcp/test_read_note_missing_identifier_integration.py index 54f643f6a..558732790 100644 --- a/test-int/mcp/test_read_note_missing_identifier_integration.py +++ b/test-int/mcp/test_read_note_missing_identifier_integration.py @@ -44,9 +44,15 @@ async def test_binary_uuid_line_read_reports_content_error( raise_on_error=False, ) + # A line range on a text file is a supported read, so the refusal is about the + # missing bytes: the API's 404 reaches a JSON caller as a structured NOT_FOUND. + # (Before text files took line ranges, this read was refused as UNSUPPORTED + # before the file was ever opened.) assert result.is_error block = result.content[0] assert block.type == "text" - assert json.loads(block.text)["error"] == "UNSUPPORTED" + refused = json.loads(block.text) + assert refused["error"] == "NOT_FOUND" + assert "legacy/legacy.txt" in refused["message"] async with db.scoped_session(session_maker) as session: assert await entity_repository.get_by_external_id(session, entity.external_id) is not None diff --git a/tests/cli/test_cli_line_scanning.py b/tests/cli/test_cli_line_scanning.py deleted file mode 100644 index cb46c20c7..000000000 --- a/tests/cli/test_cli_line_scanning.py +++ /dev/null @@ -1,58 +0,0 @@ -"""CLI scanning flags and rendered coordinates survive the MCP boundary.""" - -import json -from unittest.mock import AsyncMock, patch - -import pytest -from typer.testing import CliRunner - -from basic_memory.cli.main import app - - -@pytest.mark.parametrize("mode", ["--plain", "--json", "rich"]) -def test_cli_grep_context(mode: str) -> None: - payload = { - "current_page": 1, - "page_size": 10, - "total": 20, - "has_more": True, - "pagination_scope": "search_candidates", - "results": [ - { - "file_path": "runbook.md", - "match_count": 2, - "next_match_line": 20, - "windows": [ - { - "start_line": 3, - "end_line": 4, - "match_lines": [4], - "content": "context\nretry [safe]", - } - ], - } - ], - } - tool = AsyncMock(return_value=payload) - with ( - patch("basic_memory.mcp.tools.grep", tool), - patch( - "basic_memory.cli.commands.posix.resolve_output_mode", - return_value=mode.removeprefix("--"), - ), - ): - result = CliRunner().invoke( - app, - ["grep", "retry", "-C", "1", "--max-matches", "1"] + ([] if mode == "rich" else [mode]), - ) - assert result.exit_code == 0, result.output - assert tool.call_args.kwargs["semantic"] is False - assert tool.call_args.kwargs["context_lines"] == 1 - assert tool.call_args.kwargs["max_matches"] == 1 - if mode == "--json": - assert json.loads(result.stdout) == payload - else: - assert "runbook.md-3-context" in result.stdout - assert "runbook.md:4:retry [safe]" in result.stdout - assert "line 20" in result.stdout - assert "--page 2" in result.stdout diff --git a/tests/cli/test_cli_posix_argument_errors.py b/tests/cli/test_cli_posix_argument_errors.py new file mode 100644 index 000000000..797b222be --- /dev/null +++ b/tests/cli/test_cli_posix_argument_errors.py @@ -0,0 +1,307 @@ +"""Argument refusals of the POSIX verbs, through the `bm` CLI. + +Every case here is refused before the command reads any project data: by the +CLI's own parsing (--lines, the --meta grammar), by the bm_* tool's argument +validation, or by the tool's up-front refusals. A refusal is exit 1 with the +reason on stderr. Behavior that needs notes lives in +test-int/cli/test_posix_commands_integration.py. +""" + +import pytest +from typer.testing import CliRunner + +from basic_memory.cli.main import app as cli_app + +runner = CliRunner() + + +def assert_refused(args: list[str], message: str) -> None: + result = runner.invoke(cli_app, args) + + assert result.exit_code == 1, result.output + assert message in result.stderr + + +# --- cat (bm_read) --- + + +@pytest.mark.parametrize( + ("args", "message"), + [ + ( + ["--section", "A", "--lines", "2-"], + "section cannot be combined with start_line/end_line", + ), + ( + ["--section", "A", "--lines", "1-3"], + "section cannot be combined with start_line/end_line", + ), + (["--max-tokens", "0"], "--max-tokens: Input should be greater than or equal to 1"), + (["--lines", "0"], "--start-line: Input should be greater than or equal to 1"), + (["--lines", "0-4"], "--start-line: Input should be greater than or equal to 1"), + (["--lines", "2-1"], "end_line (1) is before start_line (2)"), + (["--lines", "abc"], '--lines must be "N-M", "N-" (to end), or "N" (one line)'), + (["--lines", "1-x"], '--lines must be "N-M"'), + ], +) +def test_cat_refuses_bad_slice_arguments(args, message): + assert_refused(["cat", "anything", *args], message) + + +# --- grep (bm_search) --- + + +@pytest.mark.parametrize( + ("args", "message"), + [ + (["--page", "0"], "--page: Input should be greater than or equal to 1"), + (["--page-size", "0"], "--page-size: Input should be greater than or equal to 1"), + (["--page-size", "101"], "--page-size: Input should be less than or equal to 100"), + (["-C", "11"], "--lines: Input should be less than or equal to 10"), + (["--max-matches", "0"], "--max-matches: Input should be greater than or equal to 1"), + (["--max-matches", "3"], "`max_matches` bounds matching lines; it needs `lines`"), + ], +) +def test_grep_refuses_bad_arguments(args, message): + assert_refused(["grep", "ok", *args], message) + + +# --- ls (bm_list) --- + + +@pytest.mark.parametrize( + ("args", "message"), + [ + (["--page", "0"], "--page: Input should be greater than or equal to 1"), + (["--page-size", "101"], "--page-size: Input should be less than or equal to 100"), + (["--sort", "bogus"], "--sort: Input should be 'title_asc'"), + ], +) +def test_ls_refuses_bad_arguments(args, message): + assert_refused(["ls", *args], message) + + +# --- find / tree (bm_find) --- + + +@pytest.mark.parametrize( + ("args", "message"), + [ + (["--page", "0"], "--page: Input should be greater than or equal to 1"), + (["--page-size", "101"], "--page-size: Input should be less than or equal to 100"), + (["--depth", "0"], "--depth: Input should be greater than or equal to 1"), + (["--depth", "11"], "--depth: Input should be less than or equal to 10"), + # The frontmatter search has no file-name glob and no depth bound, so + # both are refused rather than silently ignored. + (["--name", "*.md", "--meta", "status=active"], "`name` cannot combine with `filters`"), + (["--depth", "3", "--meta", "status=active"], "`depth` cannot combine with `filters`"), + (["--fields", "title"], "`fields` needs `filters`"), + (["--meta", "status=active", "--fields", ""], "must be non-empty"), + (["--meta", "status=active", "--fields", "title,"], "must be non-empty"), + # A malformed field path would project to null for every hit, which reads + # exactly like a field the notes do not carry. + (["--meta", "status=active", "--fields", "review..approved"], "malformed field path"), + (["--meta", "status=active", "--fields", ".owner"], "malformed field path"), + (["--meta", "status=active", "--fields", "title,a..b"], "malformed field path"), + (["--meta", "status=active", "--page", "0"], "--page: Input should be greater"), + ], +) +def test_find_refuses_bad_arguments(args, message): + assert_refused(["find", *args], message) + + +def test_tree_refuses_bad_depth(): + assert_refused(["tree", "--depth", "11"], "--depth: Input should be less than or equal to 10") + + +def test_tail_is_gone(): + """`bm tail` was removed in 1.0; `bm tool project recent` lists recent notes.""" + result = runner.invoke(cli_app, ["tail"]) + + assert result.exit_code == 2 + assert "No such command 'tail'" in result.stderr + + +# --- find --meta: predicate grammar refusals --- +# Each refusal names find's grammar before any request is built. + + +@pytest.mark.parametrize( + ("predicates", "message"), + [ + # The API has no $ne, so != is deliberately absent from the grammar. + (["status!=active"], "unsupported predicate operator in 'status!=active'"), + (["status ~= active"], "unsupported predicate operator"), + (["priority gte 3"], "unsupported predicate operator"), + (["priority in"], "unsupported predicate operator"), + (["nothing"], "unsupported predicate operator"), + # A key cannot hold whitespace or be empty, so these have no reading. + (["my key=x"], "unsupported predicate operator"), + (["=x"], "unsupported predicate operator"), + ([""], "unsupported predicate operator"), + (["status matches active"], "supported: = > >= < <= in has between"), + (["score between 0.3"], "'between' needs exactly min,max"), + (["score between 0.1,0.2,0.3"], "'between' needs exactly min,max"), + (["priority in high,,low"], "empty list element"), + (["status="], "has no value"), + (["status=active", "status=draft"], "duplicate predicate key 'status'"), + (["score>0.3", "score<0.8"], "use 'between' for ranges"), + # The alias collapses onto the same key, so the collision is still caught. + (["note_type=note", "type=spec"], "duplicate predicate key 'type'"), + ], +) +def test_find_meta_refuses_unsupported_predicates(predicates, message): + meta_args = [arg for predicate in predicates for arg in ("--meta", predicate)] + assert_refused(["find", *meta_args], message) + + +@pytest.mark.parametrize( + "predicate", + [ + # Symbol operators: the regex matches the longest SUPPORTED spelling, so + # the second operator character used to land at the head of the value. + "status==active", + "status=>active", + "status=>3", + "count>=>3", + "count<<1", + "count<=<1", + # Word operators fold the same way: everything after them is the value. + "priority in >high", + "tags has =security", + "score between >0.3,0.8", + "score between 0.3,<0.8", + # Unquoted, a value starting with an operator character reads as a typo. + "range=>=5", + ], +) +def test_find_meta_malformed_operators_refuse_instead_of_folding_into_the_value(predicate): + """REGRESSION: a mis-spelled multi-character operator is a refusal, not a value. + + 'status==active' used to parse as {"status": "=active"} and 'count>>3' as + {"count": {"$gt": ">3"}}: filters for text no note carries. + """ + assert_refused(["find", "--meta", predicate], "unsupported predicate operator") + + +def test_find_meta_malformed_operator_refusal_teaches_the_grammar_and_the_escape(): + result = runner.invoke(cli_app, ["find", "--meta", "status==active"]) + + assert result.exit_code == 1 + assert "unsupported predicate operator in 'status==active'" in result.stderr + assert "supported: = > >= < <= in has between" in result.stderr + assert 'quote the value as "=active"' in result.stderr + + +@pytest.mark.parametrize( + "predicate", + [ + # A doubled, leading or trailing dot is not a dot path. + "review..approved=true", + ".owner=null", + "owner.=x", + "status.=active", + "a..b>1", + ".score>=0.5", + "trailing.<9", + # Word operators capture the key the same way, so they fold identically. + "review..approved in a,b", + ".tags has security", + "owner. between 1,2", + ], +) +def test_find_meta_malformed_keys_refuse_before_transport(predicate): + """REGRESSION: a key that is not a dot path is refused by find, not by the API. + + The search API would refuse these with "Unsupported metadata filter key", + which names neither find nor the shape a key must have. + """ + assert_refused(["find", "--meta", predicate], "malformed predicate key") + + +def test_find_meta_malformed_key_refusal_names_the_key_and_the_grammar(): + result = runner.invoke(cli_app, ["find", "--meta", "review..approved=true"]) + + assert result.exit_code == 1 + assert "malformed predicate key 'review..approved' in 'review..approved=true'" in result.stderr + assert "dot-separated names of letters, digits, '_' or '-'" in result.stderr + assert "review.approved" in result.stderr + + +@pytest.mark.parametrize( + "predicate", + [ + # Python's JSON reader accepts these three spellings as an extension... + "score=NaN", + "score=Infinity", + "score=-Infinity", + # ...and silently overflows an oversized exponent to infinity. + "score=1e999", + "score=-1e999", + # Every operator reads its values through the same scalar reader. + "score>NaN", + "score<=Infinity", + "score in 0.5,NaN", + "marks has NaN", + "score between NaN,0.8", + "score between 0.3,1e999", + ], +) +def test_find_meta_non_finite_numbers_refuse_instead_of_failing_at_transport(predicate): + """REGRESSION: a non-finite number died in the request encoder, not the grammar.""" + assert_refused(["find", "--meta", predicate], "non-finite number") + + +def test_find_meta_non_finite_refusal_names_the_predicate_and_the_escape(): + result = runner.invoke(cli_app, ["find", "--meta", "score=NaN"]) + + assert result.exit_code == 1 + assert "predicate 'score=NaN' has a non-finite number 'NaN'" in result.stderr + assert "predicate values must be finite numbers" in result.stderr + assert 'quote the value as "NaN"' in result.stderr + + +@pytest.mark.parametrize( + "predicate", + [ + 'status="active', + 'status = "active', + 'confidence>"0.6', + # A quote that opens, closes, and opens again is still unterminated. + 'name="a"b"c', + # The list operators reach the same check through their split elements. + 'priority in "high,low', + 'tags has red,"green', + 'score between "0.3,0.8', + ], +) +def test_find_meta_dangling_quote_refuses_for_every_operator(predicate): + """REGRESSION: the scalar path used to keep the dangling quote as the value.""" + assert_refused(["find", "--meta", predicate], "unterminated quoted value") + + +def test_find_meta_unterminated_quote_refusal_teaches_both_uses_of_quoting(): + result = runner.invoke(cli_app, ["find", "--meta", 'status="active']) + + assert result.exit_code == 1 + assert "predicate 'status=\"active' has an unterminated quoted value" in result.stderr + assert 'status="active"' in result.stderr + assert 'label in "a,b",c' in result.stderr + + +@pytest.mark.parametrize( + "predicate", + [ + "score>null", + "score>=null", + "score' prefix form. Same DirectoryListResponse contract as any listing, -# so the existing renderers must handle it untouched. -LS_MOUNT_RESULT = { - "nodes": [ - _dir_node(name="main", directory_path="/main", permalink="main", type="directory"), - _dir_node( - name="research", directory_path="/research", permalink="research", type="directory" - ), - ], - "page": 1, - "page_size": 10, - "total": 2, - "has_more": False, -} - -FIND_RESULT = { - "nodes": [ - _dir_node(name="specs", directory_path="/specs", type="directory"), - _dir_node( - name="search.md", - file_path="specs/search.md", - directory_path="/specs/search.md", - title="Search [draft]", - permalink="specs/search", - ), - ], - "page": 1, - "page_size": 10, - "total": 2, - "has_more": False, -} - -FIND_RESULT_EMPTY = {"nodes": [], "page": 1, "page_size": 10, "total": 0, "has_more": False} -FIND_RESULT_MORE = {**FIND_RESULT, "has_more": True} - -# --meta flips find's payload to the search response shape (the same contract -# grep returns). --fields then *projects* each hit: the row is the note's -# identity plus a `fields` object, with null for a field the hit does not carry, -# and no note body. A projected row is therefore not a grep row with an extra -# key — spelled out here rather than spread from GREP_RESULT so this mock cannot -# drift back into promising the CLI content the tool no longer sends. -FIND_META_RESULT = GREP_RESULT -FIND_META_RESULT_MORE = {**FIND_META_RESULT, "has_more": True} - -FIND_META_FIELDS_RESULT = { - **GREP_RESULT, - "results": [ - { - "title": "Spec [draft] v2", - "permalink": "specs/spec-draft-v2", - "file_path": "specs/Spec [draft] v2.md", - "external_id": "0b3f0d1e-5f9a-4d2b-8c31-1f0b7a9c4d55", - "updated_at": "2025-01-01T00:00:00", - "fields": { - "title": "Spec [draft] v2", - "priority": "high", - "approved": True, - "missing": None, - }, - }, - { - "title": "Another Note", - "permalink": "notes/another-note", - "file_path": "notes/Another Note.md", - "external_id": "7c2a91b4-3d68-4e0f-9a15-2b6c8e4f0a37", - "updated_at": "2025-01-02T00:00:00", - "fields": { - "title": "Another Note", - "priority": None, - "approved": False, - "missing": None, - }, - }, - ], -} - -FIND_META_FIELDS_EMPTY = {**GREP_RESULT_EMPTY, "results": []} - -FIND_META_FIELDS_MORE = {**FIND_META_FIELDS_RESULT, "has_more": True} - -# tail rows: {type, title, permalink, file_path, created_at, updated_at}, newest change -# first. The second row has no updated_at, as a server before v0.24 sends it. -TAIL_RESULT = [ - { - "type": "entity", - "title": "Note A", - "permalink": "notes/note-a", - "file_path": "notes/Note A.md", - "created_at": "2024-12-30T08:00:00", - "updated_at": "2025-01-02T10:00:00", - }, - { - "type": "entity", - "title": "Note [draft] B", - "permalink": "notes/note-b", - "file_path": "notes/Note B.md", - "created_at": "2025-01-01T09:00:00", - }, -] - -# tree consumes find's payload: a FLAT page of nodes (children stripped by the -# API), so nesting must be rebuilt from directory_path segments. -TREE_RESULT = { - "nodes": [ - _dir_node(name="specs", directory_path="/specs", type="directory"), - _dir_node(name="search.md", file_path="specs/search.md", directory_path="/specs/search.md"), - # No node for specs/auth: a glob filter can return files without their - # parent directories, so "auth/" must be synthesized. - _dir_node( - name="deep.md", - file_path="specs/auth/deep.md", - directory_path="/specs/auth/deep.md", - ), - ], - "page": 1, - "page_size": 10, - "total": 3, - "has_more": False, -} - -# Rooted find: the search root itself comes back as a node and must not render -# as its own child. -TREE_ROOTED_RESULT = { - "nodes": [ - _dir_node(name="specs", directory_path="/specs", type="directory"), - _dir_node(name="intro.md", file_path="specs/intro.md", directory_path="/specs/intro.md"), - ], - "page": 1, - "page_size": 10, - "total": 2, - "has_more": False, -} - -# A directory node arriving after a file already synthesized it, plus a bare -# root node ("/") that contributes no path segment. -TREE_DIR_AFTER_FILE_RESULT = { - "nodes": [ - _dir_node(name="", directory_path="/", type="directory"), - _dir_node(name="search.md", file_path="specs/search.md", directory_path="/specs/search.md"), - _dir_node(name="specs", directory_path="/specs", type="directory"), - ], - "page": 1, - "page_size": 10, - "total": 3, - "has_more": False, -} - -TREE_RESULT_MORE = {**TREE_RESULT, "has_more": True} - -TREE_RESULT_EMPTY = {"nodes": [], "page": 1, "page_size": 10, "total": 0, "has_more": False} - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _invoke(args, **kwargs): - """Invoke the CLI wide enough that Rich tables don't truncate cell text.""" - kwargs.setdefault("env", {"COLUMNS": "240"}) - return runner.invoke(cli_app, args, **kwargs) - - -def _tty_invoke(args, **kwargs): - """Invoke the CLI as if stdout were an interactive TTY.""" - with patch("basic_memory.cli.commands.output.stdout_is_terminal", return_value=True): - return _invoke(args, **kwargs) - - -def _flattened(output: str) -> str: - # Rich wraps console output at the terminal width; collapse all whitespace - # so phrase assertions can't be split by a line break. - return " ".join(output.split()) - - -def _assert_not_json(output: str) -> None: - with pytest.raises((json.JSONDecodeError, ValueError)): - json.loads(output) - - -def test_plain_page_footer_flushes_rows_before_stderr(monkeypatch): - stdout = Mock() - stderr = io.StringIO() - monkeypatch.setattr(posix_commands, "sys", SimpleNamespace(stdout=stdout, stderr=stderr)) - - posix_commands._write_plain_page_footer(LS_RESULT_MORE) - - stdout.flush.assert_called_once_with() - assert stderr.getvalue().endswith("more available (--page)\n") - - -# Every verb with its patch target, the payload it must print, and what the -# patched callable returns. head shares cat's tool and payload; tree's JSON -# contract is still find's, but it calls find_listing, which hands back the -# listing and the root its paths are relative to from one resolution (#1421). -FIND_LISTING_TARGET = "basic_memory.mcp.tools.posix_tools.find_listing" -VERB_CASES = [ - pytest.param( - ["cat", "specs/search"], "basic_memory.mcp.tools.cat", CAT_RESULT, CAT_RESULT, id="cat" - ), - pytest.param( - ["head", "specs/search"], "basic_memory.mcp.tools.cat", CAT_RESULT, CAT_RESULT, id="head" - ), - pytest.param( - ["grep", "ranking"], "basic_memory.mcp.tools.grep", GREP_RESULT, GREP_RESULT, id="grep" - ), - pytest.param(["ls", "/specs"], "basic_memory.mcp.tools.ls", LS_RESULT, LS_RESULT, id="ls"), - pytest.param( - ["find", "/specs"], "basic_memory.mcp.tools.find", FIND_RESULT, FIND_RESULT, id="find" - ), - pytest.param(["tail"], "basic_memory.mcp.tools.tail", TAIL_RESULT, TAIL_RESULT, id="tail"), - pytest.param( - ["tree", "/specs"], - FIND_LISTING_TARGET, - TREE_RESULT, - (TREE_RESULT, "/specs"), - id="tree", - ), -] - - -# --------------------------------------------------------------------------- -# JSON contract: non-TTY auto-JSON and --json are the tool payload, verbatim -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize(("args", "target", "payload", "tool_return"), VERB_CASES) -def test_verb_non_tty_outputs_tool_payload_verbatim(args, target, payload, tool_return): - """Piped output is the MCP tool's return value, unchanged (auto-JSON).""" - with patch(target, new_callable=AsyncMock, return_value=tool_return) as mock_tool: - result = _invoke(args) - - assert result.exit_code == 0, result.output - assert json.loads(result.stdout) == payload - mock_tool.assert_called_once() - - -@pytest.mark.parametrize(("args", "target", "payload", "tool_return"), VERB_CASES) -def test_verb_json_flag_overrides_tty(args, target, payload, tool_return): - """--json wins over the interactive renderer on a TTY.""" - with patch(target, new_callable=AsyncMock, return_value=tool_return): - result = _tty_invoke([*args, "--json"]) - - assert result.exit_code == 0, result.output - assert json.loads(result.stdout) == payload - - -@patch("basic_memory.mcp.tools.cat", new_callable=AsyncMock, return_value=CAT_RESULT) -def test_json_and_plain_together_errors(mock_cat): - """The contradictory --json --plain combination is a clear non-zero error.""" - result = _tty_invoke(["cat", "specs/search", "--json", "--plain"]) - - assert result.exit_code == 1 - assert "mutually exclusive" in result.output - mock_cat.assert_not_called() - - -# --------------------------------------------------------------------------- -# cat / head rendering -# --------------------------------------------------------------------------- - - -@patch("basic_memory.mcp.tools.cat", new_callable=AsyncMock, return_value=CAT_RESULT) -def test_cat_rich_output(mock_cat): - result = _tty_invoke(["cat", "specs/search"]) - - assert result.exit_code == 0, result.output - _assert_not_json(result.output) - assert "Search Spec" in result.output - assert "ranking notes" in result.output - - -@patch("basic_memory.mcp.tools.cat", new_callable=AsyncMock, return_value=CAT_RESULT) -def test_cat_plain_output_is_exact_content(mock_cat): - """Plain stdout is the content byte-for-byte: pipes and redirects round-trip.""" - result = _tty_invoke(["cat", "specs/search", "--plain"]) - - assert result.exit_code == 0, result.output - assert result.stdout == CAT_RESULT["content"] - assert result.stderr == "" - - -@patch( - "basic_memory.mcp.tools.cat", - new_callable=AsyncMock, - return_value={**CAT_RESULT, "content": None}, -) -def test_cat_plain_output_without_content_is_empty(mock_cat): - result = _tty_invoke(["cat", "specs/search", "--plain"]) - - assert result.exit_code == 0, result.output - assert result.stdout == "" - - -@patch("basic_memory.mcp.tools.cat", new_callable=AsyncMock, return_value=CAT_SLICE_RESULT) -def test_cat_rich_slice_footer(mock_cat): - """A ranged read shows its coordinates so follow-up range reads are easy.""" - result = _tty_invoke(["cat", "specs/search", "--lines", "5-7"]) - - assert result.exit_code == 0, result.output - assert "lines 5-7 of 7" in _flattened(result.output) - - -@patch("basic_memory.mcp.tools.cat", new_callable=AsyncMock, return_value=CAT_SLICE_RESULT) -def test_cat_plain_slice_footer_goes_to_stderr(mock_cat): - """Plain mode keeps stdout content-only; slice info lands on stderr.""" - result = _tty_invoke(["cat", "specs/search", "--lines", "5-7", "--plain"]) - - assert result.exit_code == 0, result.output - assert result.stdout == CAT_SLICE_RESULT["content"] - assert "lines 5-7 of 7" in result.stderr - # The slice content carries no trailing newline, so the footer must open - # with one on stderr — otherwise it visually concatenates onto the last - # content line in a terminal while stdout stays byte-exact for pipes. - assert result.stderr.startswith("\n") - - -@patch("basic_memory.mcp.tools.cat", new_callable=AsyncMock, return_value=CAT_TRUNCATED_RESULT) -def test_cat_rich_truncation_footer(mock_cat): - """A truncated read names the section, the range, and the resume command.""" - result = _tty_invoke(["cat", "specs/search", "--section", "Decisions", "--max-tokens", "20"]) - - flat = _flattened(result.output) - assert result.exit_code == 0, result.output - assert "lines 5-40 of 80" in flat - assert "section Decisions" in flat - assert "truncated" in flat - assert "--lines 42-" in flat - - -@patch("basic_memory.mcp.tools.cat", new_callable=AsyncMock, return_value=CAT_TRUNCATED_RESULT) -def test_cat_plain_truncation_keeps_stdout_pure(mock_cat): - result = _tty_invoke(["cat", "specs/search", "--max-tokens", "20", "--plain"]) - - assert result.exit_code == 0, result.output - assert result.stdout == CAT_TRUNCATED_RESULT["content"] - assert "--lines 42-" in result.stderr - - -@patch( - "basic_memory.mcp.tools.cat", - new_callable=AsyncMock, - return_value={**CAT_RESULT, "start_line": 3, "end_line": 4}, -) -def test_cat_footer_without_total_lines(mock_cat): - """A slice payload without total_lines still reports the range, without ' of N'.""" - result = _tty_invoke(["cat", "specs/search", "--lines", "3-4"]) - - flat = _flattened(result.output) - assert result.exit_code == 0, result.output - assert "lines 3-4" in flat - assert "lines 3-4 of" not in flat - - -@patch( - "basic_memory.mcp.tools.cat", - new_callable=AsyncMock, - return_value={**CAT_RESULT, "truncated": True}, -) -def test_cat_footer_truncated_without_continue_line(mock_cat): - result = _tty_invoke(["cat", "specs/search", "--max-tokens", "20"]) - - flat = _flattened(result.output) - assert result.exit_code == 0, result.output - assert "truncated" in flat - assert "--lines" not in flat - - -@patch("basic_memory.mcp.tools.cat", new_callable=AsyncMock, return_value=CAT_RESULT) -def test_head_plain_output(mock_cat): - """head shares cat's payload and renderers.""" - result = _tty_invoke(["head", "specs/search", "--plain"]) - - assert result.exit_code == 0, result.output - assert result.stdout == CAT_RESULT["content"] - - -# --------------------------------------------------------------------------- -# cat / head argument pass-through -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize( - ("lines", "expected_start", "expected_end"), - [("5-10", 5, 10), ("5-", 5, None), ("5", 5, 5)], -) -@patch("basic_memory.mcp.tools.cat", new_callable=AsyncMock, return_value=CAT_RESULT) -def test_cat_lines_forms_parse_to_start_end(mock_cat, lines, expected_start, expected_end): - result = _invoke(["cat", "specs/search", "--lines", lines]) - - assert result.exit_code == 0, result.output - assert mock_cat.call_args.args == ("specs/search",) - assert mock_cat.call_args.kwargs["start_line"] == expected_start - assert mock_cat.call_args.kwargs["end_line"] == expected_end - - -@pytest.mark.parametrize("bad_lines", ["abc", "5-x", "-5"]) -@patch("basic_memory.mcp.tools.cat", new_callable=AsyncMock, return_value=CAT_RESULT) -def test_cat_lines_garbage_is_a_clear_error(mock_cat, bad_lines): - result = _invoke(["cat", "specs/search", "--lines", bad_lines]) - - assert result.exit_code == 1 - assert "N-M" in result.output - mock_cat.assert_not_called() - - -@patch("basic_memory.mcp.tools.cat", new_callable=AsyncMock, return_value=CAT_RESULT) -def test_cat_section_and_budget_passthrough(mock_cat): - result = _invoke( - ["cat", "specs/search", "--section", "Decisions", "--max-tokens", "200", "--no-frontmatter"] - ) - - assert result.exit_code == 0, result.output - kwargs = mock_cat.call_args.kwargs - assert kwargs["section"] == "Decisions" - assert kwargs["max_tokens"] == 200 - assert kwargs["include_frontmatter"] is False - assert kwargs["start_line"] is None - assert kwargs["end_line"] is None - - -@patch("basic_memory.mcp.tools.cat", new_callable=AsyncMock, return_value=CAT_RESULT) -def test_cat_project_passthrough(mock_cat): - uuid = "11111111-1111-1111-1111-111111111111" - result = _invoke(["cat", "specs/search", "--project", "research", "--project-id", uuid]) - - assert result.exit_code == 0, result.output - assert mock_cat.call_args.kwargs["project"] == "research" - assert mock_cat.call_args.kwargs["project_id"] == uuid - - -@patch("basic_memory.mcp.tools.cat", new_callable=AsyncMock, return_value=CAT_RESULT) -def test_head_is_a_fixed_range_cat(mock_cat): - """head -n 3 is cat with start_line=1, end_line=3.""" - result = _invoke(["head", "specs/search", "-n", "3"]) - - assert result.exit_code == 0, result.output - assert mock_cat.call_args.args == ("specs/search",) - assert mock_cat.call_args.kwargs["start_line"] == 1 - assert mock_cat.call_args.kwargs["end_line"] == 3 - - -# --------------------------------------------------------------------------- -# grep -# --------------------------------------------------------------------------- - - -@patch("basic_memory.mcp.tools.grep", new_callable=AsyncMock, return_value=GREP_RESULT) -def test_grep_rich_output(mock_grep): - result = _tty_invoke(["grep", "ranking"]) - - assert result.exit_code == 0, result.output - _assert_not_json(result.output) - # User-sourced bracketed titles survive Rich markup parsing. - assert "[draft]" in result.output - assert "Another Note" in result.output - - -@patch("basic_memory.mcp.tools.grep", new_callable=AsyncMock, return_value=GREP_RESULT) -def test_grep_plain_output(mock_grep): - result = _tty_invoke(["grep", "ranking", "--plain"]) - - assert result.exit_code == 0, result.output - assert "1. Spec [draft] v2" in result.output - assert "─" not in result.output - assert "│" not in result.output - - -@patch("basic_memory.mcp.tools.grep", new_callable=AsyncMock, return_value=GREP_RESULT_EMPTY) -def test_grep_rich_empty(mock_grep): - result = _tty_invoke(["grep", "nothing"]) - - assert result.exit_code == 0, result.output - assert "No results found" in result.output - - -@patch("basic_memory.mcp.tools.grep", new_callable=AsyncMock, return_value=GREP_RESULT) -def test_grep_semantic_and_paging_passthrough(mock_grep): - """--semantic requests ranking by meaning instead of keyword matching.""" - result = _invoke(["grep", "needle", "--semantic", "--page", "2", "--page-size", "5"]) - - assert result.exit_code == 0, result.output - assert mock_grep.call_args.args == ("needle",) - kwargs = mock_grep.call_args.kwargs - assert kwargs["semantic"] is True - assert kwargs["page"] == 2 - assert kwargs["page_size"] == 5 - - -@patch("basic_memory.mcp.tools.grep", new_callable=AsyncMock, return_value=GREP_RESULT) -def test_grep_defaults_to_keyword_matching(mock_grep): - result = _invoke(["grep", "needle"]) - - assert result.exit_code == 0, result.output - assert mock_grep.call_args.kwargs["semantic"] is False - - -# --------------------------------------------------------------------------- -# ls -# --------------------------------------------------------------------------- - - -@patch("basic_memory.mcp.tools.ls", new_callable=AsyncMock, return_value=LS_RESULT) -def test_ls_rich_output(mock_ls): - result = _tty_invoke(["ls", "/specs"]) - - flat = _flattened(result.output) - assert result.exit_code == 0, result.output - _assert_not_json(result.output) - assert "specs/" in result.output # directories get a trailing slash - assert "[draft]" in result.output # bracketed titles survive escaping - assert "specs/search-spec" in flat - assert "page 1" in flat - assert "total 2" in flat - - -@patch("basic_memory.mcp.tools.ls", new_callable=AsyncMock, return_value=LS_RESULT_MORE) -def test_ls_rich_reports_more_pages(mock_ls): - result = _tty_invoke(["ls", "/specs"]) - - assert result.exit_code == 0, result.output - assert "more available" in _flattened(result.output) - - -@patch("basic_memory.mcp.tools.ls", new_callable=AsyncMock, return_value=LS_RESULT_EMPTY) -def test_ls_rich_empty_directory(mock_ls): - result = _tty_invoke(["ls", "/empty"]) - - assert result.exit_code == 0, result.output - assert "Empty directory." in result.output - - -@patch("basic_memory.mcp.tools.ls", new_callable=AsyncMock, return_value=LS_RESULT) -def test_ls_plain_is_one_path_per_line(mock_ls): - result = _tty_invoke(["ls", "/specs", "--plain"]) - - assert result.exit_code == 0, result.output - assert result.stdout == "/specs/\n/specs/Search Spec.md\n" - - -@patch("basic_memory.mcp.tools.ls", new_callable=AsyncMock, return_value=LS_RESULT_MORE) -def test_ls_plain_reports_more_pages_on_stderr(mock_ls): - result = _tty_invoke(["ls", "/specs", "--plain"]) - - assert result.exit_code == 0, result.output - assert "more available" not in result.stdout - assert "page 1" in result.stderr - assert "total 2" in result.stderr - assert "more available (--page)" in result.stderr - - -@patch("basic_memory.mcp.tools.ls", new_callable=AsyncMock, return_value=LS_RESULT) -def test_ls_defaults_and_paging_passthrough(mock_ls): - result = _invoke(["ls", "--page", "2", "--page-size", "50"]) - - assert result.exit_code == 0, result.output - assert mock_ls.call_args.args == ("/",) # default path is the project root - assert mock_ls.call_args.kwargs["page"] == 2 - assert mock_ls.call_args.kwargs["page_size"] == 50 - - -@patch("basic_memory.mcp.tools.ls", new_callable=AsyncMock, return_value=LS_MOUNT_RESULT) -def test_ls_plain_mount_view_prints_copyable_prefixes(mock_ls): - """bm ls with no project emits one '//' line per mount, so the - prefix form the routing rules teach is copyable straight from the output.""" - result = _tty_invoke(["ls", "--plain"]) - - assert result.exit_code == 0, result.output - assert result.stdout == "/main/\n/research/\n" - - -@patch("basic_memory.mcp.tools.ls", new_callable=AsyncMock, return_value=LS_MOUNT_RESULT) -def test_ls_rich_mount_view_renders_bare_directory_nodes(mock_ls): - """Mount rows carry no title/updated_at; the rich renderer shows them fine.""" - result = _tty_invoke(["ls"]) - - flat = _flattened(result.output) - assert result.exit_code == 0, result.output - _assert_not_json(result.output) - assert "main/" in result.output - assert "research/" in result.output - assert "total 2" in flat - - -# --------------------------------------------------------------------------- -# find -# --------------------------------------------------------------------------- - - -@patch("basic_memory.mcp.tools.find", new_callable=AsyncMock, return_value=FIND_RESULT) -def test_find_rich_output(mock_find): - result = _tty_invoke(["find", "/specs", "--name", "*.md"]) - - flat = _flattened(result.output) - assert result.exit_code == 0, result.output - _assert_not_json(result.output) - assert "*.md" in result.output # the glob shows in the panel title - assert "specs/search.md" in flat - assert "[draft]" in result.output - - -@patch("basic_memory.mcp.tools.find", new_callable=AsyncMock, return_value=FIND_RESULT_EMPTY) -def test_find_rich_no_matches(mock_find): - # No --name: the panel title carries no glob suffix on this path. - result = _tty_invoke(["find", "/specs"]) - - assert result.exit_code == 0, result.output - assert "No matches." in result.output - assert "--name" not in result.output - - -@patch("basic_memory.mcp.tools.find", new_callable=AsyncMock, return_value=FIND_RESULT) -def test_find_plain_is_one_path_per_line(mock_find): - """find(1) style: file_path when present, directory_path for directories.""" - result = _tty_invoke(["find", "/specs", "--plain"]) - - assert result.exit_code == 0, result.output - assert result.stdout == "/specs\nspecs/search.md\n" - - -@patch("basic_memory.mcp.tools.find", new_callable=AsyncMock, return_value=FIND_RESULT_MORE) -def test_find_plain_reports_more_pages_on_stderr(mock_find): - result = _tty_invoke(["find", "/specs", "--plain"]) - - assert result.exit_code == 0, result.output - assert "more available" not in result.stdout - assert "page 1" in result.stderr - assert "total 2" in result.stderr - assert "more available (--page)" in result.stderr - - -@patch("basic_memory.mcp.tools.find", new_callable=AsyncMock, return_value=FIND_RESULT) -def test_find_name_and_depth_passthrough(mock_find): - result = _invoke(["find", "/specs", "--name", "*.md", "--depth", "3"]) - - assert result.exit_code == 0, result.output - assert mock_find.call_args.args == ("/specs",) - assert mock_find.call_args.kwargs["name"] == "*.md" - assert mock_find.call_args.kwargs["depth"] == 3 - - -# --------------------------------------------------------------------------- -# find --meta / --fields -# --------------------------------------------------------------------------- - - -@patch("basic_memory.mcp.tools.find", new_callable=AsyncMock, return_value=FIND_META_RESULT) -def test_find_without_meta_sends_no_predicates(mock_find): - """The plain listing call is unchanged: both new params default to None.""" - result = _invoke(["find", "/specs"]) - - assert result.exit_code == 0, result.output - assert mock_find.call_args.kwargs["meta"] is None - assert mock_find.call_args.kwargs["fields"] is None - - -@patch("basic_memory.mcp.tools.find", new_callable=AsyncMock, return_value=FIND_META_FIELDS_RESULT) -def test_find_meta_is_repeatable_and_fields_splits_on_commas(mock_find): - """--meta collects one predicate per flag; --fields is the comma form the - tool receives as a list. The CLI only splits — validation stays in the tool.""" - result = _invoke( - [ - "find", - "/specs", - "--meta", - "status=active", - "--meta", - "confidence>0.6", - "--fields", - "title, priority", - ] - ) - - assert result.exit_code == 0, result.output - assert mock_find.call_args.args == ("/specs",) - assert mock_find.call_args.kwargs["meta"] == ["status=active", "confidence>0.6"] - assert mock_find.call_args.kwargs["fields"] == ["title", "priority"] - - -@patch("basic_memory.mcp.tools.find", new_callable=AsyncMock, return_value=FIND_META_FIELDS_RESULT) -def test_find_meta_json_is_the_tool_payload_verbatim(mock_find): - """JSON stability is the tool payload's stability, projected fields included.""" - result = _invoke(["find", "--meta", "status=active", "--fields", "title", "--json"]) - - assert result.exit_code == 0, result.output - assert json.loads(result.stdout) == FIND_META_FIELDS_RESULT - - -@patch("basic_memory.mcp.tools.find", new_callable=AsyncMock, return_value=FIND_META_RESULT) -def test_find_meta_without_fields_uses_the_search_renderers(mock_find): - """A metadata payload with no projection renders like grep's results, with - the predicates as the query label.""" - result = _tty_invoke(["find", "--meta", "status=active", "--meta", "confidence>0.6", "--plain"]) - - assert result.exit_code == 0, result.output - assert "Search: status=active AND confidence>0.6" in result.stdout - assert "1. Spec [draft] v2" in result.stdout - - -@patch("basic_memory.mcp.tools.find", new_callable=AsyncMock, return_value=FIND_META_RESULT_MORE) -def test_find_meta_plain_reports_more_pages_on_stderr(mock_find): - result = _tty_invoke(["find", "--meta", "status=active", "--plain"]) - - assert result.exit_code == 0, result.output - assert "more available" not in result.stdout - assert "page 1" in result.stderr - assert "total 2" in result.stderr - assert "more available (--page)" in result.stderr - - -@patch("basic_memory.mcp.tools.find", new_callable=AsyncMock, return_value=FIND_META_RESULT) -def test_find_meta_rich_without_fields_shows_the_predicates(mock_find): - result = _tty_invoke(["find", "--meta", "status=active"]) - - flat = _flattened(result.output) - assert result.exit_code == 0, result.output - _assert_not_json(result.output) - assert "status=active" in flat - # User-sourced bracketed titles survive Rich markup parsing. - assert "[draft]" in result.output - - -@patch("basic_memory.mcp.tools.find", new_callable=AsyncMock, return_value=FIND_META_FIELDS_RESULT) -def test_find_meta_fields_plain_is_path_tab_json(mock_find): - """Plain projection output is one line per hit: path, TAB, compact JSON.""" - result = _tty_invoke( - [ - "find", - "--meta", - "status=active", - "--fields", - "title,priority,approved,missing", - "--plain", - ] - ) - - assert result.exit_code == 0, result.output - assert result.stdout == ( - 'specs/Spec [draft] v2.md\t{"title":"Spec [draft] v2","priority":"high",' - '"approved":true,"missing":null}\n' - 'notes/Another Note.md\t{"title":"Another Note","priority":null,' - '"approved":false,"missing":null}\n' - ) - - -@patch("basic_memory.mcp.tools.find", new_callable=AsyncMock, return_value=FIND_META_FIELDS_MORE) -def test_find_meta_fields_plain_reports_more_pages_on_stderr(mock_find): - result = _tty_invoke(["find", "--meta", "status=active", "--fields", "title", "--plain"]) - - assert result.exit_code == 0, result.output - assert "more available" not in result.stdout - assert "page 1" in result.stderr - assert "total 2" in result.stderr - assert "more available (--page)" in result.stderr - - -@patch("basic_memory.mcp.tools.find", new_callable=AsyncMock, return_value=FIND_META_FIELDS_RESULT) -def test_find_meta_fields_rich_adds_one_column_per_field(mock_find): - """The projected table keeps the requested field order; null renders empty - and a non-string value renders as compact JSON.""" - result = _tty_invoke( - ["find", "--meta", "status=active", "--fields", "title,priority,approved,missing"] - ) - - flat = _flattened(result.output) - assert result.exit_code == 0, result.output - _assert_not_json(result.output) - header = next(line for line in result.output.splitlines() if "priority" in line) - columns = ["Path", "title", "priority", "approved", "missing"] - assert [header.index(name) for name in columns] == sorted(header.index(n) for n in columns) - assert "specs/Spec [draft] v2.md" in flat - assert "high" in flat - assert "true" in flat - assert "false" in flat - # total/page summary comes from the search shape (current_page, not page). - assert "page 1" in flat - assert "total 2" in flat - - -@patch("basic_memory.mcp.tools.find", new_callable=AsyncMock, return_value=FIND_META_FIELDS_MORE) -def test_find_meta_fields_rich_reports_more_pages(mock_find): - """The projected table's summary reads the search shape's current_page and - points at --page when the result set continues.""" - result = _tty_invoke(["find", "--meta", "status=active", "--fields", "title"]) - - flat = _flattened(result.output) - assert result.exit_code == 0, result.output - assert "page 1" in flat - # The panel subtitle is clipped to the table width, so match its opening. - assert "more available" in flat - - -@patch("basic_memory.mcp.tools.find", new_callable=AsyncMock, return_value=FIND_META_FIELDS_EMPTY) -def test_find_meta_fields_rich_no_matches(mock_find): - result = _tty_invoke(["find", "--meta", "status=nope", "--fields", "title"]) - - assert result.exit_code == 0, result.output - assert "No matches." in result.output - - -@patch( - "basic_memory.mcp.tools.find", - new_callable=AsyncMock, - side_effect=ValueError("find: 'fields' requires 'meta' predicates"), -) -def test_find_fields_without_meta_reports_the_tool_refusal(mock_find): - """The combination rules live in the shared tool layer; the CLI passes the - flags through and reports the refusal.""" - result = _invoke(["find", "--fields", "title"]) - - assert result.exit_code == 1 - assert "Error: find: 'fields' requires 'meta' predicates" in result.stderr - assert mock_find.call_args.kwargs["fields"] == ["title"] - assert mock_find.call_args.kwargs["meta"] is None - - -# --------------------------------------------------------------------------- -# tail -# --------------------------------------------------------------------------- - - -@patch("basic_memory.mcp.tools.tail", new_callable=AsyncMock, return_value=TAIL_RESULT) -def test_tail_rich_output(mock_tail): - result = _tty_invoke(["tail"]) - - assert result.exit_code == 0, result.output - _assert_not_json(result.output) - assert "Note A" in result.output - assert "[draft]" in result.output - assert "notes/note-a" in _flattened(result.output) - - -@patch("basic_memory.mcp.tools.tail", new_callable=AsyncMock, return_value=[]) -def test_tail_rich_empty(mock_tail): - result = _tty_invoke(["tail"]) - - assert result.exit_code == 0, result.output - assert "No recent changes." in result.output - - -@patch("basic_memory.mcp.tools.tail", new_callable=AsyncMock, return_value=TAIL_RESULT) -def test_tail_plain_is_tab_separated(mock_tail): - result = _tty_invoke(["tail", "--plain"]) - - assert result.exit_code == 0, result.output - lines = result.stdout.splitlines() - assert lines[0] == "2025-01-02T10:00:00\tentity\tNote A\tnotes/note-a\tnotes/Note A.md" - assert lines[1].startswith("2025-01-01T09:00:00\tentity\tNote [draft] B\t") - - -@patch("basic_memory.mcp.tools.tail", new_callable=AsyncMock, return_value=TAIL_RESULT) -def test_tail_lines_and_timeframe_passthrough(mock_tail): - result = _invoke(["tail", "-n", "5", "--timeframe", "1d"]) - - assert result.exit_code == 0, result.output - assert mock_tail.call_args.kwargs["lines"] == 5 - assert mock_tail.call_args.kwargs["timeframe"] == "1d" - - -# --------------------------------------------------------------------------- -# tree -# --------------------------------------------------------------------------- - - -@patch(FIND_LISTING_TARGET, new_callable=AsyncMock, return_value=(TREE_RESULT, "/")) -def test_tree_plain_rebuilds_nesting_from_flat_nodes(mock_find): - """The API page is flat; nesting is rebuilt from directory_path segments, - including a synthesized parent for files whose directory node was filtered - out by the glob.""" - result = _tty_invoke(["tree", "--plain"]) - - assert result.exit_code == 0, result.output - assert result.stdout == "/\n specs/\n search.md\n auth/\n deep.md\n" - - -@patch(FIND_LISTING_TARGET, new_callable=AsyncMock, return_value=(TREE_RESULT, "/")) -def test_tree_rich_output(mock_find): - result = _tty_invoke(["tree"]) - - assert result.exit_code == 0, result.output - _assert_not_json(result.output) - assert "specs/" in result.output - assert "search.md" in result.output - assert "deep.md" in result.output - - -@patch(FIND_LISTING_TARGET, new_callable=AsyncMock, return_value=(TREE_ROOTED_RESULT, "/specs")) -def test_tree_rooted_path_skips_the_root_node(mock_find): - """The search root's own node must not render as its own child.""" - result = _tty_invoke(["tree", "/specs", "--plain"]) - - assert result.exit_code == 0, result.output - assert result.stdout == "/specs\n intro.md\n" - - -@patch(FIND_LISTING_TARGET, new_callable=AsyncMock, return_value=(TREE_DIR_AFTER_FILE_RESULT, "/")) -def test_tree_directory_node_after_synthesized_intermediate(mock_find): - """A directory listed after a file already implied it stays one directory.""" - result = _tty_invoke(["tree", "--plain"]) - - assert result.exit_code == 0, result.output - assert result.stdout == "/\n specs/\n search.md\n" - - -@patch(FIND_LISTING_TARGET, new_callable=AsyncMock, return_value=(TREE_RESULT_MORE, "/")) -def test_tree_rich_reports_more_entries(mock_find): - result = _tty_invoke(["tree"]) - - assert result.exit_code == 0, result.output - assert "more entries" in _flattened(result.output) - - -@patch(FIND_LISTING_TARGET, new_callable=AsyncMock, return_value=(TREE_RESULT_MORE, "/")) -def test_tree_plain_more_entries_note_goes_to_stderr(mock_find): - result = _tty_invoke(["tree", "--plain"]) - - assert result.exit_code == 0, result.output - assert "more entries" not in result.stdout - assert "more entries" in result.stderr - - -@patch(FIND_LISTING_TARGET, new_callable=AsyncMock, return_value=(TREE_RESULT_EMPTY, "/")) -def test_tree_rich_empty(mock_find): - result = _tty_invoke(["tree"]) - - assert result.exit_code == 0, result.output - assert "empty" in result.output - - -@patch(FIND_LISTING_TARGET, new_callable=AsyncMock, return_value=(TREE_RESULT, "/")) -def test_tree_passes_find_arguments_through(mock_find): - result = _invoke(["tree", "/specs", "--name", "*.md", "--depth", "2"]) - - assert result.exit_code == 0, result.output - assert mock_find.call_args.args == ("/specs",) - assert mock_find.call_args.kwargs["name"] == "*.md" - assert mock_find.call_args.kwargs["depth"] == 2 - - -# A qualified tree root gets back node paths that carry the project prefix, and -# the root to strip carries it too — both from the one resolution (#1421). The -# --project spelling is the other addressing frame: project-relative throughout. -TREE_QUALIFIED_RESULT = { - "nodes": [ - _dir_node(name="notes", directory_path="/second-project/notes", type="directory"), - _dir_node( - name="foo.md", - file_path="second-project/notes/foo.md", - directory_path="/second-project/notes/foo.md", - ), - ], - "page": 1, - "page_size": 10, - "total": 2, - "has_more": False, -} -TREE_RELATIVE_RESULT = { - **TREE_QUALIFIED_RESULT, - "nodes": [ - _dir_node(name="notes", directory_path="/notes", type="directory"), - _dir_node(name="foo.md", file_path="notes/foo.md", directory_path="/notes/foo.md"), - ], -} - - -@patch( - FIND_LISTING_TARGET, - new_callable=AsyncMock, - side_effect=[ - (TREE_QUALIFIED_RESULT, "second-project/notes"), - (TREE_RELATIVE_RESULT, "/notes"), - ], -) -def test_tree_qualified_path_matches_project_flag_hierarchy( - mock_find, config_manager, tmp_path_factory -): - """'bm tree /dir' and 'bm tree /dir --project ' are the - same call by rule 2, so they must render the same hierarchy — each stripping - the root in its own addressing frame, or the project segment duplicates - under the root (#1415).""" - config = config_manager.load_config() - config.projects["second-project"] = ProjectEntry( - path=str(tmp_path_factory.mktemp("second-project-cli")) - ) - config_manager.save_config(config) - - qualified = _tty_invoke(["tree", "second-project/notes", "--plain"]) - flagged = _tty_invoke(["tree", "/notes", "--project", "second-project", "--plain"]) - - assert qualified.exit_code == 0, qualified.output - assert flagged.exit_code == 0, flagged.output - assert qualified.stdout == "second-project/notes\n foo.md\n" - # Only the printed root label may differ between the equivalent spellings. - assert qualified.stdout.splitlines()[1:] == flagged.stdout.splitlines()[1:] - # The tool still receives the caller's spelling; routing stays in the tool layer. - assert mock_find.call_args_list[0].args == ("second-project/notes",) - assert mock_find.call_count == 2 - - -# --------------------------------------------------------------------------- -# Errors and routing (shared command scaffold, exercised per verb) -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize(("args", "target", "payload", "tool_return"), VERB_CASES) -def test_verb_tool_error_exits_nonzero(args, target, payload, tool_return): - """A strict-resolve miss (ToolError) becomes stderr + exit 1, per verb.""" - with patch(target, new_callable=AsyncMock, side_effect=ToolError("Entity not found: nope")): - result = _invoke(args) - - assert result.exit_code == 1 - assert "Error: Entity not found: nope" in result.output - - -@patch( - "basic_memory.mcp.tools.cat", - new_callable=AsyncMock, - side_effect=ValueError("start_line must be >= 1, got 0"), -) -def test_verb_value_error_exits_nonzero(mock_cat): - """The tool's own argument validation surfaces as a user-facing error.""" - result = _invoke(["cat", "specs/search"]) - - assert result.exit_code == 1 - assert "Error: start_line must be >= 1" in result.output - - -@patch( - "basic_memory.mcp.tools.ls", - new_callable=AsyncMock, - side_effect=UnqualifiedPathRefusedError( - "no project 'notes' — active projects: main/, research/" - ), -) -def test_verb_refusal_reaches_stderr_with_exit_1(mock_ls): - """The multi-project refusal (#1415) is a ValueError subclass, so the - existing verb error mapping delivers its self-teaching message unchanged.""" - result = _invoke(["ls", "/notes"]) - - assert result.exit_code == 1 - assert "Error: no project 'notes' — active projects: main/, research/" in result.stderr - - -@patch( - "basic_memory.mcp.tools.cat", - new_callable=AsyncMock, - side_effect=ProjectPrefixConflictError( - "path names project 'research' but project 'main' was passed — use " - "'research/' alone, or project='main' with a project-relative path" - ), -) -def test_project_flag_with_conflicting_prefix_exits_nonzero(mock_cat): - """--project plus a qualified path passes both through verbatim; the tool - layer owns the conflict decision and the CLI just reports it.""" - result = _invoke(["cat", "research/notes/foo", "--project", "main"]) - - assert result.exit_code == 1 - assert mock_cat.call_args.args == ("research/notes/foo",) - assert mock_cat.call_args.kwargs["project"] == "main" - assert "Error: path names project 'research' but project 'main' was passed" in result.stderr - - -@patch("basic_memory.mcp.tools.cat", new_callable=AsyncMock, return_value=CAT_RESULT) -def test_local_and_cloud_together_errors(mock_cat): - result = _invoke(["cat", "specs/search", "--local", "--cloud"]) - - assert result.exit_code == 1 - assert "Cannot specify both --local and --cloud" in result.output - mock_cat.assert_not_called() - - -@patch("basic_memory.mcp.tools.tail", new_callable=AsyncMock) -def test_local_flag_forces_local_routing_during_the_call(mock_tail): - """--local sets BASIC_MEMORY_FORCE_LOCAL for exactly the tool call's duration.""" - seen: dict[str, str | None] = {} - - async def capture(*args, **kwargs): - seen["force_local"] = os.environ.get("BASIC_MEMORY_FORCE_LOCAL") - return [] - - mock_tail.side_effect = capture - result = _invoke(["tail", "--local"]) - - assert result.exit_code == 0, result.output - assert seen["force_local"] == "true" - - -@patch( - FIND_LISTING_TARGET, - new_callable=AsyncMock, - return_value=(TREE_QUALIFIED_RESULT, "second-project/notes"), -) -def test_tree_routes_the_path_exactly_once(mock_find): - """tree gets the listing and the root from one resolution, so it must not - resolve the path itself. - - A CLI invocation carries no FastMCP context, so the per-request project-list - cache has nowhere to live: a second resolution is a second project-list - round trip on every cloud call, and for a workspace-qualified path a second - workspace/project index build (#1421). Patching the resolver to explode is - the check — if tree still reaches for it, this fails loudly. - """ - boom = Mock(side_effect=AssertionError("tree resolved the path a second time")) - with patch("basic_memory.mcp.project_context.resolve_project_path_route", boom): - result = _tty_invoke(["tree", "second-project/notes", "--plain"]) - - assert result.exit_code == 0, result.output - assert result.stdout == "second-project/notes\n foo.md\n" - mock_find.assert_called_once() - boom.assert_not_called() diff --git a/tests/cli/test_find_predicates.py b/tests/cli/test_find_predicates.py new file mode 100644 index 000000000..578975dd6 --- /dev/null +++ b/tests/cli/test_find_predicates.py @@ -0,0 +1,206 @@ +"""The `bm find --meta` predicate grammar and the field projection helpers. + +These are pure functions: a predicate string becomes the search API's +metadata_filters dict, and a projected field path walks typed frontmatter. The +refusals are exercised through the CLI in tests/cli/test_cli_posix_argument_errors.py +and the end-to-end matches in test-int/cli/test_posix_commands_integration.py; this +file pins the translations those rely on. +""" + +import pytest + +import basic_memory.mcp.metadata_find as metadata_find +from basic_memory.cli.commands.find_predicates import parse_meta_predicates +from basic_memory.repository.metadata_filters import ( + MetadataPath, + ParsedMetadataFilter, + parse_metadata_filters, + parse_metadata_path, +) + +# Each predicate string translates onto exactly one metadata_filters entry, in +# the grammar the search API's parse_metadata_filters already supports. +PREDICATE_GRAMMAR = [ + ("status=active", {"status": "active"}), + ("status = active", {"status": "active"}), + ("confidence>0.6", {"confidence": {"$gt": 0.6}}), + ("confidence >= 0.6", {"confidence": {"$gte": 0.6}}), + ("confidence<0.6", {"confidence": {"$lt": 0.6}}), + ("confidence<=0.6", {"confidence": {"$lte": 0.6}}), + ("priority in high,critical", {"priority": {"$in": ["high", "critical"]}}), + ("priority in high, critical", {"priority": {"$in": ["high", "critical"]}}), + ("tags has security,oauth", {"tags": ["security", "oauth"]}), + ("score between 0.3,0.8", {"score": {"$between": [0.3, 0.8]}}), + # Quoting is the documented escape for literal values, and it holds inside + # a list: the comma it protects belongs to the value, not to the list. + ('label in "a,b",c', {"label": {"$in": ["a,b", "c"]}}), + # A backslash-escaped quote stays inside the value; it neither closes the + # token nor leaves it looking unterminated. + ('label in "a\\"b",c', {"label": {"$in": ['a"b', "c"]}}), + ('note="say \\"hi\\""', {"note": 'say "hi"'}), + ('tags has "red, green"', {"tags": ["red, green"]}), + ('name in "quoted"', {"name": {"$in": ["quoted"]}}), + # An unquoted value may not start with an operator character (a mis-spelled + # operator is the far likelier reading), so quoting is how a value that + # genuinely does start with one is expressed — scalars and list elements alike. + ('range=">=5"', {"range": ">=5"}), + ('range>"<=5"', {"range": {"$gt": "<=5"}}), + ('bound in ">=5","<=9"', {"bound": {"$in": [">=5", "<=9"]}}), + ('marks has "","=b"', {"marks": ["", "=b"]}), + # Quoting is also the escape for the tokens the grammar refuses unquoted: + # the non-finite number spellings and null outside equality. + ('score="NaN"', {"score": "NaN"}), + ('score>"Infinity"', {"score": {"$gt": "Infinity"}}), + ('owner in "null","alice"', {"owner": {"$in": ["null", "alice"]}}), + # Dot-paths address nested frontmatter and pass through verbatim. + ("review.approved=true", {"review.approved": True}), + # The one alias search carries, so both surfaces accept one spelling. + ("note_type=spec", {"type": "spec"}), +] + + +@pytest.mark.parametrize(("predicate", "expected"), PREDICATE_GRAMMAR) +def test_parse_meta_predicate_grammar(predicate, expected): + assert parse_meta_predicates([predicate]) == expected + + +@pytest.mark.parametrize(("predicate", "expected"), PREDICATE_GRAMMAR) +def test_parsed_predicates_are_valid_api_metadata_filters(predicate, expected): + """Every predicate the parser accepts is a filter the search API accepts. + + parse_metadata_filters is the server-side authority; running the produced + dict through it proves the CLI grammar is a strict subset rather than a + parallel dialect that only fails at request time. + """ + assert parse_metadata_filters(parse_meta_predicates([predicate])) + + +@pytest.mark.parametrize( + ("predicate", "expected_value"), + [ + ("done=true", True), + ("done=false", False), + ("owner=null", None), + ("count=3", 3), + ("ratio=1.5", 1.5), + ("status=active", "active"), + # A JSON-quoted token forces the literal string, escaping the inference. + ('status="true"', "true"), + # Non-scalar JSON is not a filter value; the raw text stays a string. + ("shape=[1,2]", "[1,2]"), + ], +) +def test_predicate_values_are_json_scalar_inferred(predicate, expected_value): + """Values type-infer so a predicate string produces the same dict a JSON + filters caller would pass.""" + key = predicate.split("=", 1)[0] + + assert parse_meta_predicates([predicate]) == {key: expected_value} + + +def test_parse_meta_predicates_and_together(): + assert parse_meta_predicates(["status=active", "confidence>0.6"]) == { + "status": "active", + "confidence": {"$gt": 0.6}, + } + + +@pytest.mark.parametrize( + "key", + ["review..approved", ".owner", "owner.", "review.approved", "status", "note-1_a.b"], +) +def test_one_path_grammar_governs_predicates_and_filters(key): + """find's predicates and the search API accept exactly the same paths. + + parse_metadata_path owns the grammar, and both surfaces call it rather than + keeping a copy, so find cannot build a filter the repository then rejects. + """ + well_formed = parse_metadata_path(key) is not None + + if well_formed: + assert parse_meta_predicates([f"{key}=x"]) == {key: "x"} + assert parse_metadata_filters({key: "x"}) + else: + with pytest.raises(ValueError, match="malformed predicate key"): + parse_meta_predicates([f"{key}=x"]) + with pytest.raises(ValueError, match="Unsupported metadata filter key"): + parse_metadata_filters({key: "x"}) + + +@pytest.mark.parametrize( + ("key", "expected_parts"), + [ + ("status", ("status",)), + ("review.approved", ("review", "approved")), + ("a.b.c", ("a", "b", "c")), + ("note-1_a.b", ("note-1_a", "b")), + (" padded.key ", ("padded", "key")), + ], +) +def test_parse_metadata_path_yields_the_segments(key, expected_parts): + """The parse is what produces the segments a path walk consumes.""" + path = parse_metadata_path(key) + + assert path is not None + assert path.parts == expected_parts + assert path.key == key.strip() + + +@pytest.mark.parametrize( + "key", ["review..approved", ".owner", "owner.", "", " ", "..", "a..b.c", "bad key"] +) +def test_parse_metadata_path_refuses_everything_that_is_not_a_path(key): + """No segments come back for a non-path, so nothing can walk one.""" + assert parse_metadata_path(key) is None + + +def test_null_equality_reaches_the_api_as_an_is_null_clause(): + """'owner=null' is the one null spelling the grammar keeps, and it is not equality. + + The API parser turns it into an IS NULL clause; an ordinary equality clause + would be `= NULL`, which no row satisfies. + """ + filters = parse_meta_predicates(["owner=null"]) + + assert filters == {"owner": None} + assert parse_metadata_filters(filters) == [ParsedMetadataFilter(["owner"], "is_null", None)] + + +@pytest.mark.parametrize( + ("entity_metadata", "fields", "expected"), + [ + ({"title": "Alpha"}, ["title"], {"title": "Alpha"}), + ({"review": {"approved": True}}, ["review.approved"], {"review.approved": True}), + # A missing key renders as null — the row is never dropped. + ({"title": "Alpha"}, ["missing"], {"missing": None}), + # A non-dict intermediate ends the walk at null rather than raising. + ({"review": "yes"}, ["review.approved"], {"review.approved": None}), + (None, ["title"], {"title": None}), + ({"a": {"b": {"c": 1}}}, ["a.b.c", "a.b"], {"a.b.c": 1, "a.b": {"c": 1}}), + ], +) +def test_project_metadata_fields(entity_metadata, fields, expected): + # Projection takes parsed paths, not strings: the fixtures go through the + # same parse find does, which is what makes an unwalkable path unreachable + # here rather than a null that looks like data. + assert metadata_find.project_metadata_fields(entity_metadata, _paths(fields)) == expected + + +def _paths(keys: list[str]) -> list[MetadataPath]: + """Parse well-formed test paths, asserting the fixtures really are paths.""" + parsed = [parse_metadata_path(key) for key in keys] + assert all(path is not None for path in parsed), keys + return [path for path in parsed if path is not None] + + +def test_projection_metadata_preserves_indexed_defaults_without_frontmatter(): + normalized = {"title": "Plain Note", "type": "note"} + + assert metadata_find.projection_metadata("# Plain Note\n\nBody", normalized) == normalized + + +def test_projection_metadata_preserves_indexed_defaults_for_malformed_frontmatter(): + normalized = {"title": "Malformed Note", "type": "note"} + malformed = "---\ntags: [unterminated\n---\n# Malformed Note" + + assert metadata_find.projection_metadata(malformed, normalized) == normalized diff --git a/tests/cli/test_man_command.py b/tests/cli/test_man_command.py index 813044954..7a201473d 100644 --- a/tests/cli/test_man_command.py +++ b/tests/cli/test_man_command.py @@ -166,7 +166,6 @@ def fake_run(*args, **kwargs): "grep", "head", "ls", - "tail", "tree", } | NESTED_COMMAND_PATHS.keys() diff --git a/tests/cli/test_posix_tree.py b/tests/cli/test_posix_tree.py new file mode 100644 index 000000000..c7f588c92 --- /dev/null +++ b/tests/cli/test_posix_tree.py @@ -0,0 +1,52 @@ +"""`bm tree` rebuilds a hierarchy from bm_find's flat listing.""" + +import pytest + +from basic_memory.cli.commands.posix import _build_tree, _listing_root, _TreeEntry + + +def node(path: str, kind: str) -> dict[str, str]: + return {"directory_path": path, "type": kind} + + +def test_files_without_their_folder_synthesize_it(): + """A glob returns files only; their folders are drawn anyway.""" + tree = _build_tree([node("/specs/auth/deep.md", "file")], "/") + + assert tree == { + "specs": _TreeEntry( + is_dir=True, + children={"auth": _TreeEntry(is_dir=True, children={"deep.md": _TreeEntry(False)})}, + ) + } + + +def test_a_folder_listed_after_its_file_stays_one_folder(): + tree = _build_tree([node("/specs/search.md", "file"), node("/specs", "directory")], "/") + + assert list(tree) == ["specs"] + assert tree["specs"].is_dir is True + assert list(tree["specs"].children) == ["search.md"] + + +def test_the_root_is_not_its_own_child(): + tree = _build_tree([node("/specs", "directory"), node("/specs/intro.md", "file")], "/specs") + + assert tree == {"intro.md": _TreeEntry(False)} + + +@pytest.mark.parametrize( + ("path", "project", "root"), + [ + # A path that starts with the project's name answers in its permalink. + ("My Research/specs", "My Research", "my-research/specs"), + ("/My Research/specs/", "My Research", "my-research/specs"), + ("My Research", "My Research", "my-research"), + # Any other path, or no project, comes back as typed. + ("specs", "My Research", "specs"), + ("/", "My Research", "/"), + ("My Research/specs", None, "My Research/specs"), + ], +) +def test_listing_root_uses_the_spelling_bm_find_answers_in(path, project, root): + assert _listing_root(path, project) == root diff --git a/tests/implementation_mcp.py b/tests/implementation_mcp.py deleted file mode 100644 index ef7f84ebe..000000000 --- a/tests/implementation_mcp.py +++ /dev/null @@ -1,39 +0,0 @@ -"""An isolated transport harness for shared CLI implementations, not the public registry. - -The shell CLI still calls these functions directly. Existing regression tests for -that implementation keep their serialization coverage here; public MCP surface -and workflow tests use the production server and bm_* names. -""" - -import pytest -from fastmcp import FastMCP - -import basic_memory.mcp.prompts # noqa: F401 -import basic_memory.mcp.resources # noqa: F401 -from basic_memory.config import ConfigManager -from basic_memory.services.search_service import SearchService -from basic_memory.mcp import tools -from basic_memory.mcp.client_info import MCPClientInfoMiddleware -from basic_memory.mcp.empty_results import EmptyListResultMiddleware -from basic_memory.mcp.server import mcp - - -IMPLEMENTATION_NAMES = ( - "cat", - "find", - "grep", - "ls", - "man", - "tail", -) - - -@pytest.fixture -def implementation_server(config_manager: ConfigManager, search_service: SearchService) -> FastMCP: - """Bind shared implementations on an isolated server for serialization tests.""" - server = FastMCP("shared-implementation-tests", providers=[mcp]) - server.add_middleware(MCPClientInfoMiddleware()) - server.add_middleware(EmptyListResultMiddleware()) - for name in IMPLEMENTATION_NAMES: - server.tool(getattr(tools, name), name=name) - return server diff --git a/tests/markdown/test_line_scanning.py b/tests/markdown/test_line_scanning.py index b41f9d7ad..819ff44d2 100644 --- a/tests/markdown/test_line_scanning.py +++ b/tests/markdown/test_line_scanning.py @@ -59,6 +59,12 @@ def test_numbered_read_preserves_blank_lines_and_bounds_continuation() -> None: assert "next: start_line=6, end_line=6" in text +def test_an_empty_document_numbers_no_lines() -> None: + text = format_line_read("", start_line=0, end_line=0, total_lines=0) + + assert text == "Lines 0-0 of 0 (document, including frontmatter); EOF" + + @pytest.mark.parametrize(("first", "last", "total"), [(1, 0, 0), (20, 10, 10)]) def test_numbered_empty_or_past_eof(first: int, last: int, total: int) -> None: assert format_line_read("", start_line=first, end_line=last, total_lines=total).endswith( diff --git a/tests/markdown/test_sections.py b/tests/markdown/test_sections.py index 491c8188f..3e643d44c 100644 --- a/tests/markdown/test_sections.py +++ b/tests/markdown/test_sections.py @@ -464,11 +464,20 @@ def test_slice_single_line_range(): assert sliced.content == "# Auth" -def test_slice_empty_document_yields_empty_slice(): - sliced = _slice("", lines=LineRange(start=1, end=None)) +@pytest.mark.parametrize( + "arguments", + [ + {"lines": LineRange(start=1, end=None)}, + {"lines": LineRange(start=1, end=5)}, + {"max_tokens": 5}, + ], +) +def test_slice_empty_document_yields_the_empty_range(arguments): + """An empty document has no line 1, so the slice is 0-0 of 0, never 1-0.""" + sliced = _slice("", **arguments) assert sliced.content == "" - assert sliced.total_lines == 0 + assert (sliced.start_line, sliced.end_line, sliced.total_lines) == (0, 0, 0) def test_slice_empty_document_rejects_ranges_past_line_one(): diff --git a/tests/mcp/test_line_scanning.py b/tests/mcp/test_line_scanning.py index 84ccd7afa..240b210e0 100644 --- a/tests/mcp/test_line_scanning.py +++ b/tests/mcp/test_line_scanning.py @@ -1,15 +1,13 @@ -"""Exercise line reads and keyword grep through real project-scoped API clients.""" +"""Exercise line reads and line-match search through real project-scoped API clients.""" import json -from typing import Any import pytest from fastmcp import Client, FastMCP from fastmcp.exceptions import ToolError -from basic_memory.mcp.clients import KnowledgeClient, SearchClient -from basic_memory.mcp.tools import bm_read, bm_write, cat, grep -from basic_memory.schemas.search import SearchResponse +from basic_memory.mcp.clients import KnowledgeClient +from basic_memory.mcp.tools import bm_read, bm_search, bm_write from basic_memory.schemas.v2 import EntityResponseV2 @@ -20,14 +18,24 @@ async def test_scan_read_round_trip_and_token_savings(client, test_project) -> N for n in range(200) ) await bm_write(title="Scan Note", directory="test", content=body, project=test_project.name) - full = await cat("Scan Note", project=test_project.name) + full = await bm_read("Scan Note", output_format="json", project=test_project.name) + assert isinstance(full, dict) lines = full["content"].splitlines() expected_matches = [n for n, line in enumerate(lines, 1) if "retry" in line] - result = await grep("retry", context_lines=1, max_matches=2, project=test_project.name) - ordinary = await grep("retry", project=test_project.name) + result = await bm_search( + "retry", + mode="text", + lines=1, + max_matches=2, + output_format="json", + project=test_project.name, + ) + ordinary = await bm_search( + "retry", mode="text", output_format="json", project=test_project.name + ) + assert isinstance(result, dict) assert len(json.dumps(result)) < len(json.dumps(ordinary)) / 2 - assert result["pagination_scope"] == "search_candidates" row = result["results"][0] assert "content" not in row and "matched_chunk" not in row and "frontmatter" not in row assert row["match_count"] == 3 @@ -50,13 +58,6 @@ async def test_scan_read_round_trip_and_token_savings(client, test_project) -> N assert sliced["next_start_line"] == window["end_line"] + 1 assert sliced["next_end_line"] == window["end_line"] + len(window["content"].splitlines()) assert sliced["frontmatter"] is None - cat_slice = await cat( - row["file_path"], - start_line=window["start_line"], - end_line=window["end_line"], - project=test_project.name, - ) - assert cat_slice["content"] == sliced["content"] assert len(json.dumps(sliced)) < len(json.dumps(full)) / 10 numbered = await bm_read( @@ -75,7 +76,8 @@ async def test_ranges_include_frontmatter_and_have_eof_metadata(client, test_pro await bm_write( title="Bounds", directory="test", content="alpha\nbeta", project=test_project.name ) - full = await cat("Bounds", project=test_project.name) + full = await bm_read("Bounds", output_format="json", project=test_project.name) + assert isinstance(full, dict) lines = full["content"].splitlines() opening = await bm_read("Bounds", end_line=2, output_format="json", project=test_project.name) assert isinstance(opening, dict) @@ -101,7 +103,8 @@ async def test_ranges_include_frontmatter_and_have_eof_metadata(client, test_pro @pytest.mark.asyncio async def test_exact_uuid_line_read_is_one_sliced_get(client, test_project, monkeypatch) -> None: await bm_write(title="One Get", directory="test", content="alpha", project=test_project.name) - found = await grep("alpha", project=test_project.name) + found = await bm_search("alpha", mode="text", output_format="json", project=test_project.name) + assert isinstance(found, dict) identifier = found["results"][0]["external_id"] original = KnowledgeClient.get_entity calls: list[tuple[str, str | None]] = [] @@ -122,52 +125,6 @@ async def record( assert calls == [(identifier, "1-2")] -@pytest.mark.asyncio -async def test_grep_candidate_pagination_survives_no_current_match( - client, test_project, monkeypatch -) -> None: - for title in ("First", "Second"): - await bm_write(title=title, directory="test", content="retry", project=test_project.name) - original = KnowledgeClient.get_entity - - async def changed_content(self: KnowledgeClient, entity_id: str) -> EntityResponseV2: - entity = await original(self, entity_id) - # Simulate a newer accepted document while the index still matches retry. - return entity.model_copy(update={"content": "already fixed\n"}) - - monkeypatch.setattr(KnowledgeClient, "get_entity", changed_content) - first = await grep("retry", context_lines=0, page_size=1, project=test_project.name) - assert first["has_more"] is True - row = first["results"][0] - assert row["match_count"] == 0 and row["windows"] == [] - assert row["total_lines"] == 1 - assert row["next_match_line"] is None - second = await grep("retry", context_lines=0, page=2, page_size=1, project=test_project.name) - assert second["results"][0]["external_id"] != row["external_id"] - - -@pytest.mark.asyncio -async def test_grep_refuses_missing_candidate_identity(client, test_project, monkeypatch) -> None: - await bm_write(title="Legacy", directory="test", content="retry", project=test_project.name) - original = SearchClient.search - - async def old_server( - self: SearchClient, query: dict[str, Any], *, page: int, page_size: int - ) -> SearchResponse: - response = await original(self, query, page=page, page_size=page_size) - return response.model_copy( - update={ - "results": [ - row.model_copy(update={"external_id": None}) for row in response.results - ] - } - ) - - monkeypatch.setattr(SearchClient, "search", old_server) - with pytest.raises(ToolError, match="external_id"): - await grep("retry", context_lines=0, project=test_project.name) - - @pytest.mark.parametrize( "bounds", [{"start_line": 0}, {"end_line": 0}, {"start_line": 5, "end_line": 3}] ) @@ -181,34 +138,3 @@ async def test_invalid_read_bounds_fail_before_routing(bounds: dict[str, int]) - "bm_read", {"identifier": "any", "project": "any", **bounds}, raise_on_error=False ) assert result.is_error - - -@pytest.mark.parametrize( - ("semantic", "context_lines", "max_matches", "page_size", "pattern"), - [ - (True, 1, 10, 10, "retry"), - (False, -1, 10, 10, "retry"), - (False, 11, 10, 10, "retry"), - (False, 0, 0, 10, "retry"), - (False, 0, 101, 10, "retry"), - (False, 0, 10, 101, "retry"), - (False, None, 3, 10, "retry"), - (False, 0, 10, 10, "a\nb"), - ], -) -@pytest.mark.asyncio -async def test_invalid_grep_options_fail_before_routing( - semantic: bool, - context_lines: int | None, - max_matches: int, - page_size: int, - pattern: str, -) -> None: - with pytest.raises(ValueError): - await grep( - pattern, - semantic=semantic, - context_lines=context_lines, - max_matches=max_matches, - page_size=page_size, - ) diff --git a/tests/mcp/test_project_path_routing.py b/tests/mcp/test_project_path_routing.py index 5f0efdab2..e6f858680 100644 --- a/tests/mcp/test_project_path_routing.py +++ b/tests/mcp/test_project_path_routing.py @@ -25,6 +25,7 @@ from basic_memory.config_models import ProjectEntry from basic_memory.mcp.project_context import ( AddressableProject, + addressable_projects, DetectedProjectRoute, ProjectPathRoute, ProjectPrefixConflictError, @@ -42,7 +43,6 @@ from basic_memory.mcp.project_context_identifiers import ( split_project_permalink_prefix, ) -from basic_memory.mcp.tools import grep, ls from basic_memory.mcp.workspace_project_index import ( WorkspaceProjectEntry, WorkspaceProjectIndex, @@ -759,8 +759,9 @@ async def test_out_of_band_project_becomes_addressable_when_the_snapshot_ages_ou context = ContextState() config = ConfigManager().config - advertised = await ls("/", context=ctx(context)) - assert {node["name"] for node in advertised["nodes"]} == {"research"} + assert {project.name for project in await addressable_projects(context=ctx(context))} == { + "research" + } session.tenant.append( ProjectItem( @@ -782,8 +783,10 @@ async def test_out_of_band_project_becomes_addressable_when_the_snapshot_ages_ou await _age_out_session_project_list(context) - advertised = await ls("/", context=ctx(context)) - assert {node["name"] for node in advertised["nodes"]} == {"new-project", "research"} + assert {project.name for project in await addressable_projects(context=ctx(context))} == { + "new-project", + "research", + } assert await detect_project_from_identifier_prefix( "new-project/notes/x", config, context=ctx(context) ) == DetectedProjectRoute(project="new-project", project_id="new-project-external-id") @@ -842,8 +845,9 @@ async def test_project_created_mid_session_is_addressable_without_a_restart(clou context = ContextState() config = ConfigManager().config - advertised = await ls("/", context=ctx(context)) - assert {node["name"] for node in advertised["nodes"]} == {"research"} + assert {project.name for project in await addressable_projects(context=ctx(context))} == { + "research" + } # What create_memory_project does: the tenant serves the new project, and # the tool invalidates this session's project caches. @@ -858,8 +862,10 @@ async def test_project_created_mid_session_is_addressable_without_a_restart(clou ) await invalidate_project_caches(ctx(context)) - advertised = await ls("/", context=ctx(context)) - assert {node["name"] for node in advertised["nodes"]} == {"new-project", "research"} + assert {project.name for project in await addressable_projects(context=ctx(context))} == { + "new-project", + "research", + } route = await resolve_project_path_route( "new-project/notes/x", project=None, project_id=None, context=ctx(context) @@ -901,8 +907,9 @@ async def test_project_deleted_mid_session_stops_resolving(cloud_session): await detect_project_from_identifier_prefix("doomed/notes/x", config, context=ctx(context)) is None ) - advertised = await ls("/", context=ctx(context)) - assert {node["name"] for node in advertised["nodes"]} == {"research"} + assert {project.name for project in await addressable_projects(context=ctx(context))} == { + "research" + } @pytest.mark.asyncio @@ -954,13 +961,13 @@ async def test_cloud_workspace_refuses_unqualified_path(cloud_session): @pytest.mark.asyncio -async def test_cloud_workspace_refuses_pathless_call_through_the_tool(cloud_session): - """grep/tail carry no path to qualify, so the refusal is the tool's answer: +async def test_cloud_workspace_refuses_pathless_route(cloud_session): + """A pathless call has nothing to qualify, so the refusal is the answer: a silent search of one project out of several is the bug being removed.""" cloud_session("research", "engineering") with pytest.raises(UnqualifiedPathRefusedError) as excinfo: - await grep("needle") + await resolve_project_path_route("", project=None, project_id=None) assert str(excinfo.value) == ("no project specified — active projects: engineering/, research/") @@ -1237,45 +1244,43 @@ async def test_cloud_every_advertised_mount_is_addressable_under_slug_collision( itself, and so does every other mount beside it.""" cloud_session("team", "docs") - mounts = await ls() + mounts = await addressable_projects(context=None) - assert [node["name"] for node in mounts["nodes"]] == ["docs", "team"] - for node in mounts["nodes"]: - permalink = node["permalink"] + assert [project.name for project in mounts] == ["docs", "team"] + for project in mounts: + permalink = project.permalink external_id = f"{permalink}-external-id" root_route = await resolve_project_path_route(permalink, project=None, project_id=None) assert root_route == ProjectPathRoute( - project=node["name"], path="", stripped=True, project_id=external_id + project=project.name, path="", stripped=True, project_id=external_id ) path_route = await resolve_project_path_route( f"{permalink}/notes/x", project=None, project_id=None ) assert path_route == ProjectPathRoute( - project=node["name"], path="notes/x", stripped=True, project_id=external_id + project=project.name, path="notes/x", stripped=True, project_id=external_id ) @pytest.mark.asyncio async def test_cloud_every_advertised_mount_is_addressable(cloud_session): - """Round trip over the advertised list: `ls "/"` and the resolver read one - source, so every mount the root advertises routes — as a bare mount name + """Round trip over the mount set: every addressable project routes — as a bare mount name and as a path under it. A mount that listed but did not route is exactly the mismatch #1421 reported.""" cloud_session("research", "engineering", "personal-notes") - mounts = await ls() + mounts = await addressable_projects(context=None) - assert [node["name"] for node in mounts["nodes"]] == [ + assert [project.name for project in mounts] == [ "engineering", "personal-notes", "research", ] - for node in mounts["nodes"]: - permalink = node["permalink"] - assert node["directory_path"] == f"/{permalink}" + for project in mounts: + permalink = project.permalink root_route = await resolve_project_path_route(permalink, project=None, project_id=None) assert root_route.stripped is True @@ -1776,56 +1781,6 @@ async def test_cloud_refusal_points_to_other_workspaces_when_no_project_matches( ) -@pytest.mark.asyncio -async def test_cloud_root_listing_names_the_other_workspaces(cross_workspace_session): - """`ls /` lists only this workspace's projects, so it also says that other - workspaces exist and how to reach them (#1778). They are not rows: a bare - slug is not a routable path, and every row must be one.""" - cross_workspace_session( - session_projects=("research", "engineering"), - default_projects=("notes",), - ) - - mounts = await ls() - - assert [node["name"] for node in mounts["nodes"]] == ["engineering", "research"] - assert mounts["workspaces"] == ["acme", "beta"] - assert OTHER_WORKSPACES_HINT in mounts["note"] - assert "acme, beta" in mounts["note"] - - -@pytest.mark.asyncio -async def test_cloud_root_listing_survives_failed_workspace_discovery( - cross_workspace_session, monkeypatch -): - """The workspace slugs only annotate `ls /`; a control-plane failure must leave - the session's own mounts listed (#1780 review).""" - cross_workspace_session( - session_projects=("research", "engineering"), - default_projects=("notes",), - ) - - async def failing_workspaces(context=None): - raise ToolError("control plane unavailable") - - monkeypatch.setattr(project_context, "get_available_workspaces", failing_workspaces) - - mounts = await ls() - - assert [node["name"] for node in mounts["nodes"]] == ["engineering", "research"] - assert "workspaces" not in mounts - - -@pytest.mark.asyncio -async def test_cloud_single_workspace_root_listing_adds_nothing(cloud_session): - cloud_session("research", "engineering") - - mounts = await ls() - - assert "workspaces" not in mounts - assert "note" not in mounts - - @pytest.mark.asyncio async def test_cloud_single_workspace_refusal_adds_no_hint(cloud_session): """With one workspace there is nowhere else to point, so the refusal reads as diff --git a/tests/mcp/test_tool_bm_read.py b/tests/mcp/test_tool_bm_read.py index eddba68e3..00349f8bd 100644 --- a/tests/mcp/test_tool_bm_read.py +++ b/tests/mcp/test_tool_bm_read.py @@ -27,6 +27,10 @@ ) +# A source text kept verbatim beside the notes, read a range at a time. +SOURCE_TEXT = "CHAPTER 1\nLoomings.\nCall me Ishmael.\nCHAPTER 2\nThe Carpet-Bag.\n" + + async def read_text(identifier: str, **arguments: Any) -> str: result = await bm_read(identifier, **arguments) assert isinstance(result, str) @@ -45,7 +49,7 @@ async def _write(project_name: str, title: str, content: str, directory: str = " @pytest_asyncio.fixture async def non_note_files(project_repository, session_maker, test_project, test_files): - """A PNG, a PDF, a GIF and a JSON file, indexed into the test project.""" + """A PNG, a PDF, a GIF, a JSON and a plain-text file, indexed into the test project.""" gif = io.BytesIO() PILImage.new("RGB", (4, 4), "red").save(gif, format="GIF") (test_files["image"].parent / "anim.gif").write_bytes(gif.getvalue()) @@ -53,6 +57,8 @@ async def non_note_files(project_repository, session_maker, test_project, test_f PILImage.new("RGB", (4, 4), "blue").convert("P").save(palette, format="PNG") (test_files["image"].parent / "palette.png").write_bytes(palette.getvalue()) (test_files["image"].parent / "data.json").write_text('{"k": 1}') + (test_files["image"].parent / "source.txt").write_text(SOURCE_TEXT) + (test_files["image"].parent / "empty.txt").write_text("") runner = LocalProjectIndexRunner( project_repository=project_repository, session_maker=session_maker ) @@ -63,6 +69,8 @@ async def non_note_files(project_repository, session_maker, test_project, test_f "gif": "anim.gif", "json": "data.json", "palette": "palette.png", + "txt": "source.txt", + "empty": "empty.txt", } @@ -211,9 +219,68 @@ async def test_gif_is_refused_as_unsupported(client, test_project, non_note_file @pytest.mark.asyncio -async def test_slicing_a_non_note_file_is_refused(client, test_project, non_note_files): - with pytest.raises(ToolError, match="^UNSUPPORTED:.*not a Markdown note"): - await bm_read(non_note_files["json"], project=test_project.name, end_line=1) +async def test_a_text_file_reads_a_line_range(client, test_project, non_note_files): + """A .txt source slices by line like a note: same coordinates, same continuation.""" + payload = await read_json( + non_note_files["txt"], project=test_project.name, start_line=2, end_line=3 + ) + text = await read_text( + non_note_files["txt"], project=test_project.name, start_line=2, end_line=3 + ) + + assert payload["content"] == "Loomings.\nCall me Ishmael." + assert (payload["start_line"], payload["end_line"], payload["total_lines"]) == (2, 3, 5) + assert payload["has_more"] is True + assert (payload["next_start_line"], payload["next_end_line"]) == (4, 5) + assert payload["file_path"] == "source.txt" + assert payload["content_type"].startswith("text/plain") + assert text.startswith("Lines 2-3 of 5") + assert "3: Call me Ishmael." in text + + +@pytest.mark.asyncio +async def test_a_text_file_reads_its_first_and_last_lines(client, test_project, non_note_files): + head = await read_text(non_note_files["txt"], project=test_project.name, end_line=2) + tail = await read_json(non_note_files["txt"], project=test_project.name, start_line=4) + + assert head.startswith("Lines 1-2 of 5") + assert "1: CHAPTER 1" in head + assert tail["content"] == "CHAPTER 2\nThe Carpet-Bag." + assert tail["has_more"] is False + assert tail["next_start_line"] is None + + +@pytest.mark.asyncio +async def test_an_empty_text_file_reads_as_the_empty_range(client, test_project, non_note_files): + """No line 1 exists, so a range or a head read is 0-0 of 0, never 1-0.""" + payload = await read_json(non_note_files["empty"], project=test_project.name, end_line=10) + text = await read_text( + non_note_files["empty"], project=test_project.name, start_line=1, end_line=10 + ) + + assert payload["content"] == "" + assert (payload["start_line"], payload["end_line"], payload["total_lines"]) == (0, 0, 0) + assert payload["has_more"] is False + assert text == "Lines 0-0 of 0 (document, including frontmatter); EOF" + + +@pytest.mark.asyncio +async def test_a_text_file_range_past_the_end_is_refused(client, test_project, non_note_files): + with pytest.raises(ToolError, match="start_line 9 is past the end of the document"): + await bm_read(non_note_files["txt"], project=test_project.name, start_line=9) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("arguments", [{"section": "CHAPTER 1"}, {"max_tokens": 10}]) +async def test_sections_and_budgets_stay_note_only(client, test_project, non_note_files, arguments): + with pytest.raises(ToolError, match="^UNSUPPORTED:.*section and max_tokens apply only"): + await bm_read(non_note_files["txt"], project=test_project.name, **arguments) + + +@pytest.mark.asyncio +async def test_a_line_range_on_a_binary_file_is_refused(client, test_project, non_note_files): + with pytest.raises(ToolError, match="^UNSUPPORTED:.*not a text file"): + await bm_read(non_note_files["pdf"], project=test_project.name, end_line=1) # --- Refusals --- diff --git a/tests/mcp/test_tool_contracts.py b/tests/mcp/test_tool_contracts.py index 4e0a0f4d8..e057ed124 100644 --- a/tests/mcp/test_tool_contracts.py +++ b/tests/mcp/test_tool_contracts.py @@ -136,40 +136,6 @@ "output_format", "project", ], - "cat": [ - "identifier", - "start_line", - "end_line", - "section", - "max_tokens", - "include_frontmatter", - "project", - "project_id", - ], - "find": [ - "path", - "name", - "depth", - "page", - "page_size", - "meta", - "fields", - "project", - "project_id", - ], - "grep": [ - "pattern", - "semantic", - "page", - "page_size", - "project", - "project_id", - "context_lines", - "max_matches", - ], - "ls": ["path", "page", "page_size", "project", "project_id"], - "man": ["page", "query", "project", "project_id"], - "tail": ["timeframe", "lines", "project", "project_id"], } @@ -191,16 +157,7 @@ "bm_man": {"readOnlyHint": True, "destructiveHint": False}, } -# The POSIX tools are enabled by default during lifespan startup, but remain -# hidden before config is read. Optional status keeps this direct-list contract -# independent of whether another test has started the shared server. OPTIONAL_TOOL_ANNOTATIONS: dict[str, dict[str, bool]] = { - "cat": {"readOnlyHint": True, "destructiveHint": False}, - "find": {"readOnlyHint": True, "destructiveHint": False}, - "grep": {"readOnlyHint": True, "destructiveHint": False}, - "ls": {"readOnlyHint": True, "destructiveHint": False}, - "man": {"readOnlyHint": True, "destructiveHint": False}, - "tail": {"readOnlyHint": True, "destructiveHint": False}, # Listed only when enable_project_delete is on. "bm_project_delete": {"readOnlyHint": False, "destructiveHint": True}, } @@ -230,12 +187,6 @@ "bm_project_delete": tools.bm_project_delete, "bm_schema": tools.bm_schema, "bm_man": tools.bm_man, - "cat": tools.cat, - "find": tools.find, - "grep": tools.grep, - "ls": tools.ls, - "man": tools.man, - "tail": tools.tail, } diff --git a/tests/mcp/test_tool_posix.py b/tests/mcp/test_tool_posix.py deleted file mode 100644 index 1b43dcf23..000000000 --- a/tests/mcp/test_tool_posix.py +++ /dev/null @@ -1,2331 +0,0 @@ -"""Tests for the POSIX-style read-side MCP tools: cat, grep, ls, find, tail, man (#1399). - -Each tool is a thin translation over the same typed clients the canonical tools -use, so these tests run the real ASGI stack via the shared `client` fixture and -assert on the JSON shapes the canonical `output_format="json"` paths produce. -""" - -import json -import re -from datetime import datetime, timezone -from pathlib import Path -from textwrap import dedent - -import pytest -import pytest_asyncio -import yaml -from fastmcp.exceptions import ToolError - -import basic_memory.mcp.clients.knowledge as knowledge_client_module -import basic_memory.mcp.metadata_find as metadata_find -from basic_memory.mcp.clients import KnowledgeClient -import basic_memory.mcp.tools.posix_tools as posix_tools -from basic_memory import db -from basic_memory.mcp.project_context import ( - ProjectPrefixConflictError, - UnqualifiedPathRefusedError, -) -from basic_memory.mcp.tools import bm_search, bm_write, cat, find, grep, ls, man, tail -from basic_memory.models import Entity -from basic_memory.repository.metadata_filters import ( - MetadataPath, - ParsedMetadataFilter, - parse_metadata_filters, - parse_metadata_path, -) -from basic_memory.schemas.search import SearchRetrievalMode - - -@pytest.fixture -def no_project_constraint(monkeypatch): - """Clear the env project constraint so unqualified routing paths are reachable.""" - monkeypatch.delenv("BASIC_MEMORY_MCP_PROJECT", raising=False) - - -# --- cat --- - - -@pytest.mark.asyncio -async def test_cat_returns_full_note_round_trip(client, test_project): - await bm_write( - title="Cat Note", - directory="test", - content="# Cat Note\n\nline one\nline two", - project=test_project.name, - ) - - result = await cat("Cat Note", project=test_project.name) - - assert result["title"] == "Cat Note" - assert result["file_path"] == "test/Cat Note.md" - assert "line one" in result["content"] - assert "line two" in result["content"] - assert result["frontmatter"] is not None - assert result["frontmatter"]["title"] == "Cat Note" - # No range requested: the payload carries no slice bookkeeping. - assert "start_line" not in result - assert "end_line" not in result - assert "total_lines" not in result - - -@pytest.mark.asyncio -async def test_cat_include_frontmatter_toggle(client, test_project): - await bm_write( - title="Cat Frontmatter Note", - directory="test", - content="body text only", - project=test_project.name, - ) - - with_frontmatter = await cat("Cat Frontmatter Note", project=test_project.name) - without_frontmatter = await cat( - "Cat Frontmatter Note", project=test_project.name, include_frontmatter=False - ) - - assert with_frontmatter["content"].startswith("---") - assert not without_frontmatter["content"].startswith("---") - assert "body text only" in without_frontmatter["content"] - - -@pytest.mark.asyncio -async def test_cat_line_range_slices_content(client, test_project): - await bm_write( - title="Cat Range Note", - directory="test", - content="alpha\nbravo\ncharlie\ndelta", - project=test_project.name, - ) - full = await cat("Cat Range Note", project=test_project.name, include_frontmatter=False) - lines = full["content"].splitlines() - - ranged = await cat( - "Cat Range Note", - project=test_project.name, - include_frontmatter=False, - start_line=2, - end_line=3, - ) - - assert ranged["content"] == "\n".join(lines[1:3]) - assert ranged["start_line"] == 2 - assert ranged["end_line"] == 3 - assert ranged["total_lines"] == len(lines) - - -@pytest.mark.asyncio -async def test_cat_start_line_only_runs_to_end(client, test_project): - await bm_write( - title="Cat Tail Note", - directory="test", - content="alpha\nbravo\ncharlie", - project=test_project.name, - ) - full = await cat("Cat Tail Note", project=test_project.name, include_frontmatter=False) - lines = full["content"].splitlines() - - result = await cat( - "Cat Tail Note", project=test_project.name, include_frontmatter=False, start_line=2 - ) - - assert result["content"] == "\n".join(lines[1:]) - assert result["start_line"] == 2 - assert result["end_line"] == len(lines) - assert result["total_lines"] == len(lines) - - -@pytest.mark.asyncio -async def test_cat_end_line_clamped_to_total(client, test_project): - await bm_write( - title="Cat Clamp Note", - directory="test", - content="alpha\nbravo", - project=test_project.name, - ) - full = await cat("Cat Clamp Note", project=test_project.name, include_frontmatter=False) - lines = full["content"].splitlines() - - result = await cat( - "Cat Clamp Note", - project=test_project.name, - include_frontmatter=False, - start_line=1, - end_line=999, - ) - - assert result["content"] == "\n".join(lines) - assert result["end_line"] == len(lines) - assert result["total_lines"] == len(lines) - - -@pytest.mark.asyncio -async def test_cat_start_line_past_end_is_an_error(client, test_project): - """Both slicing paths reject a range that starts after the last line (#1634).""" - await bm_write( - title="Cat Past End Note", - directory="test", - content="alpha\nbravo", - project=test_project.name, - ) - body = await cat("Cat Past End Note", project=test_project.name, include_frontmatter=False) - body_lines = len(body["content"].splitlines()) - full = await cat("Cat Past End Note", project=test_project.name) - full_lines = len(full["content"].splitlines()) - - # Plain line ranges slice the payload client-side. - with pytest.raises( - ValueError, - match=f"cat: start_line 500 is past the end of the document \\({body_lines} lines\\)", - ): - await cat( - "Cat Past End Note", - project=test_project.name, - include_frontmatter=False, - start_line=500, - ) - - # max_tokens sends the range to the server-side slicer. - with pytest.raises( - ToolError, - match=f"start_line 500 is past the end of the document \\({full_lines} lines\\)", - ): - await cat("Cat Past End Note", project=test_project.name, start_line=500, max_tokens=50) - - -@pytest.mark.asyncio -async def test_cat_section_returns_exact_span(client, test_project): - await bm_write( - title="Cat Section Note", - directory="test", - content="# Guide\nintro\n## First\nalpha\n## Second\nbeta", - project=test_project.name, - ) - full = await cat("Cat Section Note", project=test_project.name) - full_lines = full["content"].splitlines() - - result = await cat("Cat Section Note", project=test_project.name, section="First") - - assert result["section"] == "Guide/First" - assert result["content"].splitlines() == ["## First", "alpha"] - assert result["total_lines"] == len(full_lines) - # Coordinates are document-absolute: they address the same lines in a - # frontmatter-included follow-up range read. - assert result["content"] == "\n".join(full_lines[result["start_line"] - 1 : result["end_line"]]) - # Slices never carry a frontmatter block. - assert result["frontmatter"] is None - assert "truncated" not in result - assert "continue_line" not in result - - -@pytest.mark.asyncio -async def test_cat_section_path_form_disambiguates(client, test_project): - await bm_write( - title="Cat Section Paths", - directory="test", - content="# Auth\n## Decisions\na\n# Ops\n## Decisions\nb", - project=test_project.name, - ) - - result = await cat("Cat Section Paths", project=test_project.name, section="Ops/Decisions") - - assert result["section"] == "Ops/Decisions" - assert result["content"].splitlines() == ["## Decisions", "b"] - - -@pytest.mark.asyncio -async def test_cat_section_bracket_form_addresses_duplicates(client, test_project): - await bm_write( - title="Cat Section Duplicates", - directory="test", - content="# Spec\n## Auth\nfirst\n## Auth\nsecond", - project=test_project.name, - ) - - result = await cat("Cat Section Duplicates", project=test_project.name, section="Auth[1]") - - assert result["section"] == "Spec/Auth[1]" - assert result["content"].splitlines() == ["## Auth", "second"] - - -@pytest.mark.asyncio -async def test_cat_unknown_section_lists_available_headings(client, test_project): - await bm_write( - title="Cat Section Missing", - directory="test", - content="# Guide\n## First\nalpha", - project=test_project.name, - ) - - with pytest.raises(ToolError, match="Available sections") as excinfo: - await cat("Cat Section Missing", project=test_project.name, section="Nope") - assert "Guide/First" in str(excinfo.value) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("kwargs", "message"), - [ - ({"section": "A", "start_line": 2}, "cannot be combined with start_line/end_line"), - ({"section": "A", "end_line": 3}, "cannot be combined with start_line/end_line"), - ({"max_tokens": 0}, "max_tokens must be >= 1"), - ({"max_tokens": -5}, "max_tokens must be >= 1"), - # A line range with max_tokens is document-absolute (frontmatter included); - # include_frontmatter=False ranges are body-relative — mixing the two would - # serve frontmatter text despite the opt-out, so the combination is rejected. - ( - {"max_tokens": 5, "start_line": 2, "include_frontmatter": False}, - "requires include_frontmatter=True", - ), - ( - {"max_tokens": 5, "end_line": 3, "include_frontmatter": False}, - "requires include_frontmatter=True", - ), - ], -) -async def test_cat_rejects_bad_slice_arguments_before_io(kwargs, message): - with pytest.raises(ValueError, match=message): - await cat("anything", **kwargs) - - -@pytest.mark.asyncio -async def test_cat_max_tokens_truncates_and_resumes(client, test_project): - await bm_write( - title="Cat Token Budget", - directory="test", - content="# One\n" + "a" * 40 + "\n# Two\n" + "b" * 40, - project=test_project.name, - ) - full = await cat("Cat Token Budget", project=test_project.name) - full_lines = full["content"].splitlines() - - truncated = await cat("Cat Token Budget", project=test_project.name, max_tokens=20) - - assert truncated["truncated"] is True - marker = truncated["content"].splitlines()[-1] - assert "truncated at max_tokens=20" in marker - assert f"continue with lines={truncated['continue_line']}-" in marker - kept_lines = truncated["content"].splitlines()[:-1] - - # Resume flow: a follow-up range read from continue_line returns exactly - # the remainder, reconstructing the document body. - rest = await cat( - "Cat Token Budget", - project=test_project.name, - start_line=truncated["continue_line"], - ) - assert kept_lines + rest["content"].splitlines() == (full_lines[truncated["start_line"] - 1 :]) - - -@pytest.mark.asyncio -async def test_cat_max_tokens_with_line_range_routes_server_side(client, test_project): - await bm_write( - title="Cat Combined Slice", - directory="test", - content="alpha\nbravo\ncharlie\ndelta", - project=test_project.name, - ) - full = await cat("Cat Combined Slice", project=test_project.name) - full_lines = full["content"].splitlines() - - result = await cat( - "Cat Combined Slice", - project=test_project.name, - start_line=2, - max_tokens=1000, - ) - - assert result["content"] == "\n".join(full_lines[1:]) - assert result["start_line"] == 2 - assert result["end_line"] == len(full_lines) - assert result["total_lines"] == len(full_lines) - assert "truncated" not in result - # The server-side slice agrees with the client-side range read byte for byte. - client_side = await cat("Cat Combined Slice", project=test_project.name, start_line=2) - assert result["content"] == client_side["content"] - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("start_line", "end_line", "message"), - [ - (0, None, "start_line must be >= 1"), - (-1, None, "start_line must be >= 1"), - (2, 1, "end_line must be >= start_line"), - (None, 0, "end_line must be >= start_line"), - ], -) -async def test_cat_rejects_bad_line_ranges(start_line, end_line, message): - with pytest.raises(ValueError, match=message): - await cat("anything", start_line=start_line, end_line=end_line) - - -@pytest.mark.asyncio -async def test_cat_unknown_identifier_raises(client, test_project): - with pytest.raises(ToolError): - await cat("no-such-note-anywhere", project=test_project.name) - - -# --- grep --- - - -@pytest.mark.asyncio -async def test_grep_keyword_finds_seeded_content(client, test_project): - await bm_write( - title="Grep Target", - directory="test", - content="# Grep Target\n\nThe posixgrepneedle hides here.", - project=test_project.name, - ) - - result = await grep("posixgrepneedle", project=test_project.name) - - assert result["current_page"] == 1 - assert isinstance(result["total_is_exact"], bool) - titles = [row["title"] for row in result["results"]] - assert "Grep Target" in titles - - -def test_grep_retrieval_mode_defaults_to_fts(): - """Keyword matching stays the default even though semantic search is on (#1685).""" - assert posix_tools._grep_retrieval_mode(False) is SearchRetrievalMode.FTS - - -def test_grep_retrieval_mode_hybrid_on_request(): - assert posix_tools._grep_retrieval_mode(True) is SearchRetrievalMode.HYBRID - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("kwargs", "message"), - [ - ({"pattern": ""}, "pattern must not be empty"), - ({"pattern": " "}, "pattern must not be empty"), - ({"pattern": "ok", "page": 0}, "page must be >= 1"), - ({"pattern": "ok", "page_size": 0}, "page_size must be >= 1"), - ], -) -async def test_grep_rejects_bad_arguments(kwargs, message): - with pytest.raises(ValueError, match=message): - await grep(**kwargs) - - -# --- ls --- - - -@pytest.mark.asyncio -async def test_ls_root_listing(client, test_graph, test_project): - # New contract (#1415): with no project addressed, ls "/" lists projects as - # mount points — this test pins the project-scoped case via project=. - result = await ls(project=test_project.name) - - assert result["total"] == 1 - assert result["has_more"] is False - assert result["nodes"][0]["name"] == "test" - assert result["nodes"][0]["type"] == "directory" - - -@pytest.mark.asyncio -async def test_ls_directory_contents(client, test_graph, test_project): - result = await ls(path="/test", project=test_project.name) - - assert result["total"] == 5 - names = {node["name"] for node in result["nodes"]} - assert names == { - "Connected Entity 1.md", - "Connected Entity 2.md", - "Deep Entity.md", - "Deeper Entity.md", - "Root.md", - } - - -@pytest.mark.asyncio -async def test_ls_empty_project(client, test_project): - # Project-scoped case (#1415): project= bypasses the mount-point view. - result = await ls(project=test_project.name) - - assert result["total"] == 0 - assert result["nodes"] == [] - assert result["has_more"] is False - - -@pytest.mark.asyncio -async def test_ls_pagination(client, test_graph, test_project): - first_page = await ls(path="/test", page_size=2, project=test_project.name) - last_page = await ls(path="/test", page=3, page_size=2, project=test_project.name) - - assert len(first_page["nodes"]) == 2 - assert first_page["has_more"] is True - assert first_page["total"] == 5 - assert len(last_page["nodes"]) == 1 - assert last_page["has_more"] is False - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("kwargs", "message"), - [ - ({"page": 0}, "page must be >= 1"), - ({"page_size": 0}, "page_size must be >= 1"), - ({"page_size": 201}, "page_size must be <= 200"), - ], -) -async def test_ls_rejects_bad_arguments(kwargs, message): - with pytest.raises(ValueError, match=message): - await ls(**kwargs) - - -# --- find --- - - -@pytest.mark.asyncio -async def test_find_glob_recurses_from_root(client, test_graph, test_project): - result = await find(name="*.md", project=test_project.name) - - assert result["total"] == 5 - names = {node["name"] for node in result["nodes"]} - assert "Root.md" in names - assert all(node["type"] == "file" for node in result["nodes"]) - - -@pytest.mark.asyncio -async def test_find_without_name_lists_everything(client, test_graph, test_project): - result = await find(project=test_project.name) - - # The default depth recurses: the /test directory plus its five files. - assert result["total"] == 6 - names = {node["name"] for node in result["nodes"]} - assert "test" in names - assert "Root.md" in names - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("kwargs", "message"), - [ - ({"depth": 0}, "depth must be between 1 and 10"), - ({"depth": 11}, "depth must be between 1 and 10"), - ({"page": 0}, "page must be >= 1"), - ({"page_size": 0}, "page_size must be >= 1"), - ({"page_size": 201}, "page_size must be <= 200"), - ], -) -async def test_find_rejects_bad_arguments(kwargs, message): - with pytest.raises(ValueError, match=message): - await find(**kwargs) - - -# --- find --meta: predicate parsing --- -# Each predicate string translates onto exactly one metadata_filters entry, in -# the grammar the search API's parse_metadata_filters already supports. - -PREDICATE_GRAMMAR = [ - ("status=active", {"status": "active"}), - ("status = active", {"status": "active"}), - ("confidence>0.6", {"confidence": {"$gt": 0.6}}), - ("confidence >= 0.6", {"confidence": {"$gte": 0.6}}), - ("confidence<0.6", {"confidence": {"$lt": 0.6}}), - ("confidence<=0.6", {"confidence": {"$lte": 0.6}}), - ("priority in high,critical", {"priority": {"$in": ["high", "critical"]}}), - ("priority in high, critical", {"priority": {"$in": ["high", "critical"]}}), - ("tags has security,oauth", {"tags": ["security", "oauth"]}), - ("score between 0.3,0.8", {"score": {"$between": [0.3, 0.8]}}), - # Quoting is the documented escape for literal values, and it holds inside - # a list: the comma it protects belongs to the value, not to the list. - ('label in "a,b",c', {"label": {"$in": ["a,b", "c"]}}), - # A backslash-escaped quote stays inside the value; it neither closes the - # token nor leaves it looking unterminated. - ('label in "a\\"b",c', {"label": {"$in": ['a"b', "c"]}}), - ('note="say \\"hi\\""', {"note": 'say "hi"'}), - ('tags has "red, green"', {"tags": ["red, green"]}), - ('name in "quoted"', {"name": {"$in": ["quoted"]}}), - # An unquoted value may not start with an operator character (a mis-spelled - # operator is the far likelier reading), so quoting is how a value that - # genuinely does start with one is expressed — scalars and list elements alike. - ('range=">=5"', {"range": ">=5"}), - ('range>"<=5"', {"range": {"$gt": "<=5"}}), - ('bound in ">=5","<=9"', {"bound": {"$in": [">=5", "<=9"]}}), - ('marks has "","=b"', {"marks": ["", "=b"]}), - # Quoting is also the escape for the tokens the grammar refuses unquoted: - # the non-finite number spellings and null outside equality. - ('score="NaN"', {"score": "NaN"}), - ('score>"Infinity"', {"score": {"$gt": "Infinity"}}), - ('owner in "null","alice"', {"owner": {"$in": ["null", "alice"]}}), - # Dot-paths address nested frontmatter and pass through verbatim. - ("review.approved=true", {"review.approved": True}), - # The one alias search_notes carries, so both surfaces accept one spelling. - ("note_type=spec", {"type": "spec"}), -] - - -@pytest.mark.parametrize(("predicate", "expected"), PREDICATE_GRAMMAR) -def test_parse_meta_predicate_grammar(predicate, expected): - assert posix_tools._parse_meta_predicates([predicate]) == expected - - -@pytest.mark.parametrize(("predicate", "expected"), PREDICATE_GRAMMAR) -def test_parsed_predicates_are_valid_api_metadata_filters(predicate, expected): - """Every predicate the parser accepts is a filter the search API accepts. - - parse_metadata_filters is the server-side authority; running the produced - dict through it proves the CLI/MCP grammar is a strict subset rather than a - parallel dialect that only fails at request time. - """ - assert parse_metadata_filters(posix_tools._parse_meta_predicates([predicate])) - - -@pytest.mark.parametrize( - ("predicate", "expected_value"), - [ - ("done=true", True), - ("done=false", False), - ("owner=null", None), - ("count=3", 3), - ("ratio=1.5", 1.5), - ("status=active", "active"), - # A JSON-quoted token forces the literal string, escaping the inference. - ('status="true"', "true"), - # Non-scalar JSON is not a filter value; the raw text stays a string. - ("shape=[1,2]", "[1,2]"), - ], -) -def test_predicate_values_are_json_scalar_inferred(predicate, expected_value): - """Values type-infer so a predicate string produces the same dict a rich - search_notes caller would pass as JSON.""" - key = predicate.split("=", 1)[0] - - assert posix_tools._parse_meta_predicates([predicate]) == {key: expected_value} - - -def test_parse_meta_predicates_and_together(): - assert posix_tools._parse_meta_predicates(["status=active", "confidence>0.6"]) == { - "status": "active", - "confidence": {"$gt": 0.6}, - } - - -@pytest.mark.parametrize( - ("predicates", "message"), - [ - # The API has no $ne, so != is deliberately absent from the grammar. - (["status!=active"], "unsupported predicate operator in 'status!=active'"), - (["status ~= active"], "unsupported predicate operator"), - (["priority gte 3"], "unsupported predicate operator"), - (["priority in"], "unsupported predicate operator"), - (["nothing"], "unsupported predicate operator"), - (["score between 0.3"], "'between' needs exactly min,max"), - (["score between 0.1,0.2,0.3"], "'between' needs exactly min,max"), - (["priority in high,,low"], "empty list element"), - # A severed quote would silently filter for values nothing carries. - (['priority in "high,low'], "unterminated quoted value"), - (['tags has red,"green'], "unterminated quoted value"), - (["status="], "has no value"), - # Non-finite numbers and null-outside-equality: both used to reach the - # server (or the request encoder) as a query nothing could answer. - (["score=NaN"], "non-finite number"), - (["score>null"], "uses null with '>'"), - (["status=active", "status=draft"], "duplicate predicate key 'status'"), - # The alias collapses onto the same key, so the collision is still caught. - (["note_type=note", "type=spec"], "duplicate predicate key 'type'"), - ], -) -def test_parse_meta_predicates_fails_fast(predicates, message): - with pytest.raises(ValueError, match=re.escape(message)): - posix_tools._parse_meta_predicates(predicates) - - -def test_unsupported_operator_names_the_supported_set(): - """The refusal teaches the whole grammar instead of just rejecting.""" - with pytest.raises(ValueError, match="supported: = > >= < <= in has between"): - posix_tools._parse_meta_predicates(["status matches active"]) - - -def test_duplicate_key_refusal_points_at_between(): - with pytest.raises(ValueError, match=re.escape("use 'between' for ranges")): - posix_tools._parse_meta_predicates(["score>0.3", "score<0.8"]) - - -@pytest.mark.parametrize( - "predicate", - [ - # Symbol operators: the regex matches the longest SUPPORTED spelling, - # so the second operator character used to land at the head of the value. - "status==active", - "status=>active", - "status=>3", - "count>=>3", - "count<<1", - "count<=<1", - # Word operators fold the same way — everything after them is the value. - "priority in >high", - "tags has =security", - "score between >0.3,0.8", - "score between 0.3,<0.8", - ], -) -def test_malformed_operators_refuse_instead_of_folding_into_the_value(predicate): - """REGRESSION: a mis-spelled multi-character operator is a refusal, not a value. - - 'status==active' used to parse as {"status": "=active"} and 'count>>3' as - {"count": {"$gt": ">3"}} — filters for text no note carries, so the caller - got an empty (or worse, a non-empty but wrong) result set where the grammar - documents an unsupported-operator error. - """ - with pytest.raises(ValueError, match="unsupported predicate operator"): - posix_tools._parse_meta_predicates([predicate]) - - -def test_malformed_operator_refusal_teaches_the_grammar_and_the_escape(): - """The refusal names the supported set and the quoting escape, in one message.""" - with pytest.raises(ValueError) as excinfo: - posix_tools._parse_meta_predicates(["status==active"]) - - message = str(excinfo.value) - assert "unsupported predicate operator in 'status==active'" in message - assert "supported: = > >= < <= in has between" in message - assert 'quote the value as "=active"' in message - - -@pytest.mark.parametrize( - "predicate", - [ - # Symbol operators: a doubled, leading or trailing dot is not a dot path. - "review..approved=true", - ".owner=null", - "owner.=x", - "a..b>1", - ".score>=0.5", - "trailing.<9", - # Word operators capture the key the same way, so they fold identically. - "review..approved in a,b", - ".tags has security", - "owner. between 1,2", - ], -) -def test_malformed_predicate_keys_refuse_before_transport(predicate): - """REGRESSION: a key that is not a dot path is refused here, not by the API. - - The key capture class admits '.' anywhere, so 'review..approved', '.owner' - and 'owner.' each parsed cleanly and travelled to the search API — which - refuses them, spending a request to answer "Unsupported metadata filter - key" in wording that names neither find nor the shape a key must have. - """ - with pytest.raises(ValueError, match="malformed predicate key"): - posix_tools._parse_meta_predicates([predicate]) - - -def test_malformed_key_refusal_names_the_key_and_the_grammar(): - """The refusal names the offending key and the shape a valid one has.""" - with pytest.raises(ValueError) as excinfo: - posix_tools._parse_meta_predicates(["review..approved=true"]) - - message = str(excinfo.value) - assert "malformed predicate key 'review..approved' in 'review..approved=true'" in message - assert "dot-separated names of letters, digits, '_' or '-'" in message - assert "review.approved" in message - - -@pytest.mark.parametrize( - "key", - ["review..approved", ".owner", "owner.", "review.approved", "status", "note-1_a.b"], -) -def test_one_path_grammar_governs_predicates_and_filters(key): - """find's predicates and the search API accept exactly the same paths. - - parse_metadata_path owns the grammar, and both surfaces call it rather than - keeping a copy — so they cannot drift into a state where find builds a - filter the repository will then reject at request time. - """ - well_formed = parse_metadata_path(key) is not None - - if well_formed: - assert posix_tools._parse_meta_predicates([f"{key}=x"]) == {key: "x"} - assert parse_metadata_filters({key: "x"}) - else: - with pytest.raises(ValueError, match="malformed predicate key"): - posix_tools._parse_meta_predicates([f"{key}=x"]) - with pytest.raises(ValueError, match="Unsupported metadata filter key"): - parse_metadata_filters({key: "x"}) - - -@pytest.mark.parametrize( - ("key", "expected_parts"), - [ - ("status", ("status",)), - ("review.approved", ("review", "approved")), - ("a.b.c", ("a", "b", "c")), - ("note-1_a.b", ("note-1_a", "b")), - (" padded.key ", ("padded", "key")), - ], -) -def test_parse_metadata_path_yields_the_segments(key, expected_parts): - """The parse is what produces the segments a path walk consumes.""" - path = parse_metadata_path(key) - - assert path is not None - assert path.parts == expected_parts - assert path.key == key.strip() - - -@pytest.mark.parametrize( - "key", ["review..approved", ".owner", "owner.", "", " ", "..", "a..b.c", "bad key"] -) -def test_parse_metadata_path_refuses_everything_that_is_not_a_path(key): - """No segments come back for a non-path, so nothing can walk one.""" - assert parse_metadata_path(key) is None - - -@pytest.mark.parametrize( - "predicate", - [ - # Python's JSON reader accepts these three spellings as an extension... - "score=NaN", - "score=Infinity", - "score=-Infinity", - # ...and silently overflows an oversized exponent to infinity. - "score=1e999", - "score=-1e999", - # Every operator reads its values through the same scalar reader. - "score>NaN", - "score<=Infinity", - "score in 0.5,NaN", - "marks has NaN", - "score between NaN,0.8", - "score between 0.3,1e999", - ], -) -def test_non_finite_numbers_refuse_instead_of_failing_at_transport(predicate): - """REGRESSION: a non-finite number died in the request encoder, not the grammar. - - json.loads builds a real float for NaN/Infinity/-Infinity and overflows - 1e999 to inf, so the parser accepted them into the filters dict. Nothing - rejected them until httpx serialized the request body and raised "Out of - range float values are not JSON compliant" — a transport failure standing in - for a predicate typo, naming neither find nor the offending predicate. - """ - with pytest.raises(ValueError, match="non-finite number"): - posix_tools._parse_meta_predicates([predicate]) - - -def test_non_finite_refusal_names_the_predicate_and_the_escape(): - """The refusal is shaped like the grammar's others: what, where, and the way out.""" - with pytest.raises(ValueError) as excinfo: - posix_tools._parse_meta_predicates(["score=NaN"]) - - message = str(excinfo.value) - assert "predicate 'score=NaN' has a non-finite number 'NaN'" in message - assert "predicate values must be finite numbers" in message - assert 'quote the value as "NaN"' in message - - -@pytest.mark.parametrize( - "predicate", - [ - 'status="active', - 'status = "active', - 'confidence>"0.6', - # A quote that opens, closes, and opens again is still unterminated. - 'name="a"b"c', - # The list operators reach the same check through their split elements. - 'priority in "high,low', - 'tags has red,"green', - 'score between "0.3,0.8', - ], -) -def test_a_dangling_quote_refuses_for_every_operator(predicate): - """REGRESSION: the scalar path used to keep the dangling quote as the value. - - 'status="active' fails json.loads, and the raw-text fallback then filtered - for the literal seven characters '"active' — a search that runs, matches - nothing, and reports an ordinary empty result with the typo buried in it. - The list operators already refused a severed quote; one shared check now - gives both paths the same answer. - """ - with pytest.raises(ValueError, match="unterminated quoted value"): - posix_tools._parse_meta_predicates([predicate]) - - -def test_unterminated_quote_refusal_teaches_both_uses_of_quoting(): - with pytest.raises(ValueError) as excinfo: - posix_tools._parse_meta_predicates(['status="active']) - - message = str(excinfo.value) - assert "predicate 'status=\"active' has an unterminated quoted value" in message - assert 'status="active"' in message - assert 'label in "a,b",c' in message - - -@pytest.mark.parametrize( - "predicate", - [ - "score>null", - "score>=null", - "score list[MetadataPath]: - """Parse well-formed test paths, asserting the fixtures really are paths.""" - parsed = [parse_metadata_path(key) for key in keys] - assert all(path is not None for path in parsed), keys - return [path for path in parsed if path is not None] - - -# --- find --meta: metadata search arm --- - - -@pytest_asyncio.fixture -async def meta_notes(client, test_project): - """Seed metadata-bearing notes across two directories (one name has a space).""" - await bm_write( - title="Alpha Spec", - directory="specs", - content="# Alpha Spec\n\nalpha body", - project=test_project.name, - metadata={ - "status": "active", - "priority": "high", - "confidence": 0.9, - "chapter_number": 63, - "tags": ["security", "oauth"], - "review": {"approved": True}, - }, - ) - await bm_write( - title="Beta Spec", - directory="specs", - content="# Beta Spec\n\nbeta body", - project=test_project.name, - metadata={ - "status": "draft", - "priority": "low", - "confidence": 0.2, - "tags": ["security"], - }, - ) - await bm_write( - title="Gamma Note", - directory="My Notes", - content="# Gamma Note\n\ngamma body", - project=test_project.name, - metadata={ - "status": "active", - "priority": "critical", - "confidence": 0.5, - "tags": ["oauth"], - }, - ) - - -@pytest_asyncio.fixture -async def diverged_permalink_notes(client, test_project): - """Seed two notes whose permalinks do not mirror their file paths. - - An explicit frontmatter `permalink:` is honored verbatim (#93), which is the - honest construction here: the shared test config sets - update_permalinks_on_move=True, so a move would NOT reproduce the split the - product's own default (False) produces in the field. - - One note lives under specs/ but permalinks under archive/; the other is its - mirror image. Any scope built from permalinks answers this pair exactly - backwards, which is what the regression below pins down. - """ - await bm_write( - title="Housed In Specs", - directory="specs", - content=dedent(""" - --- - permalink: archive/housed-in-specs - status: active - --- - - # Housed In Specs - """).strip(), - project=test_project.name, - ) - await bm_write( - title="Housed In Archive", - directory="archive", - content=dedent(""" - --- - permalink: specs/housed-in-archive - status: active - --- - - # Housed In Archive - """).strip(), - project=test_project.name, - ) - - -@pytest.mark.asyncio -async def test_find_meta_returns_the_search_response_shape(client, test_project, meta_notes): - result = await find(meta=["status=active"], project=test_project.name) - - assert set(result) == { - "results", - "current_page", - "page_size", - "total", - "total_is_exact", - "has_more", - } - assert result["total"] == 2 - assert result["total_is_exact"] is True - assert {row["title"] for row in result["results"]} == {"Alpha Spec", "Gamma Note"} - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("meta", "expected_titles"), - [ - (["status=active"], {"Alpha Spec", "Gamma Note"}), - (["confidence>0.5"], {"Alpha Spec"}), - (["confidence>=0.5"], {"Alpha Spec", "Gamma Note"}), - (["confidence<0.5"], {"Beta Spec"}), - (["confidence<=0.5"], {"Beta Spec", "Gamma Note"}), - (["priority in high,critical"], {"Alpha Spec", "Gamma Note"}), - # `has` is contains-ALL, so a two-element list narrows to one note. - (["tags has security"], {"Alpha Spec", "Beta Spec"}), - (["tags has security,oauth"], {"Alpha Spec"}), - (["confidence between 0.1,0.6"], {"Beta Spec", "Gamma Note"}), - (["review.approved=true"], {"Alpha Spec"}), - # Only Alpha Spec carries review.approved, so null is its complement. - (["review.approved=null"], {"Beta Spec", "Gamma Note"}), - (["note_type=note"], {"Alpha Spec", "Beta Spec", "Gamma Note"}), - # Repeated predicates AND together. - (["status=active", "priority=high"], {"Alpha Spec"}), - (["status=active", "priority=nonexistent"], set()), - ], -) -async def test_find_meta_operators_select_the_right_notes( - client, test_project, meta_notes, meta, expected_titles -): - result = await find(meta=meta, project=test_project.name) - - assert {row["title"] for row in result["results"]} == expected_titles - - -@pytest.mark.asyncio -async def test_find_meta_null_finds_the_notes_carrying_no_value(client, test_project, meta_notes): - """REGRESSION: 'key=null' answers "which notes have no value here?". - - It used to compile to `= NULL`, which no row satisfies, so find reported an - exact total of zero however many notes were missing the field — a wrong - answer wearing the same confident `total_is_exact` as a right one. Asserting - the complementary query in the same test keeps the null side honest: a - filter that matched nothing would pass a bare exclusion check. - """ - absent = await find(meta=["review.approved=null"], project=test_project.name) - present = await find(meta=["review.approved=true"], project=test_project.name) - - assert {row["title"] for row in absent["results"]} == {"Beta Spec", "Gamma Note"} - assert absent["total"] == 2 - assert absent["total_is_exact"] is True - assert {row["title"] for row in present["results"]} == {"Alpha Spec"} - - -@pytest.mark.asyncio -async def test_find_meta_never_returns_a_non_markdown_file( - client, test_project, meta_notes, entity_repository, search_service, session_maker -): - """REGRESSION: `find --meta` is frontmatter-only, so a PDF is never a hit. - - An indexed regular file gets an ENTITY row like any note, and it carries no - frontmatter keys at all — which is precisely what `key=null` asks for. So - the null predicate returned every PDF, image and binary in the project and - counted them into the exact total, while positive predicates hid the hole - because nothing a regular file carries could satisfy one. Both shapes are - asserted here so the constraint cannot regress to a null-only special case. - """ - now = datetime.now(timezone.utc) - async with db.scoped_session(session_maker) as session: - scan = await entity_repository.add( - session, - Entity( - project_id=test_project.id, - title="Scanned Contract", - note_type="file", - content_type="application/pdf", - file_path="specs/Scanned Contract.pdf", - permalink="specs/scanned-contract", - created_at=now, - updated_at=now, - ), - ) - await search_service.index_entity_data(scan) - - absent = await find(meta=["review.approved=null"], project=test_project.name) - present = await find(meta=["status=active"], project=test_project.name) - - assert {row["title"] for row in absent["results"]} == {"Beta Spec", "Gamma Note"} - assert absent["total"] == 2 - assert absent["total_is_exact"] is True - assert {row["title"] for row in present["results"]} == {"Alpha Spec", "Gamma Note"} - - -@pytest.mark.asyncio -@pytest.mark.parametrize("path", ["/", ""]) -async def test_find_meta_answers_at_the_project_root(client, test_project, meta_notes, path): - """Both root spellings reach the metadata search; the predicates are the whole WHERE.""" - result = await find(path, meta=["status=active"], project=test_project.name) - - assert {row["title"] for row in result["results"]} == {"Alpha Spec", "Gamma Note"} - assert result["total_is_exact"] is True - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("path", "expected_titles"), - [ - ("specs", {"Alpha Spec", "Beta Spec"}), - ("/specs", {"Alpha Spec", "Beta Spec"}), - # "./" is relative notation, the same way the directory listing reads - # it; without that, the SQL prefix "./specs/" matched nothing at all. - ("./specs", {"Alpha Spec", "Beta Spec"}), - ("./specs/", {"Alpha Spec", "Beta Spec"}), - # A spaced directory name is a file path, not a slug: it scopes verbatim. - ("My Notes", {"Gamma Note"}), - ("nonexistent", set()), - ], -) -async def test_find_meta_scopes_by_path_subtree( - client, test_project, meta_notes, path, expected_titles -): - """`path` narrows the metadata query to one directory, server-side.""" - result = await find(path, meta=["note_type=note"], project=test_project.name) - - assert {row["title"] for row in result["results"]} == expected_titles - assert result["total"] == len(expected_titles) - assert result["total_is_exact"] is True - - -@pytest.mark.asyncio -async def test_find_meta_scope_stops_at_a_directory_boundary(client, test_project, meta_notes): - """A sibling directory whose name starts with the scope stays out.""" - await bm_write( - title="Archived Spec", - directory="specs-archive", - content="# Archived Spec", - project=test_project.name, - metadata={"status": "active"}, - ) - - result = await find("specs", meta=["status=active"], project=test_project.name) - - assert {row["title"] for row in result["results"]} == {"Alpha Spec"} - - -@pytest.mark.asyncio -async def test_scope_follows_the_file_path_not_the_permalink( - client, test_project, diverged_permalink_notes -): - """REGRESSION: `find /specs --meta` scopes by where the file lives. - - A permalink stops mirroring its file path once a note pins one in - frontmatter (#93) or is moved with update_permalinks_on_move disabled (the - default). A scope built from permalink prefixes therefore answers this pair - exactly backwards: it would drop "Housed In Specs", which really is under - specs/, and admit "Housed In Archive", which is not — while still reporting - that count as exact. Scoping by the indexed file_path answers the question - the caller asked, so reintroducing a permalink-based scope fails here. - """ - result = await find("specs", meta=["status=active"], project=test_project.name) - - rows = {row["title"]: row for row in result["results"]} - assert set(rows) == {"Housed In Specs"} - assert result["total"] == 1 - assert result["total_is_exact"] is True - # The hit is the one whose *file* is under specs/, and its permalink is not. - assert rows["Housed In Specs"]["file_path"] == "specs/Housed In Specs.md" - assert rows["Housed In Specs"]["permalink"].endswith("archive/housed-in-specs") - - # The mirror image: permalinked under specs/, but housed under archive/. - archive = await find("archive", meta=["status=active"], project=test_project.name) - archive_rows = {row["title"]: row for row in archive["results"]} - assert set(archive_rows) == {"Housed In Archive"} - assert archive_rows["Housed In Archive"]["file_path"] == "archive/Housed In Archive.md" - assert archive_rows["Housed In Archive"]["permalink"].endswith("specs/housed-in-archive") - - -@pytest.mark.asyncio -async def test_find_meta_paginates_with_exact_totals(client, test_project, meta_notes): - """Scope and predicates AND inside one server-side WHERE, so the total is - the real match count and every page is reachable.""" - first = await find(meta=["status=active"], page_size=1, project=test_project.name) - second = await find(meta=["status=active"], page=2, page_size=1, project=test_project.name) - - assert first["total"] == 2 - assert first["total_is_exact"] is True - assert first["has_more"] is True - assert len(first["results"]) == 1 - assert second["has_more"] is False - assert first["results"][0]["permalink"] != second["results"][0]["permalink"] - - -@pytest.mark.asyncio -async def test_find_meta_projects_requested_fields(client, test_project, meta_notes): - """Projection reads the canonical typed frontmatter; a field a hit does - not carry renders as null instead of dropping the row.""" - result = await find( - meta=["status=active"], - fields=["title", "priority", "chapter_number", "review.approved", "tags", "missing_field"], - project=test_project.name, - ) - - projected = {row["title"]: row["fields"] for row in result["results"]} - assert projected["Alpha Spec"] == { - "title": "Alpha Spec", - "priority": "high", - "chapter_number": 63, - "review.approved": True, - "tags": ["security", "oauth"], - "missing_field": None, - } - assert projected["Gamma Note"] == { - "title": "Gamma Note", - "priority": "critical", - "chapter_number": None, - "review.approved": None, - "tags": ["oauth"], - "missing_field": None, - } - - -def test_projection_metadata_preserves_indexed_defaults_without_frontmatter(): - normalized = {"title": "Plain Note", "type": "note"} - - assert metadata_find.projection_metadata("# Plain Note\n\nBody", normalized) == normalized - - -def test_projection_metadata_preserves_indexed_defaults_for_malformed_frontmatter(): - normalized = {"title": "Malformed Note", "type": "note"} - malformed = "---\ntags: [unterminated\n---\n# Malformed Note" - - assert metadata_find.projection_metadata(malformed, normalized) == normalized - - -@pytest.mark.asyncio -async def test_find_meta_fields_returns_identity_and_projection_only( - client, test_project, meta_notes -): - """REGRESSION: a projected row carried the whole note beside the projection. - - `fields` is the entire reason to call find instead of reading every note, - and the row it produced still carried the note's `content` — up to - SearchIndexRow.CONTENT_DISPLAY_LIMIT (4000) characters of it. The 200-row - inventory the literary-analysis skill documents therefore spent, on note - bodies nobody asked for, most of what the projection exists to save. - - The assertion is the row's key set, not its size: a projected row is the - note's identity plus the fields requested, and nothing else. - """ - projected = await find(meta=["status=active"], fields=["priority"], project=test_project.name) - - assert projected["results"], "fixture should produce hits to project" - for row in projected["results"]: - assert set(row) == { - "title", - "permalink", - "file_path", - "external_id", - "updated_at", - "fields", - } - # The identity still names the note well enough to read it next. - assert row["file_path"] and row["permalink"] and row["external_id"] - # No note body reaches the caller by any key. - serialized = json.dumps(row) - assert "alpha body" not in serialized - assert "gamma body" not in serialized - - assert {row["title"]: row["fields"] for row in projected["results"]} == { - "Alpha Spec": {"priority": "high"}, - "Gamma Note": {"priority": "critical"}, - } - - -@pytest.mark.asyncio -async def test_find_meta_without_fields_still_returns_the_full_search_shape( - client, test_project, meta_notes -): - """The narrowing belongs to projection mode only. - - Without `fields` there is no projection to stand in for the hit, so `meta` - keeps answering with the search response `bm grep` renders — content - included. Pinned here so the projection change cannot quietly strip the - unprojected arm too. - """ - unprojected = await find(meta=["status=active"], project=test_project.name) - - assert unprojected["results"] - for row in unprojected["results"]: - assert "content" in row - assert "fields" not in row - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "predicate", - [ - f"score>{'9' * 400}", - f"score<=-{'9' * 400}", - f"score between 0,{'9' * 400}", - ], -) -async def test_find_meta_oversized_integer_refuses_as_a_filter_error( - client, test_project, meta_notes, predicate -): - """REGRESSION: a 400-digit comparison bound came back as a server error. - - The grammar's finite-number check reads what json.loads built, and - json.loads keeps an oversized *integer* literal as an ordinary finite - Python int — only the float spellings (NaN, Infinity, 1e999) are caught - there. So the int travelled, and _normalize_numeric raised OverflowError - server-side. OverflowError is not a ValueError, which is the only thing the - search router translates, so a predicate typo surfaced as a 500 rather than - a refusal naming the filter. - """ - with pytest.raises(ToolError, match="not a finite number"): - await find(meta=[predicate], project=test_project.name) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("field", ["review..approved", ".owner", "owner."]) -async def test_find_meta_malformed_field_path_refuses_without_reading_anything( - client, test_project, meta_notes, monkeypatch, field -): - """REGRESSION: a malformed field path returned null data at full cost. - - `.owner` walked an empty first segment and reported null for every hit even - though the notes carry `owner`, byte-identical to the null a genuinely - absent field produces — a typo the caller could not see, paid for with the - search plus the projection reads. Nothing is read now. - """ - reads = 0 - original = KnowledgeClient.get_accepted_note_contents - - async def counting_batch(self, entity_ids): - nonlocal reads - reads += 1 - return await original(self, entity_ids) - - monkeypatch.setattr(KnowledgeClient, "get_accepted_note_contents", counting_batch) - - with pytest.raises(ValueError, match=re.escape(f"malformed field path '{field}'")): - await find(meta=["status=active"], fields=[field], project=test_project.name) - - assert reads == 0 - - -@pytest.mark.asyncio -async def test_find_meta_nested_field_path_still_projects(client, test_project, meta_notes): - """The well-formed nested path the malformed spellings are typos of works.""" - result = await find( - meta=["status=active"], fields=["review.approved"], project=test_project.name - ) - - projected = {row["title"]: row["fields"] for row in result["results"]} - assert projected["Alpha Spec"] == {"review.approved": True} - assert projected["Gamma Note"] == {"review.approved": None} - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("kwargs", "message"), - [ - # The search API has no filename-glob facility, so a `name` pattern has - # no faithful translation — refuse rather than silently ignore it. - ({"name": "*.md", "meta": ["status=active"]}, "'name' cannot combine with 'meta'"), - # The file-path scope is whole-subtree; a depth bound is inexpressible. - ({"depth": 3, "meta": ["status=active"]}, "'depth' cannot combine with 'meta'"), - ({"fields": ["title"]}, "'fields' requires 'meta' predicates"), - ({"meta": ["status=active"], "fields": []}, "must be non-empty"), - ({"meta": ["status=active"], "fields": [" "]}, "must be non-empty"), - # A malformed field path walked an empty segment to null for every hit, - # which reads exactly like a field the notes do not carry — a typo - # answered with uniform, plausible, wrong data. Refuse it like a - # predicate key, through the same parse. - ({"meta": ["status=active"], "fields": ["review..approved"]}, "malformed field path"), - ({"meta": ["status=active"], "fields": [".owner"]}, "malformed field path"), - ({"meta": ["status=active"], "fields": ["owner."]}, "malformed field path"), - ({"meta": ["status=active"], "fields": ["title", "a..b"]}, "malformed field path"), - # An empty predicate list is not "no filter": it would parse to {} and - # run the metadata search with no WHERE at all, matching every note in - # the project where the caller asked for a filtered set. - ({"meta": []}, "'meta' must carry at least one predicate"), - ], -) -async def test_find_meta_combination_rules_refuse_before_io(kwargs, message): - with pytest.raises(ValueError, match=re.escape(message)): - await find(**kwargs) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("kwargs", "message"), - [ - ({"page": 0}, "page must be >= 1"), - ({"page_size": 0}, "page_size must be >= 1"), - ({"page_size": 201}, "page_size must be <= 200"), - ], -) -async def test_find_meta_rejects_bad_pagination(kwargs, message): - """The metadata arm bounds pagination exactly as the listing arm does. - - `find_listing` states those bounds for the directory arm, and the metadata - arm never reaches it — so `find` states them again on the branch it does - take. Unstated, a `page=0` would route, open a client, and come back as a - transport error from the search API instead of the refusal the plain - listing gives for the same argument. No project fixture here is the point: - the refusal lands before any I/O. - """ - with pytest.raises(ValueError, match=re.escape(message)): - await find(meta=["status=active"], **kwargs) - - -@pytest.mark.asyncio -async def test_find_meta_quoted_list_element_keeps_its_comma(client, test_project): - """The quoting escape reaches the list operators, not just the scalar ones. - - Splitting the raw value before scalar inference would sever '"a,b"' into - '"a' and 'b"' and filter for values nothing carries — no error, no hits. - """ - await bm_write( - title="Comma Note", - directory="notes", - content="# Comma Note", - project=test_project.name, - metadata={"label": "a,b"}, - ) - - result = await find(meta=['label in "a,b",c'], project=test_project.name) - - assert {row["title"] for row in result["results"]} == {"Comma Note"} - - -@pytest.mark.asyncio -async def test_find_meta_quoting_reaches_operator_prefixed_values(client, test_project): - """Quoting is the escape for a value that starts with an operator character. - - Unquoted, 'range=>=5' reads as a malformed operator and is refused; quoted, - the same value is matched literally — proved against the real stack so the - documented escape hatch is not just a parser property. - """ - await bm_write( - title="Range Note", - directory="notes", - content="# Range Note", - project=test_project.name, - metadata={"range": ">=5"}, - ) - - with pytest.raises(ValueError, match="unsupported predicate operator"): - await find(meta=["range=>=5"], project=test_project.name) - - result = await find(meta=['range=">=5"'], project=test_project.name) - - assert {row["title"] for row in result["results"]} == {"Range Note"} - - -@pytest.mark.asyncio -async def test_find_meta_empty_list_is_not_an_unfiltered_search(client, test_project, meta_notes): - """meta=[] must never widen to the whole project. - - The MCP surface can produce it directly (`"meta": []`, or `"meta": "[]"` - through coerce_list), so the refusal is asserted against the real stack — - where an unfiltered metadata search would return all three seeded notes. - """ - with pytest.raises(ValueError, match=re.escape("'meta' must carry at least one predicate")): - await find(meta=[], project=test_project.name) - - -@pytest.mark.asyncio -async def test_find_meta_fields_reads_accepted_content_in_batches( - client, test_project, meta_notes, monkeypatch -): - """Projection reads accepted Markdown in batches, not one GET per hit.""" - monkeypatch.setattr(knowledge_client_module, "MAX_ACCEPTED_CONTENT_BATCH", 1) - original = KnowledgeClient.get_accepted_note_contents - batches: list[list[str]] = [] - - async def counting_batch(self, entity_ids): - batches.append(entity_ids) - return await original(self, entity_ids) - - monkeypatch.setattr(KnowledgeClient, "get_accepted_note_contents", counting_batch) - - result = await find(meta=["status=active"], fields=["title"], project=test_project.name) - - assert [len(batch) for batch in batches] == [1, 1] - # Each row is paired with its own note's content. - assert {row["title"]: row["fields"]["title"] for row in result["results"]} == { - "Alpha Spec": "Alpha Spec", - "Gamma Note": "Gamma Note", - } - - -@pytest.mark.asyncio -async def test_find_meta_malformed_key_refuses_before_transport(client, test_project, meta_notes): - """A malformed key is refused in find's words, before a request is built. - - The API parser stays the authority on what a key may look like — find - validates against METADATA_KEY_RE, its grammar, so the two cannot - disagree — but spending a search request to be told "Unsupported metadata - filter key" told the caller neither which predicate was wrong nor what a - key must look like. Every other predicate mistake refuses here; this one - now does too. - """ - with pytest.raises(ValueError, match=re.escape("malformed predicate key 'status.'")): - await find(meta=["status.=active"], project=test_project.name) - - -@pytest.mark.asyncio -async def test_find_meta_fields_requires_hit_external_ids(client, test_project, monkeypatch): - """Projection hydrates each hit by external_id, so a server old enough to - omit it fails fast instead of silently returning unprojected rows.""" - # Import here to mirror the tool's own deferred client import. - from basic_memory.mcp.clients import SearchClient - from basic_memory.schemas.search import SearchItemType, SearchResponse, SearchResult - - async def legacy_search(self, query, page=1, page_size=10): - return SearchResponse( - results=[ - SearchResult( - title="Alpha Spec", - type=SearchItemType.ENTITY, - score=1.0, - permalink="test-project/specs/alpha-spec", - file_path="specs/Alpha Spec.md", - ) - ], - current_page=page, - page_size=page_size, - total=1, - ) - - monkeypatch.setattr(SearchClient, "search", legacy_search) - - with pytest.raises(ToolError, match="too old for `fields`"): - await find(meta=["status=active"], fields=["title"], project=test_project.name) - - -@pytest.mark.asyncio -async def test_find_without_meta_stays_the_directory_listing(client, test_graph, test_project): - """The no-meta arm is untouched: same payload with the new params defaulted - or passed as None, and no search-shape keys anywhere.""" - baseline = await find(name="*.md", project=test_project.name) - with_defaults = await find(name="*.md", meta=None, fields=None, project=test_project.name) - - assert with_defaults == baseline - assert set(baseline) == {"nodes", "page", "page_size", "total", "has_more"} - assert baseline["total"] == 5 - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("meta", "filters"), - [ - (["status=active"], {"status": "active"}), - (["confidence>0.5"], {"confidence": {"$gt": 0.5}}), - (["priority in high,critical"], {"priority": {"$in": ["high", "critical"]}}), - (["tags has security,oauth"], {"tags": ["security", "oauth"]}), - (["confidence between 0.1,0.6"], {"confidence": {"$between": [0.1, 0.6]}}), - (["review.approved=true"], {"review.approved": True}), - ], -) -async def test_find_meta_matches_bm_search_parity(client, test_project, meta_notes, meta, filters): - """PARITY: the POSIX surface and the rich surface answer the same question - identically — same filters dict, same hits — through the same real stack.""" - assert posix_tools._parse_meta_predicates(meta) == filters - - found = await find(meta=meta, project=test_project.name) - searched = await bm_search(project=test_project.name, filters=filters, output_format="json") - - assert isinstance(searched, dict) - assert found["results"] - assert {row["permalink"] for row in found["results"]} == { - row["permalink"] for row in searched["results"] - } - - -# --- tail --- - - -@pytest.mark.asyncio -async def test_tail_returns_recent_rows(client, test_graph, test_project): - rows = await tail(project=test_project.name) - - assert rows - for row in rows: - assert set(row) == { - "type", - "title", - "permalink", - "file_path", - "created_at", - "updated_at", - } - assert row["type"] == "entity" - assert isinstance(row["created_at"], str) - # tail orders by last change, so each row says when that was. - assert isinstance(row["updated_at"], str) - changed = [row["updated_at"] for row in rows] - assert changed == sorted(changed, reverse=True) - titles = {row["title"] for row in rows} - assert "Root" in titles - - -@pytest.mark.asyncio -async def test_tail_lines_caps_row_count(client, test_graph, test_project): - rows = await tail(lines=2, project=test_project.name) - - assert len(rows) <= 2 - - -@pytest.mark.asyncio -async def test_tail_custom_timeframe(client, test_graph, test_project): - rows = await tail(timeframe="1d", project=test_project.name) - - assert isinstance(rows, list) - assert {row["title"] for row in rows} >= {"Root"} - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("lines", "message"), - [ - (0, "lines must be >= 1"), - (101, "lines must be <= 100"), - ], -) -async def test_tail_rejects_bad_lines(lines, message): - with pytest.raises(ValueError, match=message): - await tail(lines=lines) - - -# --- man --- - - -@pytest.mark.asyncio -async def test_man_index_renders_the_manual(client): - result = await man() - - assert isinstance(result, str) - assert result.startswith("# Basic Memory manual") - assert "## Section 3 — MCP tools" in result - - -@pytest.mark.asyncio -async def test_man_bundled_page_by_reference(): - result = await man(page="bm-search(3)") - - assert isinstance(result, str) - assert "# bm-search(3)" in result - - -@pytest.mark.asyncio -async def test_man_page_and_query_mutually_exclusive(): - with pytest.raises(ValueError, match="not both"): - await man(page="bm-search(3)", query="search") - - -@pytest.mark.asyncio -async def test_man_reads_only_bundled_pages(client, test_project): - """A manual project is never consulted; `project` is accepted and ignored.""" - await bm_write( - title="posix-custom-guide", - directory="man", - content="# posix-custom-guide\n\nA note that is no bundled page.", - project=test_project.name, - ) - - with pytest.raises(ToolError, match="No manual entry for posix-custom-guide"): - await man(page="posix-custom-guide", project=test_project.name) - with pytest.raises(ToolError, match="No manual entry for docs/unknown-guide"): - await man(page="docs/unknown-guide") - - -@pytest.mark.asyncio -async def test_man_query_searches_the_bundled_pages(): - result = await man(query="frontmatter recursively", project="ignored") - - assert isinstance(result, dict) - titles = [row["title"] for row in result["results"]] - assert titles[:2] == ["find(1)", "bm-find(3)"] - assert result["total"] == len(titles) - assert result["results"][0]["uri"] == "memory://man/find(1)" - - -# --- project-qualified routing (#1415) --- -# Projects are mount points: '/path' inputs route to that project, an -# explicit project must agree with a path prefix, and multi-project configs -# refuse unqualified input with the active project list instead of defaulting. - - -# -- ls "/" mount-point view -- - - -@pytest.mark.asyncio -async def test_ls_root_lists_projects_as_mount_points( - client, test_project, second_project, no_project_constraint -): - """ls "/" with no project addressed is the mount table, sorted by name.""" - result = await ls() - - assert result["total"] == 2 - assert result["has_more"] is False - assert [node["name"] for node in result["nodes"]] == ["second-project", "test-project"] - for node in result["nodes"]: - assert node["type"] == "directory" - # directory_path is the copyable '/' prefix form. - assert node["directory_path"] == f"/{node['permalink']}" - - -@pytest.mark.asyncio -async def test_ls_root_mount_view_in_single_project_config( - client, test_graph, test_project, no_project_constraint -): - """The mount view is unconditional (#1415): even a single-project config - lists the mount table at "/" so in-band discovery is uniform.""" - result = await ls() - - assert result["total"] == 1 - assert result["nodes"][0]["name"] == "test-project" - assert result["nodes"][0]["directory_path"] == "/test-project" - assert result["nodes"][0]["type"] == "directory" - - -@pytest.mark.asyncio -async def test_ls_mount_view_paginates_over_project_rows( - client, test_project, second_project, no_project_constraint -): - first = await ls(page=1, page_size=1) - last = await ls(page=2, page_size=1) - - assert first["total"] == 2 - assert first["has_more"] is True - assert [node["name"] for node in first["nodes"]] == ["second-project"] - assert [node["name"] for node in last["nodes"]] == ["test-project"] - assert last["has_more"] is False - - -# -- rule 1: first-segment routing per verb -- - - -@pytest.mark.asyncio -async def test_ls_single_segment_project_name_lists_its_root( - client, test_project, second_project, no_project_constraint -): - await bm_write( - title="Second Root Note", - directory="notes", - content="# Second Root Note\n\nsecond project content", - project="second-project", - ) - - result = await ls("second-project") - - names = {node["name"] for node in result["nodes"]} - assert names == {"notes"} - - -@pytest.mark.asyncio -async def test_ls_qualified_path_routes_into_project_directory( - client, test_project, second_project, no_project_constraint -): - await bm_write( - title="Second Dir Note", - directory="notes", - content="# Second Dir Note", - project="second-project", - ) - - result = await ls("/second-project/notes") - - names = {node["name"] for node in result["nodes"]} - assert names == {"Second Dir Note.md"} - - -@pytest.mark.asyncio -async def test_find_qualified_path_routes_to_project( - client, test_project, second_project, no_project_constraint -): - await bm_write( - title="Second Find Note", - directory="notes", - content="# Second Find Note", - project="second-project", - ) - - result = await find("second-project", name="*.md") - - names = {node["name"] for node in result["nodes"]} - assert names == {"Second Find Note.md"} - - -@pytest.mark.asyncio -async def test_find_meta_qualified_path_routes_to_project( - client, test_project, second_project, no_project_constraint -): - """The meta arm shares find's route resolution, so '/dir' scopes to - that project's subtree and nothing from the other project leaks in. - - A project prefix is a mount point, not a subtree: a bare 'second-project' - strips to the empty project-relative path and answers at that project's - root, while 'second-project/notes' strips to the 'notes' subtree of the - same project. - """ - await bm_write( - title="Second Meta Note", - directory="notes", - content="# Second Meta Note", - project="second-project", - metadata={"status": "active"}, - ) - await bm_write( - title="Home Meta Note", - directory="notes", - content="# Home Meta Note", - project=test_project.name, - metadata={"status": "active"}, - ) - - await bm_write( - title="Second Root Note", - directory="specs", - content="# Second Root Note", - project="second-project", - metadata={"status": "active"}, - ) - - root = await find("second-project", meta=["status=active"]) - scoped = await find("second-project/notes", meta=["status=active"]) - - assert {row["title"] for row in root["results"]} == {"Second Meta Note", "Second Root Note"} - assert {row["title"] for row in scoped["results"]} == {"Second Meta Note"} - - -@pytest.mark.asyncio -async def test_find_meta_unqualified_refuses_in_multi_project_config( - client, test_project, second_project, no_project_constraint -): - """A metadata query is still project-scoped, so an unqualified call in a - multi-project config refuses with the active project list.""" - with pytest.raises(UnqualifiedPathRefusedError, match="no project specified"): - await find(meta=["status=active"]) - - -@pytest.mark.asyncio -async def test_find_meta_predicate_errors_precede_routing( - client, test_project, second_project, no_project_constraint -): - """Predicate parsing happens before any routing decision, so a bad operator - reports itself instead of hiding behind the multi-project refusal.""" - with pytest.raises(ValueError, match="unsupported predicate operator") as excinfo: - await find(meta=["status!=active"]) - assert not isinstance(excinfo.value, UnqualifiedPathRefusedError) - - -@pytest.mark.asyncio -async def test_cat_qualified_identifier_equals_explicit_project_read( - client, test_graph, test_project, second_project, no_project_constraint -): - """'test-project/test/root' with no project param reads the same note as - project='test-project' + 'test/root'. - - The payloads are not identical, and deliberately so: each answers in the - addressing frame its caller used, so the file_path it hands back is one the - same call shape accepts again. Content is what must match. - """ - qualified = await cat("test-project/test/root") - explicit = await cat("test/root", project=test_project.name) - - assert qualified["title"] == explicit["title"] == "Root" - assert qualified["content"] == explicit["content"] - - # Each frame's file_path round-trips in that same frame. - assert qualified["file_path"] == f"test-project/{explicit['file_path']}" - assert (await cat(qualified["file_path"]))["title"] == "Root" - assert (await cat(explicit["file_path"], project=test_project.name))["title"] == "Root" - - -@pytest.mark.asyncio -async def test_tool_output_permalink_round_trips_into_cat( - client, test_project, second_project, no_project_constraint -): - """Round trip: stored permalinks are already project-qualified, so tail's - output is a valid cat identifier with no project param anywhere — and the - mount view's prefix is that permalink's first segment.""" - await bm_write( - title="Round Trip", - directory="notes", - content="# Round Trip\n\nround trip body", - project="second-project", - ) - - rows = await tail(project="second-project") - permalink = next(row["permalink"] for row in rows if row["title"] == "Round Trip") - assert permalink == "second-project/notes/round-trip" - - mounts = await ls() - mount_prefixes = {node["directory_path"] for node in mounts["nodes"]} - assert f"/{permalink.split('/', 1)[0]}" in mount_prefixes - - result = await cat(permalink) - - assert result["title"] == "Round Trip" - assert "round trip body" in result["content"] - - -# -- rule 2: explicit project + prefix agree/conflict -- - - -@pytest.mark.asyncio -async def test_explicit_project_with_agreeing_prefix_strips( - client, test_graph, test_project, second_project, no_project_constraint -): - qualified = await ls("/test-project/test", project=test_project.name) - relative = await ls("/test", project=test_project.name) - - # Same listing, each in its caller's addressing frame (see cat's twin above). - assert qualified["total"] == relative["total"] == 5 - assert [node["name"] for node in qualified["nodes"]] == [ - node["name"] for node in relative["nodes"] - ] - assert [node["file_path"] for node in qualified["nodes"]] == [ - f"test-project/{node['file_path']}" for node in relative["nodes"] - ] - - -@pytest.mark.asyncio -async def test_explicit_project_with_conflicting_prefix_refuses( - client, test_project, second_project, no_project_constraint -): - """A disagreeing prefix is never silently resolved either way.""" - with pytest.raises( - ProjectPrefixConflictError, - match="path names project 'second-project' but project 'test-project' was passed", - ): - await cat("second-project/notes/foo", project=test_project.name) - - -# -- rule 4: multi-project unqualified refusal, self-teaching message -- - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("verb", "kwargs", "subject"), - [ - ("cat", {"identifier": "notes/foo"}, "no project 'notes'"), - ("ls", {"path": "/notes"}, "no project 'notes'"), - ("find", {"path": "/x"}, "no project 'x'"), - ("grep", {"pattern": "needle"}, "no project specified"), - ("tail", {}, "no project specified"), - ], -) -async def test_unqualified_input_refuses_in_multi_project_config( - client, test_project, second_project, no_project_constraint, verb, kwargs, subject -): - """Each verb refuses rather than silently defaulting, listing every active - project in copyable prefix form.""" - tool = {"cat": cat, "ls": ls, "find": find, "grep": grep, "tail": tail}[verb] - - with pytest.raises(UnqualifiedPathRefusedError) as excinfo: - await tool(**kwargs) - - assert str(excinfo.value) == (f"{subject} — active projects: second-project/, test-project/") - - -@pytest.mark.asyncio -async def test_grep_argument_validation_precedes_refusal( - client, test_project, second_project, no_project_constraint -): - """Bad-argument errors keep firing before any routing decision.""" - with pytest.raises(ValueError, match="pattern must not be empty"): - await grep("") - - -# -- rule 5: single-project passthrough and near-collisions -- - - -@pytest.mark.asyncio -async def test_single_project_unqualified_paths_pass_through( - client, test_graph, test_project, no_project_constraint -): - """One configured project keeps today's ergonomics: unqualified paths route - to the default project, and 'test' is not falsely stripped as a prefix of - 'test-project' (permalink comparison, not startswith).""" - listing = await ls("/test") - assert listing["total"] == 5 - - note = await cat("test/root") - assert note["title"] == "Root" - - rows = await tail() - assert {row["title"] for row in rows} >= {"Root"} - - found = await grep("Root") - assert found["results"] - - -@pytest.mark.asyncio -async def test_project_named_folder_is_reachable_double_qualified( - client, test_project, second_project, no_project_constraint -): - """Collision rule: the project wins the first segment, so a top-level folder - named like its own project is addressed by double-qualifying.""" - await bm_write( - title="Shadowed", - directory="second-project", - content="# Shadowed\n\nshadowed body", - project="second-project", - ) - - listing = await ls("second-project/second-project") - assert {node["name"] for node in listing["nodes"]} == {"Shadowed.md"} - - result = await cat("second-project/second-project/shadowed") - assert result["title"] == "Shadowed" - - -@pytest.mark.asyncio -async def test_cat_bare_project_name_is_an_error( - client, test_project, second_project, no_project_constraint -): - with pytest.raises(ValueError, match="names a project, not a note"): - await cat("second-project") - - -# -- env constraint (BASIC_MEMORY_MCP_PROJECT) -- - - -@pytest.mark.asyncio -async def test_env_constraint_counts_as_explicit_project( - client, test_graph, test_project, second_project, monkeypatch -): - """The env constraint participates exactly like the project param: no - refusal, agreeing prefixes strip, disagreeing prefixes conflict, and - ls "/" lists the constrained project's root, not the mount table.""" - monkeypatch.setenv("BASIC_MEMORY_MCP_PROJECT", test_project.name) - - rows = await tail() - assert {row["title"] for row in rows} >= {"Root"} - - stripped = await ls("/test-project/test") - assert stripped["total"] == 5 - - constrained_root = await ls() - assert {node["name"] for node in constrained_root["nodes"]} == {"test"} - - with pytest.raises(ProjectPrefixConflictError): - await cat("second-project/notes/foo") - - -# -- project_id passthrough -- - - -@pytest.mark.asyncio -async def test_project_id_routes_without_prefix_parsing( - client, test_graph, test_project, second_project, no_project_constraint -): - """project_id routes by external UUID and bypasses prefix parsing entirely, - so a multi-project config needs no qualification.""" - result = await cat("test/root", project_id=test_project.external_id) - - assert result["title"] == "Root" - - -# -- round-trip coherence: a returned path is an accepted path (#1421) -- -# The property belongs to the routing layer, not to any one verb, so these tests -# enumerate the verbs rather than naming them. A seventh posix verb that accepts -# a path has to answer for the property here before it can ship. - - -def _path_accepting_posix_verbs() -> set[str]: - """Posix verbs whose first parameter is a routable path or identifier. - - Derived from the tools themselves so a new one cannot slip past the round-trip - tests below by simply not being listed. - """ - import inspect - - verbs = {} - for verb in (cat, grep, ls, find, tail, man): - fn = getattr(verb, "fn", verb) - first = next(iter(inspect.signature(fn).parameters), None) - if first in {"path", "identifier"}: - verbs[fn.__name__] = verb - return set(verbs) - - -def test_path_accepting_verbs_are_the_ones_covered_below(): - """Pins the set the round-trip tests cover. Adding a path-accepting verb - fails here, which is the prompt to give it the same guarantee. - - One axis only, and #1435 is what showed the second: this guard passed while - `find`'s metadata arm skipped the rule, because `find` was already in the - set. ``test_every_response_qualifier_is_exercised_by_a_routed_call`` guards - the other axis — the response shapes those verbs answer with.""" - assert _path_accepting_posix_verbs() == {"cat", "ls", "find"} - - -@pytest.mark.asyncio -async def test_ls_returned_paths_route_back_to_the_same_project( - client, test_project, second_project, no_project_constraint -): - """A qualified `ls` advertises child paths; feeding one back must reach the - same project. Returning the API's project-relative '/notes' refused as - unqualified — or, with a project mounted as 'notes', opened that one.""" - await bm_write( - title="Second Root Note", - directory="notes", - content="# Second Root Note", - project="second-project", - ) - - listing = await ls("second-project") - child = next(node for node in listing["nodes"] if node["name"] == "notes") - assert child["directory_path"] == "/second-project/notes" - - # The advertised path is accepted verbatim, with no project param. - nested = await ls(child["directory_path"]) - assert {node["name"] for node in nested["nodes"]} == {"Second Root Note.md"} - - -@pytest.mark.asyncio -async def test_find_returned_paths_route_back_to_the_same_project( - client, test_project, second_project, no_project_constraint -): - """find advertises both directory_path and file_path; each must address the - project the call routed to, so `cat` and `ls` accept them unchanged.""" - await bm_write( - title="Second Root Note", - directory="notes", - content="# Second Root Note", - project="second-project", - ) - - listing = await find("second-project") - file_node = next(node for node in listing["nodes"] if node["type"] == "file") - dir_node = next(node for node in listing["nodes"] if node["type"] == "directory") - - assert file_node["file_path"].startswith("second-project/") - assert dir_node["directory_path"].startswith("/second-project") - - assert (await cat(file_node["file_path"]))["title"] == "Second Root Note" - assert (await ls(dir_node["directory_path"]))["total"] >= 1 - - -@pytest.mark.asyncio -async def test_find_meta_returned_paths_route_back_to_the_same_project( - client, test_project, second_project, no_project_constraint -): - """find's metadata arm answers with search hits, and those paths must - round-trip exactly like the listing arm's (#1435). - - The two arms of one verb shipped with different answers: `find --meta` - returned the API's project-relative 'notes/…', which `cat` then refused as - unqualified — or, where another project is mounted as 'notes', opened that - one instead. - """ - await bm_write( - title="Second Meta Note", - directory="notes", - content="# Second Meta Note", - project="second-project", - metadata={"status": "active"}, - ) - - found = await find("second-project/notes", meta=["status=active"]) - row = next(item for item in found["results"] if item["title"] == "Second Meta Note") - - assert row["file_path"] == "second-project/notes/Second Meta Note.md" - assert (await cat(row["file_path"]))["title"] == "Second Meta Note" - - # The projection narrows the row but not its address: `fields` hydrates from - # the already-re-qualified payload, so both shapes of the metadata response - # hand back the same spelling. - projected = await find("second-project/notes", meta=["status=active"], fields=["status"]) - projected_row = next( - item for item in projected["results"] if item["title"] == "Second Meta Note" - ) - assert projected_row["file_path"] == row["file_path"] - assert projected_row["fields"] == {"status": "active"} - assert (await cat(projected_row["file_path"]))["title"] == "Second Meta Note" - - -@pytest.mark.asyncio -async def test_unrouted_find_meta_keeps_project_relative_paths( - client, test_project, second_project, no_project_constraint -): - """The prefix goes back only when the call put it in the path — the same - rule the listing arm follows, so an explicit project param keeps its own - addressing frame.""" - await bm_write( - title="Second Meta Note", - directory="notes", - content="# Second Meta Note", - project="second-project", - metadata={"status": "active"}, - ) - - found = await find("notes", meta=["status=active"], project="second-project") - - assert [row["file_path"] for row in found["results"]] == ["notes/Second Meta Note.md"] - - -def _response_qualifiers() -> set[str]: - """The module's re-qualification helpers — one per routed response shape.""" - return {name for name in vars(posix_tools) if name.startswith("qualify_")} - - -@pytest.mark.asyncio -async def test_every_response_qualifier_is_exercised_by_a_routed_call( - client, test_project, second_project, no_project_constraint, monkeypatch -): - """The guard's second axis: response shapes, not verbs (#1435). - - ``test_path_accepting_verbs_are_the_ones_covered_below`` could not catch the - metadata arm — `find` was already in its set, and the gap was one arm inside - it answering with a *different* response shape. So this drives one routed - call per shape and asserts every qualifier the module defines actually - fired, and that every path those calls returned routes back. - - What that does and does not close, precisely: a new response shape needs a - new ``qualify_*`` helper, and this fails until a routed call here exercises - it; a new path-accepting verb fails the verb guard. Neither can force a new - arm to re-qualify at all — what covers that is that every shape a routed - verb can answer with already has a qualifier, so a new arm either reuses one - (and inherits the rule) or introduces one this test then reports. - """ - await bm_write( - title="Second Round Trip Note", - directory="notes", - content="# Second Round Trip Note", - project="second-project", - metadata={"status": "active"}, - ) - - fired: set[str] = set() - for name in _response_qualifiers(): - original = getattr(posix_tools, name) - - def recorder(payload, route, *, _name=name, _original=original): - fired.add(_name) - return _original(payload, route) - - monkeypatch.setattr(posix_tools, name, recorder) - - listing = await ls("second-project/notes") - tree = await find("second-project/notes") - hits = await find("second-project/notes", meta=["status=active"]) - note = await cat("second-project/notes/second-round-trip-note") - - assert fired == _response_qualifiers() - - returned_paths = {note["file_path"]} - for payload in (listing, tree): - returned_paths.update( - node["file_path"] for node in payload["nodes"] if node["type"] == "file" - ) - returned_paths.update(row["file_path"] for row in hits["results"]) - - assert returned_paths == {"second-project/notes/Second Round Trip Note.md"} - for path in returned_paths: - assert (await cat(path))["title"] == "Second Round Trip Note" - - -@pytest.mark.asyncio -async def test_unrouted_listings_keep_project_relative_paths( - client, test_graph, test_project, second_project, no_project_constraint -): - """The prefix goes back only when the call put it in the path. An explicit - project param is a different addressing frame: those paths are fed back with - the same param, so re-prefixing them would break that round trip instead.""" - listing = await ls("/test", project=test_project.name) - - assert all(not node["file_path"].startswith("test-project/") for node in listing["nodes"]) - assert (await ls("/test", project=test_project.name))["total"] == listing["total"] - - -@pytest.mark.asyncio -async def test_routed_cat_never_rewrites_note_frontmatter( - client, test_project, second_project, no_project_constraint -): - """Re-qualification touches transport metadata, never note content. - - Frontmatter is the author's own YAML and may legitimately carry keys spelled - like transport fields. Rewriting by key name anywhere in the payload turned - `file_path: imports/source.md` into `second-project/imports/source.md`, so a - routed read returned frontmatter that disagreed with both its own `content` - and the file on disk. Position, not spelling, separates the two. - """ - await bm_write( - title="Imported Note", - directory="notes", - content="# Imported Note\n\nbody", - project="second-project", - metadata={"file_path": "imports/source.md", "directory_path": "/imports"}, - ) - - payload = await cat("second-project/notes/imported-note") - - # The stored file is the authority: parse its frontmatter block and require - # the response to reproduce it exactly. - stored = (Path(second_project.path) / "notes" / "Imported Note.md").read_text() - _, _, rest = stored.partition("---\n") - block, _, _ = rest.partition("\n---") - assert yaml.safe_load(block) == payload["frontmatter"] - assert payload["frontmatter"]["file_path"] == "imports/source.md" - assert payload["frontmatter"]["directory_path"] == "/imports" - - # The transport path is still qualified, so the round trip holds. - assert payload["file_path"] == "second-project/notes/Imported Note.md" - assert (await cat(payload["file_path"]))["title"] == "Imported Note" diff --git a/tests/mcp/test_v1_registration.py b/tests/mcp/test_v1_registration.py index 9a7934d79..fa66dce1f 100644 --- a/tests/mcp/test_v1_registration.py +++ b/tests/mcp/test_v1_registration.py @@ -6,7 +6,6 @@ import pytest from fastmcp.exceptions import NotFoundError -from implementation_mcp import IMPLEMENTATION_NAMES from basic_memory.mcp import tools from basic_memory.mcp.server import mcp, set_project_delete_visibility @@ -24,7 +23,24 @@ "bm_schema", "bm_project", } -RETIRED_TOOLS = frozenset(IMPLEMENTATION_NAMES) +# Pre-1.0 tool names; none may be callable on the 1.0 server. +RETIRED_TOOLS = frozenset( + { + "build_context", + "cat", + "edit_note", + "find", + "grep", + "list_memory_projects", + "ls", + "man", + "read_note", + "recent_activity", + "search_notes", + "tail", + "write_note", + } +) def test_fresh_process_exposes_only_v1_tools() -> None: diff --git a/tests/test_man_pages.py b/tests/test_man_pages.py index 3b3a27461..05497d5dd 100644 --- a/tests/test_man_pages.py +++ b/tests/test_man_pages.py @@ -488,7 +488,7 @@ def test_render_options_includes_shared_and_global_flags() -> None: # bullets to the full set. That growth is the point, not a regression. _, grep = _cli_command(find_page(PageRef("grep", 1))) options = render_options(grep) - for shared in ("--json", "--plain", "--project", "--project-id", "--local", "--cloud"): + for shared in ("--json", "--project", "--local", "--cloud"): assert f"- **{shared}**" in options, f"{shared} missing from generated OPTIONS" @@ -504,32 +504,32 @@ def test_render_options_preserves_aliases_and_boolean_pairs() -> None: def test_render_cli_synopsis_groups_mutually_exclusive_options() -> None: # --json/--plain and --local/--cloud are rejected in combination by the CLI - # (validate_routing_flags / _validate_output_flags exit 1 for both pairs), so the + # (validate_routing_flags / validate_output_flags exit 1 for both pairs), so the # SYNOPSIS must show each pair as a single `|` alternative, not two freely- # combinable tokens. grep_path, grep = _cli_command(find_page(PageRef("grep", 1))) synopsis = render_cli_synopsis(grep_path, grep) - assert "[--json | --plain]" in synopsis assert "[--local | --cloud]" in synopsis # never the flattened, freely-combinable spelling the constraint forbids - for flattened in ("[--json]", "[--plain]", "[--local]", "[--cloud]"): + for flattened in ("[--local]", "[--cloud]"): assert flattened not in synopsis - # --project/--project-id are NOT mutually exclusive (--project-id takes - # precedence), so they stay separate tokens rather than being grouped. - _, find = _cli_command(find_page(PageRef("find", 1))) - find_synopsis = render_cli_synopsis("find", find) - assert "[--project PROJECT]" in find_synopsis - assert "[--project-id PROJECT_ID]" in find_synopsis - assert "--project |" not in find_synopsis + # No section-1 verb has --plain any more, so the output pair is pinned on a + # command shape that carries both flags. + output_pair = _FakeClickCommand( + [ + _FakeClickParam("option", "json", ["--json"], is_flag=True), + _FakeClickParam("option", "plain", ["--plain"], is_flag=True), + ] + ) + assert "[--json | --plain]" in render_cli_synopsis("t", output_pair) _, cat = _cli_command(find_page(PageRef("cat", 1))) cat_synopsis = render_cli_synopsis("cat", cat) assert "[--lines LINES | --section SECTION]" in cat_synopsis assert "[--lines LINES]" not in cat_synopsis assert "[--section SECTION]" not in cat_synopsis - # A partial pair stays usable: tail has --lines but no --section. - _, tail = _cli_command(find_page(PageRef("tail", 1))) - assert "[--lines N]" in render_cli_synopsis("tail", tail) + # A partial pair stays usable: grep has --lines but no --section. + assert "[--lines CONTEXT_LINES]" in synopsis def test_render_cli_synopsis_keeps_repeatable_options_repeatable() -> None: @@ -557,7 +557,7 @@ def test_find_synopsis_separates_listing_and_metadata_constraints() -> None: assert "--depth" not in metadata for form in (listing, metadata): assert form.startswith("bm find [PATH]") - assert "[--json | --plain]" in form + assert "[--json]" in form assert "[--local | --cloud]" in form assert all(len(line) <= 76 for line in form.splitlines()) # Both forms survive the actual page replacement/extraction boundary. @@ -612,7 +612,7 @@ def test_render_options_is_not_grouped_for_mutually_exclusive_pairs() -> None: # option as its own bullet, so each flag keeps its own description. _, grep = _cli_command(find_page(PageRef("grep", 1))) options = render_options(grep) - for flag in ("--json", "--plain", "--local", "--cloud"): + for flag in ("--local", "--cloud"): assert f"- **{flag}**" in options