From 64220d4119720cfd72fd838e88b0200c1f0a38f2 Mon Sep 17 00:00:00 2001 From: Ryan Schmukler Date: Thu, 10 Sep 2026 15:24:26 -0400 Subject: [PATCH] feat: mcpToolSearch config Every tool sent to the LLM costs context on every request: its description plus its full input schema. With a few MCP servers connected that is thousands of tokens the model rarely needs. Add `mcpToolSearch`, which trades that upfront cost for an extra round trip: "mcpToolSearch": { "deferAllWhenTotalTokensExceedPercentOfContext": 10, "includePattern": [".*"], "excludePattern": ["clojure-mcp"] } Matching MCP tools are deferred - their schemas are withheld and only a compact catalog of names and truncated descriptions is rendered into the system prompt. When the model needs one it calls the new `eca__search_tools` tool, which ranks the deferred catalog against a query, returns the matches with their input schemas, and records them on the chat so they are sent as regular tools from then on. A tool is deferred when the MCP definitions as a whole outgrow `deferAllWhenTotalTokensExceedPercentOfContext` of the model context window, or when it matches `includePattern`, and in both cases only if it does not match `excludePattern`. That limit is a percentage rather than a token count so one setting behaves sensibly on a 32k local model and on a 1M one, and it is null by default: nothing is deferred until asked for. Native tool definitions are left out of the total, so the limit tracks what MCP actually adds. Deferred tools are only withheld from the request payload; they stay resolvable and callable throughout, so a tool call that arrives before the search still executes. Every provider rebuilds the next request of a tool-call loop from the tool list its `on-tools-called` callback returns, shadowing the one the initial payload filtered, so that list is filtered too. Without it deferred schemas came back as soon as the model called any tool, which is most of a turn. Wrapped in `sync-or-async-prompt!` rather than `prompt!` because the sync path invokes the callback itself, without going through it, and covered by an integration test asserting on the continuation request. Only MCP tools can be deferred. Native ECA tools are the agent's baseline capabilities, so a catch-all pattern never takes them away - use `disabledTools` to remove one of those. `eca__search_tools` is likewise never deferred, and is only offered to the model when at least one tool is actually deferred. Pattern matching reuses the existing `disabledTools` engine rather than introducing a third dialect alongside it and the exact-match approval selectors: anchored Java regex against the builtin tool name or the `server__tool` full name, or an exact server name for all of its tools. That matcher is extracted as `tool-entry-matches?` and its regex compilation is now memoized, which also lets it warn once when an entry fails to compile - `*` is not a valid regex and was previously matched literally in silence. Configurable globally, per agent, and in agent markdown frontmatter, where the object may be abbreviated to a list or string when only `includePattern` is needed: mcpToolSearch: - github__.* The static prompt cache signature now tracks the deferrable set separately from the tool list, so changing the patterns mid-chat rebuilds the catalog. --- CHANGELOG.md | 1 + docs/config.json | 41 +++++ docs/config/agents.md | 32 ++++ docs/config/introduction.md | 5 + docs/config/tools.md | 92 ++++++++++++ .../integration/chat/mcp_remote_test.clj | 69 +++++++-- resources/prompts/tools/search_tools.md | 10 ++ src/eca/config.clj | 6 + src/eca/features/agents.clj | 61 ++++++-- src/eca/features/chat.clj | 7 +- src/eca/features/prompt.clj | 35 +++++ src/eca/features/tools.clj | 138 +++++++++++++++-- src/eca/features/tools/tool_search.clj | 116 ++++++++++++++ src/eca/features/tools/util.clj | 10 ++ src/eca/llm_api.clj | 21 ++- test/eca/features/agents_test.clj | 48 ++++++ test/eca/features/prompt_test.clj | 24 +++ test/eca/features/tools/tool_search_test.clj | 83 +++++++++++ test/eca/features/tools_test.clj | 141 +++++++++++++++++- 19 files changed, 891 insertions(+), 49 deletions(-) create mode 100644 resources/prompts/tools/search_tools.md create mode 100644 src/eca/features/tools/tool_search.clj create mode 100644 test/eca/features/tools/tool_search_test.clj diff --git a/CHANGELOG.md b/CHANGELOG.md index bde4b5679..219f05978 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Add `mcpToolSearch` config to keep MCP tool schemas out of context until the LLM loads them with the new `eca__search_tools` tool. - Recover Anthropic streaming responses interrupted by transient TLS `bad_record_mac` failures. - BREAKING: `plugins.install` now appends across config layers. Set `plugins.installMode` to `replace` beside the list to exclude inherited plugins as before. diff --git a/docs/config.json b/docs/config.json index 10d286445..01b324cbf 100644 --- a/docs/config.json +++ b/docs/config.json @@ -119,6 +119,9 @@ "type": "string" } }, + "mcpToolSearch": { + "$ref": "#/definitions/mcpToolSearch" + }, "commands": { "type": "array", "description": "Custom command prompt files.", @@ -1006,6 +1009,9 @@ "type": "string" } }, + "mcpToolSearch": { + "$ref": "#/definitions/mcpToolSearch" + }, "autoCompactPercentage": { "type": "integer", "description": "Context window usage percentage at which auto-compact triggers for this agent.", @@ -1198,6 +1204,41 @@ }, "additionalProperties": false }, + "mcpToolSearch": { + "type": "object", + "description": "Keeps matching MCP tools out of the LLM context until it loads them with the eca__search_tools tool. Deferred tools are still listed by name and short description in the system prompt. ECA native tools are never deferred.", + "markdownDescription": "Keeps matching MCP tools out of the LLM context until it loads them with the `eca__search_tools` tool. Deferred tools are still listed by name and short description in the system prompt. ECA native tools are never deferred.", + "properties": { + "deferAllWhenTotalTokensExceedPercentOfContext": { + "type": ["number", "null"], + "description": "Percentage of the model context window the MCP tool definitions may take before all of them are put behind the search tool. For example 10 defers them once they exceed 10% of the context window. Null (the default) never defers automatically, leaving includePattern in control.", + "markdownDescription": "Percentage of the model context window the MCP tool definitions may take before all of them are put behind the search tool. For example `10` defers them once they exceed 10% of the context window. `null` (the default) never defers automatically, leaving `includePattern` in control.", + "default": null, + "minimum": 0, + "maximum": 100, + "examples": [10, 5] + }, + "includePattern": { + "type": "array", + "description": "MCP tools to put behind the search tool regardless of deferAllWhenTotalTokensExceedPercentOfContext. Each entry matches an exact MCP server name (all its tools) or an anchored regex against the tool full name server__tool.", + "markdownDescription": "MCP tools to put behind the search tool regardless of `deferAllWhenTotalTokensExceedPercentOfContext`. Each entry matches an exact MCP server name (all its tools) or an anchored regex against the tool full name `server__tool`.", + "items": { + "type": "string" + }, + "examples": [[".*"], ["some-mcp__.*"]] + }, + "excludePattern": { + "type": "array", + "description": "MCP tools to keep loaded, taking precedence over both deferAllWhenTotalTokensExceedPercentOfContext and includePattern. Same matching as includePattern.", + "markdownDescription": "MCP tools to keep loaded, taking precedence over both `deferAllWhenTotalTokensExceedPercentOfContext` and `includePattern`. Same matching as `includePattern`.", + "items": { + "type": "string" + }, + "examples": [["some-mcp"], ["some-mcp__.*"]] + } + }, + "additionalProperties": false + }, "toolCallConfig": { "type": "object", "description": "Tool call configuration including approval rules and tool-specific settings.", diff --git a/docs/config/agents.md b/docs/config/agents.md index 69dda8022..7f360d419 100644 --- a/docs/config/agents.md +++ b/docs/config/agents.md @@ -117,6 +117,7 @@ Subagents can be configured in config or markdown and support/require these fiel - `variant` (optional): default model variant; ignored when unavailable for the selected model. See [Variants](variants.md#agent-default-variant). - `tools` (optional): same as ECA tool approval logic to control what tools are allowed/askable/denied. - `disabledTools` (optional): tools to hide from this agent entirely. Same matching as the global [`disabledTools`](tools.md#disabled-tools): a builtin tool name or regex (no `eca__` prefix needed), an exact MCP server name (all its tools), or a regex against the tool full name `server__tool`. +- `mcpToolSearch` (optional): MCP tools this agent loads on demand via `eca__search_tools` instead of keeping in context. See [MCP tool search](#mcp-tool-search) below. - `maxSteps` (optional): set a max limit of turns/steps that his subagent must finish and return an answer. ### Parent-scoped subagents @@ -135,6 +136,37 @@ spawnableBy: Matching uses exact resolved agent IDs. Markdown agent IDs and Markdown `spawnableBy` values are trimmed and lowercased during loading; JSON configuration values are matched against the configured agent keys exactly. +### MCP tool search + +`mcpToolSearch` mirrors the [config object](tools.md#mcp-tool-search) as a YAML mapping, so an agent can keep MCP tools out of its context until it loads them with `eca__search_tools`: + +```yaml +--- +description: Reviews pull requests +mcpToolSearch: + deferAllWhenTotalTokensExceedPercentOfContext: 10 + includePattern: + - ".*" + excludePattern: + - github__get_pull_request +--- +``` + +Since deferring without exclusions is the common case, a bare list (or a single string) is shorthand for `includePattern`: + +```yaml +mcpToolSearch: + - ".*" +``` + +```yaml +mcpToolSearch: some-mcp__.* +``` + +!!! note "Agent patterns add to the global ones" + + Both lists are unioned with the global config rather than replacing it, so an agent can defer more tools or exclude more tools, but cannot re-load a tool the global `includePattern` deferred. Use `excludePattern` on the agent for that. + When `spawnableBy` is omitted or empty, the subagent is unrestricted, preserving the default behavior. When it contains IDs: - only a listed current primary agent sees the subagent in the `spawn_agent` tool description, `/subagents`, and contextual diagnostics such as the built-in `eca-info` skill; diff --git a/docs/config/introduction.md b/docs/config/introduction.md index 3e5b35622..83f5326d9 100644 --- a/docs/config/introduction.md +++ b/docs/config/introduction.md @@ -121,6 +121,11 @@ By default ECA consider the following as the base configuration: "commands" : [], "skills": [], "disabledTools": [], + "mcpToolSearch": { + "deferAllWhenTotalTokensExceedPercentOfContext": null, + "includePattern": [], + "excludePattern": [] + }, "toolCall": { "approval": { "byDefault": "ask", diff --git a/docs/config/tools.md b/docs/config/tools.md index 8e5ecb624..100c306b9 100644 --- a/docs/config/tools.md +++ b/docs/config/tools.md @@ -353,6 +353,98 @@ Regexes must match the whole name (anchored). It can be set globally, per agent `disabledTools` removes the tool entirely from the LLM — it won't even know it exists. `toolCall.approval.deny` rules without `argsMatchers` also remove the tool from the LLM tool list, while rules with `argsMatchers` keep the tool visible and only block matching calls. +## MCP tool search + +Every tool sent to the LLM costs context: its description and full input schema are part of each request. With a few MCP servers connected that easily adds up to thousands of tokens the model rarely needs. + +MCP tool search trades that upfront cost for an extra round trip. Matching tools are *deferred*: their schemas are **not** sent to the model, only a compact catalog of names and truncated descriptions in the system prompt. When the model needs one, it calls the `eca__search_tools` tool, which loads the matching tools — from then on they are sent as regular tools and can be called normally. + +Loading is per chat and sticks for the rest of it, including every follow-up request ECA makes while the model works through a chain of tool calls. Tools you never search for stay withheld for the whole conversation. + +This is configured via `mcpToolSearch`, and is off until you turn it on: + +- `deferAllWhenTotalTokensExceedPercentOfContext`: defer **all** MCP tools once their definitions outgrow this percentage of the model's context window. `null` by default, meaning never. +- `includePattern`: MCP tools to put behind the search tool regardless of that limit. +- `excludePattern`: MCP tools to keep loaded, taking precedence over both. + +So a tool is deferred when it is over the automatic limit **or** matches `includePattern`, and does not match `excludePattern`. + +The two patterns use the same matching as [`disabledTools`](#disabled-tools) — an exact MCP server name (all its tools) or an anchored regex against the tool full name `server__tool`. + +=== "Defer once MCP gets expensive" + + ```javascript title="~/.config/eca/config.json" + { + "mcpToolSearch": { + "deferAllWhenTotalTokensExceedPercentOfContext": 10 + } + } + ``` + + On a 200k model this defers every MCP tool once their definitions pass ~20k tokens, and leaves them loaded below that. Percentage rather than a fixed token count so the same setting behaves sensibly on a 32k local model and a 1M model. + +=== "Defer all MCP tools" + + ```javascript title="~/.config/eca/config.json" + { + "mcpToolSearch": { + "includePattern": [".*"] + } + } + ``` + +=== "Defer all but one MCP server" + + ```javascript title="~/.config/eca/config.json" + { + "mcpToolSearch": { + "includePattern": [".*"], + "excludePattern": ["clojure-mcp"] + } + } + ``` + +=== "Defer one noisy MCP server" + + ```javascript title="~/.config/eca/config.json" + { + "mcpToolSearch": { + "includePattern": ["some-mcp__.*"] + } + } + ``` + +=== "Per agent" + + ```javascript title="~/.config/eca/config.json" + { + "agent": { + "plan": { + "mcpToolSearch": { + "includePattern": [".*"], + "excludePattern": ["some-mcp__read_.*"] + } + } + } + } + ``` + +Both lists are merged from the global config and the agent config, and everything here can also be set in the [agent markdown frontmatter](agents.md#mcp-tool-search). `deferAllWhenTotalTokensExceedPercentOfContext` is a single value rather than a list, so an agent's value replaces the global one; set it to `null` on the agent to opt that agent out. + +!!! info "Native tools are never deferred" + + Only MCP tools can be deferred. ECA's [native tools](../features.md#native-tools) are the agent's baseline capabilities, so a catch-all `".*"` never takes them away. Use [`disabledTools`](#disabled-tools) to remove a native tool. They are also left out of the `deferAllWhenTotalTokensExceedPercentOfContext` total, so the limit tracks what MCP actually adds. + +!!! note "Models without a known context window" + + `deferAllWhenTotalTokensExceedPercentOfContext` needs the model's context window to compute a budget. When ECA does not know it, nothing is deferred automatically — use `includePattern` if you want deferral on such a model. + +`eca__search_tools` is only offered to the model when at least one tool is actually deferred. + +!!! tip "Disabled vs Deferred" + + `disabledTools` makes a tool unusable. `mcpToolSearch` keeps it fully usable, it just costs the model one `eca__search_tools` call to load it. + ## Approval / permissions By default, ECA asks to call any non read-only tool (check the [default rules](#default-approval-rules)), but that can easily be configured in several ways via the `toolCall.approval` config: diff --git a/integration-test/integration/chat/mcp_remote_test.clj b/integration-test/integration/chat/mcp_remote_test.clj index 9c60d2f5e..67d32d44c 100644 --- a/integration-test/integration/chat/mcp_remote_test.clj +++ b/integration-test/integration/chat/mcp_remote_test.clj @@ -15,13 +15,31 @@ (def ^:private mcp-server-config {:mcpServers {"test-mcp" {:url (str "http://localhost:" mcp-mock/port "/mcp")}}}) -(defn ^:private init-with-mcp-remote! [] - (eca/start-process!) - (mcp-mock/reset-requests!) - (eca/request! (fixture/initialize-request - {:initializationOptions - (merge fixture/default-init-options mcp-server-config)})) - (eca/notify! (fixture/initialized-notification))) +(defn ^:private init-with-mcp-remote! + ([] (init-with-mcp-remote! nil)) + ([extra-config] + (eca/start-process!) + (mcp-mock/reset-requests!) + (eca/request! (fixture/initialize-request + {:initializationOptions + (merge fixture/default-init-options mcp-server-config extra-config)})) + (eca/notify! (fixture/initialized-notification)))) + +(defn ^:private await-mcp-running! [] + (eca/client-awaits-server-notification :tool/serverUpdated) ;; native + (eca/client-awaits-server-notification :tool/serverUpdated) ;; mcp starting + (eca/client-awaits-server-notification :tool/serverUpdated)) ;; mcp running + +(defn ^:private drain-chat-until-finished! + "Consumes chat notifications until the turn finishes. Used when a test asserts + on what reached the LLM rather than on the notification sequence itself." + [] + (loop [remaining 50] + (when (pos? remaining) + (let [{:keys [content]} (eca/client-awaits-server-notification :chat/contentReceived)] + (when-not (and (= "progress" (:type content)) + (= "finished" (:state content))) + (recur (dec remaining))))))) (deftest mcp-remote-server-connects (init-with-mcp-remote!) @@ -55,11 +73,7 @@ (deftest mcp-remote-tool-call-in-chat (init-with-mcp-remote!) - - ;; Wait for MCP server to be ready - (eca/client-awaits-server-notification :tool/serverUpdated) ;; native - (eca/client-awaits-server-notification :tool/serverUpdated) ;; mcp starting - (eca/client-awaits-server-notification :tool/serverUpdated) ;; mcp running + (await-mcp-running!) (testing "LLM invokes an MCP tool and ECA processes it" (mcp-mock/reset-requests!) @@ -149,6 +163,37 @@ {:name "testMcp__add"}])} req-body))))))) +(deftest mcp-deferred-tools-stay-deferred-across-the-tool-call-loop + ;; `echo` is excluded from deferral so the mocked tool call targets a tool the + ;; LLM can actually see; every other testMcp tool sits behind eca__search_tools. + (init-with-mcp-remote! {:mcpToolSearch {:includePattern ["testMcp__.*"] + :excludePattern ["testMcp__echo"]}}) + (await-mcp-running!) + + (testing "the continuation request withholds deferred tools, like the first one" + (mcp-mock/reset-requests!) + (llm.mocks/set-case! :mcp-tool-call-0) + + (eca/request! (fixture/chat-prompt-request + {:model "anthropic/claude-sonnet-4-6" + :message "Call the echo tool"})) + (drain-chat-until-finished!) + + ;; get-req-body keeps the last body, which here is the request ECA sent + ;; after running the tool - the one that used to re-send every tool. + (let [tool-names (->> (llm.mocks/get-req-body :mcp-tool-call-0) + :tools + (map :name) + set)] + (is (contains? tool-names "testMcp__echo") + "excludePattern keeps echo loaded") + (is (contains? tool-names "eca__search_tools") + "search tool is offered while something is deferred") + (is (not (contains? tool-names "testMcp__add")) + "deferred tools must not come back in the tool-call loop") + (is (not (contains? tool-names "testMcp__add-tool")) + "deferred tools must not come back in the tool-call loop")))) + (deftest mcp-remote-instructions-in-prompt (init-with-mcp-remote!) diff --git a/resources/prompts/tools/search_tools.md b/resources/prompts/tools/search_tools.md new file mode 100644 index 000000000..885a1ac97 --- /dev/null +++ b/resources/prompts/tools/search_tools.md @@ -0,0 +1,10 @@ +Load deferred tools so you can call them. + +Some tools are deferred: they are listed by name and short description in the "Deferred Tools" section of your system prompt, but their full descriptions and input schemas are not loaded, so you cannot call them yet. + +Use this tool to load the ones you need: +- Search with keywords describing the capability you want (e.g. "create pull request", "query database"), not the exact tool name. +- Omit `query` to list every deferred tool. +- Matches are loaded immediately and become callable from your next message onward, with their full input schemas returned here. +- Prefer a single search with a focused query over many searches; each loaded tool consumes context. +- If a deferred tool looks relevant to the task, load it before concluding the capability is unavailable. diff --git a/src/eca/config.clj b/src/eca/config.clj index 3d585a783..90f56ee98 100644 --- a/src/eca/config.clj +++ b/src/eca/config.clj @@ -187,6 +187,7 @@ "eca__grep" {} "eca__editor_diagnostics" {} "eca__skill" {} + "eca__search_tools" {} "eca__task" {} "eca__fetch_rule" {} "eca__spawn_agent" {}} @@ -210,6 +211,7 @@ "eca__grep" {} "eca__editor_diagnostics" {} "eca__skill" {} + "eca__search_tools" {} "eca__task" {} "eca__fetch_rule" {}} :deny {"eca__shell_command" @@ -238,6 +240,9 @@ :skills [] :extraConfigs [] :disabledTools [] + :mcpToolSearch {:deferAllWhenTotalTokensExceedPercentOfContext nil + :includePattern [] + :excludePattern []} :toolCall {:approval {:byDefault "ask" :allow {"eca__compact_chat" {} "eca__preview_file_change" {} @@ -246,6 +251,7 @@ "eca__grep" {} "eca__editor_diagnostics" {} "eca__skill" {} + "eca__search_tools" {} "eca__task" {} "eca__ask_user" {} "eca__fetch_rule" {} diff --git a/src/eca/features/agents.clj b/src/eca/features/agents.clj index 0a1fb066b..12a875b55 100644 --- a/src/eca/features/agents.clj +++ b/src/eca/features/agents.clj @@ -91,36 +91,67 @@ (logger/warn logger-tag (format "Ignoring malformed spawnableBy value: %s" (pr-str spawnable-by))) nil))) -(defn ^:private normalize-disabled-tools - "Coerces the YAML `disabledTools:` value into a vector of strings. +(defn ^:private normalize-tool-patterns + "Coerces a YAML tool pattern list (`disabledTools:`, `mcpToolSearch:` entries) + into a vector of strings. Accepts a single string or a list of strings; other shapes are ignored." - [disabled-tools] + [config-key patterns] (cond - (nil? disabled-tools) nil - (string? disabled-tools) (when-not (string/blank? disabled-tools) - [disabled-tools]) - (sequential? disabled-tools) (some->> disabled-tools - (keep (fn [entry] - (let [s (str entry)] - (when-not (string/blank? s) s)))) - vec - not-empty) + (nil? patterns) nil + (string? patterns) (when-not (string/blank? patterns) + [patterns]) + (sequential? patterns) (some->> patterns + (keep (fn [entry] + (let [s (str entry)] + (when-not (string/blank? s) s)))) + vec + not-empty) :else (do - (logger/warn logger-tag (format "Ignoring malformed disabledTools value: %s" (pr-str disabled-tools))) + (logger/warn logger-tag (format "Ignoring malformed %s value: %s" config-key (pr-str patterns))) nil))) +(defn ^:private normalize-mcp-tool-search + "Coerces the YAML `mcpToolSearch:` value into the map form ECA expects. + - Map form mirrors the config: {deferAllWhenTotalTokensExceedPercentOfContext N + includePattern [...] excludePattern [...]}. + Nested YAML keys arrive as strings, so both string and keyword keys are read. + - A bare string or list is shorthand for `includePattern`, since deferring + tools without excluding any is the common case. + - Any other shape (number, boolean, malformed) is treated as absent." + [mcp-tool-search] + (when (some? mcp-tool-search) + (let [defer-all-key :deferAllWhenTotalTokensExceedPercentOfContext + as-map (if (map? mcp-tool-search) + mcp-tool-search + {"includePattern" mcp-tool-search}) + has-entry? (fn [k] (or (contains? as-map (name k)) (contains? as-map k))) + entry (fn [k] (or (get as-map (name k)) (get as-map k))) + include (normalize-tool-patterns "mcpToolSearch.includePattern" (entry :includePattern)) + exclude (normalize-tool-patterns "mcpToolSearch.excludePattern" (entry :excludePattern)) + ;; Present but not a number (notably an explicit null) leaves it unlimited. + defer-all-percent (when-let [v (entry defer-all-key)] + (when (number? v) v))] + (not-empty + (cond-> {} + (has-entry? defer-all-key) (assoc defer-all-key defer-all-percent) + include (assoc :includePattern include) + exclude (assoc :excludePattern exclude)))))) + (defn ^:private md->agent-config - [{:keys [description mode model variant maxSteps steps tools body inherit spawnableBy disabledTools]}] + [{:keys [description mode model variant maxSteps steps tools body inherit spawnableBy + disabledTools mcpToolSearch]}] (let [agent-variant (normalize-agent-variant variant) max-steps (or maxSteps steps) tools-map (normalize-tools tools) spawnable-by (normalize-spawnable-by spawnableBy) - disabled-tools (normalize-disabled-tools disabledTools)] + disabled-tools (normalize-tool-patterns "disabledTools" disabledTools) + mcp-tool-search (normalize-mcp-tool-search mcpToolSearch)] (cond-> {} inherit (assoc :inherit (str inherit)) description (assoc :description description) spawnable-by (assoc :spawnableBy spawnable-by) disabled-tools (assoc :disabledTools disabled-tools) + mcp-tool-search (assoc :mcpToolSearch mcp-tool-search) mode (assoc :mode (if (sequential? mode) (mapv str mode) (str mode))) diff --git a/src/eca/features/chat.clj b/src/eca/features/chat.clj index e72ece3de..1857edcd0 100644 --- a/src/eca/features/chat.clj +++ b/src/eca/features/chat.clj @@ -109,7 +109,10 @@ :rules (sha {:static-rules (mapv #(select-keys % [:id :name :scope :content]) static-rules) :path-scoped-rules (mapv #(select-keys % [:id :name :scope :workspace-root :paths :enforce]) path-scoped-rules)}) :skills (sha (mapv #(select-keys % [:name :description]) skills)) - :tools (sha (sort (map :full-name all-tools)))})) + ;; Deferred tools are tracked separately because they render a catalog into + ;; the static prompt, unlike normal tools whose schemas are sent per turn. + :tools (sha {:names (sort (map :full-name all-tools)) + :deferrable (sort (map :full-name (filter :deferrable all-tools)))})})) (defn ^:private changed-system-prompt-categories "Names of system prompt categories that changed vs the cached signature. @@ -976,7 +979,7 @@ (let [breakdown (try (shared/context-breakdown {:system-prompt (f.prompt/instructions->str instructions) - :tools all-tools + :tools (f.tools/tools-for-llm all-tools) :messages (get-in @db* [:chats chat-id :messages] []) :context-limit (get-in usage [:limit :context]) :session-tokens (:session-tokens usage)}) diff --git a/src/eca/features/prompt.clj b/src/eca/features/prompt.clj index 091cb88ef..6df078227 100644 --- a/src/eca/features/prompt.clj +++ b/src/eca/features/prompt.clj @@ -255,6 +255,40 @@ [path-scoped-rules] (path-scoped-rule-sections path-scoped-rules path-scoped-rule-catalog-entry)) +(def ^:private deferred-tool-description-max-length 250) + +(defn ^:private deferred-tool-summary + "First paragraph of the tool description, truncated: the catalog is routing + metadata, the full description arrives when the tool is loaded." + [description] + (let [summary (-> (str description) + (string/split #"\R\s*\R" 2) + first + (string/replace #"\s+" " ") + string/trim)] + (if (> (count summary) deferred-tool-description-max-length) + (str (string/trimr (subs summary 0 deferred-tool-description-max-length)) "…") + summary))) + +(defn ^:private deferred-tools-section + [all-tools] + (when-let [deferred-tools (seq (filter :deferrable all-tools))] + ["## Deferred Tools" + "" + (str (format "" + (attr-str {:description (str "Tools that exist but are not loaded: their full descriptions and input schemas are unavailable and they cannot be called yet. " + "Load the ones you need with eca__search_tools, which makes them callable.")})) + "\n") + (reduce + (fn [tools-str {:keys [full-name description]}] + (str tools-str (format "\n" + (attr-str {:name full-name + :description (deferred-tool-summary description)})))) + "" + (sort-by :full-name deferred-tools)) + "" + ""])) + (defn build-static-instructions "Builds the cacheable static system-prompt prefix." [refined-contexts static-rules path-scoped-rules skills repo-map* agent-name config chat-id all-tools db] @@ -307,6 +341,7 @@ skills) "" ""]) + (deferred-tools-section all-tools) (shared/safe-selmer-render (load-builtin-prompt "additional_system_info.md") selmer-ctx "additional-system-info") (workspace-roots-section db) diff --git a/src/eca/features/tools.clj b/src/eca/features/tools.clj index 8943131a1..19a4e60d3 100644 --- a/src/eca/features/tools.clj +++ b/src/eca/features/tools.clj @@ -17,6 +17,7 @@ [eca.features.tools.shell :as f.tools.shell] [eca.features.tools.skill :as f.tools.skill] [eca.features.tools.task :as f.tools.task] + [eca.features.tools.tool-search :as f.tools.tool-search] [eca.features.tools.ask-user :as f.tools.ask-user] [eca.features.tools.util :as tools.util] [eca.logger :as logger] @@ -132,26 +133,45 @@ ([all-tools tool args db config agent-name opts] (:decision (approval-decision all-tools tool args db config agent-name opts)))) -(defn ^:private get-disabled-tools - "Returns a set of disabled tools, merging global and agent-specific." - [config agent-name] - (set (concat (get config :disabledTools []) +(defn ^:private get-tool-patterns + "Returns a set of tool patterns at `config-path`, merging global and agent-specific." + [config agent-name config-path] + (set (concat (get-in config config-path []) (if agent-name - (get-in config [:agent agent-name :disabledTools] []) + (get-in config (into [:agent agent-name] config-path) []) [])))) -(defn ^:private disabled-entry-matches? - "Matches a `disabledTools` entry against a tool, checking in order: +(defn ^:private get-disabled-tools + "Returns a set of disabled tools, merging global and agent-specific." + [config agent-name] + (get-tool-patterns config agent-name [:disabledTools])) + +(def ^:private compile-tool-pattern + "Anchored regex for a tool pattern entry, or nil when it is not a valid regex + (the entry is then matched literally). Memoized: every entry is tested against + every tool on every turn, and it keeps the warning to once per bad entry." + (memoize + (fn [entry] + (try (re-pattern entry) + (catch PatternSyntaxException _ + (logger/warn logger-tag + (format "Tool pattern '%s' is not a valid regex, matching it literally instead. Use '.*' for a wildcard." + entry)) + nil))))) + +(defn tool-entry-matches? + "Matches a tool pattern entry (`disabledTools`, `mcpToolSearch`) against a + tool, checking in order: 1. Entry as anchored regex against a eca builtin tool name (no `eca__` prefix needed). - 2. Entry as exact server name, disabling all tools of that server. + 2. Entry as exact server name, matching all tools of that server. 3. Entry as anchored regex against the tool full name `server__tool`. Invalid regexes fall back to literal equality." [entry tool] - (let [server-name (:name (:server tool)) + (let [entry (str entry) + server-name (:name (:server tool)) tool-name (:name tool) full-name (str server-name "__" tool-name) - pattern (try (re-pattern entry) - (catch PatternSyntaxException _ nil))] + pattern (compile-tool-pattern entry)] (boolean (or (and (= "eca" server-name) (if pattern @@ -162,8 +182,11 @@ (re-matches pattern full-name) (= entry full-name)))))) +(defn ^:private tool-matches-any? [tool entries] + (boolean (some #(tool-entry-matches? % tool) entries))) + (defn ^:private tool-disabled? [tool disabled-tools] - (boolean (some #(disabled-entry-matches? % tool) disabled-tools))) + (tool-matches-any? tool disabled-tools)) (defn make-tool-status-fn "Returns a function that marks tools as disabled based on config and agent. @@ -195,6 +218,7 @@ f.tools.skill/definitions f.tools.task/definitions f.tools.background/definitions + f.tools.tool-search/definitions f.tools.ask-user/definitions)) (defn tool-approval-keys @@ -238,6 +262,78 @@ [tools] (filterv #(not (contains? #{"spawn_agent" "task" "git" "ask_user"} (:name %))) tools)) +(defn ^:private get-defer-all-percent + "Percentage of the model context window the MCP tool definitions may take + before all of them are deferred without any pattern. The agent value overrides + the global one; nil (the default) leaves them unlimited." + [config agent-name] + (let [agent-config (get-in config [:agent agent-name :mcpToolSearch])] + (if (contains? agent-config :deferAllWhenTotalTokensExceedPercentOfContext) + (:deferAllWhenTotalTokensExceedPercentOfContext agent-config) + (get-in config [:mcpToolSearch :deferAllWhenTotalTokensExceedPercentOfContext])))) + +(defn ^:private tools-tokens + "Rough token cost of the tool definitions sent on every request, measured over + the same wire fields `shared/context-breakdown` reports as Tool definitions." + [tools] + (shared/estimate-tokens + (pr-str (mapv #(select-keys % [:name :description :parameters]) tools)))) + +(defn ^:private defer-all-by-total-tokens? + "True when the MCP tool definitions as a whole outgrow + `deferAllWhenTotalTokensExceedPercentOfContext` of the model context window. + False without a percent or a known context window, so an unknown model never + silently hides its tools." + [mcp-tools config agent-name db full-model] + (let [percent (get-defer-all-percent config agent-name) + context-limit (get-in db [:models full-model :limit :context])] + (boolean (and (number? percent) + (not (neg? percent)) + (number? context-limit) + (pos? context-limit) + (seq mcp-tools) + (> (tools-tokens mcp-tools) + (* context-limit (/ (double percent) 100.0))))))) + +(defn ^:private mark-deferred-tools + "Marks MCP tools kept out of the LLM context with `:deferrable`, and with + `:deferred` while the chat has not loaded them yet via `eca__search_tools`. + A tool is deferred when the MCP tool definitions as a whole outgrow + `mcpToolSearch.deferAllWhenTotalTokensExceedPercentOfContext` of the context + window, or when it matches `includePattern`, and in both cases only if it does + not match `excludePattern`. + Deferred tools stay resolvable and callable; they are simply kept out of the + tool schemas sent to the LLM. + Native ECA tools are never deferred: they are the agent's baseline capabilities, + and a catch-all pattern taking them away would break the agent. They are also + left out of the token count, so the limit tracks what MCP actually adds." + [tools {:keys [config agent-name activated db full-model]}] + (let [include-patterns (get-tool-patterns config agent-name [:mcpToolSearch :includePattern]) + exclude-patterns (get-tool-patterns config agent-name [:mcpToolSearch :excludePattern]) + mcp-tools (filterv #(not= :native (:origin %)) tools) + defer-all? (defer-all-by-total-tokens? mcp-tools config agent-name db full-model)] + (if-not (or defer-all? (seq include-patterns)) + tools + (mapv (fn [tool] + (if (and (not= :native (:origin tool)) + (or defer-all? (tool-matches-any? tool include-patterns)) + (not (tool-matches-any? tool exclude-patterns))) + (assoc tool + :deferrable true + :deferred (not (contains? activated (:full-name tool)))) + tool)) + tools)))) + +(defn deferrable-tools + "Tools kept out of the LLM context until loaded with `eca__search_tools`." + [all-tools] + (filterv :deferrable all-tools)) + +(defn tools-for-llm + "Tools whose schemas should be sent to the LLM on this turn." + [all-tools] + (remove :deferred all-tools)) + (defn resolve-tool [tool-name all-tools] (or (some #(when (= tool-name (:full-name %)) %) all-tools) @@ -287,10 +383,20 @@ ;; Apply subagent tool filtering if applicable all-tools (if subagent (filter-subagent-tools all-tools) - all-tools)] - (remove (fn [tool] - (= :deny (approval all-tools tool {} db config agent-name))) - all-tools)))) + all-tools) + all-tools (remove (fn [tool] + (= :deny (approval all-tools tool {} db config agent-name))) + all-tools) + all-tools (mark-deferred-tools all-tools + {:config config + :agent-name agent-name + :activated (tools.util/activated-deferred-tools db chat-id) + :db db + :full-model full-model})] + ;; search_tools only earns its context slot when something is actually deferred. + (if (some :deferrable all-tools) + all-tools + (remove #(= f.tools.tool-search/tool-full-name (:full-name %)) all-tools))))) (defn call-tool! [^String full-name ^Map arguments chat-id tool-call-id agent-name db* config messenger metrics call-state-fn ; thunk diff --git a/src/eca/features/tools/tool_search.clj b/src/eca/features/tools/tool_search.clj new file mode 100644 index 000000000..75c53d080 --- /dev/null +++ b/src/eca/features/tools/tool_search.clj @@ -0,0 +1,116 @@ +(ns eca.features.tools.tool-search + "The `search_tools` tool, which loads deferred tools into the conversation." + (:require + [cheshire.core :as json] + [clojure.string :as string] + [eca.features.tools.util :as tools.util] + [eca.shared :refer [multi-str]])) + +(set! *warn-on-reflection* true) + +(def tool-name "search_tools") + +(def tool-full-name (str "eca__" tool-name)) + +(def ^:private default-max-results 10) +(def ^:private max-max-results 50) + +(defn ^:private tokenize [s] + (->> (string/split (string/lower-case (str s)) #"[^a-z0-9]+") + (remove string/blank?) + set)) + +(defn ^:private score + [query-tokens {:keys [full-name description]}] + (let [name-str (string/lower-case (str full-name)) + desc-str (string/lower-case (str description)) + name-tokens (tokenize name-str) + desc-tokens (tokenize desc-str)] + (transduce + (map (fn [token] + (+ (if (contains? name-tokens token) 10 0) + (if (string/includes? name-str token) 5 0) + (if (contains? desc-tokens token) 2 0) + (if (string/includes? desc-str token) 1 0)))) + + + 0 + query-tokens))) + +(defn ^:private ->positive-int [value] + (cond + (integer? value) value + (number? value) (long value) + (string? value) (try (Long/parseLong (string/trim value)) + (catch NumberFormatException _ nil)) + :else nil)) + +(defn ^:private rank + [deferred-tools query max-results] + (let [query-tokens (tokenize query)] + (if (empty? query-tokens) + (take max-results (sort-by :full-name deferred-tools)) + (->> deferred-tools + (map #(assoc % :score (score query-tokens %))) + (filter #(pos? (:score %))) + (sort-by (juxt (comp - :score) :full-name)) + (take max-results))))) + +(defn ^:private render-tool + [{:keys [full-name description parameters]}] + (multi-str + (format "" full-name) + (string/trim (str description)) + "" + "Input schema:" + "```json" + (json/generate-string parameters {:pretty true}) + "```" + "")) + +(defn ^:private search-tools + [arguments {:keys [db* chat-id all-tools]}] + (let [query (str (get arguments "query" "")) + max-results (-> (get arguments "max_results") + ->positive-int + (or default-max-results) + (max 1) + (min max-max-results)) + deferred-tools (filter :deferrable all-tools) + matches (rank deferred-tools query max-results)] + (cond + (empty? deferred-tools) + (tools.util/single-text-content "There are no deferred tools to search." :error) + + (empty? matches) + (tools.util/single-text-content + (format (multi-str "No deferred tool matched '%s'." + "" + "Available deferred tools: %s") + query + (string/join ", " (sort (map :full-name deferred-tools))))) + + :else + (do + (tools.util/activate-deferred-tools! db* chat-id (map :full-name matches)) + (tools.util/single-text-content + (multi-str + (format "Loaded %d tool(s). They are now available and you can call them directly from the next message onward." + (count matches)) + "" + (string/join "\n\n" (map render-tool matches)))))))) + +(def definitions + {tool-name + {:description (tools.util/read-tool-description tool-name) + :parameters {:type "object" + :properties {"query" {:type "string" + :description "Keywords describing the capability you need, matched against deferred tool names and descriptions. Omit to list the deferred tools."} + "max_results" {:type "integer" + :description (format "Maximum number of tools to load (default %d, max %d)." + default-max-results max-max-results)}} + :required []} + :handler #'search-tools + :summary-fn (fn [{:keys [args]}] + (if-let [query (not-empty (str (get args "query" "")))] + (format "Searching tools for '%s'" query) + "Listing deferred tools"))}}) diff --git a/src/eca/features/tools/util.clj b/src/eca/features/tools/util.clj index bee007288..bf3000a08 100644 --- a/src/eca/features/tools/util.clj +++ b/src/eca/features/tools/util.clj @@ -62,6 +62,16 @@ [all-tools full-name] (boolean (some #(= full-name (:full-name %)) all-tools))) +(defn activated-deferred-tools + "Full names of deferred tools the chat already loaded via `eca__search_tools`." + [db chat-id] + (get-in db [:chats chat-id :activated-tools] #{})) + +(defn activate-deferred-tools! + "Marks `full-names` as loaded for the chat, so they stop being deferred." + [db* chat-id full-names] + (swap! db* update-in [:chats chat-id :activated-tools] (fnil into #{}) full-names)) + (defn selector->string [selector] (cond (keyword? selector) (name selector) diff --git a/src/eca/llm_api.clj b/src/eca/llm_api.clj index 54508f02d..16e1ebf4f 100644 --- a/src/eca/llm_api.clj +++ b/src/eca/llm_api.clj @@ -240,6 +240,12 @@ (apply dissoc merged keys-to-strip)) merged))) +(defn ^:private tools-for-request + "Drops deferred tools from a request payload. They stay resolvable and callable; + only their schemas are withheld until the model loads them with eca__search_tools." + [tools] + (some->> tools (remove :deferred) vec)) + (defn ^:private prompt! [{:keys [provider model model-capabilities instructions user-messages config variant on-message-received on-error on-prepare-tool-call on-tools-called on-reason on-usage-updated @@ -247,7 +253,7 @@ past-messages tools provider-auth sync? subagent? cancelled? prompt-cache-key] :or {on-error identity}}] (let [real-model (real-model-name model model-capabilities) - tools (when (:tools model-capabilities) tools) + tools (when (:tools model-capabilities) (tools-for-request tools)) reason? (:reason? model-capabilities) supports-image? (:image-input? model-capabilities) web-search (:web-search model-capabilities) @@ -504,6 +510,15 @@ emit-first-message-fn (fn [& args] (when (compare-and-set! first-response-received* false true) (apply on-first-response-received args))) + ;; Every provider rebuilds the next request of a tool-call loop from the + ;; tool list returned here, shadowing the one `prompt!` filtered, so it + ;; needs the same treatment or deferred schemas come back after the first + ;; tool call. Wrapped here rather than in `prompt!` because the sync path + ;; invokes the callback itself, without going through it. + on-tools-called-wrapper (fn [tool-calls] + (let [result (on-tools-called tool-calls)] + (cond-> result + (map? result) (update :tools tools-for-request)))) on-message-received-wrapper (fn [& args] (apply emit-first-message-fn args) (apply on-message-received args)) @@ -637,7 +652,7 @@ (if-let [new-result (when (seq tools-to-call) (doseq [tool-to-call tools-to-call] (on-prepare-tool-call tool-to-call)) - (call-tools-fn on-tools-called))] + (call-tools-fn on-tools-called-wrapper))] (recur new-result) (on-message-received-wrapper {:type :finish :finish-reason "stop"})))))))] (sync-prompt-with-retry* 0)) @@ -659,7 +674,7 @@ :cancelled? cancelled? :on-message-received on-message-received-wrapper :on-prepare-tool-call on-prepare-tool-call-wrapper - :on-tools-called on-tools-called + :on-tools-called on-tools-called-wrapper :on-usage-updated on-usage-updated :on-server-web-search on-server-web-search-wrapper :on-server-image-generation on-server-image-generation-wrapper diff --git a/test/eca/features/agents_test.clj b/test/eca/features/agents_test.clj index 0f3ba391e..efd7c49b3 100644 --- a/test/eca/features/agents_test.clj +++ b/test/eca/features/agents_test.clj @@ -217,6 +217,54 @@ (:disabledTools (#'agents/md->agent-config {:description "a" :disabledTools ["ok" "" nil]}))))) + (testing "mcpToolSearch map form mirrors the config shape" + (let [md (str "---\n" + "description: K8s agent\n" + "mcpToolSearch:\n" + " deferAllWhenTotalTokensExceedPercentOfContext: 10\n" + " includePattern:\n" + " - \".*\"\n" + " excludePattern:\n" + " - k8s-mcp__get_.*\n" + "---\n\n" + "Body.") + config (#'agents/md->agent-config (shared/parse-md md))] + (is (= {:deferAllWhenTotalTokensExceedPercentOfContext 10 + :includePattern [".*"] + :excludePattern ["k8s-mcp__get_.*"]} + (:mcpToolSearch config))))) + + (testing "mcpToolSearch deferAllWhenTotalTokensExceedPercentOfContext set to null leaves it unlimited" + (let [md (str "---\n" + "description: K8s agent\n" + "mcpToolSearch:\n" + " deferAllWhenTotalTokensExceedPercentOfContext: null\n" + "---\n\n" + "Body.") + config (#'agents/md->agent-config (shared/parse-md md))] + (is (= {:deferAllWhenTotalTokensExceedPercentOfContext nil} (:mcpToolSearch config))))) + + (testing "mcpToolSearch list shorthand means includePattern" + (let [md (str "---\n" + "description: K8s agent\n" + "mcpToolSearch:\n" + " - \".*\"\n" + " - other-mcp__.*\n" + "---\n\n" + "Body.") + config (#'agents/md->agent-config (shared/parse-md md))] + (is (= {:includePattern [".*" "other-mcp__.*"]} (:mcpToolSearch config))))) + + (testing "mcpToolSearch string shorthand means includePattern" + (let [config (#'agents/md->agent-config {:description "a" :mcpToolSearch ".*"})] + (is (= {:includePattern [".*"]} (:mcpToolSearch config))))) + + (testing "omitted, empty or malformed mcpToolSearch is ignored" + (is (nil? (:mcpToolSearch (#'agents/md->agent-config {:description "a"})))) + (is (nil? (:mcpToolSearch (#'agents/md->agent-config {:description "a" :mcpToolSearch 42})))) + (is (nil? (:mcpToolSearch (#'agents/md->agent-config {:description "a" :mcpToolSearch []})))) + (is (nil? (:mcpToolSearch (#'agents/md->agent-config {:description "a" :mcpToolSearch {}}))))) + (testing "tools as a YAML list normalizes to byDefault=ask + allow map (Claude form)" (let [md (str "---\n" "description: Reviewer\n" diff --git a/test/eca/features/prompt_test.clj b/test/eca/features/prompt_test.clj index 314b5711f..577180fcb 100644 --- a/test/eca/features/prompt_test.clj +++ b/test/eca/features/prompt_test.clj @@ -28,6 +28,30 @@ (is (string/includes? result "path='/tmp/has\"quote.clj'")) (is (string/includes? result "path=\"/tmp/has"both'quotes.clj\""))))) +(deftest deferred-tools-section-test + (let [static-for (fn [all-tools] + (:static (build-instructions [] [] [] [] (delay "TREE") "code" {} nil all-tools (h/db))))] + (testing "no section when nothing is deferred" + (is (not (string/includes? (static-for [{:full-name "eca__read_file" :description "Read"}]) + "")) + (is (not (string/includes? static "Lots of extra detail"))) + (is (not (string/includes? static "eca__read_file"))))) + (testing "long descriptions are truncated" + (let [static (static-for [{:full-name "some__tool" + :description (apply str (repeat 400 "a")) + :deferrable true + :deferred true}])] + (is (string/includes? static "…")) + (is (not (string/includes? static (apply str (repeat 300 "a"))))))))) + (deftest build-instructions-test (testing "Should return a map with :static and :dynamic keys" (let [result (build-instructions [] [] [] [] (delay "TREE") "code" {} nil [] (h/db))] diff --git a/test/eca/features/tools/tool_search_test.clj b/test/eca/features/tools/tool_search_test.clj new file mode 100644 index 000000000..a9bb9642e --- /dev/null +++ b/test/eca/features/tools/tool_search_test.clj @@ -0,0 +1,83 @@ +(ns eca.features.tools.tool-search-test + (:require + [clojure.string :as string] + [clojure.test :refer [deftest is testing]] + [eca.features.tools.tool-search :as f.tools.tool-search] + [eca.features.tools.util :as tools.util])) + +(def ^:private deferred-tools + [{:full-name "github__create_pull_request" + :description "Create a pull request on GitHub" + :parameters {:type "object" :properties {"title" {:type "string"}}} + :deferrable true + :deferred true} + {:full-name "postgres__run_query" + :description "Run a SQL query against the database" + :parameters {:type "object" :properties {"sql" {:type "string"}}} + :deferrable true + :deferred true}]) + +(def ^:private all-tools + (conj deferred-tools + {:full-name "eca__read_file" :description "Read a file" :parameters {}})) + +(defn ^:private search [db* arguments] + ((:handler (get f.tools.tool-search/definitions "search_tools")) + arguments + {:db* db* :chat-id "chat-1" :all-tools all-tools})) + +(deftest search-tools-test + (testing "matching tools are returned with their schema and activated for the chat" + (let [db* (atom {}) + result (search db* {"query" "pull request"}) + text (-> result :contents first :text)] + (is (false? (:error result))) + (is (string/includes? text "github__create_pull_request")) + (is (string/includes? text "\"title\"")) + (is (not (string/includes? text "postgres__run_query"))) + (is (= #{"github__create_pull_request"} + (tools.util/activated-deferred-tools @db* "chat-1"))))) + + (testing "matches on tool name too" + (let [db* (atom {})] + (search db* {"query" "run_query"}) + (is (= #{"postgres__run_query"} + (tools.util/activated-deferred-tools @db* "chat-1"))))) + + (testing "activations accumulate across searches" + (let [db* (atom {})] + (search db* {"query" "pull request"}) + (search db* {"query" "sql database"}) + (is (= #{"github__create_pull_request" "postgres__run_query"} + (tools.util/activated-deferred-tools @db* "chat-1"))))) + + (testing "blank query lists every deferred tool" + (let [db* (atom {}) + text (-> (search db* {}) :contents first :text)] + (is (string/includes? text "github__create_pull_request")) + (is (string/includes? text "postgres__run_query")))) + + (testing "max_results limits how many tools are loaded" + (let [db* (atom {})] + (search db* {"max_results" 1}) + (is (= 1 (count (tools.util/activated-deferred-tools @db* "chat-1")))))) + + (testing "non deferred tools are never returned" + (let [db* (atom {}) + text (-> (search db* {"query" "read file"}) :contents first :text)] + (is (not (string/includes? text "eca__read_file"))))) + + (testing "no match lists the available deferred tools without activating any" + (let [db* (atom {}) + result (search db* {"query" "kubernetes"}) + text (-> result :contents first :text)] + (is (false? (:error result))) + (is (string/includes? text "Available deferred tools")) + (is (empty? (tools.util/activated-deferred-tools @db* "chat-1"))))) + + (testing "errors when there is nothing deferred" + (let [db* (atom {}) + result ((:handler (get f.tools.tool-search/definitions "search_tools")) + {"query" "anything"} + {:db* db* :chat-id "chat-1" :all-tools [{:full-name "eca__read_file"}]})] + (is (true? (:error result)))))) diff --git a/test/eca/features/tools_test.clj b/test/eca/features/tools_test.clj index 336a08280..fe3884a5c 100644 --- a/test/eca/features/tools_test.clj +++ b/test/eca/features/tools_test.clj @@ -203,7 +203,146 @@ (testing "server name works per agent" (let [names (full-names {:agent {"code" {:disabledTools ["clojureMCP"]}}})] (is (not (contains? names "clojureMCP__eval"))) - (is (contains? names "eca__read_file")))))) + (is (contains? names "eca__read_file")))) + (testing "patterns are regexes, not globs" + (is (empty? (full-names {:disabledTools [".*"]}))) + ;; A bare `*` is an invalid regex, so it falls back to literal equality + ;; and matches nothing, rather than being treated as a glob wildcard. + (is (= (full-names {}) (full-names {:disabledTools ["*"]})))))) + +(def ^:private deferred-db + {:mcp-clients {"clojureMCP" + {:version "1.0.2" + :tools [{:name "eval" :description "eval clojure code" :parameters {}} + {:name "sync_deps" :description "sync deps" :parameters {}}]}}}) + +(deftest mcp-tool-search-test + (let [tools-by-name (fn [config & [db]] + (into {} (map (juxt :full-name identity)) + (f.tools/all-tools "123" "code" (or db deferred-db) config))) + include (fn [& patterns] {:mcpToolSearch {:includePattern (vec patterns)}})] + (testing "no mcpToolSearch config keeps every tool loaded and hides search_tools" + (let [tools (tools-by-name {})] + (is (not (contains? tools "eca__search_tools"))) + (is (not-any? :deferrable (vals tools))))) + (testing "included tools are deferred and search_tools is offered" + (let [tools (tools-by-name (include "clojureMCP__.*"))] + (is (match? {:deferrable true :deferred true} (get tools "clojureMCP__eval"))) + (is (match? {:deferrable true :deferred true} (get tools "clojureMCP__sync_deps"))) + (is (contains? tools "eca__search_tools")) + (is (nil? (:deferrable (get tools "eca__read_file")))))) + (testing "excludePattern takes precedence over includePattern" + (let [tools (tools-by-name {:mcpToolSearch {:includePattern [".*"] + :excludePattern ["clojureMCP__sync_deps"]}})] + (is (match? {:deferrable true} (get tools "clojureMCP__eval"))) + (is (nil? (:deferrable (get tools "clojureMCP__sync_deps")))))) + (testing "patterns are regexes, not globs" + (let [tools (tools-by-name (include "clojureMCP__.*"))] + (is (match? {:deferrable true} (get tools "clojureMCP__eval")))) + ;; A bare `*` is an invalid regex: matched literally, so nothing defers. + (let [tools (tools-by-name (include "*"))] + (is (not-any? :deferrable (vals tools))) + (is (not (contains? tools "eca__search_tools"))))) + (testing "native eca tools are never deferred, even by a catch-all pattern" + (let [tools (tools-by-name (include ".*"))] + (is (match? {:deferrable m/absent} (get tools "eca__search_tools"))) + (is (match? {:deferrable m/absent} (get tools "eca__read_file"))) + (is (match? {:deferrable m/absent} (get tools "eca__shell_command"))) + (is (every? #(nil? (:deferrable %)) + (filter #(= :native (:origin %)) (vals tools)))))) + (testing "explicitly naming a native tool still does not defer it" + (let [tools (tools-by-name (include "eca__read_file" "read_file"))] + (is (match? {:deferrable m/absent} (get tools "eca__read_file"))) + (is (not (contains? tools "eca__search_tools"))))) + (testing "search_tools is dropped when nothing actually matches" + (let [tools (tools-by-name (include "unknown-mcp__.*"))] + (is (not (contains? tools "eca__search_tools"))))) + (testing "a catch-all includePattern with no MCP servers is a no-op" + (let [tools (tools-by-name (include ".*") {})] + (is (not (contains? tools "eca__search_tools"))) + (is (contains? tools "eca__read_file")) + (is (not-any? :deferrable (vals tools))))) + (testing "agent patterns merge with the global ones" + (let [tools (tools-by-name {:mcpToolSearch {:includePattern ["clojureMCP__eval"]} + :agent {"code" {:mcpToolSearch {:includePattern ["clojureMCP__sync_deps"]}}}})] + (is (match? {:deferrable true} (get tools "clojureMCP__eval"))) + (is (match? {:deferrable true} (get tools "clojureMCP__sync_deps"))))) + (testing "an agent excludePattern applies to a global includePattern" + (let [tools (tools-by-name {:mcpToolSearch {:includePattern [".*"]} + :agent {"code" {:mcpToolSearch {:excludePattern ["clojureMCP__eval"]}}}})] + (is (nil? (:deferrable (get tools "clojureMCP__eval")))) + (is (match? {:deferrable true} (get tools "clojureMCP__sync_deps"))))) + (testing "tools already loaded by the chat stop being deferred" + (let [db (assoc-in deferred-db [:chats "123" :activated-tools] #{"clojureMCP__eval"}) + tools (tools-by-name (include "clojureMCP__.*") db)] + (is (match? {:deferrable true :deferred false} (get tools "clojureMCP__eval"))) + (is (match? {:deferrable true :deferred true} (get tools "clojureMCP__sync_deps"))))) + (testing "deferred tools stay resolvable and callable" + (let [tools (vals (tools-by-name (include "clojureMCP__.*")))] + (is (match? {:full-name "clojureMCP__eval"} + (f.tools/resolve-tool "clojureMCP__eval" tools))))) + (testing "tools-for-llm drops only the not-yet-loaded ones" + (let [tools (vals (tools-by-name (include "clojureMCP__.*"))) + sent (set (map :full-name (f.tools/tools-for-llm tools)))] + (is (not (contains? sent "clojureMCP__eval"))) + (is (contains? sent "eca__search_tools")) + (is (contains? sent "eca__read_file")))))) + +(defn ^:private defer-all-db + "MCP catalog whose two tool descriptions are `description`, on a 10k context model." + [description] + {:chats {"123" {:model "prov/model"}} + :models {"prov/model" {:limit {:context 10000}}} + :mcp-clients {"bigMCP" {:version "1.0" + :tools [{:name "one" :description description :parameters {}} + {:name "two" :description description :parameters {}}]}}}) + +(deftest mcp-tool-search-defer-all-test + ;; ~4000 tokens of definitions against a 10k context: over 10%, under 90%. + (let [big (apply str (repeat 8000 "x")) + small "tiny" + tools-by-name (fn [db config] + (into {} (map (juxt :full-name identity)) + (f.tools/all-tools "123" "code" db config))) + deferred-names (fn [db config] + (set (map :full-name (filter :deferrable (vals (tools-by-name db config))))))] + (testing "no deferAllWhenTotalTokensExceedPercentOfContext leaves even a huge catalog loaded" + (is (empty? (deferred-names (defer-all-db big) {}))) + (is (empty? (deferred-names (defer-all-db big) {:mcpToolSearch {:deferAllWhenTotalTokensExceedPercentOfContext nil}})))) + (testing "a catalog over the percent defers every MCP tool" + (let [tools (tools-by-name (defer-all-db big) {:mcpToolSearch {:deferAllWhenTotalTokensExceedPercentOfContext 10}})] + (is (match? {:deferrable true :deferred true} (get tools "bigMCP__one"))) + (is (match? {:deferrable true :deferred true} (get tools "bigMCP__two"))) + (is (contains? tools "eca__search_tools")) + (is (nil? (:deferrable (get tools "eca__read_file")))))) + (testing "a catalog under the percent stays loaded" + (is (empty? (deferred-names (defer-all-db big) {:mcpToolSearch {:deferAllWhenTotalTokensExceedPercentOfContext 90}}))) + ;; Also proves native tools are left out of the total: ECA's own definitions + ;; are well past 10% of a 10k context, so counting them would defer here. + (is (empty? (deferred-names (defer-all-db small) {:mcpToolSearch {:deferAllWhenTotalTokensExceedPercentOfContext 10}})))) + (testing "excludePattern still wins over the automatic limit" + (is (= #{"bigMCP__one"} + (deferred-names (defer-all-db big) + {:mcpToolSearch {:deferAllWhenTotalTokensExceedPercentOfContext 10 + :excludePattern ["bigMCP__two"]}})))) + (testing "includePattern still defers while under the limit" + (is (= #{"bigMCP__one"} + (deferred-names (defer-all-db small) + {:mcpToolSearch {:deferAllWhenTotalTokensExceedPercentOfContext 10 + :includePattern ["bigMCP__one"]}})))) + (testing "an unknown context window never defers automatically" + (is (empty? (deferred-names (update (defer-all-db big) :models dissoc "prov/model") + {:mcpToolSearch {:deferAllWhenTotalTokensExceedPercentOfContext 10}})))) + (testing "percent 0 defers as soon as there is any MCP tool" + (is (= #{"bigMCP__one" "bigMCP__two"} + (deferred-names (defer-all-db small) {:mcpToolSearch {:deferAllWhenTotalTokensExceedPercentOfContext 0}})))) + (testing "the agent percent overrides the global one" + (is (empty? (deferred-names (defer-all-db big) + {:mcpToolSearch {:deferAllWhenTotalTokensExceedPercentOfContext 10} + :agent {"code" {:mcpToolSearch {:deferAllWhenTotalTokensExceedPercentOfContext nil}}}}))) + (is (= #{"bigMCP__one" "bigMCP__two"} + (deferred-names (defer-all-db big) + {:agent {"code" {:mcpToolSearch {:deferAllWhenTotalTokensExceedPercentOfContext 10}}}})))))) (deftest approval-test (let [read-tool {:name "read" :server {:name "eca"} :origin :native}