From a0bcd1aacf10af283b1caa8a91762dc5c368c3d8 Mon Sep 17 00:00:00 2001 From: Kyle Date: Sun, 1 Nov 2026 23:23:32 +0800 Subject: [PATCH 01/19] feat: add tools, retrieval, MCP and subagents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 给 Agent 装上工具、知识库、MCP 和子 Agent，并同步收紧权限。 - 工具：ChatTools 提供时间、四则运算（手写递归下降解析，限制长度、 深度和数字位数）和按调用者隔离的示例工具；调用者身份通过 RuntimeContext 按类型注入，不进入工具的参数 Schema - 检索：retrieve 工具返回带 [1][2] 编号的引用；knowledge 包自带 RetrievePort，关键词检索兜底，配置了 embedding 服务时与向量检索按 RRF 融合，任一路失败退回另一路。不采用已弃用的 rag-simple - MCP：只允许 HTTP 传输，配置 stdio 时启动即失败；只注册声明为只读 的工具，因为 HTTP 接口没有人工确认通道 - 子 Agent：编程式声明 summarizer，工作区 Markdown 声明 kb-researcher；与代码或内置子 Agent 重名时启动失败（2.0.4 默认是 文件覆盖）；子 Agent 通过 ToolsConfig 拿不到联网、文件和 shell 工具 - 权限：显式传入 toolsConfig，避免读取工作区 tools.json 拉起 stdio 进程；disableWebTools 与 deny 并用，工具清单由测试精确固化 - 补充工具、检索、MCP、子 Agent、权限和端到端对话测试，README 中英同步 --- README.md | 42 ++- README_CN.md | 42 ++- src/main/java/com/yoke/api/ChatApi.java | 38 ++- .../com/yoke/config/AgentConfiguration.java | 17 +- .../com/yoke/config/HarnessAgentFactory.java | 20 +- .../yoke/config/KnowledgeConfiguration.java | 62 ++++ .../java/com/yoke/config/SharedToolkit.java | 28 ++ .../com/yoke/config/ToolConfiguration.java | 46 +++ .../java/com/yoke/config/YokeProperties.java | 123 +++++++- .../java/com/yoke/config/YokeSubagents.java | 147 +++++++++ .../com/yoke/controller/ChatController.java | 3 + .../com/yoke/knowledge/EmbeddingClient.java | 11 + .../yoke/knowledge/HybridRetrievePort.java | 96 ++++++ .../yoke/knowledge/KeywordRetrievePort.java | 130 ++++++++ .../com/yoke/knowledge/KnowledgeCorpus.java | 90 ++++++ .../yoke/knowledge/OpenAiEmbeddingClient.java | 74 +++++ src/main/java/com/yoke/knowledge/Passage.java | 11 + .../java/com/yoke/knowledge/RetrievePort.java | 20 ++ .../yoke/knowledge/VectorRetrievePort.java | 101 ++++++ .../java/com/yoke/mcp/McpConnections.java | 142 +++++++++ src/main/java/com/yoke/tool/ChatTools.java | 109 +++++++ .../com/yoke/tool/ExpressionEvaluator.java | 201 ++++++++++++ src/main/java/com/yoke/tool/RetrieveTool.java | 75 +++++ src/main/java/com/yoke/tool/UserContext.java | 13 + src/main/resources/application-local.yml | 4 + src/main/resources/application-prod.yml | 4 + src/main/resources/application-test.yml | 4 + src/main/resources/application.yml | 22 ++ src/main/resources/knowledge/api-usage.md | 13 + .../knowledge/sessions-and-storage.md | 13 + .../knowledge/tools-and-permissions.md | 16 + src/main/resources/knowledge/yoke-overview.md | 10 + .../workspace/subagents/kb-researcher.md | 12 + src/test/java/com/yoke/ArchitectureTests.java | 25 ++ src/test/java/com/yoke/ProfilesTests.java | 12 + .../java/com/yoke/ToolPermissionTests.java | 57 +++- .../java/com/yoke/ToolsEndToEndTests.java | 127 ++++++++ .../AgentConfigurationConflictTests.java | 53 ++++ .../yoke/config/AgentConfigurationTests.java | 12 +- .../config/KnowledgeConfigurationTests.java | 81 +++++ .../yoke/config/RedisConfigurationTests.java | 4 +- .../com/yoke/config/SharedToolkitTests.java | 89 ++++++ .../com/yoke/config/SubagentContextTests.java | 44 +++ .../java/com/yoke/config/SubagentTests.java | 290 ++++++++++++++++++ .../com/yoke/config/ToolFilterWarnTests.java | 84 +++++ .../com/yoke/config/YokePropertiesTests.java | 97 +++++- .../knowledge/HybridRetrievePortTests.java | 124 ++++++++ .../knowledge/KeywordRetrievePortTests.java | 52 ++++ .../yoke/knowledge/KnowledgeCorpusTests.java | 65 ++++ .../knowledge/OpenAiEmbeddingClientTests.java | 127 ++++++++ .../knowledge/VectorRetrievePortTests.java | 109 +++++++ .../com/yoke/mcp/McpConnectionsTests.java | 233 ++++++++++++++ .../java/com/yoke/mcp/McpStartupTests.java | 82 +++++ .../java/com/yoke/support/McpTestServer.java | 169 ++++++++++ .../java/com/yoke/support/ScriptedModel.java | 94 ++++++ .../java/com/yoke/support/TestAgents.java | 35 +++ .../java/com/yoke/tool/ChatToolsTests.java | 184 +++++++++++ .../yoke/tool/ExpressionEvaluatorTests.java | 128 ++++++++ .../java/com/yoke/tool/RetrieveToolTests.java | 95 ++++++ .../com/yoke/tool/ToolConversationTests.java | 177 +++++++++++ 60 files changed, 4342 insertions(+), 46 deletions(-) create mode 100644 src/main/java/com/yoke/config/KnowledgeConfiguration.java create mode 100644 src/main/java/com/yoke/config/SharedToolkit.java create mode 100644 src/main/java/com/yoke/config/ToolConfiguration.java create mode 100644 src/main/java/com/yoke/config/YokeSubagents.java create mode 100644 src/main/java/com/yoke/knowledge/EmbeddingClient.java create mode 100644 src/main/java/com/yoke/knowledge/HybridRetrievePort.java create mode 100644 src/main/java/com/yoke/knowledge/KeywordRetrievePort.java create mode 100644 src/main/java/com/yoke/knowledge/KnowledgeCorpus.java create mode 100644 src/main/java/com/yoke/knowledge/OpenAiEmbeddingClient.java create mode 100644 src/main/java/com/yoke/knowledge/Passage.java create mode 100644 src/main/java/com/yoke/knowledge/RetrievePort.java create mode 100644 src/main/java/com/yoke/knowledge/VectorRetrievePort.java create mode 100644 src/main/java/com/yoke/mcp/McpConnections.java create mode 100644 src/main/java/com/yoke/tool/ChatTools.java create mode 100644 src/main/java/com/yoke/tool/ExpressionEvaluator.java create mode 100644 src/main/java/com/yoke/tool/RetrieveTool.java create mode 100644 src/main/java/com/yoke/tool/UserContext.java create mode 100644 src/main/resources/knowledge/api-usage.md create mode 100644 src/main/resources/knowledge/sessions-and-storage.md create mode 100644 src/main/resources/knowledge/tools-and-permissions.md create mode 100644 src/main/resources/knowledge/yoke-overview.md create mode 100644 src/main/resources/workspace/subagents/kb-researcher.md create mode 100644 src/test/java/com/yoke/ToolsEndToEndTests.java create mode 100644 src/test/java/com/yoke/config/AgentConfigurationConflictTests.java create mode 100644 src/test/java/com/yoke/config/KnowledgeConfigurationTests.java create mode 100644 src/test/java/com/yoke/config/SharedToolkitTests.java create mode 100644 src/test/java/com/yoke/config/SubagentContextTests.java create mode 100644 src/test/java/com/yoke/config/SubagentTests.java create mode 100644 src/test/java/com/yoke/config/ToolFilterWarnTests.java create mode 100644 src/test/java/com/yoke/knowledge/HybridRetrievePortTests.java create mode 100644 src/test/java/com/yoke/knowledge/KeywordRetrievePortTests.java create mode 100644 src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java create mode 100644 src/test/java/com/yoke/knowledge/OpenAiEmbeddingClientTests.java create mode 100644 src/test/java/com/yoke/knowledge/VectorRetrievePortTests.java create mode 100644 src/test/java/com/yoke/mcp/McpConnectionsTests.java create mode 100644 src/test/java/com/yoke/mcp/McpStartupTests.java create mode 100644 src/test/java/com/yoke/support/McpTestServer.java create mode 100644 src/test/java/com/yoke/support/ScriptedModel.java create mode 100644 src/test/java/com/yoke/support/TestAgents.java create mode 100644 src/test/java/com/yoke/tool/ChatToolsTests.java create mode 100644 src/test/java/com/yoke/tool/ExpressionEvaluatorTests.java create mode 100644 src/test/java/com/yoke/tool/RetrieveToolTests.java create mode 100644 src/test/java/com/yoke/tool/ToolConversationTests.java diff --git a/README.md b/README.md index 1094b85..9357e48 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ A yoke is the wooden frame that makes oxen and horses pull in the direction you - One endpoint for multi-turn conversations: `POST /api/chat`, plus `POST /api/chat/stream` that streams the reasoning process over SSE. - A temporary `HarnessAgent` is built for every request and closed afterwards; conversation history lives in Redis, so it survives restarts and is shared by all replicas. History is isolated per `(userId, sessionId)`, and long conversations are compacted automatically. -- The agent has no file, shell or web tools and sub-agents are switched off, and a logging middleware records a timing trace for every call. +- Tools, knowledge and sub-agents: calculator, clock and per-user notes tools, a `retrieve` tool over a built-in knowledge base that returns `[1][2]`-numbered citations, external MCP servers over HTTP, and two sub-agents (one declared in code, one declared as a workspace Markdown file). The agent still has no file, shell or web tools, and a logging middleware records a timing trace for every call. - Models are plugged in through the official AgentScope starters: OpenAI-compatible APIs (works with DashScope, DeepSeek and others), Anthropic, Gemini and Ollama. Switching is a dependency and configuration change. - `local` / `test` / `prod` profiles. In production, a missing API key fails the startup instead of shipping with an empty key. - Virtual threads are enabled by default, for both Tomcat and Spring's default thread pool. @@ -105,8 +105,8 @@ Every stream ends with either `done` or `error`. A blank `message` returns 400 b Configuration is split by profile. The `local` profile is active when none is specified; switch with the `SPRING_PROFILES_ACTIVE=prod` environment variable or the `--spring.profiles.active=prod` command-line argument. -- `application.yml`: Shared by all profiles: model, agent, virtual threads, log format, workspace, key prefix and compaction -- `application-local.yml`: Local development: API key from the environment (may be empty), Redis at `redis://127.0.0.1:6379` (the one from `docker-compose.yml`), `com.yoke` logs at debug +- `application.yml`: Shared by all profiles: model, agent, virtual threads, log format, workspace, key prefix, compaction, knowledge-base embedding and the (commented) MCP example +- `application-local.yml`: Local development: API keys from the environment (may be empty), Redis at `redis://127.0.0.1:6379` (the one from `docker-compose.yml`), `com.yoke` logs at debug - `application-test.yml`: Automated tests: a fake API key, the model is never really called; the Redis address is injected by the tests - `application-prod.yml`: Production: the API key and the Redis address (`YOKE_REDIS_URI`) must come from the environment, startup fails without them; graceful shutdown @@ -129,8 +129,31 @@ Yoke's own settings use the `yoke.*` prefix: | `yoke.compaction.trigger-messages` | Compact the conversation once it reaches this many messages, default 30 | | `yoke.compaction.keep-messages` | Messages kept verbatim after a compaction, default 10. Values `<= 0`, or a keep count not smaller than the trigger count, fall back to the defaults | | `yoke.workspace` | `HarnessAgent` workspace for local files (offloaded messages, memory), default `~/.yoke/workspace`; it stays on the machine and is not shared between replicas | +| `yoke.knowledge.embedding.model-name` / `api-key` / `base-url` / `dimensions` | Optional embedding service (OpenAI-compatible `/embeddings`) for vector retrieval. Vector retrieval is enabled only when both `model-name` and `api-key` (`YOKE_EMBEDDING_API_KEY`) are set; otherwise retrieval uses keywords only. If the service fails at runtime, retrieval falls back to keywords automatically | +| `yoke.mcp.servers[]` | External MCP servers, none by default. Each entry has `name`, `transport` (`sse` or `streamable-http` only), `url`, `timeout` (default 30s) and optional `headers`. `stdio` is rejected and the application fails at startup; so is a server that cannot be reached | -The agent is built without file, shell or web tools, and sub-agents are disabled as a whole: reading the 2.0.4 source shows that a sub-agent inherits only the file and shell switches, not the web switch, so leaving sub-agents on would let the model spawn one that can fetch arbitrary URLs. What remains is limited to the memory and session tools (`memory_save` only appends to the fixed `MEMORY.md` and `memory/.md` in the workspace, without a size limit), `load_skill_through_path` and `wait_async_results`. A `MiddlewareBase` bean is attached to every new agent; `LoggingMiddleware` logs agent, model and tool timings without tool arguments or outputs. OpenTelemetry tracing is not attached by default: add an OpenTelemetry SDK and exporter and register an `OtelTracingMiddleware` bean when you need it. +The agent is built without file, shell or web tools. Reading the 2.0.4 source shows that a sub-agent inherits only the file and shell switches, not the web switch, so the web tools are also denied through a `ToolsConfig` deny list (on top of `disableWebTools()` for the main agent), which is applied to the main agent, the built-in `general-purpose` sub-agent and every declared sub-agent. Passing an explicit `ToolsConfig` also stops 2.0.4 from reading `tools.json` from the workspace, which could otherwise declare `stdio` MCP servers (processes started on the host). The model sees exactly these tools: Yoke's own (`get_current_time`, `calculate`, `list_my_notes`, `retrieve`), the framework's memory, session, skill-loading, async-result, sub-agent and background-task tools (`memory_save` only appends to the fixed `MEMORY.md` and `memory/.md` in the workspace, without a size limit), plus whatever the configured MCP servers publish. `ToolPermissionTests` asserts this list as an exact set, so a framework upgrade that adds a tool fails the build. A `MiddlewareBase` bean is attached to every new agent; `LoggingMiddleware` logs agent, model and tool timings without tool arguments or outputs. OpenTelemetry tracing is not attached by default: add an OpenTelemetry SDK and exporter and register an `OtelTracingMiddleware` bean when you need it. + +## Tools, Knowledge and Sub-agents + +- **Tools** (`com.yoke.tool`): plain classes with `@Tool` methods, registered once into a shared `Toolkit`. `calculate` uses a hand-written recursive-descent parser (no script engine, input length and nesting limits, division by zero reported as text). `list_my_notes` takes a `UserContext` parameter without `@ToolParam`: the framework injects it by type from the request's `RuntimeContext`, so the model never sees or forges the user id. +- **Retrieval**: the `retrieve` tool searches `src/main/resources/knowledge/*.md` and returns passages numbered `[1]`, `[2]` with their source, which the model is asked to cite. It is the only retrieval channel; the framework's `knowledge()` auto-injection is deliberately not used. Retrieval is layered behind the application's own `RetrievePort` (no framework types): BM25 keyword search is always available, and with an embedding service configured a vector channel is added and the two are fused with RRF (constant 60); if one channel fails the other is used. Note that with embeddings enabled, user queries and the knowledge-base text are sent to the embedding provider. The framework's own `rag-simple` module is not used: the `core/rag` package is `@Deprecated(forRemoval = true)` in 2.0.4 and the module drags in Qdrant, Milvus, Elasticsearch, PostgreSQL, PDFBox, POI, Tika and more. +- **MCP**: only HTTP transports are accepted. Example: + + ```yaml + yoke: + mcp: + servers: + - name: docs + transport: streamable-http # or sse + url: https://mcp.example.com/mcp + timeout: 30s + headers: + Authorization: Bearer ${MCP_DOCS_TOKEN} + ``` + + Tools appear under the names the server publishes. In 2.0.4 an MCP tool runs without confirmation only if the server marks it `readOnlyHint`, and Yoke's HTTP API has no human confirmation channel (the turn would otherwise end silently at the tool call with an empty reply). So only tools declared read-only are registered; the others are skipped (their names are logged at INFO) and neither the main agent nor any sub-agent ever sees them. +- **Sub-agents**: `summarizer` is declared in code (`YokeSubagents`) with no tools; `kb-researcher` is declared by `src/main/resources/workspace/subagents/kb-researcher.md`, copied into `yoke.workspace/subagents/` at startup, and may only use `retrieve`. 2.0.4 silently lets a workspace file override a code declaration with the same id (and `general-purpose.md` would replace the built-in one), so Yoke scans the files actually present in `yoke.workspace/subagents/` (including leftovers and hand-placed ones, not just the shipped ones) and fails at startup on any such clash. To use another model, replace `agentscope-openai-spring-boot-starter` in `build.gradle` with the matching starter (dashscope, anthropic, gemini or ollama) and change `agentscope.model.provider`. @@ -141,11 +164,14 @@ src/main/java/com/yoke/ ├── YokeApplication.java Application entry point ├── api/ HTTP contract (ChatApi: paths, validation and docs live here) ├── codec/ Conversion between domain types and framework types (MessageCodec, EventCodec) -├── config/ Agent and Redis configuration (HarnessAgentFactory, YokeProperties) +├── config/ Agent, Redis, tool, knowledge and sub-agent configuration (HarnessAgentFactory, YokeProperties, YokeSubagents) ├── controller/ Contract implementation (ChatController) +├── knowledge/ Retrieval port and implementations: keyword, vector, RRF fusion (no framework types) +├── mcp/ Connections to external MCP servers (HTTP transports only) ├── middleware/ Agent middlewares (LoggingMiddleware) -└── model/ Request, response and stream event objects (ChatRequest, ChatResponse, ChatEvent) -src/main/resources/ application.yml and the per-profile configuration +├── model/ Request, response and stream event objects (ChatRequest, ChatResponse, ChatEvent) +└── tool/ Tools exposed to the model (ChatTools, RetrieveTool, UserContext) +src/main/resources/ application.yml, the per-profile configuration, knowledge/*.md and workspace/subagents/*.md docker-compose.yml Local Redis, also used by the integration tests docs/brand/ Logo ``` @@ -156,7 +182,7 @@ docs/brand/ Logo ## Contributing -Issues and pull requests are welcome. AgentScope message and event types must not leak out of `codec` (and `middleware`, whose hook signatures need them); `ArchitectureTests` enforces this. Before submitting, run `./gradlew build` and make sure all tests pass. Code comments are written in Chinese. +Issues and pull requests are welcome. AgentScope message and event types must not leak out of `codec` (and `middleware`, whose hook signatures need them), and the `knowledge` package must stay free of AgentScope types; `ArchitectureTests` enforces this. Before submitting, run `./gradlew build` and make sure all tests pass. Code comments are written in Chinese. ## License diff --git a/README_CN.md b/README_CN.md index ea6cad0..8354246 100644 --- a/README_CN.md +++ b/README_CN.md @@ -21,7 +21,7 @@ Yoke（轭）是套在牛马身上、让它们听使唤拉车的木架；Yoke - 一个接口完成多轮对话：`POST /api/chat`；另有 `POST /api/chat/stream`，通过 SSE 边推理边推送。 - 每次请求新建临时 `HarnessAgent`，用完即关闭；对话历史保存在 Redis 里，重启不丢、多副本共享，按 `(userId, sessionId)` 隔离，对话过长时自动压缩。 -- Agent 不开放文件、命令和联网工具，子 Agent 整体关闭，日志中间件为每次调用记录耗时轨迹。 +- 工具、知识库与子 Agent：计算器、时钟、按用户隔离的笔记工具，基于内置知识库的 `retrieve` 工具（返回带 `[1][2]` 编号的引用），通过 HTTP 接入外部 MCP 服务，以及两个子 Agent（一个在代码里声明，一个由工作区 Markdown 文件声明）。Agent 仍不开放文件、命令和联网工具，日志中间件为每次调用记录耗时轨迹。 - 通过 AgentScope 官方 starter 接入各种模型：OpenAI 兼容协议（可对接 DashScope、DeepSeek 等）、Anthropic、Gemini、Ollama，换依赖和配置即可。 - local / test / prod 多环境配置；生产环境缺少密钥时启动即失败，避免带着空密钥上线。 - 默认开启虚拟线程，Tomcat 和 Spring 默认线程池都使用虚拟线程。 @@ -108,8 +108,8 @@ data:{"sessionId":"demo","text":"北京今天晴，25 度。","usage":{"inputTok 配置按环境拆分，没有显式指定环境时默认激活 `local`；部署时用环境变量 `SPRING_PROFILES_ACTIVE=prod` 或命令行参数 `--spring.profiles.active=prod` 切换。 -- `application.yml`：各环境通用：模型、Agent、虚拟线程、日志格式、工作区、键前缀和压缩配置 -- `application-local.yml`：本地开发：密钥读环境变量（可为空），Redis 连 `redis://127.0.0.1:6379`（即 `docker-compose.yml` 里的那个），`com.yoke` 日志为 debug +- `application.yml`：各环境通用：模型、Agent、虚拟线程、日志格式、工作区、键前缀、压缩、知识库向量化配置和（注释形式的）MCP 示例 +- `application-local.yml`：本地开发：各密钥读环境变量（可为空），Redis 连 `redis://127.0.0.1:6379`（即 `docker-compose.yml` 里的那个），`com.yoke` 日志为 debug - `application-test.yml`：自动化测试：假密钥，不会真正调用模型；Redis 地址由测试注入 - `application-prod.yml`：生产：密钥和 Redis 地址（`YOKE_REDIS_URI`）必须由环境变量提供，缺失则启动失败；优雅停机 @@ -132,8 +132,31 @@ Yoke 自己的配置使用 `yoke.*` 前缀： | `yoke.compaction.trigger-messages` | 对话消息达到这个条数就压缩，默认 30 | | `yoke.compaction.keep-messages` | 压缩后原样保留的最近条数，默认 10。小于等于 0，或保留条数不小于触发条数时，回落到默认值 | | `yoke.workspace` | `HarnessAgent` 的工作区，放本地文件（转存的消息、记忆），默认 `~/.yoke/workspace`；只在本机，副本之间不共享 | +| `yoke.knowledge.embedding.model-name` / `api-key` / `base-url` / `dimensions` | 可选的向量化服务（OpenAI 兼容的 `/embeddings`），用于向量检索。`model-name` 和 `api-key`（`YOKE_EMBEDDING_API_KEY`）都有值才启用向量检索，否则只用关键词检索；运行中向量化服务出错时自动退回关键词检索 | +| `yoke.mcp.servers[]` | 外部 MCP 服务，默认一个也不接。每项有 `name`、`transport`（只能是 `sse` 或 `streamable-http`）、`url`、`timeout`（默认 30 秒）和可选的 `headers`。配置 `stdio` 会让应用启动失败；服务连不上同样启动失败 | -Agent 创建时不开放文件、命令和联网工具，并且整体关闭子 Agent：读 2.0.4 源码发现子 Agent 只继承文件和 shell 的关闭、不继承联网工具的关闭，开着子 Agent 就能让模型派生出能抓取任意 URL 的子 Agent。关闭后剩下的只有记忆和会话类工具（`memory_save` 只往工作区固定的 `MEMORY.md` 与 `memory/<日期>.md` 追加，内容没有大小限制）、`load_skill_through_path` 和 `wait_async_results`。容器里的 `MiddlewareBase` Bean 会挂到每个新建的 Agent 上；`LoggingMiddleware` 记录 Agent、模型、工具的耗时，不记录工具的参数和输出。OpenTelemetry 追踪默认不挂载：需要时自行引入 OpenTelemetry SDK 和导出器，再注册一个 `OtelTracingMiddleware` Bean。 +Agent 创建时不开放文件、命令和联网工具。读 2.0.4 源码发现子 Agent 只继承文件和 shell 的关闭、不继承联网工具的关闭，所以联网工具除了主 Agent 的 `disableWebTools()`，还用 `ToolsConfig` 的 deny 名单禁掉，它同时作用于主 Agent、内置的 `general-purpose` 子 Agent 和所有声明的子 Agent。显式传入 `ToolsConfig` 还会让 2.0.4 不再去读工作区里的 `tools.json`（那个文件可以声明 `stdio` MCP 服务，也就是在宿主机上启动进程）。模型看到的工具恰好是：Yoke 自己的（`get_current_time`、`calculate`、`list_my_notes`、`retrieve`），框架的记忆、会话、技能加载、异步结果、子 Agent 和后台任务类工具（`memory_save` 只往工作区固定的 `MEMORY.md` 与 `memory/<日期>.md` 追加，内容没有大小限制），再加上所配置的 MCP 服务公布的工具。`ToolPermissionTests` 用精确集合断言这份清单，框架升级新增了工具会让构建失败。容器里的 `MiddlewareBase` Bean 会挂到每个新建的 Agent 上；`LoggingMiddleware` 记录 Agent、模型、工具的耗时，不记录工具的参数和输出。OpenTelemetry 追踪默认不挂载：需要时自行引入 OpenTelemetry SDK 和导出器，再注册一个 `OtelTracingMiddleware` Bean。 + +## 工具、知识库与子 Agent + +- **工具**（`com.yoke.tool`）：带 `@Tool` 方法的普通类，启动时一次性注册进共享的 `Toolkit`。`calculate` 用手写的递归下降解析器（不用脚本引擎，输入长度和嵌套深度有上限，除零以文本形式报告）。`list_my_notes` 的 `UserContext` 参数不带 `@ToolParam`：框架按类型从本次请求的 `RuntimeContext` 里注入，模型既看不到也伪造不了用户标识。 +- **检索**：`retrieve` 工具检索 `src/main/resources/knowledge/*.md`，返回带 `[1]`、`[2]` 编号和来源的片段，并要求模型据此引用。它是唯一的检索通道，有意不同时使用框架的 `knowledge()` 自动注入。检索分层放在应用自己的 `RetrievePort` 后面（不含框架类型）：BM25 关键词检索永远可用；配置了向量化服务就再加一路向量检索，两路用 RRF（常数 60）融合，任何一路失败就用另一路。注意开启向量化后，用户的检索问题和知识库语料会发送给 embedding 服务提供方。没有使用框架自带的 `rag-simple`：`core/rag` 包在 2.0.4 里已 `@Deprecated(forRemoval = true)`，而且该模块会拖进 Qdrant、Milvus、Elasticsearch、PostgreSQL、PDFBox、POI、Tika 等一大批依赖。 +- **MCP**：只接受 HTTP 传输。示例： + + ```yaml + yoke: + mcp: + servers: + - name: docs + transport: streamable-http # 或 sse + url: https://mcp.example.com/mcp + timeout: 30s + headers: + Authorization: Bearer ${MCP_DOCS_TOKEN} + ``` + + 工具以服务端公布的名字出现。2.0.4 里 MCP 工具只有服务端声明了 `readOnlyHint` 才会免确认执行，而 Yoke 的 HTTP 接口没有人工确认通道（否则一轮对话会在工具调用处无声结束、回复为空）。所以只注册声明了只读的工具，其余跳过（工具名记在 INFO 日志里），主 Agent 和子 Agent 都看不到它们。 +- **子 Agent**：`summarizer` 在代码里声明（`YokeSubagents`），没有任何工具；`kb-researcher` 由 `src/main/resources/workspace/subagents/kb-researcher.md` 声明，启动时拷贝到 `yoke.workspace/subagents/`，只能用 `retrieve`。2.0.4 对同 id 冲突是静默的：工作区文件会覆盖代码里的声明（`general-purpose.md` 还会替换内置的那个），所以 Yoke 会扫描 `yoke.workspace/subagents/` 里实际存在的全部文件（包括遗留的、手放的，不只是随应用发布的那几个），遇到任何这类重名都直接启动失败。 要换成其他模型，把 `build.gradle` 里的 `agentscope-openai-spring-boot-starter` 换成对应的 starter（dashscope、anthropic、gemini、ollama），并修改 `agentscope.model.provider`。 @@ -144,11 +167,14 @@ src/main/java/com/yoke/ ├── YokeApplication.java 应用入口 ├── api/ HTTP 接口契约（ChatApi，路径、校验和文档都在这里） ├── codec/ 领域类型与框架类型之间的转换（MessageCodec、EventCodec） -├── config/ Agent 与 Redis 配置（HarnessAgentFactory、YokeProperties） +├── config/ Agent、Redis、工具、知识库与子 Agent 配置（HarnessAgentFactory、YokeProperties、YokeSubagents） ├── controller/ 接口实现（ChatController） +├── knowledge/ 检索端口及实现：关键词、向量、RRF 融合（不含框架类型） +├── mcp/ 到外部 MCP 服务的连接（只支持 HTTP 传输） ├── middleware/ Agent 中间件（LoggingMiddleware） -└── model/ 请求、响应与流式事件对象（ChatRequest、ChatResponse、ChatEvent） -src/main/resources/ application.yml 及各环境配置 +├── model/ 请求、响应与流式事件对象（ChatRequest、ChatResponse、ChatEvent） +└── tool/ 暴露给模型的工具（ChatTools、RetrieveTool、UserContext） +src/main/resources/ application.yml、各环境配置、knowledge/*.md 和 workspace/subagents/*.md docker-compose.yml 本地 Redis，集成测试也用它 docs/brand/ logo ``` @@ -160,7 +186,7 @@ docs/brand/ logo ## 参与贡献 -欢迎提交 Issue 和 Pull Request。提交前请运行 `./gradlew build` 确保全部测试通过；代码注释使用中文。AgentScope 的消息和事件类型不能泄漏到 `codec` 之外（`middleware` 除外，它的钩子签名本身就要用这些类型），`ArchitectureTests` 会检查这一点。 +欢迎提交 Issue 和 Pull Request。提交前请运行 `./gradlew build` 确保全部测试通过；代码注释使用中文。AgentScope 的消息和事件类型不能泄漏到 `codec` 之外（`middleware` 除外，它的钩子签名本身就要用这些类型），`knowledge` 包不能出现任何 AgentScope 类型，`ArchitectureTests` 会检查这些。 ## 许可证 diff --git a/src/main/java/com/yoke/api/ChatApi.java b/src/main/java/com/yoke/api/ChatApi.java index 840bcf8..25baea2 100644 --- a/src/main/java/com/yoke/api/ChatApi.java +++ b/src/main/java/com/yoke/api/ChatApi.java @@ -38,13 +38,37 @@ *

实现说明：暂时没有鉴权，{@code userId} 由调用方自报，不可信，不要把它当成身份校验； * 同一个会话的并发请求不会被串行化，由调用方保证同一会话同一时间只发一个请求。 * - *

工具权限：Agent 不开放文件读写、命令执行和联网工具，也整体关闭了子 Agent。 - * 读 2.0.4 源码发现子 Agent 只继承文件和 shell 的关闭、不继承联网工具的关闭， - * 开着子 Agent 就能让模型通过 {@code agent_spawn} 派生出能联网的子 Agent，所以整体关闭。 - * 关闭后模型仍能用的只有 {@code memory_save}、{@code memory_get}、{@code memory_search}、{@code session_history}、 - * {@code session_list}、{@code session_search}、{@code load_skill_through_path} 和 {@code wait_async_results}： - * 联网路径已经没有；{@code memory_save} 是唯一写盘的，只往工作区里固定的 {@code MEMORY.md} 和 - * {@code memory/<日期>.md} 追加，路径不由模型控制，但追加的内容没有大小限制。 + *

模型可以使用的工具（精确清单由 ToolPermissionTests 守着）： + *

    + *
  • Yoke 自带：{@code get_current_time}（带时区偏移的当前时间）、{@code calculate}（四则运算，手写解析器，不执行任何代码）、 + * {@code list_my_notes}（当前用户的笔记，演示数据）、{@code retrieve}（在 Yoke 自己的知识库里检索）；
    • + *
  • 框架内置：{@code memory_save}、{@code memory_get}、{@code memory_search}、{@code session_history}、 + * {@code session_list}、{@code session_search}、{@code load_skill_through_path}、{@code wait_async_results}， + * 以及子 Agent 和后台任务相关的 {@code agent_spawn}、{@code agent_send}、{@code agent_list}、 + * {@code task_output}、{@code task_cancel}、{@code task_list}；
    • + *
  • 配置了 {@code yoke.mcp.servers} 时，再加上 MCP 服务公布的工具，工具名是服务端公布的原名，不加前缀。
    • + *
+ * 不开放文件读写、命令执行和联网（{@code web_fetch}、{@code web_search}）：主 Agent 同时用 {@code disableWebTools} 和工具配置的 deny 名单，子 Agent 只靠 deny 名单（它们不继承 {@code disableWebTools}）。其中 {@code memory_save} 是唯一写盘的， + * 只往工作区里固定的 {@code MEMORY.md} 和 {@code memory/<日期>.md} 追加，路径不由模型控制，但追加的内容没有大小限制。 + * + *

业务上下文不经过模型：{@code list_my_notes} 要知道「当前用户是谁」，userId 由控制器放进 RuntimeContext， + * 工具按类型取用，不是模型传的参数，模型看不到也伪造不了。这个 userId 本身仍是调用方自报的（见上面的实现说明）。 + * + *

引用格式：{@code retrieve} 返回的资料带 {@code [1]}、{@code [2]} 这样的编号和来源文档标题， + * 模型回答时按相同编号标注出处，调用方可以据此回溯。检索是工具化的，只有模型认为需要时才会调用， + * 没有同时开启框架的 knowledge 自动注入。没有配置向量化服务时只用关键词检索，向量化服务出错时自动退回关键词检索。 + * + *

子 Agent：{@code summarizer}（代码里声明，只做摘要，没有任何业务工具）、{@code kb-researcher} + * （工作区文件 {@code workspace/subagents/kb-researcher.md} 声明，只能用 {@code retrieve}）和框架内置的 + * {@code general-purpose}（继承 Yoke 的业务工具）。2.0.4 里子 Agent 不继承父 Agent 的联网工具关闭，所以联网工具靠工具配置里的 + * deny 名单在所有层（主 Agent、内置子 Agent、声明的子 Agent）一并禁掉；子 Agent 同样没有文件读写和命令执行，也不能再派生子 Agent。 + * + *

MCP 工具的授权：2.0.4 对 MCP 工具只放行服务端声明了 {@code readOnlyHint} 的，其余需要用户确认， + * 而本接口没有人工确认通道，否则一轮对话会在工具调用处无声结束、调用方拿到空回复。 + * 所以注册时就只放行声明了只读的工具，其余跳过（启动日志里记工具名），模型根本看不到它们， + * 这对主 Agent 和子 Agent 都成立。MCP 服务配置的服务连不上时，应用启动失败。 + * + *

隐私：配置了向量化服务（{@code yoke.knowledge.embedding}）后，用户的检索问题和知识库语料会发送给该服务的提供方。 * *

已知限制：{@code HarnessAgent.close()} 会等待框架里全局的异步写入静默 * （{@code awaitMirrorQuiescence} 和 {@code awaitQuiescence}，各最长 5 秒），它在每个请求的清理回调里执行， diff --git a/src/main/java/com/yoke/config/AgentConfiguration.java b/src/main/java/com/yoke/config/AgentConfiguration.java index ba445e6..43672b7 100644 --- a/src/main/java/com/yoke/config/AgentConfiguration.java +++ b/src/main/java/com/yoke/config/AgentConfiguration.java @@ -1,9 +1,11 @@ package com.yoke.config; import java.nio.file.Path; +import java.util.List; import io.agentscope.core.middleware.MiddlewareBase; import io.agentscope.core.model.Model; import io.agentscope.harness.agent.DistributedStore; +import io.agentscope.harness.agent.subagent.SubagentDeclaration; import io.agentscope.spring.boot.properties.AgentProperties; import io.agentscope.spring.boot.properties.AgentscopeProperties; @@ -19,7 +21,9 @@ * （agentscope.agent.enabled=false），但名称、提示词、迭代次数仍沿用 agentscope.agent.* 的配置。 * *

对话历史存在 {@link DistributedStore}（Redis）里，见 {@link RedisConfiguration}； - * 容器里所有 {@link MiddlewareBase} Bean 都会按顺序挂到每个新建的 Agent 上，一个都没有也不影响启动。 + * 工具来自 {@link SharedToolkit}（见 {@link ToolConfiguration}），子 Agent 见 {@link YokeSubagents}。 + * + *

容器里所有 {@link MiddlewareBase} Bean 都会按顺序挂到每个新建的 Agent 上，一个都没有也不影响启动。 */ @Configuration(proxyBeanMethods = false) public class AgentConfiguration { @@ -30,16 +34,23 @@ public HarnessAgentFactory harnessAgentFactory( DistributedStore distributedStore, AgentscopeProperties properties, YokeProperties yoke, + SharedToolkit toolkit, ObjectProvider middlewares) { AgentProperties agent = properties.getAgent(); + Path workspace = Path.of(yoke.workspace()); + // 子 Agent：代码里声明的 + 工作区文件声明的，id 不许重复；文件在启动时拷进工作区，由 Harness 从那里加载 + List subagents = YokeSubagents.programmatic(); + YokeSubagents.prepareWorkspace(workspace, subagents); return new HarnessAgentFactory( agent.getName(), agent.getSysPrompt(), agent.getMaxIters(), model, distributedStore, - Path.of(yoke.workspace()), + workspace, yoke.compaction(), - middlewares.orderedStream().toList()); + middlewares.orderedStream().toList(), + toolkit.toolkit(), + subagents); } } diff --git a/src/main/java/com/yoke/config/HarnessAgentFactory.java b/src/main/java/com/yoke/config/HarnessAgentFactory.java index 0259ae3..c9841fa 100644 --- a/src/main/java/com/yoke/config/HarnessAgentFactory.java +++ b/src/main/java/com/yoke/config/HarnessAgentFactory.java @@ -5,9 +5,11 @@ import io.agentscope.core.middleware.MiddlewareBase; import io.agentscope.core.model.Model; +import io.agentscope.core.tool.Toolkit; import io.agentscope.harness.agent.DistributedStore; import io.agentscope.harness.agent.HarnessAgent; import io.agentscope.harness.agent.memory.compaction.CompactionConfig; +import io.agentscope.harness.agent.subagent.SubagentDeclaration; import lombok.RequiredArgsConstructor; /** @@ -29,6 +31,9 @@ public class HarnessAgentFactory { private final Path workspace; private final YokeProperties.Compaction compaction; private final List middlewares; + /** 共享的 Toolkit 模板，里面是 Yoke 自己的工具和 MCP 工具；启动后不再修改，Harness 的 build() 会先拷贝再往副本里加内置工具。 */ + private final Toolkit toolkit; + private final List subagents; public HarnessAgent create() { return HarnessAgent.builder() @@ -41,13 +46,20 @@ public HarnessAgent create() { .distributedStore(distributedStore) .compaction(compactionConfig()) .middlewares(middlewares) - // 权限取舍：对外服务的 Agent 不给读写文件和执行命令的能力，联网搜索同理 + // 共享同一个模板是安全的：2.0.4 的 build() 先 toolkit.copy() 再注册内置工具，模板不会被污染（见 SharedToolkitTests） + .toolkit(toolkit) + .subagents(subagents) + // 权限取舍：对外服务的 Agent 不给读写文件和执行命令的能力 .disableFilesystemTools() .disableShellTool() + // 联网工具两道防线并用：disableWebTools 管主 Agent（按开关，框架改名也有效）， + // ToolsConfig.deny 管所有层——子 Agent 只继承文件和 shell 的关闭、不继承 web 的关闭 + // （2.0.4 的 HarnessAgentBuilderSupport 只传这两个），会自己注册 web_fetch / web_search，只能靠 deny 按名字移除， + // 它会同时传给内置的 general-purpose 和所有声明的子 Agent。 + // 代价：主 Agent 构建时 web 工具未注册，ToolFilter 对 deny 里的名字每次打 WARN（实测两条），在 application.yml 里调低了级别。 + // 显式传 toolsConfig 还让 2.0.4 不再去读工作区里的 tools.json（里面能声明 stdio MCP 服务） .disableWebTools() - // 子 Agent 只继承文件和 shell 的关闭，不继承 web 的关闭（2.0.4 的 HarnessAgentBuilderSupport 只传这两个）， - // 开着就能让模型通过 agent_spawn 派生出带 web_fetch 的子 Agent 联网；第 03 轮专门引入子 Agent 时再按需打开 - .disableSubagents() + .toolsConfig(YokeSubagents.toolsConfig()) .build(); } diff --git a/src/main/java/com/yoke/config/KnowledgeConfiguration.java b/src/main/java/com/yoke/config/KnowledgeConfiguration.java new file mode 100644 index 0000000..3b0851c --- /dev/null +++ b/src/main/java/com/yoke/config/KnowledgeConfiguration.java @@ -0,0 +1,62 @@ +package com.yoke.config; + +import java.net.http.HttpClient; +import java.time.Duration; +import java.util.List; + +import com.yoke.knowledge.EmbeddingClient; +import com.yoke.knowledge.HybridRetrievePort; +import com.yoke.knowledge.KeywordRetrievePort; +import com.yoke.knowledge.KnowledgeCorpus; +import com.yoke.knowledge.OpenAiEmbeddingClient; +import com.yoke.knowledge.Passage; +import com.yoke.knowledge.RetrievePort; +import com.yoke.knowledge.VectorRetrievePort; +import lombok.extern.slf4j.Slf4j; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.JdkClientHttpRequestFactory; +import org.springframework.web.client.RestClient; + +/** + * 知识库检索的装配：语料来自 {@code classpath:knowledge/*.md}。 + * + *

检索链路分层，由里到外： + *

    + *
  1. 关键词检索（{@link KeywordRetrievePort}）永远可用，是兜底；
    2. + *
  2. 配了向量化服务（{@code yoke.knowledge.embedding}）时，再加一路向量检索， + * 两路用 RRF 融合（{@link HybridRetrievePort}），任何一路失败退回另一路。
    3. + *
+ * 所以没有 embedding 密钥时应用照常启动，检索只是退化成纯关键词。 + */ +@Slf4j +@Configuration(proxyBeanMethods = false) +public class KnowledgeConfiguration { + + static final String CORPUS_PATTERN = "classpath*:knowledge/*.md"; + + @Bean + public RetrievePort retrievePort(YokeProperties yoke) { + List passages = KnowledgeCorpus.loadFromClasspath(CORPUS_PATTERN); + log.info("知识库已加载 {} 个片段", passages.size()); + KeywordRetrievePort keyword = new KeywordRetrievePort(passages); + YokeProperties.Knowledge.Embedding embedding = yoke.knowledge().embedding(); + if (!embedding.enabled()) { + log.info("未配置 yoke.knowledge.embedding，检索只使用关键词"); + return keyword; + } + // 连接和读取都设超时：向量化服务挂了要快速失败，让上层降级到关键词，而不是拖住整个工具调用 + JdkClientHttpRequestFactory requestFactory = new JdkClientHttpRequestFactory( + HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build()); + requestFactory.setReadTimeout(Duration.ofSeconds(15)); + RestClient client = RestClient.builder() + .baseUrl(embedding.baseUrl()) + .defaultHeader("Authorization", "Bearer " + embedding.apiKey()) + .requestFactory(requestFactory) + .build(); + EmbeddingClient embeddings = new OpenAiEmbeddingClient(client, embedding.modelName(), embedding.dimensions()); + log.info("检索使用向量加关键词双路融合，向量模型：{}", embedding.modelName()); + return new HybridRetrievePort(List.of(new VectorRetrievePort(passages, embeddings), keyword)); + } +} diff --git a/src/main/java/com/yoke/config/SharedToolkit.java b/src/main/java/com/yoke/config/SharedToolkit.java new file mode 100644 index 0000000..a50db1d --- /dev/null +++ b/src/main/java/com/yoke/config/SharedToolkit.java @@ -0,0 +1,28 @@ +package com.yoke.config; + +import io.agentscope.core.tool.Toolkit; + +/** + * 启动时装配好的 Toolkit 模板（Yoke 自己的工具加 MCP 工具），所有请求的 Agent 共用这一个。 + * + *

能共用，依据是 2.0.4 的源码：{@code HarnessAgent.Builder.build()} 一上来就 {@code toolkit.copy()}， + * Harness 自己的内置工具（记忆、会话、子 Agent 等）注册在副本上，传进去的模板不会被污染； + * {@code Toolkit} 内部的注册表用的是 ConcurrentHashMap，只读的 {@code copy()} 可以并发。 + * 前提是模板在启动之后不再被修改，并且里面的工具实例无状态、线程安全（副本按引用共享工具实例）。 + * 官方 starter 把 Toolkit 注册成原型作用域 Bean，是因为它的 ReActAgent 直接使用传入的 Toolkit、不做拷贝。 + * + *

单独包一层类型，是为了不和官方 starter 注册的原型 {@code Toolkit} Bean 在按类型注入时撞车。 + * MCP 连接由 {@code McpConnections} 持有，模板和副本都不拥有连接。 + */ +public final class SharedToolkit { + + private final Toolkit toolkit; + + public SharedToolkit(Toolkit toolkit) { + this.toolkit = toolkit; + } + + public Toolkit toolkit() { + return toolkit; + } +} diff --git a/src/main/java/com/yoke/config/ToolConfiguration.java b/src/main/java/com/yoke/config/ToolConfiguration.java new file mode 100644 index 0000000..e8e70b6 --- /dev/null +++ b/src/main/java/com/yoke/config/ToolConfiguration.java @@ -0,0 +1,46 @@ +package com.yoke.config; + +import com.yoke.knowledge.RetrievePort; +import com.yoke.mcp.McpConnections; +import com.yoke.tool.ChatTools; +import com.yoke.tool.RetrieveTool; +import io.agentscope.core.tool.Toolkit; + +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * 工具装配：Yoke 自己的工具、检索工具和外部 MCP 工具汇总到一个 Toolkit 模板里。 + * + *

三个来源的工具类都是无状态的，可以安全地被所有请求的 Toolkit 副本共享。 + * 没有配置 MCP 服务时 {@link McpConnections} 是空的，不影响启动。 + */ +@Configuration(proxyBeanMethods = false) +public class ToolConfiguration { + + @Bean + public ChatTools chatTools() { + return new ChatTools(); + } + + /** 没有 {@link RetrievePort} 时检索工具仍然注册，调用时返回「未配置」的提示语，而不是让工具凭空消失。 */ + @Bean + public RetrieveTool retrieveTool(ObjectProvider port) { + return new RetrieveTool(port.getIfAvailable()); + } + + @Bean(destroyMethod = "close") + public McpConnections mcpConnections(YokeProperties yoke) { + return McpConnections.connect(yoke.mcp().servers()); + } + + @Bean + public SharedToolkit sharedToolkit(ChatTools chatTools, RetrieveTool retrieveTool, McpConnections mcp) { + Toolkit template = new Toolkit(); + template.registerTool(chatTools); + template.registerTool(retrieveTool); + mcp.registerInto(template); + return new SharedToolkit(template); + } +} diff --git a/src/main/java/com/yoke/config/YokeProperties.java b/src/main/java/com/yoke/config/YokeProperties.java index a9f4fcf..b5f384b 100644 --- a/src/main/java/com/yoke/config/YokeProperties.java +++ b/src/main/java/com/yoke/config/YokeProperties.java @@ -1,5 +1,12 @@ package com.yoke.config; +import java.net.URI; +import java.time.Duration; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + import jakarta.validation.Valid; import jakarta.validation.constraints.NotBlank; import org.slf4j.Logger; @@ -7,6 +14,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.bind.DefaultValue; +import org.springframework.util.StringUtils; import org.springframework.validation.annotation.Validated; /** @@ -15,13 +23,17 @@ * @param redis Redis 连接与键前缀 * @param compaction 对话压缩的触发与保留条数 * @param workspace HarnessAgent 的工作区目录，放压缩前转存的消息、记忆等本地文件 + * @param knowledge 知识库检索（可选的向量化配置） + * @param mcp 要接入的外部 MCP 服务，默认一个也不接 */ @Validated @ConfigurationProperties(prefix = "yoke") public record YokeProperties( @Valid @DefaultValue Redis redis, @DefaultValue Compaction compaction, - @NotBlank String workspace) { + @NotBlank String workspace, + @Valid @DefaultValue Knowledge knowledge, + @Valid @DefaultValue Mcp mcp) { /** * Redis 配置。 @@ -68,4 +80,113 @@ public record Compaction( } } } + + /** + * 知识库检索配置。 + * + * @param embedding 向量化服务；不配置时只用关键词检索 + */ + public record Knowledge(@DefaultValue Embedding embedding) { + + /** + * 向量化服务（OpenAI 兼容的 {@code /embeddings} 接口），{@code modelName} 和 {@code apiKey} 都有值才启用。 + * + * @param modelName 向量模型名，例如 text-embedding-3-small；为空表示不启用向量检索 + * @param baseUrl 接口根地址，默认 OpenAI 官方；用 DashScope 等兼容服务时改它 + * @param apiKey 密钥，只从环境变量或外部配置提供，不要写进仓库 + * @param dimensions 向量维度，可选；不传用模型默认值 + */ + public record Embedding( + String modelName, + @DefaultValue("https://api.openai.com/v1") String baseUrl, + String apiKey, + Integer dimensions) { + + public boolean enabled() { + return StringUtils.hasText(modelName) && StringUtils.hasText(apiKey); + } + + /** 密钥不进日志和调试输出。 */ + @Override + public String toString() { + return "Embedding[modelName=" + modelName + ", baseUrl=" + baseUrl + ", apiKey=" + + (StringUtils.hasText(apiKey) ? "***" : null) + ", dimensions=" + dimensions + "]"; + } + } + } + + /** + * 外部 MCP 服务配置。 + * + * @param servers 要接入的服务列表，默认为空：不配置就不接任何 MCP，也不影响启动 + */ + public record Mcp(@DefaultValue List<@Valid McpServer> servers) { + + public Mcp { + servers = servers == null ? List.of() : List.copyOf(servers); + long distinct = servers.stream().map(McpServer::name).distinct().count(); + if (distinct != servers.size()) { + throw new IllegalArgumentException("yoke.mcp.servers 里的 name 不能重复：每个服务的工具靠它区分和释放"); + } + } + } + + /** + * 一个 MCP 服务。 + * + *

只允许 HTTP 传输（{@code sse} 或 {@code streamable-http}）。{@code stdio} 会让服务端按配置里的命令行 + * 去启动本地进程：配置一旦被改动或被注入，就等于远程代码执行；多副本、容器里也没有可靠的进程管理。 + * 所以填 {@code stdio} 或任何别的值，应用启动时就失败，而不是等到运行时。 + * + * @param name 服务名，在列表里唯一，用于日志和释放 + * @param transport 传输方式，{@code sse} 或 {@code streamable-http} + * @param url 服务地址 + * @param timeout 单次请求超时，默认 30 秒 + * @param headers 每个请求附带的 HTTP 头，通常放认证令牌；可选 + */ + public record McpServer( + @NotBlank String name, + @NotBlank String transport, + @NotBlank String url, + @DefaultValue("30s") Duration timeout, + @DefaultValue Map headers) { + + public static final String SSE = "sse"; + public static final String STREAMABLE_HTTP = "streamable-http"; + private static final Set ALLOWED = Set.of(SSE, STREAMABLE_HTTP); + + public McpServer { + String normalized = transport == null ? "" : transport.trim().toLowerCase(Locale.ROOT); + if (!ALLOWED.contains(normalized)) { + throw new IllegalArgumentException("yoke.mcp.servers[" + name + "].transport=" + transport + + " 不被允许：运行态只支持 HTTP 传输（sse 或 streamable-http）。" + + "stdio 会按配置启动本地进程，存在命令注入风险，所以被拒绝"); + } + transport = normalized; + if (name == null || name.isBlank() || url == null || url.isBlank()) { + throw new IllegalArgumentException("yoke.mcp.servers 的 name 和 url 不能为空"); + } + if (!url.startsWith("http://") && !url.startsWith("https://")) { + throw new IllegalArgumentException("yoke.mcp.servers[" + name + "].url 必须以 http:// 或 https:// 开头"); + } + if (timeout == null || timeout.isZero() || timeout.isNegative()) { + timeout = Duration.ofSeconds(30); + } + headers = headers == null ? Map.of() : Map.copyOf(headers); + } + + /** 地址和请求头里可能带令牌：toString 只留协议、主机和端口，请求头只列名字。 */ + @Override + public String toString() { + String origin; + try { + URI uri = URI.create(url); + origin = uri.getScheme() + "://" + uri.getHost() + (uri.getPort() > 0 ? ":" + uri.getPort() : ""); + } catch (IllegalArgumentException e) { + origin = ""; + } + return "McpServer[name=" + name + ", transport=" + transport + ", url=" + origin + + ", timeout=" + timeout + ", headers=" + headers.keySet() + "]"; + } + } } diff --git a/src/main/java/com/yoke/config/YokeSubagents.java b/src/main/java/com/yoke/config/YokeSubagents.java new file mode 100644 index 0000000..bef2f7c --- /dev/null +++ b/src/main/java/com/yoke/config/YokeSubagents.java @@ -0,0 +1,147 @@ +package com.yoke.config; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import io.agentscope.harness.agent.subagent.SubagentDeclaration; +import lombok.extern.slf4j.Slf4j; + +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; + +/** + * Yoke 的子 Agent 声明，两种来源并存： + * + *

    + *
  • 编程式：{@link #programmatic()}，在代码里用 {@link SubagentDeclaration} 声明，例如 {@code summarizer}；
    • + *
  • 工作区文件：{@code src/main/resources/workspace/subagents/*.md}，文件名即 agent id， + * 启动时拷贝到 {@code yoke.workspace/subagents/}，由 HarnessAgent 按 2.0.4 的约定从工作区加载。
    • + *
+ * + *

同一个 id 同时出现在两处时，2.0.4 不会报错：默认的动态加载（DynamicSubagentsMiddleware）里 + * 「动态声明胜出」，工作区文件静默覆盖代码里的声明；框架内置的 {@code general-purpose} 同理， + * 一个叫 {@code general-purpose.md} 的文件就能把它换掉。配置错位却毫无提示很危险， + * 所以 Yoke 在启动时自己检查，冲突直接启动失败，见 {@link #requireNoConflict}。 + * + *

权限：子 Agent 不继承父 Agent 的 {@code disableWebTools}，只继承文件和 shell 的关闭。 + * 因此子 Agent 的联网禁用靠传给 Builder 的 {@code ToolsConfig.deny}， + * 它会同时传给内置的 general-purpose 子 Agent 和所有声明的子 Agent，见 {@link #toolsConfig()}。 + */ +@Slf4j +public final class YokeSubagents { + + public static final String SUMMARIZER = "summarizer"; + + /** + * 「一个工具都不要」的占位名。2.0.4 里声明的 {@code tools} 为空表示继承父 Agent 的全部工具， + * 没有办法直接表达「零个」；放一个不存在的名字，白名单非空，其余工具就都被过滤掉了。 + */ + static final String NO_TOOLS = "__no_tools__"; + + /** 要对所有 Agent（含全部子 Agent）禁掉的联网工具。 */ + static final List DENIED_TOOLS = List.of("web_fetch", "web_search"); + + /** 工作区里放子 Agent 声明文件的目录名，2.0.4 约定为 {@code /subagents/}。 */ + static final String WORKSPACE_DIR = "subagents"; + + private YokeSubagents() { + } + + /** 编程式声明的子 Agent。 */ + public static List programmatic() { + return List.of(SubagentDeclaration.builder() + .name(SUMMARIZER) + .description("把一大段文字压缩成要点摘要。需要概括长文本、长工具结果或历史讨论时委派给它；它不会调用任何工具。") + .inlineAgentsBody("你是摘要助手。只做一件事：把收到的内容概括成不超过 5 条的要点，保持原意，不添加原文没有的信息。") + .steps(2) + .tools(List.of(NO_TOOLS)) + .build()); + } + + /** + * 传给 {@code HarnessAgent.Builder.toolsConfig} 的工具配置：只含一份 deny 名单。 + * + *

传入显式配置还有一个作用：2.0.4 在没有显式配置时会去读工作区里的 {@code tools.json}， + * 其中可以声明要启动的 stdio MCP 服务。显式传入后不再读取那个文件，堵住这条绕过 MCP 传输限制的路。 + */ + public static io.agentscope.harness.agent.tools.ToolsConfig toolsConfig() { + io.agentscope.harness.agent.tools.ToolsConfig config = new io.agentscope.harness.agent.tools.ToolsConfig(); + config.setDeny(DENIED_TOOLS); + return config; + } + + /** 把 classpath 里的声明文件拷到工作区；每次启动覆盖，保证文件内容与发布的版本一致。 */ + public static Set installWorkspaceFiles(Path workspace) { + try { + Resource[] resources = new PathMatchingResourcePatternResolver() + .getResources("classpath:workspace/" + WORKSPACE_DIR + "/*.md"); + Path target = workspace.resolve(WORKSPACE_DIR); + Files.createDirectories(target); + for (Resource resource : resources) { + Files.write(target.resolve(resource.getFilename()), resource.getInputStream().readAllBytes()); + } + return Arrays.stream(resources).map(YokeSubagents::agentId).collect(Collectors.toUnmodifiableSet()); + } catch (IOException e) { + throw new UncheckedIOException("安装子 Agent 声明文件失败：" + workspace, e); + } + } + + /** + * 启动时的完整准备：安装随应用发布的声明文件，再扫描工作区里实际存在的全部声明文件做冲突检查。 + * + *

只检查 classpath 里的文件不够：{@code yoke.workspace} 是持久目录，里面手放的、或旧版本遗留的 + * {@code summarizer.md}、{@code general-purpose.md} 在 2.0.4 里「文件赢」，会静默替换代码声明和内置子 Agent。 + * 所以以目录里实际的文件名为准；冲突直接启动失败。其余多出来的文件只记警告（它们会作为子 Agent 被加载）。 + */ + public static void prepareWorkspace(Path workspace, List declarations) { + Set shipped = installWorkspaceFiles(workspace); + Set actual = existingFileIds(workspace); + requireNoConflict(declarations, actual); + actual.stream().filter(id -> !shipped.contains(id)) + .forEach(id -> log.warn("工作区 {}/ 里有非随应用发布的子 Agent 声明文件：{}.md，它会被加载为子 Agent", WORKSPACE_DIR, id)); + } + + /** 工作区 {@code subagents/} 目录下全部 {@code .md} 文件对应的 id。 */ + static Set existingFileIds(Path workspace) { + Path dir = workspace.resolve(WORKSPACE_DIR); + if (!Files.isDirectory(dir)) { + return Set.of(); + } + try (var files = Files.list(dir)) { + return files.filter(Files::isRegularFile).map(path -> path.getFileName().toString()) + .filter(name -> name.endsWith(".md")) + .map(name -> name.substring(0, name.length() - ".md".length())) + .collect(Collectors.toCollection(java.util.TreeSet::new)); + } catch (IOException e) { + throw new UncheckedIOException("扫描子 Agent 声明目录失败：" + dir, e); + } + } + + /** 框架内置的子 Agent id，声明文件不许占用。 */ + static final String BUILT_IN = "general-purpose"; + + /** 同一个 id 不允许同时由代码和文件声明，文件也不许占用框架内置的 id。 */ + public static void requireNoConflict(List declarations, Set fileIds) { + Set taken = declarations.stream().map(SubagentDeclaration::getName) + .collect(Collectors.toCollection(java.util.TreeSet::new)); + taken.add(BUILT_IN); + Set clashes = fileIds.stream().filter(taken::contains) + .collect(Collectors.toCollection(java.util.TreeSet::new)); + if (!clashes.isEmpty()) { + throw new IllegalStateException("子 Agent id 重复：" + clashes + " 同时被 workspace/subagents/*.md 和代码或框架内置声明占用。" + + "2.0.4 对此不报错，工作区文件会静默覆盖另一处，请改名或删掉其中一处"); + } + } + + private static String agentId(Resource resource) { + String name = resource.getFilename(); + return name.substring(0, name.length() - ".md".length()); + } +} diff --git a/src/main/java/com/yoke/controller/ChatController.java b/src/main/java/com/yoke/controller/ChatController.java index d4563f6..15086eb 100644 --- a/src/main/java/com/yoke/controller/ChatController.java +++ b/src/main/java/com/yoke/controller/ChatController.java @@ -10,6 +10,7 @@ import com.yoke.model.ChatEvent; import com.yoke.model.ChatRequest; import com.yoke.model.ChatResponse; +import com.yoke.tool.UserContext; import io.agentscope.core.agent.RuntimeContext; import io.agentscope.harness.agent.HarnessAgent; import lombok.RequiredArgsConstructor; @@ -78,6 +79,8 @@ private static RuntimeContext contextOf(String userId, String sessionId) { return RuntimeContext.builder() .userId(userId) .sessionId(sessionId) + // 工具方法里不带 @ToolParam 的 UserContext 参数按类型从这里注入，模型看不到也伪造不了 + .put(UserContext.class, new UserContext(userId)) .build(); } diff --git a/src/main/java/com/yoke/knowledge/EmbeddingClient.java b/src/main/java/com/yoke/knowledge/EmbeddingClient.java new file mode 100644 index 0000000..1be4895 --- /dev/null +++ b/src/main/java/com/yoke/knowledge/EmbeddingClient.java @@ -0,0 +1,11 @@ +package com.yoke.knowledge; + +import java.util.List; + +import reactor.core.publisher.Mono; + +/** 文本向量化：一批文本进，同样顺序、同样数量的向量出。 */ +public interface EmbeddingClient { + + Mono> embed(List texts); +} diff --git a/src/main/java/com/yoke/knowledge/HybridRetrievePort.java b/src/main/java/com/yoke/knowledge/HybridRetrievePort.java new file mode 100644 index 0000000..717bf6c --- /dev/null +++ b/src/main/java/com/yoke/knowledge/HybridRetrievePort.java @@ -0,0 +1,96 @@ +package com.yoke.knowledge; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +/** + * 多路检索 + RRF（Reciprocal Rank Fusion）融合，同时承担降级。 + * + *

每一路各自取一批候选，按名次融合：一个片段的得分是它在各路里 {@code 1 / (60 + 名次)} 之和， + * 60 是 RRF 论文里的常数。只看名次不看原始分数，所以向量相似度和 BM25 这种量纲完全不同的分数也能直接融合。 + * + *

降级规则： + *

    + *
  • 某一路失败（比如向量化服务超时）：记一条警告，用剩下的路继续；
    • + *
  • 所有路都失败：把错误抛出去，让调用方区分「检索坏了」和「没有命中」；
    • + *
  • 所有路都成功但都为空：返回空列表。
    • + *
+ * 融合只发生在成功的路之间，所以向量路挂掉时结果自然退化成纯关键词检索。 + */ +public final class HybridRetrievePort implements RetrievePort { + + private static final Logger log = LoggerFactory.getLogger(HybridRetrievePort.class); + + /** RRF 常数。 */ + static final int RRF_K = 60; + + /** 每一路多取的候选倍数：融合需要比最终 topK 更宽的候选池。 */ + private static final int CANDIDATE_FACTOR = 3; + + private final List channels; + + public HybridRetrievePort(List channels) { + if (channels.isEmpty()) { + throw new IllegalArgumentException("至少需要一路检索"); + } + this.channels = List.copyOf(channels); + } + + @Override + public Mono> retrieve(String query, int topK) { + int candidates = Math.max(topK * CANDIDATE_FACTOR, 10); + return Flux.fromIterable(channels) + .concatMap(channel -> channel.retrieve(query, candidates) + .defaultIfEmpty(List.of()) + .map(Outcome::success) + .onErrorResume(error -> { + log.warn("一路检索失败，降级使用其余检索：{} - {}", channel.getClass().getSimpleName(), error.toString()); + return Mono.just(Outcome.failure(error)); + })) + .collectList() + .flatMap(outcomes -> { + List> succeeded = outcomes.stream() + .filter(outcome -> outcome.error() == null) + .map(Outcome::passages) + .toList(); + if (succeeded.isEmpty()) { + return Mono.error(outcomes.getFirst().error()); + } + return Mono.just(fuse(succeeded, topK)); + }); + } + + static List fuse(List> rankings, int topK) { + Map scores = new LinkedHashMap<>(); + Map byId = new LinkedHashMap<>(); + for (List ranking : rankings) { + for (int rank = 0; rank < ranking.size(); rank++) { + Passage passage = ranking.get(rank); + byId.putIfAbsent(passage.id(), passage); + scores.merge(passage.id(), 1.0 / (RRF_K + rank + 1), Double::sum); + } + } + List ids = new ArrayList<>(scores.keySet()); + // 稳定排序：得分相同时保持首次出现的先后 + ids.sort((a, b) -> Double.compare(scores.get(b), scores.get(a))); + return ids.stream().limit(topK).map(byId::get).toList(); + } + + private record Outcome(List passages, Throwable error) { + + static Outcome success(List passages) { + return new Outcome(passages, null); + } + + static Outcome failure(Throwable error) { + return new Outcome(List.of(), error); + } + } +} diff --git a/src/main/java/com/yoke/knowledge/KeywordRetrievePort.java b/src/main/java/com/yoke/knowledge/KeywordRetrievePort.java new file mode 100644 index 0000000..6b4d3b3 --- /dev/null +++ b/src/main/java/com/yoke/knowledge/KeywordRetrievePort.java @@ -0,0 +1,130 @@ +package com.yoke.knowledge; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import reactor.core.publisher.Mono; + +/** + * 进程内的关键词检索（BM25），不依赖任何外部服务，是检索链路最里层的兜底。 + * + *

分词不引入中文分词库：连续的汉字按相邻两字切（「检索」切成「检索」，「知识库」切成「知识」「识库」）， + * 英文和数字按单词切并转小写。对这种几十篇文档的小语料足够用，代价是召回里会混进一些字面重叠的噪声。 + * + *

语料在构造时建好索引，之后只读，线程安全。 + */ +public final class KeywordRetrievePort implements RetrievePort { + + private static final double K1 = 1.2; + private static final double B = 0.75; + + private final List passages; + private final List> termFrequencies; + private final Map documentFrequency = new HashMap<>(); + private final double averageLength; + + public KeywordRetrievePort(List passages) { + this.passages = List.copyOf(passages); + this.termFrequencies = new ArrayList<>(); + long totalLength = 0; + for (Passage passage : this.passages) { + Map frequencies = new HashMap<>(); + List tokens = tokenize(passage.source() + " " + passage.text()); + tokens.forEach(token -> frequencies.merge(token, 1, Integer::sum)); + frequencies.keySet().forEach(token -> documentFrequency.merge(token, 1, Integer::sum)); + termFrequencies.add(frequencies); + totalLength += tokens.size(); + } + this.averageLength = this.passages.isEmpty() ? 1 : Math.max(1, (double) totalLength / this.passages.size()); + } + + @Override + public Mono> retrieve(String query, int topK) { + return Mono.fromSupplier(() -> search(query, topK)); + } + + private List search(String query, int topK) { + if (query == null || query.isBlank() || topK <= 0 || passages.isEmpty()) { + return List.of(); + } + Set queryTerms = new HashSet<>(tokenize(query)); + record Scored(Passage passage, double score) { + } + List scored = new ArrayList<>(); + for (int i = 0; i < passages.size(); i++) { + double score = score(queryTerms, termFrequencies.get(i)); + if (score > 0) { + scored.add(new Scored(passages.get(i), score)); + } + } + return scored.stream() + .sorted(Comparator.comparingDouble(Scored::score).reversed()) + .limit(topK) + .map(Scored::passage) + .toList(); + } + + private double score(Set queryTerms, Map frequencies) { + double length = frequencies.values().stream().mapToInt(Integer::intValue).sum(); + double score = 0; + for (String term : queryTerms) { + Integer tf = frequencies.get(term); + if (tf == null) { + continue; + } + int df = documentFrequency.getOrDefault(term, 0); + double idf = Math.log(1 + (passages.size() - df + 0.5) / (df + 0.5)); + score += idf * tf * (K1 + 1) / (tf + K1 * (1 - B + B * length / averageLength)); + } + return score; + } + + /** 汉字按相邻两字切；单个孤立汉字也保留；字母数字按连续串切并转小写。 */ + static List tokenize(String text) { + List tokens = new ArrayList<>(); + String lower = text.toLowerCase(Locale.ROOT); + int i = 0; + while (i < lower.length()) { + int codePoint = lower.codePointAt(i); + int width = Character.charCount(codePoint); + if (isHan(codePoint)) { + int end = i; + while (end < lower.length() && isHan(lower.codePointAt(end))) { + end += Character.charCount(lower.codePointAt(end)); + } + String run = lower.substring(i, end); + int runLength = run.codePointCount(0, run.length()); + if (runLength == 1) { + tokens.add(run); + } else { + int[] points = run.codePoints().toArray(); + for (int j = 0; j + 1 < points.length; j++) { + tokens.add(new String(points, j, 2)); + } + } + i = end; + } else if (Character.isLetterOrDigit(codePoint)) { + int end = i; + while (end < lower.length() && Character.isLetterOrDigit(lower.codePointAt(end)) + && !isHan(lower.codePointAt(end))) { + end += Character.charCount(lower.codePointAt(end)); + } + tokens.add(lower.substring(i, end)); + i = end; + } else { + i += width; + } + } + return tokens; + } + + private static boolean isHan(int codePoint) { + return Character.UnicodeScript.of(codePoint) == Character.UnicodeScript.HAN; + } +} diff --git a/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java b/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java new file mode 100644 index 0000000..f3de9f6 --- /dev/null +++ b/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java @@ -0,0 +1,90 @@ +package com.yoke.knowledge; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; + +/** + * 把 {@code classpath*:knowledge/*.md} 切成片段。 + * + *

切法：文档第一个一级标题（{@code # }）当作文档标题，之后每个二级标题（{@code ## }）起一个片段， + * 片段正文超过 {@link #MAX_CHUNK_LENGTH} 个字符时按段落再切。每个片段正文前面带上「文档标题 / 小节标题」， + * 这样单看一个片段也知道它在讲什么。 + */ +public final class KnowledgeCorpus { + + static final int MAX_CHUNK_LENGTH = 500; + + private KnowledgeCorpus() { + } + + /** 读取 classpath 下的全部知识文档并切片；没有文档时返回空列表。 */ + public static List loadFromClasspath(String pattern) { + try { + Resource[] resources = new PathMatchingResourcePatternResolver().getResources(pattern); + List passages = new ArrayList<>(); + Arrays.stream(resources) + .sorted(Comparator.comparing(Resource::getFilename, Comparator.nullsLast(Comparator.naturalOrder()))) + .forEach(resource -> { + try { + String markdown = new String(resource.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + passages.addAll(split(resource.getFilename(), markdown)); + } catch (IOException e) { + throw new UncheckedIOException("读取知识文档失败：" + resource.getFilename(), e); + } + }); + return passages; + } catch (IOException e) { + throw new UncheckedIOException("扫描知识文档失败：" + pattern, e); + } + } + + /** 把一篇 Markdown 切成片段；{@code fileName} 只在文档没有一级标题时充当标题。 */ + static List split(String fileName, String markdown) { + String title = fileName == null ? "未命名文档" : fileName; + String section = null; + StringBuilder body = new StringBuilder(); + List passages = new ArrayList<>(); + String docId = fileName == null ? "doc" : fileName; + + for (String line : markdown.split("\\R")) { + if (line.startsWith("# ")) { + title = line.substring(2).trim(); + } else if (line.startsWith("## ")) { + flush(passages, docId, title, section, body); + section = line.substring(3).trim(); + } else { + body.append(line).append('\n'); + } + } + flush(passages, docId, title, section, body); + return passages; + } + + private static void flush(List passages, String docId, String title, String section, StringBuilder body) { + String text = body.toString().strip(); + body.setLength(0); + if (text.isEmpty()) { + return; + } + String heading = section == null ? title : title + " / " + section; + StringBuilder chunk = new StringBuilder(); + for (String paragraph : text.split("\\n\\s*\\n")) { + if (chunk.length() > 0 && chunk.length() + paragraph.length() > MAX_CHUNK_LENGTH) { + passages.add(new Passage(docId + "#" + passages.size(), title, heading + "\n" + chunk.toString().strip())); + chunk.setLength(0); + } + chunk.append(paragraph.strip()).append("\n\n"); + } + if (chunk.length() > 0) { + passages.add(new Passage(docId + "#" + passages.size(), title, heading + "\n" + chunk.toString().strip())); + } + } +} diff --git a/src/main/java/com/yoke/knowledge/OpenAiEmbeddingClient.java b/src/main/java/com/yoke/knowledge/OpenAiEmbeddingClient.java new file mode 100644 index 0000000..0c0c8a0 --- /dev/null +++ b/src/main/java/com/yoke/knowledge/OpenAiEmbeddingClient.java @@ -0,0 +1,74 @@ +package com.yoke.knowledge; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import org.springframework.http.MediaType; +import org.springframework.web.client.RestClient; + +/** + * 调用 OpenAI 兼容的 {@code POST {baseUrl}/embeddings} 接口。 + * + *

为什么自己写而不用框架的 {@code OpenAITextEmbedding}：它在 rag-simple 模块里。2.0.4 里 agentscope-core 的 + * {@code core/rag} 包（{@code Knowledge}、{@code Document}、{@code RetrieveConfig}）已整体标记为 + * {@code @Deprecated(forRemoval = true, since = "2.0.0")}；rag-simple 的 pom 还把 Qdrant client、Milvus sdk、 + * elasticsearch-java、postgresql、pgvector、pdfbox、poi-ooxml、tika-core、tika-parsers-standard-package、 + * dashscope-sdk-java、openai-java、httpclient5、commons-lang3、model-dashscope、model-ollama 都作为普通依赖 + * （无 scope、非 optional）带进来；而这个接口只是一个 JSON 请求。 + * + *

隐私：用户的检索问题和知识库语料会发送给配置的 embedding 服务提供方。 + * RestClient 是阻塞的，所以放到 boundedElastic 上执行，不占用响应式线程。 + */ +public final class OpenAiEmbeddingClient implements EmbeddingClient { + + private final RestClient client; + private final String modelName; + private final Integer dimensions; + + public OpenAiEmbeddingClient(RestClient client, String modelName, Integer dimensions) { + this.client = client; + this.modelName = modelName; + this.dimensions = dimensions; + } + + @Override + public Mono> embed(List texts) { + return Mono.fromCallable(() -> request(texts)).subscribeOn(Schedulers.boundedElastic()); + } + + @SuppressWarnings("unchecked") + private List request(List texts) { + Map body = new HashMap<>(); + body.put("model", modelName); + body.put("input", texts); + body.put("encoding_format", "float"); + if (dimensions != null) { + body.put("dimensions", dimensions); + } + Map response = client.post() + .uri("/embeddings") + .contentType(MediaType.APPLICATION_JSON) + .body(body) + .retrieve() + .body(Map.class); + if (response == null || !(response.get("data") instanceof List data) || data.size() != texts.size()) { + throw new IllegalStateException("向量化接口返回的条数与请求不一致"); + } + List> items = new ArrayList<>(); + data.forEach(item -> items.add((Map) item)); + // 接口按 index 标识顺序，不假定返回顺序一定和请求一致 + items.sort(Comparator.comparing(item -> ((Number) item.get("index")).intValue())); + List vectors = new ArrayList<>(); + for (Map item : items) { + List values = (List) item.get("embedding"); + vectors.add(values.stream().mapToDouble(Number::doubleValue).toArray()); + } + return vectors; + } +} diff --git a/src/main/java/com/yoke/knowledge/Passage.java b/src/main/java/com/yoke/knowledge/Passage.java new file mode 100644 index 0000000..cb8af03 --- /dev/null +++ b/src/main/java/com/yoke/knowledge/Passage.java @@ -0,0 +1,11 @@ +package com.yoke.knowledge; + +/** + * 知识库里的一个片段，也是检索的返回单位。 + * + * @param id 片段的稳定标识（文档名加序号），多路检索融合时靠它判断是不是同一个片段 + * @param source 片段所属文档的标题，引用时展示给用户 + * @param text 片段正文 + */ +public record Passage(String id, String source, String text) { +} diff --git a/src/main/java/com/yoke/knowledge/RetrievePort.java b/src/main/java/com/yoke/knowledge/RetrievePort.java new file mode 100644 index 0000000..d9a5290 --- /dev/null +++ b/src/main/java/com/yoke/knowledge/RetrievePort.java @@ -0,0 +1,20 @@ +package com.yoke.knowledge; + +import java.util.List; + +import reactor.core.publisher.Mono; + +/** + * 检索端口：给一个问题，返回最相关的若干片段。 + * + *

这是 Yoke 自己的接口，不依赖 AgentScope：2.0.4 里框架的 rag 包已经整体标记为「将移除」 + * （{@code @Deprecated(forRemoval = true)}，注释写明「在应用层集成检索」），所以检索放在应用层， + * 以后换向量库、换检索算法都不碰 Agent 和工具的代码。 + * + *

约定：结果按相关度从高到低排列，最多 {@code topK} 条，没有命中返回空列表； + * 实现失败时通过 Mono 的错误信号表达，由调用方决定怎么降级。 + */ +public interface RetrievePort { + + Mono> retrieve(String query, int topK); +} diff --git a/src/main/java/com/yoke/knowledge/VectorRetrievePort.java b/src/main/java/com/yoke/knowledge/VectorRetrievePort.java new file mode 100644 index 0000000..5f29730 --- /dev/null +++ b/src/main/java/com/yoke/knowledge/VectorRetrievePort.java @@ -0,0 +1,101 @@ +package com.yoke.knowledge; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +/** + * 向量检索：片段向量放在内存里，查询时用余弦相似度取前 K 个。 + * + *

片段向量在第一次检索时才算（懒索引），启动不依赖向量化服务；索引失败时这次检索报错， + * 下一次检索会重试，不会把失败结果缓存下来。索引成功后只读，线程安全。 + */ +public final class VectorRetrievePort implements RetrievePort { + + private static final Logger log = LoggerFactory.getLogger(VectorRetrievePort.class); + + /** 一次向量化请求最多带多少个片段。 */ + static final int BATCH_SIZE = 16; + + private final List passages; + private final EmbeddingClient embeddings; + + private Mono> index; + + public VectorRetrievePort(List passages, EmbeddingClient embeddings) { + this.passages = List.copyOf(passages); + this.embeddings = embeddings; + } + + @Override + public Mono> retrieve(String query, int topK) { + if (query == null || query.isBlank() || topK <= 0 || passages.isEmpty()) { + return Mono.just(List.of()); + } + return index().flatMap(vectors -> embeddings.embed(List.of(query)).map(queryVectors -> { + double[] queryVector = queryVectors.getFirst(); + record Scored(Passage passage, double score) { + } + List scored = new ArrayList<>(); + for (int i = 0; i < passages.size(); i++) { + scored.add(new Scored(passages.get(i), cosine(queryVector, vectors.get(i)))); + } + return scored.stream() + .sorted(Comparator.comparingDouble(Scored::score).reversed()) + .limit(topK) + .map(Scored::passage) + .toList(); + })); + } + + /** 同一时刻只会有一个在建的索引；失败后清掉，让下一次检索重试。 */ + private synchronized Mono> index() { + if (index == null) { + index = buildIndex() + .doOnError(e -> { + log.warn("知识库向量索引构建失败，下次检索会重试：{}", e.getMessage()); + resetIndex(); + }) + .cache(); + } + return index; + } + + private synchronized void resetIndex() { + index = null; + } + + private Mono> buildIndex() { + List> batches = new ArrayList<>(); + for (int from = 0; from < passages.size(); from += BATCH_SIZE) { + batches.add(passages.subList(from, Math.min(from + BATCH_SIZE, passages.size())).stream() + .map(Passage::text).toList()); + } + return Flux.fromIterable(batches) + .concatMap(embeddings::embed) + .collectList() + .map(lists -> lists.stream().flatMap(List::stream).toList()) + .filter(vectors -> vectors.size() == passages.size()) + .switchIfEmpty(Mono.error(new IllegalStateException("向量化结果的数量与片段数量不一致"))); + } + + static double cosine(double[] a, double[] b) { + if (a.length != b.length) { + throw new IllegalStateException("向量维度不一致：" + a.length + " 与 " + b.length); + } + double dot = 0; + double normA = 0; + double normB = 0; + for (int i = 0; i < a.length; i++) { + dot += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + return normA == 0 || normB == 0 ? 0 : dot / (Math.sqrt(normA) * Math.sqrt(normB)); + } +} diff --git a/src/main/java/com/yoke/mcp/McpConnections.java b/src/main/java/com/yoke/mcp/McpConnections.java new file mode 100644 index 0000000..8f9b748 --- /dev/null +++ b/src/main/java/com/yoke/mcp/McpConnections.java @@ -0,0 +1,142 @@ +package com.yoke.mcp; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +import com.yoke.config.YokeProperties; +import io.agentscope.core.tool.Toolkit; +import io.agentscope.core.tool.mcp.McpClientBuilder; +import io.agentscope.core.tool.mcp.McpClientWrapper; +import io.modelcontextprotocol.spec.McpSchema; +import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Mono; + +/** + * 持有到外部 MCP 服务的长连接，负责建连和释放。 + * + *

连接在应用启动时一次建好，所有请求共享（连接内部是线程安全的异步客户端）；工具注册到共享的 Toolkit 模板里， + * 每个请求的 Toolkit 是模板的副本，副本不拥有连接，所以请求结束不会关掉它们。 + * + *

生命周期约定： + *

    + *
  • 建连顺序即配置顺序；任何一个服务连不上，启动失败，并先把已经连上的按相反顺序关闭；
    • + *
  • 关闭同样按相反顺序，每个连接各自 try/catch：一个关不掉不影响其余的；
    • + *
  • 启动失败时关闭过程中的异常只作为 suppressed 挂到主异常上，不会掩盖真正的失败原因。
    • + *
+ * + *

只支持 HTTP 传输，非法的传输方式在配置绑定阶段就已被拒绝，见 {@link YokeProperties.McpServer}。 + * 请求元数据（trace id、工具调用 id）不透传给 MCP 服务：它们不是完全可信的外部方。 + */ +@Slf4j +public final class McpConnections implements AutoCloseable { + + /** 连接对象的创建方式，抽出来是为了让测试能注入失败的连接。 */ + @FunctionalInterface + public interface ClientFactory extends Function { + } + + private final List clients = new ArrayList<>(); + + private McpConnections() { + } + + /** 按配置连接所有服务；没有配置时返回一个空的连接集合。 */ + public static McpConnections connect(List servers) { + return connect(servers, McpConnections::buildClient); + } + + public static McpConnections connect(List servers, ClientFactory factory) { + McpConnections connections = new McpConnections(); + for (YokeProperties.McpServer server : servers) { + try { + McpClientWrapper client = factory.apply(server); + // initialize 会完成握手并缓存服务公布的工具清单；失败就让整个启动失败 + connections.clients.add(client); + client.initialize().block(initTimeout(server)); + log.info("MCP 服务已连接：{}（{}）", server.name(), server.transport()); + } catch (RuntimeException e) { + RuntimeException primary = new IllegalStateException("无法连接 MCP 服务 " + server + + "，请确认服务已启动且地址、传输方式正确（原因：" + e.getClass().getSimpleName() + "）", e); + connections.closeSuppressing(primary); + throw primary; + } + } + return connections; + } + + /** + * 把已连接服务的工具注册进 Toolkit，工具名就是服务端公布的原名。 + * + *

只注册服务端声明了 {@code readOnlyHint} 的工具，其余跳过，模型根本看不到它们。原因：2.0.4 对 MCP 工具 + * 只放行只读的，其余都要用户确认；HTTP 接口没有人工确认通道，把它们暴露给模型只会让一轮对话在工具调用处无声结束， + * 调用方拿到空回复。与其这样，不如一开始就不暴露。跳过时只记工具名到 INFO 日志，不记参数。 + * 用 {@code disableTools} 而不是 {@code enableTools}：后者传空列表在 2.0.4 里表示「全部启用」。 + */ + public void registerInto(Toolkit toolkit) { + for (McpClientWrapper client : clients) { + List skipped = client.listTools().block().stream() + .filter(tool -> tool.annotations() == null || !Boolean.TRUE.equals(tool.annotations().readOnlyHint())) + .map(McpSchema.Tool::name).toList(); + if (!skipped.isEmpty()) { + log.info("MCP 服务 {} 的以下工具未声明只读，不暴露给模型：{}", client.getName(), skipped); + } + toolkit.registration().mcpClient(client).disableTools(skipped).apply(); + } + } + + public List names() { + return clients.stream().map(McpClientWrapper::getName).toList(); + } + + @Override + public void close() { + RuntimeException failure = null; + for (int i = clients.size() - 1; i >= 0; i--) { + McpClientWrapper client = clients.get(i); + try { + client.close(); + } catch (RuntimeException e) { + log.warn("关闭 MCP 连接失败：{}", client.getName(), e); + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + } + clients.clear(); + if (failure != null) { + throw failure; + } + } + + /** 建连失败后的清理：关闭异常只挂在主异常上，不抛出。 */ + private void closeSuppressing(RuntimeException primary) { + try { + close(); + } catch (RuntimeException closeFailure) { + primary.addSuppressed(closeFailure); + } + } + + private static Duration initTimeout(YokeProperties.McpServer server) { + return server.timeout().plusSeconds(5); + } + + private static McpClientWrapper buildClient(YokeProperties.McpServer server) { + McpClientBuilder builder = McpClientBuilder.create(server.name()) + .timeout(server.timeout()) + .initializationTimeout(server.timeout()) + .propagateMeta(false); + if (YokeProperties.McpServer.SSE.equals(server.transport())) { + builder.sseTransport(server.url()); + } else { + builder.streamableHttpTransport(server.url()); + } + // 请求头只对 HTTP 传输生效，必须在选定传输之后设置，否则会被静默忽略 + builder.headers(server.headers()); + return Mono.defer(builder::buildAsync).block(); + } +} diff --git a/src/main/java/com/yoke/tool/ChatTools.java b/src/main/java/com/yoke/tool/ChatTools.java new file mode 100644 index 0000000..5bb3dc4 --- /dev/null +++ b/src/main/java/com/yoke/tool/ChatTools.java @@ -0,0 +1,109 @@ +package com.yoke.tool; + +import java.time.Clock; +import java.time.DateTimeException; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Map; + +import io.agentscope.core.tool.Tool; +import io.agentscope.core.tool.ToolParam; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Yoke 自带的几个通用工具。 + * + *

纯 POJO，不依赖 Spring：启动时只实例化一次，注册进共享的 Toolkit 模板，之后每个请求的 Toolkit 副本 + * 共享同一个实例（{@code Toolkit.copy()} 按引用共享工具对象）。所以不能保存任何请求态字段， + * 工具方法只依赖入参和不可变的字段，天然线程安全。 + * + *

description 写的是「什么时候该用它」而不是功能罗列：模型靠它决定要不要调用。 + * 参数名用 {@code @ToolParam(name = ...)} 显式声明，不依赖编译参数。 + */ +public class ChatTools { + + private static final Logger log = LoggerFactory.getLogger(ChatTools.class); + + private static final DateTimeFormatter TIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss xxx"); + + /** 笔记工具单次最多返回的条数。 */ + static final int MAX_NOTES = 20; + + /** 演示用的内存假数据：真实系统里这里是按 userId 查库。 */ + private static final Map> NOTES = Map.of( + "default", List.of("周五前把第 03 篇教程初稿写完", "给 Redis 加上密码", "读一遍 HarnessAgent 的子 Agent 文档"), + "alice", List.of("和产品对齐检索的引用格式", "评估 MCP 服务的鉴权方式")); + + private final Clock clock; + + public ChatTools() { + this(Clock.systemUTC()); + } + + /** 时钟可注入，测试里用固定时间，不依赖真实的当前时刻。 */ + public ChatTools(Clock clock) { + this.clock = clock; + } + + @Tool(name = "get_current_time", + description = "当用户问现在几点、今天几号、星期几，或者需要知道当前日期才能回答时调用。" + + "模型自己不知道当前时间，不要凭印象回答。返回带时区偏移的日期时间。") + public String getCurrentTime( + @ToolParam(name = "timezone", required = false, + description = "IANA 时区名，例如 Asia/Shanghai、America/New_York；不传时用 UTC") String timezone) { + ZoneId zone; + try { + zone = timezone == null || timezone.isBlank() ? ZoneId.of("UTC") : ZoneId.of(timezone.trim()); + } catch (DateTimeException e) { + return "无法识别的时区：" + abbreviate(timezone) + "。请使用 IANA 时区名，例如 Asia/Shanghai。"; + } + ZonedDateTime now = ZonedDateTime.ofInstant(Instant.now(clock), zone); + return now.format(TIME_FORMAT) + "（" + zone.getId() + "，" + now.getDayOfWeek() + "）"; + } + + @Tool(name = "calculate", + description = "当用户要做加减乘除运算、算价格或比例、核对数字时调用，不要心算大数或小数。" + + "只支持数字、+ - * /、括号和小数，不支持函数和变量。表达式不合法时返回错误原因，请据此修正后重试。") + public String calculate( + @ToolParam(name = "expression", description = "算术表达式，例如 (1.5 + 2) * 4 / 3") String expression) { + try { + return ExpressionEvaluator.evaluate(expression); + } catch (ExpressionEvaluator.InvalidExpressionException e) { + return "无法计算：" + e.getMessage(); + } + } + + /** + * 演示「业务上下文不经过模型」：{@code user} 没有 {@code @ToolParam}，由框架按类型从 RuntimeContext 注入， + * 不进工具的 JSON Schema。取不到时（没人放进上下文）拒绝返回，而不是回落到某个默认用户。 + */ + @Tool(name = "list_my_notes", + description = "当用户问「我的笔记」「我记了什么」「我有哪些待办」时调用。只返回当前用户自己的笔记，无需也不能指定用户。") + public String listMyNotes( + UserContext user, + @ToolParam(name = "limit", required = false, description = "最多返回几条，默认 5，上限 20") Integer limit) { + if (user == null || user.userId() == null || user.userId().isBlank()) { + log.warn("list_my_notes 被调用但 RuntimeContext 里没有 UserContext，拒绝返回数据"); + return "无法确定当前用户，已拒绝返回笔记。"; + } + int max = limit == null ? 5 : Math.clamp(limit, 1, MAX_NOTES); + List notes = NOTES.getOrDefault(user.userId(), List.of()); + if (notes.isEmpty()) { + return "当前用户（" + user.userId() + "）还没有笔记。"; + } + StringBuilder out = new StringBuilder("当前用户（").append(user.userId()).append("）的笔记："); + List shown = notes.subList(0, Math.min(max, notes.size())); + for (int i = 0; i < shown.size(); i++) { + out.append("\n").append(i + 1).append(". ").append(shown.get(i)); + } + return out.toString(); + } + + private static String abbreviate(String text) { + return text.length() <= 40 ? text : text.substring(0, 40) + "…"; + } +} diff --git a/src/main/java/com/yoke/tool/ExpressionEvaluator.java b/src/main/java/com/yoke/tool/ExpressionEvaluator.java new file mode 100644 index 0000000..36cc227 --- /dev/null +++ b/src/main/java/com/yoke/tool/ExpressionEvaluator.java @@ -0,0 +1,201 @@ +package com.yoke.tool; + +import java.math.BigDecimal; +import java.math.MathContext; +import java.util.function.Supplier; + +/** + * 四则运算表达式求值器，手写的递归下降解析器。 + * + *

为什么不用脚本引擎或 eval：表达式来自模型，模型的输入又可能来自用户甚至外部网页， + * 任何能执行代码的求值手段都等于把代码执行权交了出去。这里的文法只有数字、{@code + - * /}、 + * 括号和一元正负号，没有变量、函数、幂运算，也就没有能被滥用的东西。 + * + * 

+ * expr   := term (('+' | '-') term)*
+ * term   := factor (('*' | '/') factor)*
+ * factor := ('+' | '-') factor | number | '(' expr ')'
+ *
+ * + *

防护：输入长度、括号嵌套深度、单个数字的位数都有上限，除数为 0 明确报错； + * 用 {@link BigDecimal} 避免浮点误差（0.1 + 0.2 得 0.3）。没有幂运算，结果的位数最多是输入位数的量级， + * 不会像 {@code 9^9^9} 那样撑爆内存。无状态，线程安全；每次求值新建一个解析实例。 + */ +public final class ExpressionEvaluator { + + /** 表达式最大长度（字符数）。 */ + public static final int MAX_LENGTH = 200; + + /** 括号最大嵌套深度，避免病态输入把递归栈撑爆。 */ + static final int MAX_DEPTH = 32; + + /** 单个数字字面量最多的字符数（含小数点）。 */ + static final int MAX_NUMBER_LENGTH = 30; + + private static final MathContext DIVISION_PRECISION = MathContext.DECIMAL64; + + private ExpressionEvaluator() { + } + + /** 表达式不合法（语法、除零、超限）。消息直接给模型看，所以写成清晰的中文。 */ + public static final class InvalidExpressionException extends RuntimeException { + + InvalidExpressionException(String message) { + super(message); + } + } + + /** + * 求值并格式化为不带科学计数法、不带多余 0 的字符串。 + * + * @throws InvalidExpressionException 表达式为空、过长、含非法字符、语法错误或除数为 0 + */ + public static String evaluate(String expression) { + if (expression == null || expression.isBlank()) { + throw new InvalidExpressionException("表达式不能为空"); + } + if (expression.length() > MAX_LENGTH) { + throw new InvalidExpressionException("表达式太长（最多 " + MAX_LENGTH + " 个字符）"); + } + BigDecimal result = new Parser(expression).parse(); + BigDecimal normalized = result.signum() == 0 ? BigDecimal.ZERO : result.stripTrailingZeros(); + return normalized.toPlainString(); + } + + private static final class Parser { + + private final String text; + private int position; + private int depth; + + Parser(String text) { + this.text = text; + } + + BigDecimal parse() { + BigDecimal value = expression(); + skipSpaces(); + if (position < text.length()) { + throw unexpected(); + } + return value; + } + + private BigDecimal expression() { + BigDecimal value = term(); + while (true) { + skipSpaces(); + if (accept('+')) { + value = value.add(term()); + } else if (accept('-')) { + value = value.subtract(term()); + } else { + return value; + } + } + } + + private BigDecimal term() { + BigDecimal value = factor(); + while (true) { + skipSpaces(); + if (accept('*')) { + value = value.multiply(factor()); + } else if (accept('/')) { + BigDecimal divisor = factor(); + if (divisor.signum() == 0) { + throw new InvalidExpressionException("除数不能为 0"); + } + value = value.divide(divisor, DIVISION_PRECISION); + } else { + return value; + } + } + } + + private BigDecimal factor() { + skipSpaces(); + if (accept('+')) { + return enter(this::factor); + } + if (accept('-')) { + return enter(this::factor).negate(); + } + if (accept('(')) { + BigDecimal inner = enter(this::expression); + skipSpaces(); + if (!accept(')')) { + throw new InvalidExpressionException("缺少右括号 ')'"); + } + return inner; + } + return number(); + } + + /** 每进入一层括号或一元符号就加深一层，超过上限直接拒绝。 */ + private BigDecimal enter(Supplier rule) { + if (++depth > MAX_DEPTH) { + throw new InvalidExpressionException("括号或正负号嵌套太深（最多 " + MAX_DEPTH + " 层）"); + } + try { + return rule.get(); + } finally { + depth--; + } + } + + private BigDecimal number() { + int start = position; + boolean seenDot = false; + while (position < text.length()) { + char c = text.charAt(position); + if (c >= '0' && c <= '9') { + position++; + } else if (c == '.' && !seenDot) { + seenDot = true; + position++; + } else { + break; + } + } + String literal = text.substring(start, position); + if (literal.isEmpty() || literal.equals(".")) { + throw unexpected(); + } + if (literal.length() > MAX_NUMBER_LENGTH) { + throw new InvalidExpressionException("数字太长（最多 " + MAX_NUMBER_LENGTH + " 位）"); + } + if (literal.endsWith(".") || literal.startsWith(".")) { + throw new InvalidExpressionException("数字格式不对：" + literal + "（小数点两侧都要有数字）"); + } + return new BigDecimal(literal); + } + + private boolean accept(char expected) { + if (position < text.length() && text.charAt(position) == expected) { + position++; + return true; + } + return false; + } + + private void skipSpaces() { + while (position < text.length() && text.charAt(position) == ' ') { + position++; + } + } + + private InvalidExpressionException unexpected() { + if (position >= text.length()) { + return new InvalidExpressionException("表达式不完整，结尾处缺少数字"); + } + char c = text.charAt(position); + return new InvalidExpressionException( + "第 " + (position + 1) + " 个字符 '" + printable(c) + "' 不合法，只支持数字、+ - * / 和括号"); + } + + private static String printable(char c) { + return Character.isISOControl(c) ? "\\u" + String.format("%04x", (int) c) : String.valueOf(c); + } + } +} diff --git a/src/main/java/com/yoke/tool/RetrieveTool.java b/src/main/java/com/yoke/tool/RetrieveTool.java new file mode 100644 index 0000000..b1add38 --- /dev/null +++ b/src/main/java/com/yoke/tool/RetrieveTool.java @@ -0,0 +1,75 @@ +package com.yoke.tool; + +import java.util.List; + +import com.yoke.knowledge.Passage; +import com.yoke.knowledge.RetrievePort; +import io.agentscope.core.tool.Tool; +import io.agentscope.core.tool.ToolParam; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import reactor.core.publisher.Mono; + +/** + * 「工具化检索」：把知识库检索做成一个模型自己决定何时调用的工具，而不是每轮都自动往提示词里塞资料。 + * + *

只走这一条通道，不同时使用 HarnessAgent / ReActAgent 的 {@code knowledge()} 自动注入： + * 两条通道同时开，同一个问题会被检索两次，两份资料挤进上下文还可能互相矛盾，引用编号也对不上； + * 而且 2.0.4 里框架的 rag 包已标记为将移除。工具化的另一个好处是闲聊不触发检索，省 token。 + * + *

返回值带 {@code [1][2]} 编号，模型回答时按编号引用，用户就能回头核对出处。 + * 无状态，线程安全；{@code port} 为空表示没有配置知识库，工具返回提示语而不是抛异常。 + */ +public class RetrieveTool { + + private static final Logger log = LoggerFactory.getLogger(RetrieveTool.class); + + static final int DEFAULT_TOP_K = 3; + + /** 单次最多返回几条：模型传多大都不会超过，避免一次把上下文塞满。 */ + static final int MAX_TOP_K = 8; + + static final int MAX_QUERY_LENGTH = 200; + + private final RetrievePort port; + + public RetrieveTool(RetrievePort port) { + this.port = port; + } + + @Tool(name = "retrieve", + description = "当问题涉及 Yoke 自身的设计、配置、接口或使用方法，而你不确定答案时调用，先查资料再回答；" + + "闲聊、通用常识和计算不要调用。返回带 [1][2] 编号的资料片段，回答时请用相同的编号标注引用来源。" + + "查不到时如实告诉用户，不要编造。") + public Mono retrieve( + @ToolParam(name = "query", description = "要查的问题或关键词，用完整的短句，例如「会话历史存在哪里」") String query, + @ToolParam(name = "top_k", required = false, description = "返回几条，默认 3，上限 8") Integer topK) { + if (port == null) { + return Mono.just("知识库尚未配置，无法检索，请直接根据已有信息回答并说明没有查资料。"); + } + if (query == null || query.isBlank()) { + return Mono.just("检索失败：query 不能为空。"); + } + String trimmed = query.length() > MAX_QUERY_LENGTH ? query.substring(0, MAX_QUERY_LENGTH) : query; + int limit = topK == null ? DEFAULT_TOP_K : Math.clamp(topK, 1, MAX_TOP_K); + return port.retrieve(trimmed, limit) + .map(RetrieveTool::format) + .onErrorResume(error -> { + // 细节只写日志，不回给模型：异常信息里可能带上游服务的地址 + log.warn("知识库检索失败", error); + return Mono.just("检索服务暂时不可用，请稍后再试，或直接说明无法查证。"); + }); + } + + static String format(List passages) { + if (passages.isEmpty()) { + return "没有检索到相关资料。请如实告诉用户知识库里没有查到，不要编造。"; + } + StringBuilder out = new StringBuilder("检索到 ").append(passages.size()).append(" 条相关资料，回答时用 [编号] 标注引用："); + for (int i = 0; i < passages.size(); i++) { + Passage passage = passages.get(i); + out.append("\n\n[").append(i + 1).append("] 来源：").append(passage.source()).append('\n').append(passage.text()); + } + return out.toString(); + } +} diff --git a/src/main/java/com/yoke/tool/UserContext.java b/src/main/java/com/yoke/tool/UserContext.java new file mode 100644 index 0000000..acd9483 --- /dev/null +++ b/src/main/java/com/yoke/tool/UserContext.java @@ -0,0 +1,13 @@ +package com.yoke.tool; + +/** + * 当前请求的用户上下文，由控制器放进 {@code RuntimeContext}，工具方法里把它声明成不带 @ToolParam 的参数， + * 框架就按类型从 {@code RuntimeContext} 里注入。 + * + *

它不会出现在工具的 JSON Schema 里：模型既看不到这个参数，也没法在调用时伪造 userId。 + * 凡是「属于谁」的业务信息都应该走这条路，而不是让模型当参数传。 + * + * @param userId 当前用户标识，目前由调用方自报（还没有鉴权） + */ +public record UserContext(String userId) { +} diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index baef002..12f1341 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -4,6 +4,10 @@ agentscope: api-key: ${OPENAI_API_KEY:} yoke: + knowledge: + embedding: + # 没有设置环境变量时为空，检索只用关键词 + api-key: ${YOKE_EMBEDDING_API_KEY:} redis: # 对应 docker-compose.yml 里的 Redis，本地研发先 docker compose up -d uri: ${YOKE_REDIS_URI:redis://127.0.0.1:6379} diff --git a/src/main/resources/application-prod.yml b/src/main/resources/application-prod.yml index e64d1d2..0c19e38 100644 --- a/src/main/resources/application-prod.yml +++ b/src/main/resources/application-prod.yml @@ -7,6 +7,10 @@ agentscope: api-key: ${OPENAI_API_KEY} yoke: + knowledge: + embedding: + # 可选：不设置时检索只用关键词，不影响启动（与 OPENAI_API_KEY 不同，缺失不是错误） + api-key: ${YOKE_EMBEDDING_API_KEY:} redis: # 生产环境必须通过环境变量提供 Redis 连接串（含密码时用 redis://:密码@主机:端口），没有设置时启动即失败 uri: ${YOKE_REDIS_URI} diff --git a/src/main/resources/application-test.yml b/src/main/resources/application-test.yml index f86f233..f6565d4 100644 --- a/src/main/resources/application-test.yml +++ b/src/main/resources/application-test.yml @@ -4,5 +4,9 @@ agentscope: api-key: sk-test yoke: + knowledge: + embedding: + # 测试环境故意留空：保证测试不会访问向量化服务，检索只走关键词 + api-key: # 测试用独立的工作区；Redis 连接串由测试基类注入（Testcontainers 拉起的容器），这里故意不写 workspace: ${java.io.tmpdir}/yoke-test-workspace diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index a2a4df1..83dafab 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -10,6 +10,10 @@ spring: enabled: true logging: + level: + # 主 Agent 同时用了 disableWebTools() 和 ToolsConfig.deny：web 工具没注册，ToolFilter 每次构建对 deny 里的名字打 WARN， + # 摘要子 Agent 的占位白名单名也一样。这些是预期内的噪音，调到 ERROR；配置写错的风险由 SubagentTests 守着 + io.agentscope.harness.agent.tools.ToolFilter: ERROR pattern: console: "%red(%d{yyyy-MM-dd HH:mm:ss}) %green([%thread]) %highlight(%-5level) %green(%X{traceId:-},%X{spanId:-}) %boldMagenta(%logger{50}) - %cyan(%msg%n)" @@ -23,6 +27,24 @@ yoke: # 对话消息达到 trigger-messages 条就压缩，压缩后原样保留最近 keep-messages 条；<=0 的非法值回落到 30 / 10 trigger-messages: 30 keep-messages: 10 + knowledge: + # 知识库检索（retrieve 工具）：语料是 classpath:knowledge/*.md。 + # embedding 的 model-name 和 api-key 都有值才启用向量检索（与关键词检索融合）；任意一个为空就只用关键词检索， + # 向量化服务出错时也会自动退回关键词。密钥见各环境文件，base-url 可换成其他 OpenAI 兼容服务 + embedding: + model-name: ${YOKE_EMBEDDING_MODEL:} + base-url: ${YOKE_EMBEDDING_BASE_URL:https://api.openai.com/v1} + # 外部 MCP 服务，默认一个也不接。只允许 HTTP 传输（sse 或 streamable-http），配置 stdio 会让应用启动失败； + # 连不上也会启动失败。接入后服务端公布的工具以原名出现在模型的工具清单里； + # 只有服务端声明了 readOnlyHint 的工具会被注册，其余需要用户确认，而 HTTP 接口没有确认通道，所以不暴露给模型。示例： + # mcp: + # servers: + # - name: docs + # transport: streamable-http + # url: https://mcp.example.com/mcp + # timeout: 30s + # headers: + # Authorization: Bearer ${MCP_DOCS_TOKEN} # Agent 与模型由 AgentScope 官方 starter 根据下面的配置自动创建 # 这里只放各环境通用的配置，密钥和环境差异见 application-{profile}.yml diff --git a/src/main/resources/knowledge/api-usage.md b/src/main/resources/knowledge/api-usage.md new file mode 100644 index 0000000..cb4b02b --- /dev/null +++ b/src/main/resources/knowledge/api-usage.md @@ -0,0 +1,13 @@ +# 接口使用 + +## 普通对话 +向 POST /api/chat 发送 JSON，字段有可选的 userId、可选的 sessionId 和必填的 message。message 缺失、为空或纯空白会返回 400。响应里包含回复文本和本次对话的 sessionId，后续请求带上它就能延续对话。 + +## 流式对话 +POST /api/chat/stream 以 SSE 的形式边推理边推送事件。事件名有 textDelta、toolCall、toolResult、hint、done 和 error。一次对话以 done 或 error 结尾，二者必有其一且只会出现一个。toolCall 和 toolResult 用相同的 id 对应。 + +## 并发的限制 +同一个会话的并发请求不会被串行化，由调用方保证同一会话同一时间只发一个请求。目前也没有鉴权，userId 由调用方自报，不可信，不要把它当成身份校验。 + +## 运行环境 +应用默认使用 local 环境，其他环境通过 SPRING_PROFILES_ACTIVE 切换。prod 环境必须提供 OPENAI_API_KEY 和 YOKE_REDIS_URI，缺失就启动失败，这是有意为之，避免带着空配置上线。 diff --git a/src/main/resources/knowledge/sessions-and-storage.md b/src/main/resources/knowledge/sessions-and-storage.md new file mode 100644 index 0000000..6493169 --- /dev/null +++ b/src/main/resources/knowledge/sessions-and-storage.md @@ -0,0 +1,13 @@ +# 会话与存储 + +## 对话历史存在哪里 +对话历史不放在 Agent 实例里，而是放在 Redis。由于每个请求都会新建 Agent，新的实例要靠共享存储才能接上之前的对话。Yoke 使用 AgentScope 的 RedisDistributedStore，按 userId 和 sessionId 组成的会话槽位保存历史，所以服务重启之后历史仍然在。 + +## 会话标识的规则 +请求里的 userId 和 sessionId 必须成对非空才对应一个会话。只传 sessionId 时归属 default 用户；只传 userId 时为该用户新建一个会话，新的会话 ID 在响应里返回；都不传时在 default 用户下新建会话。标识只能包含字母、数字、下划线、点和短横线，长度 1 到 64。 + +## 启动时检查 Redis +应用启动时会 ping 一次 Redis，连不上就直接启动失败，而不是等到第一个请求才发现历史存不下来。连接串通过 yoke.redis.uri 配置，生产环境来自环境变量 YOKE_REDIS_URI，没有默认值。 + +## 对话压缩 +对话消息达到 yoke.compaction.trigger-messages 条（默认 30）就会触发压缩，压缩后原样保留最近 yoke.compaction.keep-messages 条（默认 10）。压缩前的消息会转存到工作区目录。 diff --git a/src/main/resources/knowledge/tools-and-permissions.md b/src/main/resources/knowledge/tools-and-permissions.md new file mode 100644 index 0000000..a541e7a --- /dev/null +++ b/src/main/resources/knowledge/tools-and-permissions.md @@ -0,0 +1,16 @@ +# 工具与权限 + +## 模型能用哪些工具 +Yoke 自带的工具有 get_current_time、calculate、list_my_notes 和 retrieve。get_current_time 返回带时区偏移的当前时间；calculate 做四则运算，支持括号和小数；list_my_notes 返回当前用户的笔记；retrieve 在 Yoke 自己的知识库里检索资料。此外还有 HarnessAgent 内置的记忆和会话类工具，例如 memory_save、memory_search 和 session_history。 + +## 默认关闭了哪些能力 +对外服务的 Agent 不开放文件读写、命令执行和联网工具，也就是 read_file、write_file、execute、web_fetch、web_search 这一类都看不到。子 Agent 不会继承联网工具的关闭，所以 Yoke 通过工具配置的 deny 名单把 web_fetch 和 web_search 对所有子 Agent 一并禁掉。 + +## 业务上下文不经过模型 +list_my_notes 需要知道当前用户是谁，但 userId 不是模型传的参数：它由控制器放进 RuntimeContext，工具方法里不带 @ToolParam 的 UserContext 参数由框架按类型注入。模型既看不到这个参数，也没法伪造。 + +## 检索如何降级 +retrieve 工具背后是检索端口。配置了 embedding 模型时，向量检索和关键词检索并行，用 RRF 融合排名；没有配置或向量检索失败时，自动退回纯关键词检索。返回的资料带 [1][2] 编号，回答时应按编号标注引用。 + +## 接入 MCP 服务 +通过 yoke.mcp.servers 可以接入外部 MCP 服务，传输方式只允许 sse 和 streamable-http 两种 HTTP 传输。配置 stdio 会在启动时直接失败，因为 stdio 会让服务端按配置去启动本地进程。 diff --git a/src/main/resources/knowledge/yoke-overview.md b/src/main/resources/knowledge/yoke-overview.md new file mode 100644 index 0000000..82d6268 --- /dev/null +++ b/src/main/resources/knowledge/yoke-overview.md @@ -0,0 +1,10 @@ +# Yoke 概览 + +## Yoke 是什么 +Yoke 是一个可以直接运行的 Spring Boot 单体 Agent 服务，定位词是 AgentHarness。它基于 Spring Boot 4.1、AgentScope Java 2.0.4、JDK 25 和 Gradle 构建，是单模块的应用，不是发布给别人依赖的库。 + +## 每次请求新建一个 Agent +Yoke 不使用单例 Agent。每个对话请求都会由 HarnessAgentFactory 新建一个 HarnessAgent，请求结束时关闭它。这样做的原因是 AgentScope 2.0.4 的 HarnessAgent.Builder 不能重复使用：同一个 Builder 多次 build 会不断追加内部中间件。所以工厂只共享不可变的部分，例如模型、存储和配置，Builder 每次都现建。 + +## 虚拟线程 +应用通过 spring.threads.virtual.enabled 一个开关，让 Tomcat 请求线程和 Spring 默认任务线程池都使用虚拟线程，不需要单独配置线程池。 diff --git a/src/main/resources/workspace/subagents/kb-researcher.md b/src/main/resources/workspace/subagents/kb-researcher.md new file mode 100644 index 0000000..63190d1 --- /dev/null +++ b/src/main/resources/workspace/subagents/kb-researcher.md @@ -0,0 +1,12 @@ +--- +description: 专门查证 Yoke 自身设计、配置和接口细节的研究员。需要核对「Yoke 怎么做某件事」又不想占用主对话的上下文时，把问题委派给它；它只会查知识库，并返回带 [编号] 引用的结论。 +maxIters: 4 +tools: [retrieve] +--- + +你是 Yoke 知识库研究员。 + +工作方式： +1. 先用 retrieve 工具查资料，必要时换个说法再查一次，最多查两次。 +2. 只根据检索到的资料作答，资料里没有的内容就明确说「知识库里没有查到」，不要凭印象补充。 +3. 回答要简短，每个结论后面用 [1]、[2] 这样的编号标注出处，编号与检索结果里的编号一致。 diff --git a/src/test/java/com/yoke/ArchitectureTests.java b/src/test/java/com/yoke/ArchitectureTests.java index c62fa72..ad83950 100644 --- a/src/test/java/com/yoke/ArchitectureTests.java +++ b/src/test/java/com/yoke/ArchitectureTests.java @@ -58,4 +58,29 @@ public boolean test(JavaClass input) { static final ArchRule 框架的消息和事件类型只能出现在codec和middleware里 = noClasses() .that().resideOutsideOfPackages("com.yoke.codec..", "com.yoke.middleware..") .should().dependOnClassesThat().resideInAnyPackage("io.agentscope.core.message..", "io.agentscope.core.event.."); + + /** + * 检索端口是应用层自己的接口：不依赖 AgentScope（2.0.4 里框架的 rag 包已标记为将移除）， + * 以后换向量库、换检索算法都不影响 Agent 和工具。向量化客户端、关键词检索、融合与降级都在这个包里。 + */ + @ArchTest + static final ArchRule 知识库检索包不依赖框架 = noClasses().that().resideInAPackage("com.yoke.knowledge..") + .should().dependOnClassesThat().resideInAPackage("io.agentscope.."); + + @ArchTest + static final ArchRule 知识库检索包不反向依赖工具_配置和接口层 = noClasses().that().resideInAPackage("com.yoke.knowledge..") + .should().dependOnClassesThat().resideInAnyPackage("com.yoke.tool..", "com.yoke.mcp..", "com.yoke.config..", + "com.yoke.controller..", "com.yoke.api.."); + + /** 工具类只允许用框架的工具注解和 Toolkit 一族（io.agentscope.core.tool），不碰 Agent、模型、消息。 */ + @ArchTest + static final ArchRule 工具类只依赖框架的tool包 = noClasses().that().resideInAPackage("com.yoke.tool..") + .should().dependOnClassesThat(DescribedPredicate.and(resideInAPackage("io.agentscope.."), + DescribedPredicate.not(resideInAPackage("io.agentscope.core.tool..")))); + + /** MCP 连接只用框架的 tool 包（McpClientBuilder、Toolkit 都在那里）。 */ + @ArchTest + static final ArchRule MCP连接只依赖框架的tool包 = noClasses().that().resideInAPackage("com.yoke.mcp..") + .should().dependOnClassesThat(DescribedPredicate.and(resideInAPackage("io.agentscope.."), + DescribedPredicate.not(resideInAPackage("io.agentscope.core.tool..")))); } diff --git a/src/test/java/com/yoke/ProfilesTests.java b/src/test/java/com/yoke/ProfilesTests.java index 9dfd835..759efbb 100644 --- a/src/test/java/com/yoke/ProfilesTests.java +++ b/src/test/java/com/yoke/ProfilesTests.java @@ -47,6 +47,14 @@ private static Object yamlValue(String file, String key) throws IOException { assertThat(yamlValue("application-test.yml", "yoke.redis.uri")).isNull(); } + @Test + void 向量化密钥在三个环境里的写法一致_都不会让缺失变成启动失败() throws IOException { + assertThat(yamlValue("application-local.yml", "yoke.knowledge.embedding.api-key")).isEqualTo("${YOKE_EMBEDDING_API_KEY:}"); + assertThat(yamlValue("application-prod.yml", "yoke.knowledge.embedding.api-key")).isEqualTo("${YOKE_EMBEDDING_API_KEY:}"); + // test 环境故意留空，保证测试不访问向量化服务 + assertThat(yamlValue("application-test.yml", "yoke.knowledge.embedding.api-key")).isIn("", null); + } + @Nested @SpringBootTest(properties = "OPENAI_API_KEY=sk-test") class 默认环境 { @@ -75,6 +83,10 @@ class 测试环境 { assertThat(environment.getProperty("logging.level.com.yoke")).isNull(); assertThat(environment.getProperty("yoke.redis.uri")).startsWith("redis://"); assertThat(environment.getProperty("yoke.redis.key-prefix")).isEqualTo("yoke:"); + assertThat(environment.getProperty("logging.level.io.agentscope.harness.agent.tools.ToolFilter")).isEqualTo("ERROR"); + // 测试环境不启用向量检索，也不接任何 MCP 服务 + assertThat(environment.getProperty("yoke.knowledge.embedding.api-key", "")).isEmpty(); + assertThat(environment.getProperty("yoke.mcp.servers[0].name")).isNull(); } } diff --git a/src/test/java/com/yoke/ToolPermissionTests.java b/src/test/java/com/yoke/ToolPermissionTests.java index 229ba22..f03968d 100644 --- a/src/test/java/com/yoke/ToolPermissionTests.java +++ b/src/test/java/com/yoke/ToolPermissionTests.java @@ -24,8 +24,11 @@ import static org.assertj.core.api.Assertions.assertThat; /** - * 权限取舍：对外服务的 Agent 看不到文件读写、命令执行和联网工具。 - * 以「什么都不关的 HarnessAgent 比线上多出哪些工具」作为参照，避免硬编码工具名后框架改名而悄悄失效。 + * 权限取舍：对外服务的 Agent 看不到文件读写、命令执行和联网工具；模型可见的工具清单是精确的集合。 + * + *

清单 = 什么都不关的 HarnessAgent 的内置工具，减去被关掉的那一批，加上 Yoke 自己的工具。 + * 用精确集合断言，框架升级新增了内置工具、或者有人不小心多注册了工具，这里都会失败提醒。 + * 子 Agent 已经在第 03 轮打开（agent_spawn 等在清单里），它们自己能看到的工具另见 SubagentTests。 */ @SpringBootTest @ActiveProfiles("test") @@ -48,31 +51,61 @@ private static Set toolsOfUnrestrictedAgent(Path workspace) { } /** - * 被关掉的工具：文件读写、命令执行、联网，以及子 Agent 相关的（子 Agent 不继承联网工具的关闭）。 + * 被关掉的工具：文件读写、命令执行、联网。 * 名字来自上面参照 Agent 的实测清单，由本类的第二个测试守着。 */ private static final Set FORBIDDEN = Set.of( "read_file", "write_file", "edit_file", "list_files", "glob_files", "grep_files", - "execute", "web_search", "web_fetch", - "agent_spawn", "agent_send", "agent_list", "task_cancel", "task_list", "task_output"); + "execute", "web_search", "web_fetch"); - @Test - void 模型看不到文件_命令_联网和子Agent工具() { - controller.chat(new ChatRequest("tools-" + UUID.randomUUID(), "你有哪些工具")).block(); + /** + * 保留下来的框架内置工具，逐个写明（2.0.4 实测）：记忆、会话、技能加载、异步结果、子 Agent 和后台任务。 + * 写成字面量而不是从参照 Agent 推导，是为了让框架升级新增了任何内置工具时这里失败，逼着人来决定放不放行。 + */ + private static final Set KEPT_BUILT_IN = Set.of( + "memory_save", "memory_get", "memory_search", + "session_history", "session_list", "session_search", + "load_skill_through_path", "wait_async_results", + "agent_spawn", "agent_send", "agent_list", + "task_output", "task_cancel", "task_list"); + + /** Yoke 自己注册的工具；默认配置下没有 MCP 工具。 */ + private static final Set OWN_TOOLS = Set.of("get_current_time", "calculate", "list_my_notes", "retrieve"); + private Set exposedTools() { + controller.chat(new ChatRequest("tools-" + UUID.randomUUID(), "你有哪些工具")).block(); // 回复之后框架还会异步发起记忆整理之类不带工具的模型调用，所以取最后一次带工具的调用 - Set exposed = model.receivedTools.stream().filter(tools -> !tools.isEmpty()) + return model.receivedTools.stream().filter(tools -> !tools.isEmpty()) .reduce((first, second) -> second).map(TreeSet::new).orElseThrow(); + } + + @Test + void 模型看不到文件_命令_联网工具() { + Set exposed = exposedTools(); assertThat(exposed).as("模型确实拿到了工具清单").isNotEmpty(); assertThat(exposed).doesNotContainAnyElementsOf(FORBIDDEN); } + @Test + void 模型可见的工具清单是内置保留项加Yoke自己的工具_不多不少() { + Set expected = new TreeSet<>(KEPT_BUILT_IN); + expected.addAll(OWN_TOOLS); + + assertThat(exposedTools()).isEqualTo(expected); + } + + @Test + void 内置保留项加被关掉的项恰好是不加限制时的全部内置工具(@TempDir Path workspace) { + Set all = new TreeSet<>(KEPT_BUILT_IN); + all.addAll(FORBIDDEN); + + assertThat(toolsOfUnrestrictedAgent(workspace)).isEqualTo(all); + } + @Test void 被关掉的工具恰好是不加限制时多出来的那一批(@TempDir Path workspace) { - controller.chat(new ChatRequest("tools-" + UUID.randomUUID(), "你有哪些工具")).block(); - Set exposed = model.receivedTools.stream().filter(tools -> !tools.isEmpty()) - .reduce((first, second) -> second).map(TreeSet::new).orElseThrow(); + Set exposed = exposedTools(); Set removed = toolsOfUnrestrictedAgent(workspace); removed.removeAll(exposed); diff --git a/src/test/java/com/yoke/ToolsEndToEndTests.java b/src/test/java/com/yoke/ToolsEndToEndTests.java new file mode 100644 index 0000000..1293707 --- /dev/null +++ b/src/test/java/com/yoke/ToolsEndToEndTests.java @@ -0,0 +1,127 @@ +package com.yoke; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +import com.yoke.controller.ChatController; +import com.yoke.model.ChatEvent; +import com.yoke.model.ChatRequest; +import com.yoke.support.McpTestServer; +import com.yoke.support.RedisIntegrationTest; +import com.yoke.support.ScriptedModel; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.http.codec.ServerSentEvent; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * 整个应用上下文（真实 Redis、真实的 Spring 装配）加脚本化的假模型：证明工具、知识库、UserContext 在真实装配下是连着的。 + * 单元层面的测试各自把依赖手工接好，接线错误（比如检索工具没拿到检索端口）只有这一层能发现。 + */ +@SpringBootTest +@ActiveProfiles("test") +@Import(ToolsEndToEndTests.Scripted.class) +class ToolsEndToEndTests extends RedisIntegrationTest { + + @TestConfiguration(proxyBeanMethods = false) + static class Scripted { + + @Bean + ScriptedModel scriptedModel() { + return new ScriptedModel(call -> { + if (call.systemText().startsWith("You are a memory")) { + return ScriptedModel.text("[]"); + } + String question = call.messages().stream().filter(m -> m.getRole() == io.agentscope.core.message.MsgRole.USER) + .map(io.agentscope.core.message.Msg::getTextContent).reduce("", (a, b) -> b); + if (call.hasToolResult()) { + return ScriptedModel.text("据资料：" + call.lastToolResultText()); + } + if (question.contains("回显")) { + return ScriptedModel.toolUse("m1", "echo", Map.of("text", "来自端到端")); + } + if (question.contains("笔记")) { + return ScriptedModel.toolUse("n1", "list_my_notes", Map.of()); + } + return ScriptedModel.toolUse("r1", "retrieve", Map.of("query", question, "top_k", 2)); + }); + } + } + + /** 真实的进程内 MCP 服务，应用上下文按配置连上它；先于上下文启动，类结束时关闭。 */ + static final McpTestServer MCP = new McpTestServer(); + + @DynamicPropertySource + static void mcpProperties(DynamicPropertyRegistry registry) { + registry.add("yoke.mcp.servers[0].name", () -> "demo"); + registry.add("yoke.mcp.servers[0].transport", () -> "streamable-http"); + registry.add("yoke.mcp.servers[0].url", MCP::url); + } + + @AfterAll + static void stopMcp() { + MCP.close(); + } + + @Autowired + ChatController controller; + + private List stream(ChatRequest request) { + return controller.stream(request).map(ServerSentEvent::data).collectList().block(Duration.ofSeconds(60)); + } + + private static String toolOutput(List events) { + return events.stream().filter(ChatEvent.ToolResult.class::isInstance).map(ChatEvent.ToolResult.class::cast) + .map(ChatEvent.ToolResult::output).findFirst().orElseThrow(); + } + + @Test + void 检索工具在真实装配下连着知识库_返回带编号的资料() { + List events = stream(new ChatRequest("e2e-r-" + System.nanoTime(), "对话历史存在哪里")); + + ChatEvent.ToolCall call = events.stream().filter(ChatEvent.ToolCall.class::isInstance) + .map(ChatEvent.ToolCall.class::cast).findFirst().orElseThrow(); + ChatEvent.ToolResult result = events.stream().filter(ChatEvent.ToolResult.class::isInstance) + .map(ChatEvent.ToolResult.class::cast).findFirst().orElseThrow(); + assertThat(call.name()).isEqualTo("retrieve"); + assertThat(call.arguments()).contains("对话历史存在哪里").contains("\"top_k\":2"); + assertThat(result.id()).isEqualTo(call.id()); + assertThat(events.indexOf(call)).as("toolCall 先于 toolResult").isLessThan(events.indexOf(result)); + assertThat(result.output()).contains("[1] 来源：").contains("Redis").doesNotContain("尚未配置"); + assertThat(events.getLast()).isInstanceOf(ChatEvent.Done.class); + } + + @Test + void 配置的只读MCP工具在真实装配下经过完整的ReAct循环被调用() { + List events = stream(new ChatRequest("e2e-m-" + System.nanoTime(), "请回显一下")); + + ChatEvent.ToolCall call = events.stream().filter(ChatEvent.ToolCall.class::isInstance) + .map(ChatEvent.ToolCall.class::cast).findFirst().orElseThrow(); + ChatEvent.ToolResult result = events.stream().filter(ChatEvent.ToolResult.class::isInstance) + .map(ChatEvent.ToolResult.class::cast).findFirst().orElseThrow(); + assertThat(call.name()).isEqualTo("echo"); + assertThat(result.id()).isEqualTo(call.id()); + assertThat(result.output()).contains("echo: 来自端到端"); + assertThat(events.indexOf(call)).isLessThan(events.indexOf(result)); + assertThat(((ChatEvent.Done) events.getLast()).text()).contains("echo: 来自端到端"); + assertThat(MCP.writeNoteCalls).hasValue(0); + } + + @Test + void 请求里的userId在真实装配下传到了工具() { + List alice = stream(new ChatRequest("e2e-a-" + System.nanoTime(), "我的笔记", "alice")); + + assertThat(toolOutput(alice)).contains("alice").contains("MCP"); + } +} diff --git a/src/test/java/com/yoke/config/AgentConfigurationConflictTests.java b/src/test/java/com/yoke/config/AgentConfigurationConflictTests.java new file mode 100644 index 0000000..158d61d --- /dev/null +++ b/src/test/java/com/yoke/config/AgentConfigurationConflictTests.java @@ -0,0 +1,53 @@ +package com.yoke.config; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import io.agentscope.core.model.Model; +import io.agentscope.core.tool.Toolkit; +import io.agentscope.harness.agent.DistributedStore; +import io.agentscope.spring.boot.properties.AgentscopeProperties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.springframework.boot.test.context.runner.ApplicationContextRunner; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +/** 真实的装配入口 AgentConfiguration：工作区里已有同名声明文件时，应用上下文启动失败。 */ +class AgentConfigurationConflictTests { + + private ApplicationContextRunner runner(Path workspace) { + return new ApplicationContextRunner() + .withBean(Model.class, () -> mock(Model.class)) + .withBean(DistributedStore.class, () -> mock(DistributedStore.class)) + .withBean(AgentscopeProperties.class, AgentscopeProperties::new) + .withBean(SharedToolkit.class, () -> new SharedToolkit(new Toolkit())) + .withBean(YokeProperties.class, () -> new YokeProperties(new YokeProperties.Redis("redis://127.0.0.1:6379", "yoke:"), + new YokeProperties.Compaction(30, 10), workspace.toString(), + new YokeProperties.Knowledge(new YokeProperties.Knowledge.Embedding(null, null, null, null)), + new YokeProperties.Mcp(List.of()))) + .withUserConfiguration(AgentConfiguration.class); + } + + @Test + void 工作区里遗留了同名的summarizer文件时启动失败(@TempDir Path workspace) throws Exception { + Files.createDirectories(workspace.resolve("subagents")); + Files.writeString(workspace.resolve("subagents/summarizer.md"), "---\ndescription: 遗留\n---\n旧提示词"); + + runner(workspace).run(context -> { + assertThat(context).hasFailed(); + assertThat(context.getStartupFailure()).hasStackTraceContaining("子 Agent id 重复").hasStackTraceContaining("summarizer"); + }); + } + + @Test + void 工作区干净时正常启动并装好声明文件(@TempDir Path workspace) { + runner(workspace).run(context -> { + assertThat(context).hasNotFailed(); + assertThat(workspace.resolve("subagents/kb-researcher.md")).exists(); + }); + } +} diff --git a/src/test/java/com/yoke/config/AgentConfigurationTests.java b/src/test/java/com/yoke/config/AgentConfigurationTests.java index 2323ef9..65f16cd 100644 --- a/src/test/java/com/yoke/config/AgentConfigurationTests.java +++ b/src/test/java/com/yoke/config/AgentConfigurationTests.java @@ -1,7 +1,10 @@ package com.yoke.config; +import java.util.List; + import io.agentscope.core.middleware.MiddlewareBase; import io.agentscope.core.model.Model; +import io.agentscope.core.tool.Toolkit; import io.agentscope.harness.agent.DistributedStore; import io.agentscope.spring.boot.properties.AgentscopeProperties; import org.junit.jupiter.api.Test; @@ -38,7 +41,14 @@ AgentscopeProperties agentscopeProperties() { @Bean YokeProperties yokeProperties() { return new YokeProperties(new YokeProperties.Redis("redis://127.0.0.1:6379", "yoke:"), - new YokeProperties.Compaction(30, 10), "/tmp/yoke-test"); + new YokeProperties.Compaction(30, 10), "/tmp/yoke-test", + new YokeProperties.Knowledge(new YokeProperties.Knowledge.Embedding(null, null, null, null)), + new YokeProperties.Mcp(List.of())); + } + + @Bean + SharedToolkit sharedToolkit() { + return new SharedToolkit(new Toolkit()); } } diff --git a/src/test/java/com/yoke/config/KnowledgeConfigurationTests.java b/src/test/java/com/yoke/config/KnowledgeConfigurationTests.java new file mode 100644 index 0000000..a97c40a --- /dev/null +++ b/src/test/java/com/yoke/config/KnowledgeConfigurationTests.java @@ -0,0 +1,81 @@ +package com.yoke.config; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; + +import com.sun.net.httpserver.HttpServer; +import com.yoke.knowledge.HybridRetrievePort; +import com.yoke.knowledge.KeywordRetrievePort; +import com.yoke.knowledge.Passage; +import com.yoke.knowledge.RetrievePort; +import org.junit.jupiter.api.Test; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +import static org.assertj.core.api.Assertions.assertThat; + +class KnowledgeConfigurationTests { + + @Configuration(proxyBeanMethods = false) + @EnableConfigurationProperties(YokeProperties.class) + @Import(KnowledgeConfiguration.class) + static class Context { + } + + private final ApplicationContextRunner runner = new ApplicationContextRunner() + .withUserConfiguration(Context.class) + .withPropertyValues("yoke.redis.uri=redis://127.0.0.1:6379", "yoke.workspace=/tmp/yoke"); + + @Test + void 没有配置向量化服务时只用关键词检索且应用照常启动() { + runner.run(context -> { + assertThat(context).hasNotFailed(); + RetrievePort port = context.getBean(RetrievePort.class); + assertThat(port).isInstanceOf(KeywordRetrievePort.class); + assertThat(port.retrieve("对话历史存在哪里", 3).block()).isNotEmpty(); + }); + } + + @Test + void 只配了模型名没有密钥时同样不启用向量检索() { + runner.withPropertyValues("yoke.knowledge.embedding.model-name=text-embedding-3-small") + .run(context -> assertThat(context.getBean(RetrievePort.class)).isInstanceOf(KeywordRetrievePort.class)); + } + + @Test + void 配了向量化服务时使用双路融合() { + runner.withPropertyValues("yoke.knowledge.embedding.model-name=text-embedding-3-small", + "yoke.knowledge.embedding.api-key=sk-test") + .run(context -> assertThat(context.getBean(RetrievePort.class)).isInstanceOf(HybridRetrievePort.class)); + } + + @Test + void 向量化服务不可用时检索自动退回关键词() throws IOException { + // 本地起一个永远返回 500 的向量化服务：向量路失败，关键词路必须兜住 + HttpServer broken = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + broken.createContext("/", exchange -> { + byte[] body = "{}".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(500, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + broken.start(); + try { + runner.withPropertyValues("yoke.knowledge.embedding.model-name=text-embedding-3-small", + "yoke.knowledge.embedding.api-key=sk-test", + "yoke.knowledge.embedding.base-url=http://127.0.0.1:" + broken.getAddress().getPort() + "/v1") + .run(context -> { + RetrievePort port = context.getBean(RetrievePort.class); + assertThat(port).isInstanceOf(HybridRetrievePort.class); + assertThat(port.retrieve("对话历史存在哪里", 3).block()).extracting(Passage::text) + .anySatisfy(text -> assertThat(text).contains("Redis")); + }); + } finally { + broken.stop(0); + } + } +} diff --git a/src/test/java/com/yoke/config/RedisConfigurationTests.java b/src/test/java/com/yoke/config/RedisConfigurationTests.java index 67ce696..f33cb1a 100644 --- a/src/test/java/com/yoke/config/RedisConfigurationTests.java +++ b/src/test/java/com/yoke/config/RedisConfigurationTests.java @@ -8,7 +8,9 @@ class RedisConfigurationTests { private static YokeProperties withUri(String uri) { - return new YokeProperties(new YokeProperties.Redis(uri, "yoke:"), new YokeProperties.Compaction(30, 10), "/tmp/yoke"); + return new YokeProperties(new YokeProperties.Redis(uri, "yoke:"), new YokeProperties.Compaction(30, 10), "/tmp/yoke", + new YokeProperties.Knowledge(new YokeProperties.Knowledge.Embedding(null, null, null, null)), + new YokeProperties.Mcp(java.util.List.of())); } private static String allMessages(Throwable error) { diff --git a/src/test/java/com/yoke/config/SharedToolkitTests.java b/src/test/java/com/yoke/config/SharedToolkitTests.java new file mode 100644 index 0000000..78c8e5e --- /dev/null +++ b/src/test/java/com/yoke/config/SharedToolkitTests.java @@ -0,0 +1,89 @@ +package com.yoke.config; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import com.yoke.support.ScriptedModel; +import com.yoke.support.TestAgents; +import com.yoke.tool.ChatTools; +import com.yoke.tool.RetrieveTool; +import io.agentscope.core.agent.RuntimeContext; +import io.agentscope.core.tool.Toolkit; +import io.agentscope.harness.agent.HarnessAgent; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * 守住「所有请求共用同一个 Toolkit 模板」这个前提：它成立靠的是 2.0.4 的 HarnessAgent.Builder.build() 会先拷贝 Toolkit。 + * 将来框架升级后如果不再拷贝，Harness 的内置工具会写进模板，这里会失败。 + */ +class SharedToolkitTests { + + private static Toolkit template() { + Toolkit toolkit = new Toolkit(); + toolkit.registerTool(new ChatTools()); + toolkit.registerTool(new RetrieveTool(null)); + return toolkit; + } + + private static void chat(HarnessAgentFactory factory, String session) { + try (HarnessAgent agent = factory.create()) { + agent.call("hi", RuntimeContext.builder().userId("u").sessionId(session).build()).block(); + } + } + + @Test + void 连续多次请求之后模板里仍然只有Yoke自己的工具(@TempDir Path workspace) { + Toolkit template = template(); + Set before = new TreeSet<>(template.getToolNames()); + ScriptedModel model = new ScriptedModel(call -> ScriptedModel.text("ok")); + HarnessAgentFactory factory = TestAgents.factory(model, workspace, template); + + for (int i = 0; i < 3; i++) { + chat(factory, "s" + i); + } + + assertThat(before).containsExactlyInAnyOrder("get_current_time", "calculate", "list_my_notes", "retrieve"); + assertThat(new TreeSet<>(template.getToolNames())).as("Harness 的内置工具没有写进共享模板").isEqualTo(before); + List> perCall = model.calls.stream().map(ScriptedModel.Call::toolNames) + .filter(names -> !names.isEmpty()).>map(TreeSet::new).toList(); + assertThat(perCall).hasSize(3); + assertThat(perCall.get(1)).isEqualTo(perCall.get(0)); + assertThat(perCall.get(2)).isEqualTo(perCall.get(0)); + assertThat(perCall.get(0)).contains("agent_spawn", "memory_save", "calculate"); + } + + @Test + void 并发地用同一个模板构建Agent_每个Agent看到的工具清单都完整一致(@TempDir Path workspace) throws Exception { + ScriptedModel model = new ScriptedModel(call -> ScriptedModel.text("ok")); + HarnessAgentFactory factory = TestAgents.factory(model, workspace, template()); + int workers = 16; + + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> jobs = new ArrayList<>(); + for (int i = 0; i < workers; i++) { + String session = "c" + i; + jobs.add(() -> { + chat(factory, session); + return null; + }); + } + for (var future : executor.invokeAll(jobs, 120, TimeUnit.SECONDS)) { + future.get(); + } + } + + List> perCall = model.calls.stream().map(ScriptedModel.Call::toolNames) + .filter(names -> !names.isEmpty()).>map(TreeSet::new).toList(); + assertThat(perCall).hasSize(workers); + assertThat(perCall).allSatisfy(tools -> assertThat(tools).isEqualTo(perCall.getFirst())); + } +} diff --git a/src/test/java/com/yoke/config/SubagentContextTests.java b/src/test/java/com/yoke/config/SubagentContextTests.java new file mode 100644 index 0000000..e4bebc3 --- /dev/null +++ b/src/test/java/com/yoke/config/SubagentContextTests.java @@ -0,0 +1,44 @@ +package com.yoke.config; + +import java.nio.file.Path; +import java.util.Map; + +import com.yoke.controller.ChatController; +import com.yoke.model.ChatRequest; +import com.yoke.support.ScriptedModel; +import com.yoke.support.TestAgents; +import com.yoke.tool.ChatTools; +import io.agentscope.core.tool.Toolkit; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.assertj.core.api.Assertions.assertThat; + +/** 子 Agent 里调用依赖 UserContext 的工具时，上下文有没有一路传下去。 */ +class SubagentContextTests { + + @Test + void 通用子Agent里调用list_my_notes能拿到发起请求的用户(@TempDir Path workspace) { + ScriptedModel model = new ScriptedModel(call -> { + if (call.systemText().startsWith("You are a memory")) { + return ScriptedModel.text("[]"); + } + boolean parent = call.systemText().startsWith("你是测试助手"); + if (parent) { + return call.hasToolResult() ? ScriptedModel.text("主 Agent 收尾") + : ScriptedModel.toolUse("t1", "agent_spawn", Map.of("agent_id", "general-purpose", "task", "查笔记")); + } + return call.hasToolResult() ? ScriptedModel.text("子：" + call.lastToolResultText()) + : ScriptedModel.toolUse("t2", "list_my_notes", Map.of()); + }); + Toolkit toolkit = new Toolkit(); + toolkit.registerTool(new ChatTools()); + ChatController controller = new ChatController(TestAgents.factory(model, workspace, toolkit)); + + controller.chat(new ChatRequest("sub-ctx", "让子 Agent 查我的笔记", "alice")).block(); + + String subReply = model.calls.stream().filter(call -> !call.systemText().startsWith("你是测试助手")) + .filter(call -> call.hasToolResult()).map(ScriptedModel.Call::lastToolResultText).findFirst().orElseThrow(); + assertThat(subReply).contains("alice"); + } +} diff --git a/src/test/java/com/yoke/config/SubagentTests.java b/src/test/java/com/yoke/config/SubagentTests.java new file mode 100644 index 0000000..620b56b --- /dev/null +++ b/src/test/java/com/yoke/config/SubagentTests.java @@ -0,0 +1,290 @@ +package com.yoke.config; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +import com.yoke.support.ScriptedModel; +import com.yoke.support.TestAgents; +import com.yoke.tool.ChatTools; +import com.yoke.tool.RetrieveTool; +import io.agentscope.core.agent.RuntimeContext; +import io.agentscope.core.tool.Toolkit; +import io.agentscope.harness.agent.HarnessAgent; +import io.agentscope.harness.agent.subagent.SubagentDeclaration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * 子 Agent 的声明与权限。最关键的是安全回归：2.0.4 的子 Agent 不继承父 Agent 的 {@code disableWebTools}， + * 所以这里用「记录工具清单的假模型」真实驱动每一种子 Agent 跑一次，枚举它实际能看到的工具。 + */ +class SubagentTests { + + /** 对外服务的 Agent 任何一层都不该看到的工具：联网、文件读写、命令执行。 */ + private static final Set NEVER = Set.of( + "web_fetch", "web_search", + "read_file", "write_file", "edit_file", "list_files", "glob_files", "grep_files", + "execute"); + + private static final String PARENT_PROMPT = "你是测试助手"; + + private static Toolkit template() { + Toolkit toolkit = new Toolkit(); + toolkit.registerTool(new ChatTools()); + toolkit.registerTool(new RetrieveTool(null)); + return toolkit; + } + + /** 主 Agent 先委派给 agentId，子 Agent 回一句话，主 Agent 收尾。 */ + private static ScriptedModel delegatingTo(String agentId) { + return new ScriptedModel(call -> { + if (call.systemText().startsWith("You are a memory")) { + return ScriptedModel.text("[]"); + } + if (!call.systemText().startsWith(PARENT_PROMPT)) { + return ScriptedModel.text("子 Agent 的回复"); + } + if (!call.hasToolResult() && !call.tools().isEmpty()) { + return ScriptedModel.toolUse("t1", "agent_spawn", Map.of("agent_id", agentId, "task", "做点事")); + } + return ScriptedModel.text("主 Agent 收尾"); + }); + } + + private static void run(ScriptedModel model, Path workspace) { + try (HarnessAgent agent = TestAgents.factory(model, workspace, template()).create()) { + agent.call("hi", RuntimeContext.builder().userId("u").sessionId("s").build()).block(); + } + } + + /** 子 Agent 自己发起的模型调用里出现过的全部工具名（主 Agent 和记忆整理的调用不算）。 */ + private static Set toolsSeenBySubagent(ScriptedModel model) { + Set seen = new TreeSet<>(); + model.calls.stream() + .filter(call -> !call.systemText().startsWith(PARENT_PROMPT) && !call.systemText().startsWith("You are a memory")) + .forEach(call -> seen.addAll(call.toolNames())); + return seen; + } + + private static Set toolsSeenByParent(ScriptedModel model) { + Set seen = new TreeSet<>(); + model.calls.stream().filter(call -> call.systemText().startsWith(PARENT_PROMPT)).forEach(call -> seen.addAll(call.toolNames())); + return seen; + } + + @Test + void 摘要子Agent拿不到联网_文件_命令_也拿不到Yoke自己的任何工具(@TempDir Path workspace) { + ScriptedModel model = delegatingTo("summarizer"); + run(model, workspace); + + Set seen = toolsSeenBySubagent(model); + + assertThat(seen).as("子 Agent 确实被调用了").isNotEmpty(); + assertThat(seen).doesNotContainAnyElementsOf(NEVER); + // 唯一剩下的是技能加载器，父 Agent 也有它；没有 retrieve、calculate、记忆、会话类工具，也不能再派生子 Agent + assertThat(seen).containsExactly("load_skill_through_path"); + } + + @Test + void 工作区文件声明的子Agent只能用白名单里的检索工具(@TempDir Path workspace) { + ScriptedModel model = delegatingTo("kb-researcher"); + run(model, workspace); + + Set seen = toolsSeenBySubagent(model); + + assertThat(seen).doesNotContainAnyElementsOf(NEVER); + assertThat(seen).containsExactlyInAnyOrder("load_skill_through_path", "retrieve"); + } + + @Test + void 内置的通用子Agent同样拿不到联网_文件_命令_也不能再派生子Agent(@TempDir Path workspace) { + ScriptedModel model = delegatingTo("general-purpose"); + run(model, workspace); + + Set seen = toolsSeenBySubagent(model); + + assertThat(seen).as("子 Agent 确实被调用了").isNotEmpty(); + assertThat(seen).doesNotContainAnyElementsOf(NEVER); + assertThat(seen).doesNotContain("agent_spawn", "agent_send"); + // 它继承父 Agent 的业务工具，这是它「通用」的本意 + assertThat(seen).contains("calculate", "get_current_time", "retrieve"); + } + + @Test + void 声明文件忘了写tools白名单时也拿不到联网工具_靠的是父Agent的deny名单(@TempDir Path workspace) throws IOException { + // 2.0.4 里 tools 为空表示继承父 Agent 的全部工具；这是最容易写错的地方，所以要有第二道防线 + TestAgents.factory(new ScriptedModel(c -> ScriptedModel.text("x")), workspace, template()); + Files.writeString(workspace.resolve("subagents/open.md"), """ + --- + description: 忘了写 tools 白名单的子 Agent + --- + + 你是一个什么限制都没写的子 Agent。 + """); + ScriptedModel model = delegatingTo("open"); + + run(model, workspace); + + Set seen = toolsSeenBySubagent(model); + assertThat(seen).as("子 Agent 确实被调用了").isNotEmpty(); + assertThat(seen).contains("calculate"); + assertThat(seen).doesNotContainAnyElementsOf(NEVER); + } + + @Test + void 主Agent自己的工具清单也没有联网_文件_命令(@TempDir Path workspace) { + ScriptedModel model = delegatingTo("summarizer"); + run(model, workspace); + + Set seen = toolsSeenByParent(model); + + assertThat(seen).doesNotContainAnyElementsOf(NEVER); + assertThat(seen).contains("agent_spawn", "calculate", "retrieve"); + } + + @Test + void 工作区里的tools_json不会被读取_也就不能借它启动stdio进程(@TempDir Path workspace) throws IOException { + Path marker = workspace.resolve("spawned-by-tools-json"); + Files.writeString(workspace.resolve("tools.json"), """ + {"mcpServers": {"evil": {"transport": "stdio", "command": "touch", "args": ["%s"], "initializationTimeout": "PT1S"}}} + """.formatted(marker)); + + run(delegatingTo("summarizer"), workspace); + + assertThat(marker).as("Yoke 显式传了 toolsConfig，2.0.4 就不再读 tools.json").doesNotExist(); + } + + @Test + void 对照实验_不传toolsConfig时2_0_4确实会读tools_json并启动进程(@TempDir Path workspace) throws Exception { + Path marker = workspace.resolve("spawned-by-tools-json"); + Files.writeString(workspace.resolve("tools.json"), """ + {"mcpServers": {"evil": {"transport": "stdio", "command": "touch", "args": ["%s"], "initializationTimeout": "PT1S"}}} + """.formatted(marker)); + + try (HarnessAgent agent = HarnessAgent.builder().name("plain").model(new ScriptedModel(c -> ScriptedModel.text("x"))) + .workspace(workspace).stateStore(new io.agentscope.core.state.InMemoryAgentStateStore()).build()) { + // 构造即触发：这就是要堵的口子 + } catch (RuntimeException ignored) { + // 握手失败无所谓，进程已经被拉起过 + } + + assertThat(marker).as("没有显式 toolsConfig 时，工作区里的 tools.json 能让 2.0.4 启动任意本地进程").exists(); + } + + // ---- 声明与冲突 ---- + + @Test + void 编程式声明的摘要子Agent只做摘要_迭代次数很小() { + SubagentDeclaration summarizer = YokeSubagents.programmatic().stream() + .filter(d -> d.getName().equals(YokeSubagents.SUMMARIZER)).findFirst().orElseThrow(); + + assertThat(summarizer.getSteps()).isLessThanOrEqualTo(3); + assertThat(summarizer.getTools()).containsExactly(YokeSubagents.NO_TOOLS); + assertThat(summarizer.getInlineAgentsBody()).contains("摘要"); + assertThat(summarizer.getDescription()).contains("摘要"); + } + + @Test + void 声明文件随应用安装到工作区_并且每次启动覆盖被改动的内容(@TempDir Path workspace) throws IOException { + Set ids = YokeSubagents.installWorkspaceFiles(workspace); + Path installed = workspace.resolve("subagents/kb-researcher.md"); + + assertThat(ids).containsExactly("kb-researcher"); + assertThat(installed).exists(); + String original = Files.readString(installed); + assertThat(original).contains("tools: [retrieve]"); + + Files.writeString(installed, original.replace("tools: [retrieve]", "tools: []")); + YokeSubagents.installWorkspaceFiles(workspace); + + assertThat(Files.readString(installed)).isEqualTo(original); + } + + @Test + void 同一个id既在代码里又在文件里时Yoke启动即失败() { + List declarations = YokeSubagents.programmatic(); + + assertThatThrownBy(() -> YokeSubagents.requireNoConflict(declarations, Set.of("summarizer", "other"))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("summarizer").hasMessageContaining("静默"); + // 没有冲突时放行 + YokeSubagents.requireNoConflict(declarations, Set.of("kb-researcher")); + } + + @Test + void 声明文件不许占用框架内置的general_purpose() { + assertThatThrownBy(() -> YokeSubagents.requireNoConflict(YokeSubagents.programmatic(), Set.of("general-purpose"))) + .isInstanceOf(IllegalStateException.class).hasMessageContaining("general-purpose"); + } + + @Test + void 实测2_0_4的同id冲突行为_工作区文件静默覆盖代码声明而不是报错(@TempDir Path workspace) throws IOException { + // 这是对框架行为的记录性测试：Yoke 自己的检查在它之前拦截，这里绕开检查，看框架本身怎么处理 + ScriptedModel model = new ScriptedModel(call -> ScriptedModel.text("ok")); + var factory = TestAgents.factory(model, workspace, template()); + Files.writeString(workspace.resolve("subagents/summarizer.md"), """ + --- + description: 来自文件的同名摘要助手 + --- + + 文件版本。 + """); + + try (HarnessAgent agent = factory.create()) { + agent.call("hi", RuntimeContext.builder().userId("u").sessionId("s").build()).block(); + } + + String parentPrompt = model.calls.stream().filter(c -> c.systemText().startsWith(PARENT_PROMPT)).findFirst().orElseThrow().systemText(); + assertThat(parentPrompt).contains("来自文件的同名摘要助手").doesNotContain("把一大段文字压缩成要点摘要"); + } + + @Test + void 随应用发布的声明文件与代码里的声明没有重名(@TempDir Path workspace) { + // 防止有人在 resources/workspace/subagents 下加了与代码同名的文件 + Set fileIds = YokeSubagents.installWorkspaceFiles(workspace); + + YokeSubagents.requireNoConflict(YokeSubagents.programmatic(), fileIds); + + assertThat(fileIds).contains("kb-researcher"); + } + + @Test + void 工作区里已有的同名声明文件会让启动失败_即使它不是随应用发布的(@TempDir Path workspace) throws IOException { + // yoke.workspace 是持久目录：手放的、旧版本遗留的 summarizer.md 在 2.0.4 里会静默替换代码声明 + Files.createDirectories(workspace.resolve("subagents")); + Files.writeString(workspace.resolve("subagents/summarizer.md"), "---\ndescription: 遗留文件\n---\n旧提示词"); + + assertThatThrownBy(() -> YokeSubagents.prepareWorkspace(workspace, YokeSubagents.programmatic())) + .isInstanceOf(IllegalStateException.class).hasMessageContaining("summarizer"); + } + + @Test + void 工作区里已有的general_purpose文件同样让启动失败(@TempDir Path workspace) throws IOException { + Files.createDirectories(workspace.resolve("subagents")); + Files.writeString(workspace.resolve("subagents/general-purpose.md"), "---\ndescription: 想替换内置子 Agent\n---\n提示词"); + + assertThatThrownBy(() -> YokeSubagents.prepareWorkspace(workspace, YokeSubagents.programmatic())) + .isInstanceOf(IllegalStateException.class).hasMessageContaining("general-purpose"); + // 同样的预置文件走工厂（TestAgents 与线上同一条准备流程）也一样失败 + assertThatThrownBy(() -> TestAgents.factory(new ScriptedModel(c -> ScriptedModel.text("x")), workspace, template())) + .isInstanceOf(IllegalStateException.class); + } + + @Test + void 工作区里多出来的无冲突文件只告警不失败(@TempDir Path workspace) throws IOException { + Files.createDirectories(workspace.resolve("subagents")); + Files.writeString(workspace.resolve("subagents/extra.md"), "---\ndescription: 额外\n---\n提示词"); + + YokeSubagents.prepareWorkspace(workspace, YokeSubagents.programmatic()); + + assertThat(YokeSubagents.existingFileIds(workspace)).containsExactlyInAnyOrder("extra", "kb-researcher"); + } +} diff --git a/src/test/java/com/yoke/config/ToolFilterWarnTests.java b/src/test/java/com/yoke/config/ToolFilterWarnTests.java new file mode 100644 index 0000000..e12520f --- /dev/null +++ b/src/test/java/com/yoke/config/ToolFilterWarnTests.java @@ -0,0 +1,84 @@ +package com.yoke.config; + +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import com.yoke.support.ScriptedModel; +import com.yoke.support.TestAgents; +import com.yoke.tool.ChatTools; +import com.yoke.tool.RetrieveTool; +import io.agentscope.core.agent.RuntimeContext; +import io.agentscope.core.tool.Toolkit; +import io.agentscope.harness.agent.HarnessAgent; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.LoggerFactory; + +import static org.assertj.core.api.Assertions.assertThat; + +/** 实测 ToolFilter 对「deny/allow 里尚未注册的工具名」打的 WARN，用 logback 内存 appender 抓取。 */ +class ToolFilterWarnTests { + + private static List warnsDuring(Runnable action) { + Logger logger = (Logger) LoggerFactory.getLogger("io.agentscope.harness.agent.tools.ToolFilter"); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + // 同一个 JVM 里别的 Spring 测试会按 application.yml 把这个 logger 调成 ERROR，这里显式调回 WARN 再恢复 + Level original = logger.getLevel(); + logger.setLevel(Level.WARN); + try { + action.run(); + } finally { + logger.setLevel(original); + logger.detachAppender(appender); + } + return appender.list.stream().filter(e -> e.getLevel() == Level.WARN).map(ILoggingEvent::getFormattedMessage).toList(); + } + + private static final String DENY_WEB_SEARCH = "tools.json 'deny' references unknown tool 'web_search' (not currently registered)."; + private static final String DENY_WEB_FETCH = "tools.json 'deny' references unknown tool 'web_fetch' (not currently registered)."; + + private List warnsWhenDelegatingTo(String agentId, Path workspace) { + ScriptedModel model = new ScriptedModel(call -> { + if (call.systemText().startsWith("You are a memory")) { + return ScriptedModel.text("[]"); + } + if (call.systemText().startsWith("你是测试助手")) { + return call.hasToolResult() ? ScriptedModel.text("收尾") + : ScriptedModel.toolUse("t1", "agent_spawn", Map.of("agent_id", agentId, "task", "x")); + } + return ScriptedModel.text("子回复"); + }); + Toolkit toolkit = new Toolkit(); + toolkit.registerTool(new ChatTools()); + toolkit.registerTool(new RetrieveTool(null)); + return warnsDuring(() -> { + try (HarnessAgent agent = TestAgents.factory(model, workspace, toolkit).create()) { + agent.call("hi", RuntimeContext.builder().userId("u").sessionId("s-" + agentId).build()).block(); + } + }); + } + + /** + * 同时使用 disableWebTools() 和 deny 的代价：主 Agent 构建时 web 工具根本没注册，ToolFilter 对 deny 里的两个名字各打一条 WARN。 + * 子 Agent 自己会注册 web 工具再被 deny 移除，不触发。这里把这个代价固化下来（实测）； + * application.yml 把这个 logger 的级别调到 ERROR 以免每个请求两条噪音。 + */ + @Test + void 主Agent构建时每次产生两条deny未注册的警告_子Agent构建不产生(@TempDir Path workspace) { + assertThat(warnsWhenDelegatingTo("general-purpose", workspace)).containsExactlyInAnyOrder(DENY_WEB_SEARCH, DENY_WEB_FETCH); + assertThat(warnsWhenDelegatingTo("kb-researcher", workspace)).containsExactlyInAnyOrder(DENY_WEB_SEARCH, DENY_WEB_FETCH); + } + + @Test + void 摘要子Agent的占位名走allow分支_每次派生多一条警告(@TempDir Path workspace) { + assertThat(warnsWhenDelegatingTo("summarizer", workspace)).containsExactlyInAnyOrder(DENY_WEB_SEARCH, DENY_WEB_FETCH, + "tools.json 'allow' references unknown tool '__no_tools__' (not currently registered)."); + } +} diff --git a/src/test/java/com/yoke/config/YokePropertiesTests.java b/src/test/java/com/yoke/config/YokePropertiesTests.java index 36fd58f..b999574 100644 --- a/src/test/java/com/yoke/config/YokePropertiesTests.java +++ b/src/test/java/com/yoke/config/YokePropertiesTests.java @@ -11,6 +11,7 @@ import org.springframework.boot.context.properties.source.MapConfigurationPropertySource; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; class YokePropertiesTests { @@ -84,7 +85,8 @@ private static Map required() { @Test void 只按条数触发压缩且token阈值不会先触发() { HarnessAgentFactory factory = new HarnessAgentFactory( - "Yoke", "prompt", 10, null, null, null, new YokeProperties.Compaction(30, 10), List.of()); + "Yoke", "prompt", 10, null, null, null, new YokeProperties.Compaction(30, 10), List.of(), + null, List.of()); CompactionConfig config = factory.compactionConfig(); @@ -95,4 +97,97 @@ private static Map required() { // 0 才改用 keepMessages，默认 -1 是动态 token assertThat(config.getKeepTokens()).isZero(); } + + // ---- 知识库与 MCP 配置 ---- + + @Test + void 不配置任何MCP服务和向量化服务时使用空默认值() { + YokeProperties properties = bind(required()); + + assertThat(properties.mcp().servers()).isEmpty(); + assertThat(properties.knowledge().embedding().enabled()).isFalse(); + assertThat(properties.knowledge().embedding().baseUrl()).isEqualTo("https://api.openai.com/v1"); + } + + @Test + void MCP服务按列表绑定_传输方式忽略大小写_超时有默认值() { + Map properties = required(); + properties.put("yoke.mcp.servers[0].name", "docs"); + properties.put("yoke.mcp.servers[0].transport", "Streamable-HTTP"); + properties.put("yoke.mcp.servers[0].url", "https://mcp.example.com/mcp"); + properties.put("yoke.mcp.servers[0].headers.Authorization", "Bearer t"); + properties.put("yoke.mcp.servers[1].name", "legacy"); + properties.put("yoke.mcp.servers[1].transport", "sse"); + properties.put("yoke.mcp.servers[1].url", "http://10.0.0.5:8080/sse"); + properties.put("yoke.mcp.servers[1].timeout", "5s"); + + List servers = bind(properties).mcp().servers(); + + assertThat(servers).extracting(YokeProperties.McpServer::name).containsExactly("docs", "legacy"); + assertThat(servers.get(0).transport()).isEqualTo("streamable-http"); + assertThat(servers.get(0).timeout()).isEqualTo(java.time.Duration.ofSeconds(30)); + assertThat(servers.get(0).headers()).containsEntry("Authorization", "Bearer t"); + assertThat(servers.get(1).timeout()).isEqualTo(java.time.Duration.ofSeconds(5)); + } + + @Test + void 配置stdio传输时绑定失败_报错说明原因() { + Map properties = required(); + properties.put("yoke.mcp.servers[0].name", "local-fs"); + properties.put("yoke.mcp.servers[0].transport", "stdio"); + properties.put("yoke.mcp.servers[0].url", "http://unused"); + + assertThatThrownBy(() -> bind(properties)) + .hasStackTraceContaining("stdio").hasStackTraceContaining("只支持 HTTP 传输") + .hasStackTraceContaining("local-fs"); + } + + @Test + void 未知的传输方式和缺失的传输方式同样被拒绝() { + assertThatThrownBy(() -> new YokeProperties.McpServer("x", "websocket", "http://h", null, null)) + .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("只支持 HTTP 传输"); + assertThatThrownBy(() -> new YokeProperties.McpServer("x", null, "http://h", null, null)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new YokeProperties.McpServer("x", " ", "http://h", null, null)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void 地址必须是http或https() { + assertThatThrownBy(() -> new YokeProperties.McpServer("x", "sse", "file:///etc/passwd", null, null)) + .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("http://"); + assertThatThrownBy(() -> new YokeProperties.McpServer("x", "sse", "npx -y some-server", null, null)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void 服务名不能重复() { + YokeProperties.McpServer one = new YokeProperties.McpServer("same", "sse", "http://a/sse", null, null); + YokeProperties.McpServer two = new YokeProperties.McpServer("same", "sse", "http://b/sse", null, null); + + assertThatThrownBy(() -> new YokeProperties.Mcp(List.of(one, two))) + .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("不能重复"); + } + + @Test + void MCP服务的toString不泄漏地址里的令牌和请求头的值() { + YokeProperties.McpServer server = new YokeProperties.McpServer("docs", "sse", + "https://mcp.example.com:8443/sse?token=query-secret", null, Map.of("Authorization", "Bearer header-secret")); + + String text = server.toString(); + + assertThat(text).contains("docs").contains("https://mcp.example.com:8443").contains("Authorization"); + assertThat(text).doesNotContain("query-secret").doesNotContain("header-secret"); + } + + @Test + void 向量化配置的toString不泄漏密钥() { + YokeProperties.Knowledge.Embedding embedding = new YokeProperties.Knowledge.Embedding( + "text-embedding-3-small", "https://api.openai.com/v1", "sk-very-secret", 512); + + assertThat(embedding.toString()).contains("text-embedding-3-small").doesNotContain("sk-very-secret"); + assertThat(embedding.enabled()).isTrue(); + assertThat(new YokeProperties.Knowledge.Embedding("m", "u", " ", null).enabled()).isFalse(); + assertThat(new YokeProperties.Knowledge.Embedding(null, "u", "k", null).enabled()).isFalse(); + } } diff --git a/src/test/java/com/yoke/knowledge/HybridRetrievePortTests.java b/src/test/java/com/yoke/knowledge/HybridRetrievePortTests.java new file mode 100644 index 0000000..6601951 --- /dev/null +++ b/src/test/java/com/yoke/knowledge/HybridRetrievePortTests.java @@ -0,0 +1,124 @@ +package com.yoke.knowledge; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Mono; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class HybridRetrievePortTests { + + private static Passage p(String id) { + return new Passage(id, "文档", "正文 " + id); + } + + private static RetrievePort returning(Passage... passages) { + return (query, topK) -> Mono.just(List.of(passages)); + } + + private static RetrievePort failing() { + return (query, topK) -> Mono.error(new IllegalStateException("向量化服务超时")); + } + + @Test + void RRF按名次融合_两路都靠前的片段排第一() { + // 向量路：A B C；关键词路：B D A。B 在两路分别是第 2 和第 1 名，A 是第 1 和第 3 名 + RetrievePort hybrid = new HybridRetrievePort(List.of(returning(p("A"), p("B"), p("C")), returning(p("B"), p("D"), p("A")))); + + List result = hybrid.retrieve("q", 4).block(); + + // B: 1/62 + 1/61；A: 1/61 + 1/63；C: 1/63；D: 1/62 + assertThat(result).extracting(Passage::id).containsExactly("B", "A", "D", "C"); + } + + @Test + void RRF得分使用常数60() { + assertThat(HybridRetrievePort.RRF_K).isEqualTo(60); + // 第 1 名对第 2 名的优势是 1/61 - 1/62；只有一路时直接保持原顺序 + assertThat(HybridRetrievePort.fuse(List.of(List.of(p("X"), p("Y"))), 2)).extracting(Passage::id).containsExactly("X", "Y"); + } + + @Test + void 融合后只保留topK条() { + RetrievePort hybrid = new HybridRetrievePort(List.of(returning(p("A"), p("B")), returning(p("C"), p("D")))); + + assertThat(hybrid.retrieve("q", 3).block()).hasSize(3); + } + + @Test + void 同一个片段在两路里重复出现只算一条() { + RetrievePort hybrid = new HybridRetrievePort(List.of(returning(p("A")), returning(p("A")))); + + assertThat(hybrid.retrieve("q", 5).block()).extracting(Passage::id).containsExactly("A"); + } + + @Test + void 一路失败时退回另一路() { + RetrievePort hybrid = new HybridRetrievePort(List.of(failing(), returning(p("K1"), p("K2")))); + + assertThat(hybrid.retrieve("q", 5).block()).extracting(Passage::id).containsExactly("K1", "K2"); + } + + @Test + void 关键词路失败时退回向量路() { + RetrievePort hybrid = new HybridRetrievePort(List.of(returning(p("V1")), failing())); + + assertThat(hybrid.retrieve("q", 5).block()).extracting(Passage::id).containsExactly("V1"); + } + + @Test + void 一路为空另一路有结果时返回有结果的那一路() { + RetrievePort hybrid = new HybridRetrievePort(List.of(returning(), returning(p("K1")))); + + assertThat(hybrid.retrieve("q", 5).block()).extracting(Passage::id).containsExactly("K1"); + } + + @Test + void 两路都成功但都为空时返回空列表() { + RetrievePort hybrid = new HybridRetrievePort(List.of(returning(), returning())); + + assertThat(hybrid.retrieve("q", 5).block()).isEmpty(); + } + + @Test + void 所有路都失败时抛出错误而不是假装没有命中() { + RetrievePort hybrid = new HybridRetrievePort(List.of(failing(), failing())); + + assertThatThrownBy(() -> hybrid.retrieve("q", 5).block()) + .isInstanceOf(IllegalStateException.class).hasMessageContaining("超时"); + } + + @Test + void 失败的一路不会拖累成功的一路被调用() { + AtomicInteger calls = new AtomicInteger(); + RetrievePort counting = (query, topK) -> { + calls.incrementAndGet(); + return Mono.just(List.of(p("K"))); + }; + + new HybridRetrievePort(List.of(failing(), counting)).retrieve("q", 3).block(); + + assertThat(calls).hasValue(1); + } + + @Test + void 每一路取的候选比最终topK更宽() { + AtomicInteger requested = new AtomicInteger(); + RetrievePort spy = (query, topK) -> { + requested.set(topK); + return Mono.just(List.of()); + }; + + new HybridRetrievePort(List.of(spy)).retrieve("q", 3).block(); + + assertThat(requested.get()).isGreaterThan(3); + } + + @Test + void 没有任何检索通道时构造即失败() { + assertThatThrownBy(() -> new HybridRetrievePort(List.of())).isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/src/test/java/com/yoke/knowledge/KeywordRetrievePortTests.java b/src/test/java/com/yoke/knowledge/KeywordRetrievePortTests.java new file mode 100644 index 0000000..cdc8612 --- /dev/null +++ b/src/test/java/com/yoke/knowledge/KeywordRetrievePortTests.java @@ -0,0 +1,52 @@ +package com.yoke.knowledge; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class KeywordRetrievePortTests { + + private static final Passage REDIS = new Passage("a#0", "会话与存储", "对话历史放在 Redis 里，重启之后仍然在。"); + private static final Passage TOOLS = new Passage("b#0", "工具与权限", "calculate 工具做四则运算，支持括号和小数。"); + private static final Passage VIRTUAL = new Passage("c#0", "概览", "应用使用虚拟线程处理请求。"); + + private final KeywordRetrievePort port = new KeywordRetrievePort(List.of(REDIS, TOOLS, VIRTUAL)); + + @Test + void 中文问题能命中字面相关的片段() { + assertThat(port.retrieve("对话历史存在哪里", 3).block()).first().isEqualTo(REDIS); + assertThat(port.retrieve("四则运算怎么算", 3).block()).first().isEqualTo(TOOLS); + } + + @Test + void 英文和数字按单词匹配且不区分大小写() { + assertThat(port.retrieve("REDIS", 3).block()).containsExactly(REDIS); + assertThat(port.retrieve("calculate", 3).block()).containsExactly(TOOLS); + } + + @Test + void 没有任何字面重叠时返回空列表() { + assertThat(port.retrieve("量子纠缠", 3).block()).isEmpty(); + } + + @Test + void 结果数量不超过topK() { + assertThat(port.retrieve("的 工具 对话 虚拟 线程 Redis 括号", 2).block()).hasSizeLessThanOrEqualTo(2); + } + + @Test + void 空问题_非正topK和空语料都返回空列表() { + assertThat(port.retrieve(" ", 3).block()).isEmpty(); + assertThat(port.retrieve(null, 3).block()).isEmpty(); + assertThat(port.retrieve("Redis", 0).block()).isEmpty(); + assertThat(new KeywordRetrievePort(List.of()).retrieve("Redis", 3).block()).isEmpty(); + } + + @Test + void 分词把连续汉字按相邻两字切并保留英文单词() { + assertThat(KeywordRetrievePort.tokenize("知识库 Redis 检索")).containsExactly("知识", "识库", "redis", "检索"); + assertThat(KeywordRetrievePort.tokenize("库")).containsExactly("库"); + } +} diff --git a/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java b/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java new file mode 100644 index 0000000..8f644c7 --- /dev/null +++ b/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java @@ -0,0 +1,65 @@ +package com.yoke.knowledge; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class KnowledgeCorpusTests { + + @Test + void 一级标题当文档标题_二级标题切片段_片段带上完整标题() { + List passages = KnowledgeCorpus.split("demo.md", "# 文档标题\n\n## 第一节\n内容一\n\n## 第二节\n内容二\n"); + + assertThat(passages).hasSize(2); + assertThat(passages.get(0).source()).isEqualTo("文档标题"); + assertThat(passages.get(0).text()).startsWith("文档标题 / 第一节\n").contains("内容一"); + assertThat(passages.get(1).text()).startsWith("文档标题 / 第二节\n").contains("内容二"); + assertThat(passages).extracting(Passage::id).doesNotHaveDuplicates(); + } + + @Test + void 过长的小节按段落再切() { + String paragraph = "字".repeat(KnowledgeCorpus.MAX_CHUNK_LENGTH - 50); + List passages = KnowledgeCorpus.split("long.md", "# 长文\n## 一节\n" + paragraph + "\n\n" + paragraph + "\n"); + + assertThat(passages).hasSize(2); + } + + @Test + void 没有一级标题时用文件名当标题() { + assertThat(KnowledgeCorpus.split("x.md", "## 节\n内容").getFirst().source()).isEqualTo("x.md"); + } + + @Test + void 空小节不产生片段() { + assertThat(KnowledgeCorpus.split("x.md", "# 标题\n## 空节\n## 有内容\n正文")).hasSize(1); + } + + @Test + void 随应用发布的知识文档被完整加载() { + List passages = KnowledgeCorpus.loadFromClasspath("classpath*:knowledge/*.md"); + + assertThat(passages).extracting(Passage::source) + .contains("Yoke 概览", "会话与存储", "工具与权限", "接口使用"); + assertThat(passages).extracting(Passage::id).doesNotHaveDuplicates(); + } + + @Test + void 没有匹配的文档时返回空列表() { + assertThat(KnowledgeCorpus.loadFromClasspath("classpath*:no-such-dir/*.md")).isEmpty(); + } + + @Test + void 关键词检索在真实语料上能回答关于Yoke的问题() { + KeywordRetrievePort port = new KeywordRetrievePort(KnowledgeCorpus.loadFromClasspath("classpath*:knowledge/*.md")); + + assertThat(port.retrieve("对话历史存在哪里", 3).block()).first() + .satisfies(passage -> assertThat(passage.text()).contains("Redis")); + assertThat(port.retrieve("为什么不允许 stdio 传输", 3).block()).first() + .satisfies(passage -> assertThat(passage.text()).contains("stdio")); + assertThat(port.retrieve("流式接口有哪些事件", 3).block()).first() + .satisfies(passage -> assertThat(passage.text()).contains("textDelta")); + } +} diff --git a/src/test/java/com/yoke/knowledge/OpenAiEmbeddingClientTests.java b/src/test/java/com/yoke/knowledge/OpenAiEmbeddingClientTests.java new file mode 100644 index 0000000..6adfcb4 --- /dev/null +++ b/src/test/java/com/yoke/knowledge/OpenAiEmbeddingClientTests.java @@ -0,0 +1,127 @@ +package com.yoke.knowledge; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import org.springframework.web.client.RestClient; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** 用 JDK 自带的 HttpServer 模拟 OpenAI 兼容的 /embeddings 接口，不访问外网。 */ +class OpenAiEmbeddingClientTests { + + private HttpServer server; + private final List requestBodies = new CopyOnWriteArrayList<>(); + private final List authHeaders = new CopyOnWriteArrayList<>(); + private volatile int status = 200; + private volatile String responseBody; + private volatile boolean computeFromRequest; + + @BeforeEach + void start() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/v1/embeddings", exchange -> { + String request = new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8); + requestBodies.add(request); + authHeaders.add(exchange.getRequestHeaders().getFirst("Authorization")); + byte[] body = (computeFromRequest ? vectorsFor(request) : responseBody).getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + server.start(); + } + + @AfterEach + void stop() { + server.stop(0); + } + + private OpenAiEmbeddingClient client(Integer dimensions) { + RestClient rest = RestClient.builder() + .baseUrl("http://127.0.0.1:" + server.getAddress().getPort() + "/v1") + .defaultHeader("Authorization", "Bearer sk-test") + .build(); + return new OpenAiEmbeddingClient(rest, "text-embedding-3-small", dimensions); + } + + @Test + void 请求体按OpenAI协议组装_返回按index排好序的向量() { + // 故意乱序返回，客户端必须按 index 还原 + responseBody = """ + {"data":[{"index":1,"embedding":[0.0,1.0]},{"index":0,"embedding":[1.0,0.5]}]}"""; + + List vectors = client(256).embed(List.of("第一段", "第二段")).block(); + + assertThat(vectors).hasSize(2); + assertThat(vectors.get(0)).containsExactly(1.0, 0.5); + assertThat(vectors.get(1)).containsExactly(0.0, 1.0); + assertThat(requestBodies.getFirst()).contains("\"model\":\"text-embedding-3-small\"") + .contains("\"input\":[\"第一段\",\"第二段\"]").contains("\"dimensions\":256") + .contains("\"encoding_format\":\"float\""); + assertThat(authHeaders.getFirst()).isEqualTo("Bearer sk-test"); + } + + @Test + void 不指定维度时请求里不带dimensions() { + responseBody = """ + {"data":[{"index":0,"embedding":[1.0]}]}"""; + + client(null).embed(List.of("x")).block(); + + assertThat(requestBodies.getFirst()).doesNotContain("dimensions"); + } + + @Test + void 返回条数与请求不一致时报错() { + responseBody = """ + {"data":[{"index":0,"embedding":[1.0]}]}"""; + + assertThatThrownBy(() -> client(null).embed(List.of("a", "b")).block()).hasMessageContaining("不一致"); + } + + @Test + void 上游返回错误状态码时报错() { + status = 503; + responseBody = "{}"; + + assertThatThrownBy(() -> client(null).embed(List.of("a")).block()).isInstanceOf(RuntimeException.class); + } + + @Test + void 向量检索端到端_经过真实HTTP调用的向量化() { + // 按请求里的文本现算向量：含「甲」的是 [1,0]，其余是 [0,1] + computeFromRequest = true; + List passages = List.of(new Passage("a", "A", "甲方"), new Passage("b", "B", "乙方")); + VectorRetrievePort port = new VectorRetrievePort(passages, client(null)); + + assertThat(port.retrieve("甲", 1).block()).extracting(Passage::id).containsExactly("a"); + assertThat(port.retrieve("乙", 1).block()).extracting(Passage::id).containsExactly("b"); + // 1 次建索引 + 2 次查询 + assertThat(requestBodies).hasSize(3); + } + + private static String vectorsFor(String requestBody) { + try { + var input = new com.fasterxml.jackson.databind.ObjectMapper().readTree(requestBody).get("input"); + StringBuilder data = new StringBuilder("{\"data\":["); + for (int i = 0; i < input.size(); i++) { + String vector = input.get(i).asText().contains("甲") ? "[1.0,0.0]" : "[0.0,1.0]"; + data.append(i == 0 ? "" : ",").append("{\"index\":").append(i).append(",\"embedding\":").append(vector).append("}"); + } + return data.append("]}").toString(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/src/test/java/com/yoke/knowledge/VectorRetrievePortTests.java b/src/test/java/com/yoke/knowledge/VectorRetrievePortTests.java new file mode 100644 index 0000000..0c97633 --- /dev/null +++ b/src/test/java/com/yoke/knowledge/VectorRetrievePortTests.java @@ -0,0 +1,109 @@ +package com.yoke.knowledge; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Mono; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class VectorRetrievePortTests { + + /** 假的向量化：文本里出现哪个关键字就在对应维度上打 1，足够让余弦相似度可预测。 */ + static class KeywordEmbedding implements EmbeddingClient { + final AtomicInteger calls = new AtomicInteger(); + final List batchSizes = new ArrayList<>(); + boolean failNext; + + @Override + public synchronized Mono> embed(List texts) { + calls.incrementAndGet(); + batchSizes.add(texts.size()); + if (failNext) { + failNext = false; + return Mono.error(new IllegalStateException("上游 503")); + } + return Mono.just(texts.stream().map(text -> new double[] { + text.contains("Redis") ? 1 : 0, text.contains("工具") ? 1 : 0, text.contains("线程") ? 1 : 0, 0.01}).toList()); + } + } + + private static final List PASSAGES = List.of( + new Passage("a", "存储", "历史放在 Redis"), + new Passage("b", "工具", "calculate 工具做运算"), + new Passage("c", "线程", "使用虚拟线程")); + + @Test + void 按余弦相似度返回最接近的片段() { + VectorRetrievePort port = new VectorRetrievePort(PASSAGES, new KeywordEmbedding()); + + assertThat(port.retrieve("Redis 在哪", 1).block()).extracting(Passage::id).containsExactly("a"); + assertThat(port.retrieve("哪个工具", 2).block()).extracting(Passage::id).first().isEqualTo("b"); + } + + @Test + void 片段向量只算一次_之后每次检索只多一次查询向量化() { + KeywordEmbedding embedding = new KeywordEmbedding(); + VectorRetrievePort port = new VectorRetrievePort(PASSAGES, embedding); + + port.retrieve("Redis", 1).block(); + port.retrieve("工具", 1).block(); + port.retrieve("线程", 1).block(); + + // 1 次建索引（3 个片段一批）+ 3 次查询 + assertThat(embedding.calls).hasValue(4); + assertThat(embedding.batchSizes.getFirst()).isEqualTo(3); + } + + @Test + void 片段多时分批向量化() { + List many = new ArrayList<>(); + for (int i = 0; i < VectorRetrievePort.BATCH_SIZE + 4; i++) { + many.add(new Passage("p" + i, "文档", "Redis " + i)); + } + KeywordEmbedding embedding = new KeywordEmbedding(); + + new VectorRetrievePort(many, embedding).retrieve("Redis", 1).block(); + + assertThat(embedding.batchSizes.subList(0, 2)).containsExactly(VectorRetrievePort.BATCH_SIZE, 4); + } + + @Test + void 建索引失败时报错且下一次检索会重试而不是缓存失败() { + KeywordEmbedding embedding = new KeywordEmbedding(); + embedding.failNext = true; + VectorRetrievePort port = new VectorRetrievePort(PASSAGES, embedding); + + assertThatThrownBy(() -> port.retrieve("Redis", 1).block()).hasMessageContaining("503"); + + assertThat(port.retrieve("Redis", 1).block()).extracting(Passage::id).containsExactly("a"); + } + + @Test + void 向量化返回条数不对时报错() { + EmbeddingClient broken = texts -> Mono.just(List.of(new double[] {1})); + VectorRetrievePort port = new VectorRetrievePort(PASSAGES, broken); + + assertThatThrownBy(() -> port.retrieve("Redis", 1).block()).hasMessageContaining("不一致"); + } + + @Test + void 空问题和空语料不触发任何向量化调用() { + KeywordEmbedding embedding = new KeywordEmbedding(); + + assertThat(new VectorRetrievePort(PASSAGES, embedding).retrieve(" ", 3).block()).isEmpty(); + assertThat(new VectorRetrievePort(List.of(), embedding).retrieve("Redis", 3).block()).isEmpty(); + assertThat(embedding.calls).hasValue(0); + } + + @Test + void 余弦相似度对零向量安全() { + assertThat(VectorRetrievePort.cosine(new double[] {0, 0}, new double[] {1, 1})).isZero(); + assertThat(VectorRetrievePort.cosine(new double[] {1, 0}, new double[] {1, 0})).isEqualTo(1.0); + assertThatThrownBy(() -> VectorRetrievePort.cosine(new double[] {1}, new double[] {1, 2})) + .hasMessageContaining("维度"); + } +} diff --git a/src/test/java/com/yoke/mcp/McpConnectionsTests.java b/src/test/java/com/yoke/mcp/McpConnectionsTests.java new file mode 100644 index 0000000..3a79008 --- /dev/null +++ b/src/test/java/com/yoke/mcp/McpConnectionsTests.java @@ -0,0 +1,233 @@ +package com.yoke.mcp; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.yoke.config.YokeProperties; +import com.yoke.support.McpTestServer; +import io.agentscope.core.message.TextBlock; +import io.agentscope.core.message.ToolUseBlock; +import io.agentscope.core.tool.ToolCallParam; +import io.agentscope.core.tool.Toolkit; +import io.agentscope.core.tool.mcp.McpClientWrapper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Mono; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** 客户端是真实的 McpClientBuilder，服务端是 MCP 官方 SDK 搭的进程内服务，走完整的 Streamable HTTP 握手。 */ +class McpConnectionsTests { + + private McpTestServer server; + + @BeforeEach + void start() { + server = new McpTestServer(); + } + + @AfterEach + void stop() { + server.close(); + } + + private YokeProperties.McpServer server(String name, Map headers) { + return new YokeProperties.McpServer(name, "streamable-http", server.url(), Duration.ofSeconds(10), headers); + } + + @Test + void 真实握手后工具以服务端公布的原名注册_不加前缀() { + try (McpConnections connections = McpConnections.connect(List.of(server("demo", Map.of())))) { + Toolkit toolkit = new Toolkit(); + connections.registerInto(toolkit); + + assertThat(toolkit.getToolNames()).containsExactly("echo"); + assertThat(connections.names()).containsExactly("demo"); + } + } + + @Test + void 未声明只读的工具不会注册进Toolkit_只读的才会() { + try (McpConnections connections = McpConnections.connect(List.of(server("demo", Map.of())))) { + Toolkit toolkit = new Toolkit(); + connections.registerInto(toolkit); + + // 服务端公布了 echo（只读）和 write_note（未声明），后者模型根本看不到 + assertThat(toolkit.getToolNames()).contains("echo").doesNotContain("write_note"); + assertThat(toolkit.getToolSchemas()).extracting(schema -> schema.getName()).doesNotContain("write_note"); + assertThat(server.writeNoteCalls).hasValue(0); + } + } + + @Test + void SSE传输同样可以真实握手并注册工具() { + try (McpTestServer sseServer = new McpTestServer(McpTestServer.Transport.SSE)) { + YokeProperties.McpServer config = new YokeProperties.McpServer("legacy", "sse", sseServer.url(), + Duration.ofSeconds(10), Map.of()); + try (McpConnections connections = McpConnections.connect(List.of(config))) { + Toolkit toolkit = new Toolkit(); + connections.registerInto(toolkit); + + assertThat(toolkit.getToolNames()).containsExactly("echo"); + } + } + } + + @Test + void 注册进Toolkit的MCP工具可以被真实调用() { + try (McpConnections connections = McpConnections.connect(List.of(server("demo", Map.of())))) { + Toolkit toolkit = new Toolkit(); + connections.registerInto(toolkit); + Map input = Map.of("text", "你好"); + ToolUseBlock use = ToolUseBlock.builder().id("c1").name("echo").input(input).content("{\"text\":\"你好\"}").build(); + + var result = toolkit.callTool(ToolCallParam.builder().toolUseBlock(use).input(input).build()).block(); + + assertThat(result.getOutput()).filteredOn(TextBlock.class::isInstance) + .extracting(block -> ((TextBlock) block).getText()).containsExactly("echo: 你好"); + } + } + + @Test + void 配置里的请求头会带到服务端() { + try (McpConnections ignored = McpConnections.connect(List.of(server("demo", Map.of("Authorization", "Bearer secret-token"))))) { + assertThat(server.receivedHeaders).isNotEmpty() + .allSatisfy(headers -> assertThat(headers).containsEntry("authorization", "Bearer secret-token")); + } + } + + @Test + void 工具副本不拥有连接_副本被丢弃后连接仍可用() { + try (McpConnections connections = McpConnections.connect(List.of(server("demo", Map.of())))) { + Toolkit template = new Toolkit(); + connections.registerInto(template); + Toolkit copy = template.copy(); + assertThat(copy.getToolNames()).containsExactly("echo"); + + copy.closeMcpClients(); + + Map input = Map.of("text", "还在"); + ToolUseBlock use = ToolUseBlock.builder().id("c2").name("echo").input(input).content("{\"text\":\"还在\"}").build(); + var result = template.callTool(ToolCallParam.builder().toolUseBlock(use).input(input).build()).block(); + assertThat(((TextBlock) result.getOutput().getFirst()).getText()).isEqualTo("echo: 还在"); + } + } + + @Test + void 没有配置任何服务时是空连接且可以安全关闭() { + McpConnections connections = McpConnections.connect(List.of()); + + assertThat(connections.names()).isEmpty(); + connections.registerInto(new Toolkit()); + connections.close(); + } + + // ---- 释放顺序与失败处理（用记录型的假连接，不需要网络）---- + + /** 记录 close 调用顺序的假连接；initialize 可以指定成功或失败。 */ + static final class FakeClient extends McpClientWrapper { + + private final List closeLog; + private final Mono initialize; + private final RuntimeException closeFailure; + + FakeClient(String name, List closeLog, Mono initialize, RuntimeException closeFailure) { + super(name); + this.closeLog = closeLog; + this.initialize = initialize; + this.closeFailure = closeFailure; + } + + @Override + public Mono initialize() { + return initialize; + } + + @Override + public Mono> listTools() { + return Mono.just(List.of()); + } + + @Override + public Mono callTool(String toolName, Map arguments) { + return Mono.empty(); + } + + @Override + public Mono callTool(String toolName, Map arguments, + Map meta) { + return Mono.empty(); + } + + @Override + public void close() { + closeLog.add(name); + if (closeFailure != null) { + throw closeFailure; + } + } + } + + private static YokeProperties.McpServer cfg(String name) { + return new YokeProperties.McpServer(name, "sse", "http://127.0.0.1:1/sse", Duration.ofSeconds(1), Map.of()); + } + + @Test + void 关闭按连接的相反顺序进行() { + List closed = new ArrayList<>(); + McpConnections connections = McpConnections.connect(List.of(cfg("a"), cfg("b"), cfg("c")), + s -> new FakeClient(s.name(), closed, Mono.empty(), null)); + + connections.close(); + + assertThat(closed).containsExactly("c", "b", "a"); + } + + @Test + void 某个连接关不掉不影响其余连接被关闭_且异常不被吞掉() { + List closed = new ArrayList<>(); + McpConnections connections = McpConnections.connect(List.of(cfg("a"), cfg("b"), cfg("c")), + s -> new FakeClient(s.name(), closed, Mono.empty(), + s.name().equals("b") ? new IllegalStateException("b 关闭失败") : null)); + + assertThatThrownBy(connections::close).hasMessage("b 关闭失败"); + + assertThat(closed).containsExactly("c", "b", "a"); + } + + @Test + void 建连失败时已连上的按相反顺序关闭_并以原始失败为主异常() { + List closed = new ArrayList<>(); + RuntimeException closeFailure = new IllegalStateException("b 关闭时也失败了"); + + assertThatThrownBy(() -> McpConnections.connect(List.of(cfg("a"), cfg("b"), cfg("c")), s -> switch (s.name()) { + case "a" -> new FakeClient("a", closed, Mono.empty(), null); + case "b" -> new FakeClient("b", closed, Mono.empty(), closeFailure); + default -> new FakeClient("c", closed, Mono.error(new IllegalStateException("连接被拒绝")), null); + })) + // 主异常是「c 连不上」，不是关闭 b 时的失败 + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("无法连接 MCP 服务").hasMessageContaining("name=c") + .hasRootCauseMessage("连接被拒绝") + // 关闭 b 的失败只作为 suppressed 挂着，没有掩盖主因 + .satisfies(e -> assertThat(e.getSuppressed()).containsExactly(closeFailure)); + + // 失败的 c 自己也要关（它可能已建立了一半的连接），然后 b、a + assertThat(closed).containsExactly("c", "b", "a"); + } + + @Test + void 连不上真实地址时启动失败_且错误信息不含地址里的令牌() { + YokeProperties.McpServer unreachable = new YokeProperties.McpServer("bad", "streamable-http", + "http://127.0.0.1:1/mcp?token=super-secret", Duration.ofSeconds(2), Map.of()); + + assertThatThrownBy(() -> McpConnections.connect(List.of(unreachable))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("无法连接 MCP 服务").hasMessageContaining("name=bad") + .satisfies(e -> assertThat(e.getMessage()).doesNotContain("super-secret")); + } +} diff --git a/src/test/java/com/yoke/mcp/McpStartupTests.java b/src/test/java/com/yoke/mcp/McpStartupTests.java new file mode 100644 index 0000000..867d5e0 --- /dev/null +++ b/src/test/java/com/yoke/mcp/McpStartupTests.java @@ -0,0 +1,82 @@ +package com.yoke.mcp; + +import com.yoke.config.KnowledgeConfiguration; +import com.yoke.config.ToolConfiguration; +import com.yoke.config.SharedToolkit; +import com.yoke.config.YokeProperties; +import com.yoke.support.McpTestServer; +import org.junit.jupiter.api.Test; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +import static org.assertj.core.api.Assertions.assertThat; + +/** 从配置到容器的整条启动链路：只允许 HTTP 传输，默认不接任何 MCP，连不上就启动失败。 */ +class McpStartupTests { + + @Configuration(proxyBeanMethods = false) + @EnableConfigurationProperties(YokeProperties.class) + @Import({ToolConfiguration.class, KnowledgeConfiguration.class}) + static class Context { + } + + private final ApplicationContextRunner runner = new ApplicationContextRunner() + .withUserConfiguration(Context.class) + .withPropertyValues("yoke.redis.uri=redis://127.0.0.1:6379", "yoke.workspace=/tmp/yoke"); + + @Test + void 默认不配置任何MCP服务_应用正常启动且工具里没有MCP工具() { + runner.run(context -> { + assertThat(context).hasNotFailed(); + assertThat(context.getBean(McpConnections.class).names()).isEmpty(); + assertThat(context.getBean(SharedToolkit.class).toolkit().getToolNames()) + .containsExactlyInAnyOrder("get_current_time", "calculate", "list_my_notes", "retrieve"); + }); + } + + @Test + void 配置stdio传输时应用启动即失败() { + runner.withPropertyValues( + "yoke.mcp.servers[0].name=local-fs", + "yoke.mcp.servers[0].transport=stdio", + "yoke.mcp.servers[0].url=http://unused") + .run(context -> { + assertThat(context).hasFailed(); + assertThat(context.getStartupFailure()).hasStackTraceContaining("只支持 HTTP 传输") + .hasStackTraceContaining("stdio"); + }); + } + + @Test + void 服务连不上时应用启动即失败() { + runner.withPropertyValues( + "yoke.mcp.servers[0].name=down", + "yoke.mcp.servers[0].transport=streamable-http", + "yoke.mcp.servers[0].url=http://127.0.0.1:1/mcp", + "yoke.mcp.servers[0].timeout=2s") + .run(context -> { + assertThat(context).hasFailed(); + assertThat(context.getStartupFailure()).hasStackTraceContaining("无法连接 MCP 服务"); + }); + } + + @Test + void 配置了可用的HTTP服务时其工具出现在每个请求的Toolkit里() { + try (McpTestServer server = new McpTestServer()) { + runner.withPropertyValues( + "yoke.mcp.servers[0].name=demo", + "yoke.mcp.servers[0].transport=streamable-http", + "yoke.mcp.servers[0].url=" + server.url()) + .run(context -> { + assertThat(context).hasNotFailed(); + // 服务端公布的原名，不加前缀 + assertThat(context.getBean(SharedToolkit.class).toolkit().getToolNames()) + .contains("echo").doesNotContain("write_note"); + assertThat(context.getBean(McpConnections.class).names()).containsExactly("demo"); + }); + } + } +} diff --git a/src/test/java/com/yoke/support/McpTestServer.java b/src/test/java/com/yoke/support/McpTestServer.java new file mode 100644 index 0000000..2c06b63 --- /dev/null +++ b/src/test/java/com/yoke/support/McpTestServer.java @@ -0,0 +1,169 @@ +package com.yoke.support; + +import java.io.File; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; + +import io.modelcontextprotocol.json.McpJsonMapper; +import io.modelcontextprotocol.server.McpServer; +import io.modelcontextprotocol.server.McpServerFeatures; +import io.modelcontextprotocol.server.McpSyncServer; +import io.modelcontextprotocol.server.transport.HttpServletSseServerTransportProvider; +import io.modelcontextprotocol.server.transport.HttpServletStreamableServerTransportProvider; +import io.modelcontextprotocol.spec.McpSchema; +import jakarta.servlet.http.HttpServletRequest; +import org.apache.catalina.Context; +import org.apache.catalina.startup.Tomcat; + +/** + * 进程内的最小 MCP 服务：用 MCP 官方 SDK 的 Streamable HTTP 服务端传输挂在嵌入式 Tomcat 上（两者都已在类路径里， + * 不引入新依赖），公布 {@code echo}（只读）和 {@code write_note}（未声明只读）两个工具；也可以起成旧的 SSE 传输。测试里的客户端是真实的 McpClientBuilder，走完整的握手、列工具、调用流程。 + */ +public final class McpTestServer implements AutoCloseable { + + public static final String PATH = "/mcp"; + + public enum Transport { STREAMABLE_HTTP, SSE } + + /** 服务端收到的请求头（小写名到值），用来证明配置里的请求头确实被带上了。 */ + public final List> receivedHeaders = new CopyOnWriteArrayList<>(); + + private final Tomcat tomcat; + private final McpSyncServer server; + private final int port; + + private final Transport transport; + + public McpTestServer() { + this(Transport.STREAMABLE_HTTP); + } + + public McpTestServer(Transport transport) { + this.transport = transport; + try { + port = freePort(); + File base = Files.createTempDirectory("mcp-test-tomcat").toFile(); + tomcat = new Tomcat(); + tomcat.setBaseDir(base.getAbsolutePath()); + tomcat.setPort(port); + tomcat.getConnector(); + + jakarta.servlet.http.HttpServlet servlet; + McpSchema.ServerCapabilities capabilities = McpSchema.ServerCapabilities.builder().tools(false).build(); + if (transport == Transport.SSE) { + HttpServletSseServerTransportProvider provider = HttpServletSseServerTransportProvider.builder() + .jsonMapper(McpJsonMapper.getDefault()) + // 不设 baseUrl：服务端通告相对路径的消息端点。AgentScope 把完整的 SSE 地址当作客户端的 baseUri， + // 服务端若通告绝对地址（http://主机:端口/...），会因为「与 baseUri 不匹配」而被客户端拒绝 + .sseEndpoint("/sse") + .messageEndpoint("/mcp/message") + .build(); + server = McpServer.sync(provider).serverInfo("yoke-test-mcp", "1.0.0") + .capabilities(capabilities).tools(echoTool(), writeNoteTool()).build(); + servlet = provider; + } else { + HttpServletStreamableServerTransportProvider provider = HttpServletStreamableServerTransportProvider.builder() + .jsonMapper(McpJsonMapper.getDefault()) + .mcpEndpoint(PATH) + .build(); + server = McpServer.sync(provider).serverInfo("yoke-test-mcp", "1.0.0") + .capabilities(capabilities).tools(echoTool(), writeNoteTool()).build(); + servlet = provider; + } + + Context context = tomcat.addContext("", base.getAbsolutePath()); + Tomcat.addServlet(context, "mcp", servlet).setAsyncSupported(true); + context.addServletMappingDecoded("/*", "mcp"); + context.addFilterDef(filterDef()); + context.addFilterMap(filterMap()); + tomcat.start(); + } catch (Exception e) { + throw new IllegalStateException("启动测试用 MCP 服务失败", e); + } + } + + public String url() { + return "http://127.0.0.1:" + port + (transport == Transport.SSE ? "/sse" : PATH); + } + + /** {@code write_note} 被真正执行的次数；它没有声明只读，用来证明需要授权的 MCP 工具不会被悄悄执行。 */ + public final AtomicInteger writeNoteCalls = new AtomicInteger(); + + private McpServerFeatures.SyncToolSpecification writeNoteTool() { + McpSchema.JsonSchema schema = new McpSchema.JsonSchema("object", + Map.of("text", Map.of("type", "string")), List.of("text"), null, null, null); + McpSchema.Tool tool = McpSchema.Tool.builder() + .name("write_note") + .description("写一条笔记（有副作用，没有声明只读）") + .inputSchema(schema) + .build(); + return McpServerFeatures.SyncToolSpecification.builder() + .tool(tool) + .callHandler((exchange, request) -> { + writeNoteCalls.incrementAndGet(); + return new McpSchema.CallToolResult("已写入", false); + }) + .build(); + } + + private static McpServerFeatures.SyncToolSpecification echoTool() { + McpSchema.JsonSchema schema = new McpSchema.JsonSchema("object", + Map.of("text", Map.of("type", "string", "description", "要回显的文字")), List.of("text"), null, null, null); + McpSchema.Tool tool = McpSchema.Tool.builder() + .name("echo") + .description("原样回显收到的文字，用来验证 MCP 连通") + .inputSchema(schema) + // 声明只读：2.0.4 对只读的 MCP 工具自动放行，其余都要求显式授权 + .annotations(new McpSchema.ToolAnnotations("echo", true, false, true, false, false)) + .build(); + return McpServerFeatures.SyncToolSpecification.builder() + .tool(tool) + .callHandler((exchange, request) -> new McpSchema.CallToolResult( + "echo: " + request.arguments().get("text"), false)) + .build(); + } + + private org.apache.tomcat.util.descriptor.web.FilterDef filterDef() { + org.apache.tomcat.util.descriptor.web.FilterDef def = new org.apache.tomcat.util.descriptor.web.FilterDef(); + def.setFilterName("headers"); + def.setFilter((jakarta.servlet.Filter) (request, response, chain) -> { + // JSON-RPC 请求体按 UTF-8；Servlet 在 Content-Type 不带 charset 时默认按 ISO-8859-1 解码，中文会乱码 + request.setCharacterEncoding("UTF-8"); + if (request instanceof HttpServletRequest http) { + Map headers = new java.util.HashMap<>(); + http.getHeaderNames().asIterator().forEachRemaining(name -> headers.put(name.toLowerCase(), http.getHeader(name))); + receivedHeaders.add(headers); + } + chain.doFilter(request, response); + }); + return def; + } + + private static org.apache.tomcat.util.descriptor.web.FilterMap filterMap() { + org.apache.tomcat.util.descriptor.web.FilterMap map = new org.apache.tomcat.util.descriptor.web.FilterMap(); + map.setFilterName("headers"); + map.addURLPattern("/*"); + return map; + } + + @Override + public void close() { + try { + server.closeGracefully(); + tomcat.stop(); + tomcat.destroy(); + } catch (Exception e) { + throw new IllegalStateException("关闭测试用 MCP 服务失败", e); + } + } + + private static int freePort() throws java.io.IOException { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } +} diff --git a/src/test/java/com/yoke/support/ScriptedModel.java b/src/test/java/com/yoke/support/ScriptedModel.java new file mode 100644 index 0000000..aa565a0 --- /dev/null +++ b/src/test/java/com/yoke/support/ScriptedModel.java @@ -0,0 +1,94 @@ +package com.yoke.support; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Function; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.agentscope.core.message.ContentBlock; +import io.agentscope.core.message.Msg; +import io.agentscope.core.message.MsgRole; +import io.agentscope.core.message.TextBlock; +import io.agentscope.core.message.ToolResultBlock; +import io.agentscope.core.message.ToolUseBlock; +import io.agentscope.core.model.ChatResponse; +import io.agentscope.core.model.GenerateOptions; +import io.agentscope.core.model.Model; +import io.agentscope.core.model.ToolSchema; +import reactor.core.publisher.Flux; + +/** + * 按脚本回复的假模型，不产生任何网络调用：每次调用把「消息 + 工具清单」交给脚本函数，脚本返回要回复的内容块。 + * 同时把每次调用原样记录下来，测试可以据此断言「模型看到了哪些工具、拿到了什么工具结果」。 + */ +public class ScriptedModel implements Model { + + /** 一次模型调用的现场。 */ + public record Call(List messages, List tools) { + + public List toolNames() { + return tools.stream().map(ToolSchema::getName).toList(); + } + + /** 第一条系统消息的文本，用来区分是主 Agent 还是哪个子 Agent 在调用。 */ + public String systemText() { + return messages.stream().filter(m -> m.getRole() == MsgRole.SYSTEM).findFirst() + .map(Msg::getTextContent).orElse(""); + } + + /** 最后一个工具结果块的文本；没有则返回 null。 */ + public String lastToolResultText() { + String found = null; + for (Msg message : messages) { + for (ContentBlock block : message.getContent()) { + if (block instanceof ToolResultBlock result) { + found = result.getOutput().stream() + .filter(TextBlock.class::isInstance).map(b -> ((TextBlock) b).getText()) + .reduce("", String::concat); + } + } + } + return found; + } + + public boolean hasToolResult() { + return lastToolResultText() != null; + } + } + + public final List calls = new CopyOnWriteArrayList<>(); + + private final Function> script; + + public ScriptedModel(Function> script) { + this.script = script; + } + + public static List text(String text) { + return List.of(TextBlock.builder().text(text).build()); + } + + public static List toolUse(String id, String name, Map input) { + // 流式聚合时框架以 content 里的原始 JSON 为准解析入参，所以两处都要给 + try { + String json = new ObjectMapper().writeValueAsString(input); + return List.of(ToolUseBlock.builder().id(id).name(name).input(input).content(json).build()); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public Flux stream(List messages, List tools, GenerateOptions options) { + Call call = new Call(List.copyOf(messages), tools == null ? List.of() : List.copyOf(tools)); + calls.add(call); + return Flux.just(ChatResponse.builder().content(script.apply(call)).build()); + } + + @Override + public String getModelName() { + return "scripted"; + } +} diff --git a/src/test/java/com/yoke/support/TestAgents.java b/src/test/java/com/yoke/support/TestAgents.java new file mode 100644 index 0000000..835fe8b --- /dev/null +++ b/src/test/java/com/yoke/support/TestAgents.java @@ -0,0 +1,35 @@ +package com.yoke.support; + +import java.nio.file.Path; +import java.util.List; + +import com.yoke.config.HarnessAgentFactory; +import com.yoke.config.YokeProperties; +import com.yoke.config.YokeSubagents; +import io.agentscope.core.model.Model; +import io.agentscope.core.state.InMemoryAgentStateStore; +import io.agentscope.core.tool.Toolkit; +import io.agentscope.harness.agent.DistributedStore; +import io.agentscope.harness.agent.filesystem.remote.store.InMemoryStore; + +/** 不依赖 Redis 的 Agent 装配：会话状态放内存，用来测试工具和子 Agent 的行为。 */ +public final class TestAgents { + + private TestAgents() { + } + + public static DistributedStore inMemoryStore() { + return DistributedStore.builder() + .agentStateStore(new InMemoryAgentStateStore()) + .baseStore(new InMemoryStore()) + .build(); + } + + /** 与线上同一个工厂，只把模型、存储、工具换成测试用的；子 Agent 声明文件会真实地装进工作区。 */ + public static HarnessAgentFactory factory(Model model, Path workspace, Toolkit toolkitTemplate) { + var subagents = YokeSubagents.programmatic(); + YokeSubagents.prepareWorkspace(workspace, subagents); + return new HarnessAgentFactory("Yoke", "你是测试助手", 6, model, inMemoryStore(), workspace, + new YokeProperties.Compaction(30, 10), List.of(), toolkitTemplate, subagents); + } +} diff --git a/src/test/java/com/yoke/tool/ChatToolsTests.java b/src/test/java/com/yoke/tool/ChatToolsTests.java new file mode 100644 index 0000000..719a056 --- /dev/null +++ b/src/test/java/com/yoke/tool/ChatToolsTests.java @@ -0,0 +1,184 @@ +package com.yoke.tool; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.agentscope.core.agent.RuntimeContext; +import io.agentscope.core.message.ToolResultBlock; +import io.agentscope.core.message.TextBlock; +import io.agentscope.core.model.ToolSchema; +import io.agentscope.core.tool.ToolCallParam; +import io.agentscope.core.tool.Toolkit; +import org.junit.jupiter.api.Test; +import io.agentscope.core.message.ToolUseBlock; + +import static org.assertj.core.api.Assertions.assertThat; + +class ChatToolsTests { + + private static final Clock FIXED = Clock.fixed(Instant.parse("2026-11-01T07:30:00Z"), ZoneOffset.UTC); + + private final ChatTools tools = new ChatTools(FIXED); + + // ---- get_current_time ---- + + @Test + void 当前时间带时区偏移和时区名() { + // 美东 2026-11-01 凌晨 2 点结束夏令时：07:30 UTC 时已经是标准时间 -05:00（不是夏令时的 -04:00） + assertThat(tools.getCurrentTime("Asia/Shanghai")).startsWith("2026-11-01 15:30:00 +08:00（Asia/Shanghai，SUNDAY）"); + assertThat(tools.getCurrentTime("America/New_York")).startsWith("2026-11-01 02:30:00 -05:00"); + } + + @Test + void 不传时区默认UTC() { + assertThat(tools.getCurrentTime(null)).startsWith("2026-11-01 07:30:00 +00:00（UTC"); + assertThat(tools.getCurrentTime(" ")).startsWith("2026-11-01 07:30:00 +00:00"); + } + + @Test + void 非法时区返回提示而不是抛异常() { + assertThat(tools.getCurrentTime("Mars/Olympus")).contains("无法识别的时区").contains("Mars/Olympus"); + // 过长的输入在回显时被截断，不会原样灌进上下文 + assertThat(tools.getCurrentTime("x".repeat(500)).length()).isLessThan(200); + } + + // ---- calculate ---- + + @Test + void 计算返回结果() { + assertThat(tools.calculate("(1.5 + 2) * 4 / 7")).isEqualTo("2"); + } + + @Test + void 非法表达式返回清晰的错误文本而不是抛异常() { + assertThat(tools.calculate("1 / 0")).isEqualTo("无法计算：除数不能为 0"); + assertThat(tools.calculate("System.exit(0)")).startsWith("无法计算："); + assertThat(tools.calculate(null)).isEqualTo("无法计算：表达式不能为空"); + } + + // ---- list_my_notes ---- + + @Test + void 笔记只返回当前用户自己的() { + String mine = tools.listMyNotes(new UserContext("alice"), null); + + assertThat(mine).contains("alice").contains("MCP"); + assertThat(mine).doesNotContain("Redis"); + assertThat(tools.listMyNotes(new UserContext("default"), null)).contains("Redis").doesNotContain("MCP"); + } + + @Test + void 没有上下文时拒绝而不是回落到默认用户() { + assertThat(tools.listMyNotes(null, 5)).contains("无法确定当前用户").doesNotContain("Redis"); + assertThat(tools.listMyNotes(new UserContext(" "), 5)).contains("无法确定当前用户"); + } + + @Test + void 笔记条数被钳制在合法范围内() { + // 默认只给 5 条，演示数据只有 3 条，全部返回 + assertThat(tools.listMyNotes(new UserContext("default"), null).lines().count()).isEqualTo(1 + 3); + // limit 为 1 只返回一条；0 和负数按 1 处理；超大值不会越界 + assertThat(tools.listMyNotes(new UserContext("default"), 1).lines().count()).isEqualTo(1 + 1); + assertThat(tools.listMyNotes(new UserContext("default"), 0).lines().count()).isEqualTo(1 + 1); + assertThat(tools.listMyNotes(new UserContext("default"), -9).lines().count()).isEqualTo(1 + 1); + assertThat(tools.listMyNotes(new UserContext("default"), Integer.MAX_VALUE).lines().count()).isEqualTo(1 + 3); + } + + @Test + void 没有笔记的用户得到明确提示() { + assertThat(tools.listMyNotes(new UserContext("nobody"), null)).contains("还没有笔记"); + } + + // ---- 经过真实 Toolkit 的注入 ---- + + private static Toolkit toolkit(ChatTools tools) { + Toolkit toolkit = new Toolkit(); + toolkit.registerTool(tools); + return toolkit; + } + + private static String call(Toolkit toolkit, String name, Map input, RuntimeContext context) { + // Toolkit 校验入参时读的是 content 里的原始 JSON，所以和 input 一起给 + ToolUseBlock use = ToolUseBlock.builder().id("c1").name(name).input(input).content(json(input)).build(); + ToolResultBlock result = toolkit.callTool(ToolCallParam.builder() + .toolUseBlock(use).input(input).runtimeContext(context).build()).block(); + return result.getOutput().stream().filter(TextBlock.class::isInstance) + .map(block -> ((TextBlock) block).getText()).reduce("", String::concat); + } + + private static String json(Map input) { + try { + return new ObjectMapper().writeValueAsString(input); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException(e); + } + } + + @Test + void 模型看到的工具清单里没有UserContext参数() { + List schemas = toolkit(tools).getToolSchemas(); + + ToolSchema notes = schemas.stream().filter(s -> s.getName().equals("list_my_notes")).findFirst().orElseThrow(); + @SuppressWarnings("unchecked") + Map properties = (Map) notes.getParameters().get("properties"); + // 只有模型该填的 limit；userId、UserContext 都不在 Schema 里 + assertThat(properties.keySet()).containsExactly("limit"); + assertThat(notes.getParameters().toString()).doesNotContainIgnoringCase("userId").doesNotContain("UserContext"); + } + + @Test + void 工具里拿到的是RuntimeContext里的用户而不是模型传的() { + Toolkit toolkit = toolkit(tools); + RuntimeContext alice = RuntimeContext.builder().userId("alice").put(UserContext.class, new UserContext("alice")).build(); + + // 模型即便硬塞一个 userId 参数，也不会生效 + String result = call(toolkit, "list_my_notes", Map.of("limit", 5, "userId", "default", "user", Map.of("userId", "default")), alice); + + assertThat(result).contains("alice").contains("MCP").doesNotContain("Redis"); + } + + @Test + void 换一个用户拿到不同的笔记() { + Toolkit toolkit = toolkit(tools); + RuntimeContext alice = RuntimeContext.builder().put(UserContext.class, new UserContext("alice")).build(); + RuntimeContext other = RuntimeContext.builder().put(UserContext.class, new UserContext("default")).build(); + + assertThat(call(toolkit, "list_my_notes", Map.of(), alice)).contains("MCP"); + assertThat(call(toolkit, "list_my_notes", Map.of(), other)).contains("Redis"); + } + + @Test + void 上下文里没有放UserContext时工具拒绝服务() { + String result = call(toolkit(tools), "list_my_notes", Map.of(), RuntimeContext.builder().userId("alice").build()); + + assertThat(result).contains("无法确定当前用户"); + } + + @Test + void 并发调用互不串用户() throws Exception { + Toolkit toolkit = toolkit(tools); + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> jobs = new java.util.ArrayList<>(); + for (int i = 0; i < 200; i++) { + boolean alice = i % 2 == 0; + jobs.add(() -> { + RuntimeContext context = RuntimeContext.builder() + .put(UserContext.class, new UserContext(alice ? "alice" : "default")).build(); + String out = call(toolkit, "list_my_notes", Map.of(), context); + return alice ? out.contains("MCP") && !out.contains("Redis") : out.contains("Redis") && !out.contains("MCP"); + }); + } + for (var future : executor.invokeAll(jobs, 30, TimeUnit.SECONDS)) { + assertThat(future.get()).isTrue(); + } + } + } +} diff --git a/src/test/java/com/yoke/tool/ExpressionEvaluatorTests.java b/src/test/java/com/yoke/tool/ExpressionEvaluatorTests.java new file mode 100644 index 0000000..b3ad6f4 --- /dev/null +++ b/src/test/java/com/yoke/tool/ExpressionEvaluatorTests.java @@ -0,0 +1,128 @@ +package com.yoke.tool; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class ExpressionEvaluatorTests { + + @ParameterizedTest + @CsvSource(delimiter = '|', value = { + "1+2|3", + "1 + 2 * 3|7", + "(1 + 2) * 3|9", + "10 / 4|2.5", + "0.1 + 0.2|0.3", + "-3 + 5|2", + "2 * -3|-6", + "--4|4", + "((2))|2", + "100|100", + "1/3|0.3333333333333333", + "7 - 10|-3", + "1.50 * 2|3", + "0 * 5|0" + }) + void 合法表达式按四则运算优先级求值(String expression, String expected) { + assertThat(ExpressionEvaluator.evaluate(expression)).isEqualTo(expected); + } + + @Test + void 除数为0明确报错() { + assertThatThrownBy(() -> ExpressionEvaluator.evaluate("1 / 0")) + .isInstanceOf(ExpressionEvaluator.InvalidExpressionException.class) + .hasMessageContaining("除数不能为 0"); + assertThatThrownBy(() -> ExpressionEvaluator.evaluate("5 / (2 - 2)")) + .hasMessageContaining("除数不能为 0"); + } + + @ParameterizedTest + @ValueSource(strings = { + "", " ", "+", "1 +", "* 2", "(1 + 2", "1 + 2)", "1 2", "1..2", ".5", "5.", "abc", "1 + a", + "Math.sqrt(4)", "System.exit(0)", "1; 2", "2 ** 3", "2 ^ 3", "1e3", "0x10", "１＋２", "(１)", "1\t+ 2" + }) + void 非法表达式一律拒绝且不会执行任何代码(String expression) { + assertThatThrownBy(() -> ExpressionEvaluator.evaluate(expression)) + .isInstanceOf(ExpressionEvaluator.InvalidExpressionException.class); + } + + @Test + void null也只是非法表达式() { + assertThatThrownBy(() -> ExpressionEvaluator.evaluate(null)) + .isInstanceOf(ExpressionEvaluator.InvalidExpressionException.class); + } + + @Test + void 超长输入被拒绝() { + String tooLong = "1+".repeat(ExpressionEvaluator.MAX_LENGTH) + "1"; + + assertThatThrownBy(() -> ExpressionEvaluator.evaluate(tooLong)) + .hasMessageContaining("太长"); + // 恰好在上限内的长表达式仍可计算 + String atLimit = "1" + "+1".repeat((ExpressionEvaluator.MAX_LENGTH - 1) / 2); + assertThat(atLimit.length()).isLessThanOrEqualTo(ExpressionEvaluator.MAX_LENGTH); + assertThat(ExpressionEvaluator.evaluate(atLimit)).isEqualTo(String.valueOf(1 + (ExpressionEvaluator.MAX_LENGTH - 1) / 2)); + } + + @Test + void 括号嵌套过深被拒绝而不是栈溢出() { + String deep = "(".repeat(ExpressionEvaluator.MAX_DEPTH + 1) + "1" + ")".repeat(ExpressionEvaluator.MAX_DEPTH + 1); + + assertThatThrownBy(() -> ExpressionEvaluator.evaluate(deep)).hasMessageContaining("嵌套太深"); + String fine = "(".repeat(ExpressionEvaluator.MAX_DEPTH) + "1" + ")".repeat(ExpressionEvaluator.MAX_DEPTH); + assertThat(ExpressionEvaluator.evaluate(fine)).isEqualTo("1"); + } + + @Test + void 连续的一元符号同样受深度限制() { + String minuses = "-".repeat(ExpressionEvaluator.MAX_DEPTH + 1) + "1"; + + assertThatThrownBy(() -> ExpressionEvaluator.evaluate(minuses)).hasMessageContaining("嵌套太深"); + } + + @Test + void 超长数字被拒绝() { + String number = "9".repeat(ExpressionEvaluator.MAX_NUMBER_LENGTH + 1); + + assertThatThrownBy(() -> ExpressionEvaluator.evaluate(number + " + 1")).hasMessageContaining("数字太长"); + } + + @Test + void 大数相乘用高精度不溢出() { + assertThat(ExpressionEvaluator.evaluate("99999999999 * 99999999999")).isEqualTo("9999999999800000000001"); + } + + @Test + void 错误位置指向第一个非法字符() { + assertThatThrownBy(() -> ExpressionEvaluator.evaluate("1 + a")) + .hasMessageContaining("第 5 个字符 'a'"); + } + + @Test + void 链式除以极小数结果很大但不会失控() { + // 12 次除以 1e-10，等于乘以 1e120；输入在长度上限内，结果是 121 位的整数，仍然是精确的十进制文本 + String expression = "1" + "/0.0000000001".repeat(12); + assertThat(expression.length()).isLessThanOrEqualTo(ExpressionEvaluator.MAX_LENGTH); + + String result = ExpressionEvaluator.evaluate(expression); + + assertThat(result).isEqualTo("1" + "0".repeat(120)); + } + + @Test + void 链式乘以极小数结果的小数位随输入线性增长_仍能正常输出() { + // 6 个 25 位小数相乘：小数位数为各因子之和（150 位），输入长度上限保证了它不会无限增长 + String factor = "0.0000000000000000000000001"; + String expression = String.join("*", java.util.Collections.nCopies(6, factor)); + assertThat(expression.length()).isLessThanOrEqualTo(ExpressionEvaluator.MAX_LENGTH); + + String result = ExpressionEvaluator.evaluate(expression); + + assertThat(result).startsWith("0.").endsWith("1").hasSize(2 + 150); + assertThat(result.chars().filter(c -> c == '1').count()).isEqualTo(1); + } +} diff --git a/src/test/java/com/yoke/tool/RetrieveToolTests.java b/src/test/java/com/yoke/tool/RetrieveToolTests.java new file mode 100644 index 0000000..6801400 --- /dev/null +++ b/src/test/java/com/yoke/tool/RetrieveToolTests.java @@ -0,0 +1,95 @@ +package com.yoke.tool; + +import java.util.ArrayList; +import java.util.List; + +import com.yoke.knowledge.Passage; +import com.yoke.knowledge.RetrievePort; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Mono; + +import static org.assertj.core.api.Assertions.assertThat; + +class RetrieveToolTests { + + private static final List HITS = List.of( + new Passage("a", "会话与存储", "会话与存储 / 对话历史\n历史放在 Redis"), + new Passage("b", "工具与权限", "工具与权限 / 模型能用哪些工具\n有 calculate")); + + @Test + void 结果带编号和来源_方便模型按编号引用() { + RetrieveTool tool = new RetrieveTool((query, topK) -> Mono.just(HITS)); + + String result = tool.retrieve("历史", 3).block(); + + assertThat(result).contains("检索到 2 条相关资料") + .contains("[1] 来源：会话与存储").contains("历史放在 Redis") + .contains("[2] 来源：工具与权限").contains("有 calculate"); + assertThat(result.indexOf("[1]")).isLessThan(result.indexOf("[2]")); + } + + @Test + void topK不传用默认值_传大了被钳制到上限_传小了至少为1() { + List requested = new ArrayList<>(); + RetrieveTool tool = new RetrieveTool((query, topK) -> { + requested.add(topK); + return Mono.just(List.of()); + }); + + tool.retrieve("q", null).block(); + tool.retrieve("q", 1000).block(); + tool.retrieve("q", Integer.MAX_VALUE).block(); + tool.retrieve("q", 0).block(); + tool.retrieve("q", -5).block(); + tool.retrieve("q", 5).block(); + + assertThat(requested).containsExactly(RetrieveTool.DEFAULT_TOP_K, RetrieveTool.MAX_TOP_K, RetrieveTool.MAX_TOP_K, 1, 1, 5); + } + + @Test + void 过长的问题被截断后再检索() { + List queries = new ArrayList<>(); + RetrieveTool tool = new RetrieveTool((query, topK) -> { + queries.add(query); + return Mono.just(List.of()); + }); + + tool.retrieve("问".repeat(5000), 3).block(); + + assertThat(queries.getFirst()).hasSize(RetrieveTool.MAX_QUERY_LENGTH); + } + + @Test + void 没有命中时明确告诉模型不要编造() { + RetrieveTool tool = new RetrieveTool((query, topK) -> Mono.just(List.of())); + + assertThat(tool.retrieve("量子纠缠", 3).block()).contains("没有检索到").contains("不要编造"); + } + + @Test + void 未配置检索端口时返回提示语而不是抛异常() { + RetrieveTool tool = new RetrieveTool(null); + + assertThat(tool.retrieve("任何问题", 3).block()).contains("知识库尚未配置"); + } + + @Test + void 空问题返回提示而不触发检索() { + RetrievePort unreachable = (query, topK) -> { + throw new AssertionError("不应该被调用"); + }; + RetrieveTool tool = new RetrieveTool(unreachable); + + assertThat(tool.retrieve(" ", 3).block()).contains("query 不能为空"); + assertThat(tool.retrieve(null, 3).block()).contains("query 不能为空"); + } + + @Test + void 检索失败时只返回通用提示_不泄漏上游细节() { + RetrieveTool tool = new RetrieveTool((query, topK) -> Mono.error(new IllegalStateException("连接 10.0.0.8:443 被拒绝"))); + + String result = tool.retrieve("q", 3).block(); + + assertThat(result).contains("暂时不可用").doesNotContain("10.0.0.8"); + } +} diff --git a/src/test/java/com/yoke/tool/ToolConversationTests.java b/src/test/java/com/yoke/tool/ToolConversationTests.java new file mode 100644 index 0000000..3a331b4 --- /dev/null +++ b/src/test/java/com/yoke/tool/ToolConversationTests.java @@ -0,0 +1,177 @@ +package com.yoke.tool; + +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Map; + +import com.yoke.controller.ChatController; +import com.yoke.model.ChatEvent; +import com.yoke.model.ChatRequest; +import com.yoke.model.ChatResponse; +import com.yoke.support.McpTestServer; +import com.yoke.support.ScriptedModel; +import com.yoke.support.TestAgents; +import com.yoke.config.YokeProperties; +import com.yoke.mcp.McpConnections; +import io.agentscope.core.tool.Toolkit; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.springframework.http.codec.ServerSentEvent; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * 带工具的完整一轮对话：模型（脚本）先决定调用工具，真实的 HarnessAgent ReAct 循环执行工具、把结果回填， + * 模型再据此作答。控制器、EventCodec 都是真实的；不调用真实模型，也不需要 Redis。 + */ +class ToolConversationTests { + + private static Toolkit template() { + Toolkit toolkit = new Toolkit(); + toolkit.registerTool(new ChatTools()); + toolkit.registerTool(new RetrieveTool(null)); + return toolkit; + } + + private static ScriptedModel script(String tool, Map input) { + return new ScriptedModel(call -> { + if (call.systemText().startsWith("You are a memory")) { + return ScriptedModel.text("[]"); + } + if (!call.hasToolResult()) { + return ScriptedModel.toolUse("call-1", tool, input); + } + return ScriptedModel.text("工具说：" + call.lastToolResultText()); + }); + } + + private static List stream(ChatController controller, ChatRequest request) { + return controller.stream(request).map(ServerSentEvent::data).collectList().block(Duration.ofSeconds(30)); + } + + @Test + void 流式对话里工具被真实执行_结果回填给模型_事件里有调用和结果(@TempDir Path workspace) { + ScriptedModel model = script("calculate", Map.of("expression", "(1 + 2) * 4")); + ChatController controller = new ChatController(TestAgents.factory(model, workspace, template())); + + List events = stream(controller, new ChatRequest("s-calc", "帮我算 (1+2)*4")); + + ChatEvent.ToolCall call = events.stream().filter(ChatEvent.ToolCall.class::isInstance) + .map(ChatEvent.ToolCall.class::cast).findFirst().orElseThrow(); + ChatEvent.ToolResult result = events.stream().filter(ChatEvent.ToolResult.class::isInstance) + .map(ChatEvent.ToolResult.class::cast).findFirst().orElseThrow(); + assertThat(call.name()).isEqualTo("calculate"); + assertThat(call.arguments()).contains("(1 + 2) * 4"); + assertThat(result.id()).isEqualTo(call.id()); + assertThat(result.name()).isEqualTo("calculate"); + assertThat(result.state()).isEqualTo("success"); + assertThat(result.output()).contains("12"); + // 事件顺序：先调用，后结果，最后以 done 结束 + assertThat(events.indexOf(call)).isLessThan(events.indexOf(result)); + assertThat(events.getLast()).isInstanceOf(ChatEvent.Done.class); + // 结果真的回填给了模型：第二次调用的输入里带着工具结果，模型据此作答 + assertThat(((ChatEvent.Done) events.getLast()).text()).contains("工具说：").contains("12"); + assertThat(model.calls.stream().filter(c -> c.hasToolResult()).findFirst().orElseThrow().lastToolResultText()).contains("12"); + } + + @Test + void 非流式对话同样走工具并拿到最终回复(@TempDir Path workspace) { + ScriptedModel model = script("calculate", Map.of("expression", "1 / 0")); + ChatController controller = new ChatController(TestAgents.factory(model, workspace, template())); + + ChatResponse response = controller.chat(new ChatRequest("s-err", "算 1/0")).block(Duration.ofSeconds(30)); + + // 工具把错误作为文本返回给模型，由模型转述；而不是让整次对话失败 + assertThat(response.reply()).contains("无法计算：除数不能为 0"); + } + + @Test + void 请求里的userId经由RuntimeContext注入到工具_模型看不到也伪造不了(@TempDir Path workspace) { + // 模型调用 list_my_notes 时还硬塞了一个 userId=default，试图读别人的笔记 + ScriptedModel model = script("list_my_notes", Map.of("limit", 5, "userId", "default")); + ChatController controller = new ChatController(TestAgents.factory(model, workspace, template())); + + List alice = stream(controller, new ChatRequest("s-a", "我的笔记", "alice")); + List fallback = stream(controller, new ChatRequest("s-d", "我的笔记")); + + assertThat(resultOf(alice)).contains("alice").contains("MCP").doesNotContain("Redis"); + assertThat(resultOf(fallback)).contains("default").contains("Redis").doesNotContain("MCP"); + } + + @Test + void 带编号的检索结果经过完整一轮对话回到模型手里(@TempDir Path workspace) { + var port = new com.yoke.knowledge.KeywordRetrievePort( + com.yoke.knowledge.KnowledgeCorpus.loadFromClasspath("classpath*:knowledge/*.md")); + Toolkit toolkit = new Toolkit(); + toolkit.registerTool(new RetrieveTool(port)); + ScriptedModel model = script("retrieve", Map.of("query", "对话历史存在哪里", "top_k", 2)); + ChatController controller = new ChatController(TestAgents.factory(model, workspace, toolkit)); + + List events = stream(controller, new ChatRequest("s-r", "历史存哪")); + + String output = resultOf(events); + assertThat(output).contains("[1] 来源：").contains("[2] 来源：").contains("Redis").doesNotContain("[3]"); + } + + @Test + void MCP工具以服务端公布的原名出现在模型可见的工具清单里并能被调用(@TempDir Path workspace) { + try (McpTestServer server = new McpTestServer(); + McpConnections mcp = McpConnections.connect(List.of(new YokeProperties.McpServer( + "demo", "streamable-http", server.url(), Duration.ofSeconds(10), Map.of())))) { + Toolkit toolkit = template(); + mcp.registerInto(toolkit); + ScriptedModel model = script("echo", Map.of("text", "你好 MCP")); + ChatController controller = new ChatController(TestAgents.factory(model, workspace, toolkit)); + + List events = stream(controller, new ChatRequest("s-mcp", "回显一下")); + + // 模型看到的是原名 echo，没有 demo__ 之类的前缀 + assertThat(model.calls.getFirst().toolNames()).contains("echo").noneMatch(name -> name.contains("__") && name.contains("echo")); + assertThat(resultOf(events)).contains("echo: 你好 MCP"); + } + } + + @Test + void 未声明只读的MCP工具不会暴露给模型_主Agent和子Agent都看不到_即使模型硬调也不会执行(@TempDir Path workspace) { + // 2.0.4 只放行声明了 readOnlyHint 的 MCP 工具，其余要用户确认；HTTP 接口没有确认通道，所以注册时就不暴露 + try (McpTestServer server = new McpTestServer(); + McpConnections mcp = McpConnections.connect(List.of(new YokeProperties.McpServer( + "demo", "streamable-http", server.url(), Duration.ofSeconds(10), Map.of())))) { + Toolkit toolkit = template(); + mcp.registerInto(toolkit); + + // 主 Agent：清单里有 echo、没有 write_note；模型幻觉出对 write_note 的调用，只得到“工具不存在”的错误 + ScriptedModel parent = script("write_note", Map.of("text", "危险写入")); + List events = stream(new ChatController(TestAgents.factory(parent, workspace, toolkit)), + new ChatRequest("s-hidden", "写一条笔记")); + assertThat(parent.calls.getFirst().toolNames()).contains("echo").doesNotContain("write_note"); + assertThat(server.writeNoteCalls).hasValue(0); + assertThat(events.getLast()).isInstanceOf(ChatEvent.Done.class); + + // 子 Agent（内置 general-purpose 继承父 Agent 的工具）：同样只有 echo + ScriptedModel delegating = new ScriptedModel(call -> { + if (call.systemText().startsWith("You are a memory")) { + return ScriptedModel.text("[]"); + } + if (call.systemText().startsWith("你是测试助手")) { + return call.hasToolResult() ? ScriptedModel.text("收尾") + : ScriptedModel.toolUse("t1", "agent_spawn", Map.of("agent_id", "general-purpose", "task", "x")); + } + return ScriptedModel.text("子回复"); + }); + stream(new ChatController(TestAgents.factory(delegating, workspace, toolkit)), new ChatRequest("s-hidden-sub", "委派")); + var subCalls = delegating.calls.stream() + .filter(c -> !c.systemText().startsWith("你是测试助手") && !c.systemText().startsWith("You are a memory")).toList(); + assertThat(subCalls).isNotEmpty(); + assertThat(subCalls).allSatisfy(c -> assertThat(c.toolNames()).contains("echo").doesNotContain("write_note")); + assertThat(server.writeNoteCalls).hasValue(0); + } + } + + private static String resultOf(List events) { + return events.stream().filter(ChatEvent.ToolResult.class::isInstance).map(ChatEvent.ToolResult.class::cast) + .map(ChatEvent.ToolResult::output).findFirst().orElseThrow(); + } +} From 22e1299e28f081ffb2fd9ba5a2c5dcec0ea400b7 Mon Sep 17 00:00:00 2001 From: Kyle Date: Mon, 9 Nov 2026 09:35:49 +0800 Subject: [PATCH 02/19] fix: align knowledge docs and harden MCP startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 知识库文档同步 AG-UI 流式协议、get_weather 和被禁用的工具 - MCP 启动失败不再把原异常作为 cause，避免地址里的令牌进日志 - listTools 加超时；MCP 工具与已有工具重名时启动失败 - 判空写法统一为 StrUtil、Objects、Optional --- .../java/com/yoke/config/YokeProperties.java | 17 +++++++------ .../yoke/knowledge/HybridRetrievePort.java | 3 ++- .../yoke/knowledge/KeywordRetrievePort.java | 6 +++-- .../com/yoke/knowledge/KnowledgeCorpus.java | 7 +++--- .../yoke/knowledge/OpenAiEmbeddingClient.java | 5 ++-- .../yoke/knowledge/VectorRetrievePort.java | 6 +++-- .../java/com/yoke/mcp/McpConnections.java | 25 ++++++++++++++----- src/main/java/com/yoke/tool/ChatTools.java | 9 ++++--- .../com/yoke/tool/ExpressionEvaluator.java | 3 ++- src/main/java/com/yoke/tool/RetrieveTool.java | 9 ++++--- src/main/resources/knowledge/api-usage.md | 2 +- .../knowledge/tools-and-permissions.md | 4 +-- .../yoke/knowledge/KnowledgeCorpusTests.java | 2 +- .../com/yoke/mcp/McpConnectionsTests.java | 19 +++++++++++++- 14 files changed, 81 insertions(+), 36 deletions(-) diff --git a/src/main/java/com/yoke/config/YokeProperties.java b/src/main/java/com/yoke/config/YokeProperties.java index 2fbdafb..b6e1768 100644 --- a/src/main/java/com/yoke/config/YokeProperties.java +++ b/src/main/java/com/yoke/config/YokeProperties.java @@ -5,9 +5,11 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; +import cn.hutool.core.util.StrUtil; import jakarta.validation.Valid; import jakarta.validation.constraints.NotBlank; import lombok.extern.slf4j.Slf4j; @@ -15,7 +17,6 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.bind.DefaultValue; -import org.springframework.util.StringUtils; import org.springframework.validation.annotation.Validated; /** @@ -104,14 +105,14 @@ public record Embedding( Integer dimensions) { public boolean enabled() { - return StringUtils.hasText(modelName) && StringUtils.hasText(apiKey); + return StrUtil.isNotBlank(modelName) && StrUtil.isNotBlank(apiKey); } /** 密钥不进日志和调试输出。 */ @Override public String toString() { return "Embedding[modelName=" + modelName + ", baseUrl=" + baseUrl + ", apiKey=" - + (StringUtils.hasText(apiKey) ? "***" : null) + ", dimensions=" + dimensions + "]"; + + (StrUtil.isNotBlank(apiKey) ? "***" : null) + ", dimensions=" + dimensions + "]"; } } } @@ -124,7 +125,7 @@ public String toString() { public record Mcp(@DefaultValue List<@Valid McpServer> servers) { public Mcp { - servers = servers == null ? List.of() : List.copyOf(servers); + servers = Optional.ofNullable(servers).map(List::copyOf).orElse(List.of()); long distinct = servers.stream().map(McpServer::name).distinct().count(); if (distinct != servers.size()) { throw new IllegalArgumentException("yoke.mcp.servers 里的 name 不能重复：每个服务的工具靠它区分和释放"); @@ -157,23 +158,23 @@ public record McpServer( private static final Set ALLOWED = Set.of(SSE, STREAMABLE_HTTP); public McpServer { - String normalized = transport == null ? "" : transport.trim().toLowerCase(Locale.ROOT); + String normalized = Optional.ofNullable(transport).map(t -> t.trim().toLowerCase(Locale.ROOT)).orElse(""); if (!ALLOWED.contains(normalized)) { throw new IllegalArgumentException("yoke.mcp.servers[" + name + "].transport=" + transport + " 不被允许：运行态只支持 HTTP 传输（sse 或 streamable-http）。" + "stdio 会按配置启动本地进程，存在命令注入风险，所以被拒绝"); } transport = normalized; - if (name == null || name.isBlank() || url == null || url.isBlank()) { + if (StrUtil.isBlank(name) || StrUtil.isBlank(url)) { throw new IllegalArgumentException("yoke.mcp.servers 的 name 和 url 不能为空"); } if (!url.startsWith("http://") && !url.startsWith("https://")) { throw new IllegalArgumentException("yoke.mcp.servers[" + name + "].url 必须以 http:// 或 https:// 开头"); } - if (timeout == null || timeout.isZero() || timeout.isNegative()) { + if (Objects.isNull(timeout) || timeout.isZero() || timeout.isNegative()) { timeout = Duration.ofSeconds(30); } - headers = headers == null ? Map.of() : Map.copyOf(headers); + headers = Optional.ofNullable(headers).map(Map::copyOf).orElse(Map.of()); } /** 地址和请求头里可能带令牌：toString 只留协议、主机和端口，请求头只列名字。 */ diff --git a/src/main/java/com/yoke/knowledge/HybridRetrievePort.java b/src/main/java/com/yoke/knowledge/HybridRetrievePort.java index 717bf6c..1de08fa 100644 --- a/src/main/java/com/yoke/knowledge/HybridRetrievePort.java +++ b/src/main/java/com/yoke/knowledge/HybridRetrievePort.java @@ -4,6 +4,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,7 +58,7 @@ public Mono> retrieve(String query, int topK) { .collectList() .flatMap(outcomes -> { List> succeeded = outcomes.stream() - .filter(outcome -> outcome.error() == null) + .filter(outcome -> Objects.isNull(outcome.error())) .map(Outcome::passages) .toList(); if (succeeded.isEmpty()) { diff --git a/src/main/java/com/yoke/knowledge/KeywordRetrievePort.java b/src/main/java/com/yoke/knowledge/KeywordRetrievePort.java index 6b4d3b3..e788157 100644 --- a/src/main/java/com/yoke/knowledge/KeywordRetrievePort.java +++ b/src/main/java/com/yoke/knowledge/KeywordRetrievePort.java @@ -7,8 +7,10 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; +import cn.hutool.core.util.StrUtil; import reactor.core.publisher.Mono; /** @@ -50,7 +52,7 @@ public Mono> retrieve(String query, int topK) { } private List search(String query, int topK) { - if (query == null || query.isBlank() || topK <= 0 || passages.isEmpty()) { + if (StrUtil.isBlank(query) || topK <= 0 || passages.isEmpty()) { return List.of(); } Set queryTerms = new HashSet<>(tokenize(query)); @@ -75,7 +77,7 @@ private double score(Set queryTerms, Map frequencies) { double score = 0; for (String term : queryTerms) { Integer tf = frequencies.get(term); - if (tf == null) { + if (Objects.isNull(tf)) { continue; } int df = documentFrequency.getOrDefault(term, 0); diff --git a/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java b/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java index f3de9f6..5024fb0 100644 --- a/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java +++ b/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java @@ -7,6 +7,7 @@ import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.Optional; import org.springframework.core.io.Resource; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; @@ -48,11 +49,11 @@ public static List loadFromClasspath(String pattern) { /** 把一篇 Markdown 切成片段；{@code fileName} 只在文档没有一级标题时充当标题。 */ static List split(String fileName, String markdown) { - String title = fileName == null ? "未命名文档" : fileName; + String title = Optional.ofNullable(fileName).orElse("未命名文档"); String section = null; StringBuilder body = new StringBuilder(); List passages = new ArrayList<>(); - String docId = fileName == null ? "doc" : fileName; + String docId = Optional.ofNullable(fileName).orElse("doc"); for (String line : markdown.split("\\R")) { if (line.startsWith("# ")) { @@ -74,7 +75,7 @@ private static void flush(List passages, String docId, String title, St if (text.isEmpty()) { return; } - String heading = section == null ? title : title + " / " + section; + String heading = Optional.ofNullable(section).map(name -> title + " / " + name).orElse(title); StringBuilder chunk = new StringBuilder(); for (String paragraph : text.split("\\n\\s*\\n")) { if (chunk.length() > 0 && chunk.length() + paragraph.length() > MAX_CHUNK_LENGTH) { diff --git a/src/main/java/com/yoke/knowledge/OpenAiEmbeddingClient.java b/src/main/java/com/yoke/knowledge/OpenAiEmbeddingClient.java index 0c0c8a0..f56f290 100644 --- a/src/main/java/com/yoke/knowledge/OpenAiEmbeddingClient.java +++ b/src/main/java/com/yoke/knowledge/OpenAiEmbeddingClient.java @@ -5,6 +5,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @@ -48,7 +49,7 @@ private List request(List texts) { body.put("model", modelName); body.put("input", texts); body.put("encoding_format", "float"); - if (dimensions != null) { + if (Objects.nonNull(dimensions)) { body.put("dimensions", dimensions); } Map response = client.post() @@ -57,7 +58,7 @@ private List request(List texts) { .body(body) .retrieve() .body(Map.class); - if (response == null || !(response.get("data") instanceof List data) || data.size() != texts.size()) { + if (Objects.isNull(response) || !(response.get("data") instanceof List data) || data.size() != texts.size()) { throw new IllegalStateException("向量化接口返回的条数与请求不一致"); } List> items = new ArrayList<>(); diff --git a/src/main/java/com/yoke/knowledge/VectorRetrievePort.java b/src/main/java/com/yoke/knowledge/VectorRetrievePort.java index 5f29730..b4ed617 100644 --- a/src/main/java/com/yoke/knowledge/VectorRetrievePort.java +++ b/src/main/java/com/yoke/knowledge/VectorRetrievePort.java @@ -3,7 +3,9 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.List; +import java.util.Objects; +import cn.hutool.core.util.StrUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import reactor.core.publisher.Flux; @@ -34,7 +36,7 @@ public VectorRetrievePort(List passages, EmbeddingClient embeddings) { @Override public Mono> retrieve(String query, int topK) { - if (query == null || query.isBlank() || topK <= 0 || passages.isEmpty()) { + if (StrUtil.isBlank(query) || topK <= 0 || passages.isEmpty()) { return Mono.just(List.of()); } return index().flatMap(vectors -> embeddings.embed(List.of(query)).map(queryVectors -> { @@ -55,7 +57,7 @@ record Scored(Passage passage, double score) { /** 同一时刻只会有一个在建的索引；失败后清掉，让下一次检索重试。 */ private synchronized Mono> index() { - if (index == null) { + if (Objects.isNull(index)) { index = buildIndex() .doOnError(e -> { log.warn("知识库向量索引构建失败，下次检索会重试：{}", e.getMessage()); diff --git a/src/main/java/com/yoke/mcp/McpConnections.java b/src/main/java/com/yoke/mcp/McpConnections.java index 8f9b748..5868a49 100644 --- a/src/main/java/com/yoke/mcp/McpConnections.java +++ b/src/main/java/com/yoke/mcp/McpConnections.java @@ -3,6 +3,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.function.Function; import com.yoke.config.YokeProperties; @@ -37,6 +38,9 @@ public final class McpConnections implements AutoCloseable { public interface ClientFactory extends Function { } + /** 读取工具清单的等待上限：服务端卡住时不能让启动无限挂起。 */ + private static final Duration LIST_TOOLS_TIMEOUT = Duration.ofSeconds(10); + private final List clients = new ArrayList<>(); private McpConnections() { @@ -57,8 +61,9 @@ public static McpConnections connect(List servers, Cli client.initialize().block(initTimeout(server)); log.info("MCP 服务已连接：{}（{}）", server.name(), server.transport()); } catch (RuntimeException e) { + // 不把原异常作为 cause：它的消息里可能带着地址中的 token，只留类名 RuntimeException primary = new IllegalStateException("无法连接 MCP 服务 " + server - + "，请确认服务已启动且地址、传输方式正确（原因：" + e.getClass().getSimpleName() + "）", e); + + "，请确认服务已启动且地址、传输方式正确（原因：" + e.getClass().getSimpleName() + "）"); connections.closeSuppressing(primary); throw primary; } @@ -72,13 +77,17 @@ public static McpConnections connect(List servers, Cli *

只注册服务端声明了 {@code readOnlyHint} 的工具，其余跳过，模型根本看不到它们。原因：2.0.4 对 MCP 工具 * 只放行只读的，其余都要用户确认；HTTP 接口没有人工确认通道，把它们暴露给模型只会让一轮对话在工具调用处无声结束， * 调用方拿到空回复。与其这样，不如一开始就不暴露。跳过时只记工具名到 INFO 日志，不记参数。 + * 暴露的工具名若与 Toolkit 里已有的工具重名（例如内置的 {@code calculate}），启动失败，避免外部服务悄悄顶替内置工具。 * 用 {@code disableTools} 而不是 {@code enableTools}：后者传空列表在 2.0.4 里表示「全部启用」。 */ public void registerInto(Toolkit toolkit) { for (McpClientWrapper client : clients) { - List skipped = client.listTools().block().stream() - .filter(tool -> tool.annotations() == null || !Boolean.TRUE.equals(tool.annotations().readOnlyHint())) - .map(McpSchema.Tool::name).toList(); + List tools = client.listTools().block(LIST_TOOLS_TIMEOUT); + List skipped = tools.stream().filter(McpConnections::notReadOnly).map(McpSchema.Tool::name).toList(); + tools.stream().map(McpSchema.Tool::name).filter(name -> !skipped.contains(name)) + .filter(toolkit.getToolNames()::contains).findFirst().ifPresent(name -> { + throw new IllegalStateException("MCP 服务 " + client.getName() + " 的工具 " + name + " 与已有工具重名"); + }); if (!skipped.isEmpty()) { log.info("MCP 服务 {} 的以下工具未声明只读，不暴露给模型：{}", client.getName(), skipped); } @@ -86,6 +95,10 @@ public void registerInto(Toolkit toolkit) { } } + private static boolean notReadOnly(McpSchema.Tool tool) { + return Objects.isNull(tool.annotations()) || !Boolean.TRUE.equals(tool.annotations().readOnlyHint()); + } + public List names() { return clients.stream().map(McpClientWrapper::getName).toList(); } @@ -99,7 +112,7 @@ public void close() { client.close(); } catch (RuntimeException e) { log.warn("关闭 MCP 连接失败：{}", client.getName(), e); - if (failure == null) { + if (Objects.isNull(failure)) { failure = e; } else { failure.addSuppressed(e); @@ -107,7 +120,7 @@ public void close() { } } clients.clear(); - if (failure != null) { + if (Objects.nonNull(failure)) { throw failure; } } diff --git a/src/main/java/com/yoke/tool/ChatTools.java b/src/main/java/com/yoke/tool/ChatTools.java index 5bb3dc4..cfcf4e9 100644 --- a/src/main/java/com/yoke/tool/ChatTools.java +++ b/src/main/java/com/yoke/tool/ChatTools.java @@ -8,7 +8,10 @@ import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import cn.hutool.core.util.StrUtil; import io.agentscope.core.tool.Tool; import io.agentscope.core.tool.ToolParam; import org.slf4j.Logger; @@ -57,7 +60,7 @@ public String getCurrentTime( description = "IANA 时区名，例如 Asia/Shanghai、America/New_York；不传时用 UTC") String timezone) { ZoneId zone; try { - zone = timezone == null || timezone.isBlank() ? ZoneId.of("UTC") : ZoneId.of(timezone.trim()); + zone = ZoneId.of(Optional.ofNullable(timezone).filter(StrUtil::isNotBlank).map(String::trim).orElse("UTC")); } catch (DateTimeException e) { return "无法识别的时区：" + abbreviate(timezone) + "。请使用 IANA 时区名，例如 Asia/Shanghai。"; } @@ -86,11 +89,11 @@ public String calculate( public String listMyNotes( UserContext user, @ToolParam(name = "limit", required = false, description = "最多返回几条，默认 5，上限 20") Integer limit) { - if (user == null || user.userId() == null || user.userId().isBlank()) { + if (Objects.isNull(user) || StrUtil.isBlank(user.userId())) { log.warn("list_my_notes 被调用但 RuntimeContext 里没有 UserContext，拒绝返回数据"); return "无法确定当前用户，已拒绝返回笔记。"; } - int max = limit == null ? 5 : Math.clamp(limit, 1, MAX_NOTES); + int max = Optional.ofNullable(limit).map(value -> Math.clamp(value, 1, MAX_NOTES)).orElse(5); List notes = NOTES.getOrDefault(user.userId(), List.of()); if (notes.isEmpty()) { return "当前用户（" + user.userId() + "）还没有笔记。"; diff --git a/src/main/java/com/yoke/tool/ExpressionEvaluator.java b/src/main/java/com/yoke/tool/ExpressionEvaluator.java index 36cc227..92738f0 100644 --- a/src/main/java/com/yoke/tool/ExpressionEvaluator.java +++ b/src/main/java/com/yoke/tool/ExpressionEvaluator.java @@ -3,6 +3,7 @@ import java.math.BigDecimal; import java.math.MathContext; import java.util.function.Supplier; +import cn.hutool.core.util.StrUtil; /** * 四则运算表达式求值器，手写的递归下降解析器。 @@ -51,7 +52,7 @@ public static final class InvalidExpressionException extends RuntimeException { * @throws InvalidExpressionException 表达式为空、过长、含非法字符、语法错误或除数为 0 */ public static String evaluate(String expression) { - if (expression == null || expression.isBlank()) { + if (StrUtil.isBlank(expression)) { throw new InvalidExpressionException("表达式不能为空"); } if (expression.length() > MAX_LENGTH) { diff --git a/src/main/java/com/yoke/tool/RetrieveTool.java b/src/main/java/com/yoke/tool/RetrieveTool.java index b1add38..4bdc1b3 100644 --- a/src/main/java/com/yoke/tool/RetrieveTool.java +++ b/src/main/java/com/yoke/tool/RetrieveTool.java @@ -1,7 +1,10 @@ package com.yoke.tool; import java.util.List; +import java.util.Objects; +import java.util.Optional; +import cn.hutool.core.util.StrUtil; import com.yoke.knowledge.Passage; import com.yoke.knowledge.RetrievePort; import io.agentscope.core.tool.Tool; @@ -44,14 +47,14 @@ public RetrieveTool(RetrievePort port) { public Mono retrieve( @ToolParam(name = "query", description = "要查的问题或关键词，用完整的短句，例如「会话历史存在哪里」") String query, @ToolParam(name = "top_k", required = false, description = "返回几条，默认 3，上限 8") Integer topK) { - if (port == null) { + if (Objects.isNull(port)) { return Mono.just("知识库尚未配置，无法检索，请直接根据已有信息回答并说明没有查资料。"); } - if (query == null || query.isBlank()) { + if (StrUtil.isBlank(query)) { return Mono.just("检索失败：query 不能为空。"); } String trimmed = query.length() > MAX_QUERY_LENGTH ? query.substring(0, MAX_QUERY_LENGTH) : query; - int limit = topK == null ? DEFAULT_TOP_K : Math.clamp(topK, 1, MAX_TOP_K); + int limit = Optional.ofNullable(topK).map(value -> Math.clamp(value, 1, MAX_TOP_K)).orElse(DEFAULT_TOP_K); return port.retrieve(trimmed, limit) .map(RetrieveTool::format) .onErrorResume(error -> { diff --git a/src/main/resources/knowledge/api-usage.md b/src/main/resources/knowledge/api-usage.md index cb4b02b..fa12b58 100644 --- a/src/main/resources/knowledge/api-usage.md +++ b/src/main/resources/knowledge/api-usage.md @@ -4,7 +4,7 @@ 向 POST /api/chat 发送 JSON，字段有可选的 userId、可选的 sessionId 和必填的 message。message 缺失、为空或纯空白会返回 400。响应里包含回复文本和本次对话的 sessionId，后续请求带上它就能延续对话。 ## 流式对话 -POST /api/chat/stream 以 SSE 的形式边推理边推送事件。事件名有 textDelta、toolCall、toolResult、hint、done 和 error。一次对话以 done 或 error 结尾，二者必有其一且只会出现一个。toolCall 和 toolResult 用相同的 id 对应。 +POST /api/chat/stream 说 AG-UI 协议，以 SSE 的形式边推理边推送标准事件。请求体是 AG-UI 的 RunAgentInput，其中 threadId 就是会话 ID，服务端只采用最后一条 user 消息。事件类型写在每条 data 的 type 字段里，常见的有 RUN_STARTED、TEXT_MESSAGE_CONTENT、TOOL_CALL_START、TOOL_CALL_RESULT、RUN_FINISHED 和 RUN_ERROR。框架提示和 token 用量以 CUSTOM 事件发出，name 分别是 hint 和 token_usage。RUN_FINISHED 与 RUN_ERROR 二者只会出现一个，没有 done 事件，完整回复由客户端拼接各个 TEXT_MESSAGE_CONTENT 得到。TOOL_CALL_START 和 TOOL_CALL_RESULT 用相同的 toolCallId 对应。 ## 并发的限制 同一个会话的并发请求不会被串行化，由调用方保证同一会话同一时间只发一个请求。目前也没有鉴权，userId 由调用方自报，不可信，不要把它当成身份校验。 diff --git a/src/main/resources/knowledge/tools-and-permissions.md b/src/main/resources/knowledge/tools-and-permissions.md index a541e7a..fb9c1aa 100644 --- a/src/main/resources/knowledge/tools-and-permissions.md +++ b/src/main/resources/knowledge/tools-and-permissions.md @@ -1,10 +1,10 @@ # 工具与权限 ## 模型能用哪些工具 -Yoke 自带的工具有 get_current_time、calculate、list_my_notes 和 retrieve。get_current_time 返回带时区偏移的当前时间；calculate 做四则运算，支持括号和小数；list_my_notes 返回当前用户的笔记；retrieve 在 Yoke 自己的知识库里检索资料。此外还有 HarnessAgent 内置的记忆和会话类工具，例如 memory_save、memory_search 和 session_history。 +Yoke 自带的工具有 get_current_time、calculate、list_my_notes、get_weather 和 retrieve。get_current_time 返回带时区偏移的当前时间；get_weather 查询城市的实时天气；calculate 做四则运算，支持括号和小数；list_my_notes 返回当前用户的笔记；retrieve 在 Yoke 自己的知识库里检索资料。此外还有 HarnessAgent 内置的 memory_save 等工具；memory_get、memory_search 和 session_* 这类会读到其他用户数据的工具已被禁用，模型看不到。 ## 默认关闭了哪些能力 -对外服务的 Agent 不开放文件读写、命令执行和联网工具，也就是 read_file、write_file、execute、web_fetch、web_search 这一类都看不到。子 Agent 不会继承联网工具的关闭，所以 Yoke 通过工具配置的 deny 名单把 web_fetch 和 web_search 对所有子 Agent 一并禁掉。 +对外服务的 Agent 不开放文件读写、命令执行和联网工具，也就是 read_file、write_file、execute、web_fetch、web_search 这一类都看不到。子 Agent 不会继承联网工具的关闭，所以 Yoke 通过工具配置的 deny 名单把 web_fetch、web_search 以及上述记忆和会话读取类工具对所有子 Agent 一并禁掉。 ## 业务上下文不经过模型 list_my_notes 需要知道当前用户是谁，但 userId 不是模型传的参数：它由控制器放进 RuntimeContext，工具方法里不带 @ToolParam 的 UserContext 参数由框架按类型注入。模型既看不到这个参数，也没法伪造。 diff --git a/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java b/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java index 8f644c7..3e98262 100644 --- a/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java +++ b/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java @@ -60,6 +60,6 @@ class KnowledgeCorpusTests { assertThat(port.retrieve("为什么不允许 stdio 传输", 3).block()).first() .satisfies(passage -> assertThat(passage.text()).contains("stdio")); assertThat(port.retrieve("流式接口有哪些事件", 3).block()).first() - .satisfies(passage -> assertThat(passage.text()).contains("textDelta")); + .satisfies(passage -> assertThat(passage.text()).contains("TEXT_MESSAGE_CONTENT")); } } diff --git a/src/test/java/com/yoke/mcp/McpConnectionsTests.java b/src/test/java/com/yoke/mcp/McpConnectionsTests.java index 3a79008..0dd4485 100644 --- a/src/test/java/com/yoke/mcp/McpConnectionsTests.java +++ b/src/test/java/com/yoke/mcp/McpConnectionsTests.java @@ -50,6 +50,22 @@ private YokeProperties.McpServer server(String name, Map headers } } + @Test + void MCP工具与已有工具重名时启动失败_不让外部服务顶替内置工具() { + try (McpConnections connections = McpConnections.connect(List.of(server("demo", Map.of())))) { + Toolkit toolkit = new Toolkit(); + toolkit.registerTool(new Object() { + @io.agentscope.core.tool.Tool(name = "echo", description = "内置同名工具") + public String echo() { + return "builtin"; + } + }); + + assertThatThrownBy(() -> connections.registerInto(toolkit)) + .isInstanceOf(IllegalStateException.class).hasMessageContaining("echo").hasMessageContaining("重名"); + } + } + @Test void 未声明只读的工具不会注册进Toolkit_只读的才会() { try (McpConnections connections = McpConnections.connect(List.of(server("demo", Map.of())))) { @@ -212,7 +228,8 @@ private static YokeProperties.McpServer cfg(String name) { // 主异常是「c 连不上」，不是关闭 b 时的失败 .isInstanceOf(IllegalStateException.class) .hasMessageContaining("无法连接 MCP 服务").hasMessageContaining("name=c") - .hasRootCauseMessage("连接被拒绝") + // 原异常不作为 cause 传出（消息里可能带着地址里的令牌），只在主异常消息里留类名 + .hasNoCause().hasMessageContaining("IllegalStateException").hasMessageNotContaining("连接被拒绝") // 关闭 b 的失败只作为 suppressed 挂着，没有掩盖主因 .satisfies(e -> assertThat(e.getSuppressed()).containsExactly(closeFailure)); From 5f408f1364cdfc9d7a20d23d4fe379cd2196be77 Mon Sep 17 00:00:00 2001 From: Kyle Date: Mon, 9 Nov 2026 09:37:11 +0800 Subject: [PATCH 03/19] fix: run retrieval channels in parallel and drop irrelevant vector hits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 混合检索改为 flatMapSequential：向量与关键词两路同时发起，向量服务慢或挂了不再拖住关键词兜底 - 向量检索加最低相似度（默认 0.3），问无关问题时返回空，而不是凑够 K 条无关资料 --- .../com/yoke/knowledge/HybridRetrievePort.java | 2 +- .../com/yoke/knowledge/VectorRetrievePort.java | 13 +++++++++++++ .../yoke/knowledge/HybridRetrievePortTests.java | 15 +++++++++++++++ .../yoke/knowledge/VectorRetrievePortTests.java | 10 ++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/yoke/knowledge/HybridRetrievePort.java b/src/main/java/com/yoke/knowledge/HybridRetrievePort.java index 1de08fa..e6e3269 100644 --- a/src/main/java/com/yoke/knowledge/HybridRetrievePort.java +++ b/src/main/java/com/yoke/knowledge/HybridRetrievePort.java @@ -48,7 +48,7 @@ public HybridRetrievePort(List channels) { public Mono> retrieve(String query, int topK) { int candidates = Math.max(topK * CANDIDATE_FACTOR, 10); return Flux.fromIterable(channels) - .concatMap(channel -> channel.retrieve(query, candidates) + .flatMapSequential(channel -> channel.retrieve(query, candidates) .defaultIfEmpty(List.of()) .map(Outcome::success) .onErrorResume(error -> { diff --git a/src/main/java/com/yoke/knowledge/VectorRetrievePort.java b/src/main/java/com/yoke/knowledge/VectorRetrievePort.java index b4ed617..50da189 100644 --- a/src/main/java/com/yoke/knowledge/VectorRetrievePort.java +++ b/src/main/java/com/yoke/knowledge/VectorRetrievePort.java @@ -24,14 +24,26 @@ public final class VectorRetrievePort implements RetrievePort { /** 一次向量化请求最多带多少个片段。 */ static final int BATCH_SIZE = 16; + /** + * 默认最低相似度：低于它的片段视为不相关，不返回。问和知识库无关的问题时应当检索不到， + * 而不是凑够 K 条无关资料让模型去引用。不同向量模型的分数分布不同，换模型时按需调整。 + */ + static final double DEFAULT_MIN_SCORE = 0.3; + private final List passages; private final EmbeddingClient embeddings; + private final double minScore; private Mono> index; public VectorRetrievePort(List passages, EmbeddingClient embeddings) { + this(passages, embeddings, DEFAULT_MIN_SCORE); + } + + public VectorRetrievePort(List passages, EmbeddingClient embeddings, double minScore) { this.passages = List.copyOf(passages); this.embeddings = embeddings; + this.minScore = minScore; } @Override @@ -48,6 +60,7 @@ record Scored(Passage passage, double score) { scored.add(new Scored(passages.get(i), cosine(queryVector, vectors.get(i)))); } return scored.stream() + .filter(item -> item.score() >= minScore) .sorted(Comparator.comparingDouble(Scored::score).reversed()) .limit(topK) .map(Scored::passage) diff --git a/src/test/java/com/yoke/knowledge/HybridRetrievePortTests.java b/src/test/java/com/yoke/knowledge/HybridRetrievePortTests.java index 6601951..3912a0a 100644 --- a/src/test/java/com/yoke/knowledge/HybridRetrievePortTests.java +++ b/src/test/java/com/yoke/knowledge/HybridRetrievePortTests.java @@ -34,6 +34,21 @@ private static RetrievePort failing() { assertThat(result).extracting(Passage::id).containsExactly("B", "A", "D", "C"); } + @Test + void 两路检索并行执行_总耗时取决于最慢的一路而不是两路之和() { + RetrievePort slow = (query, topK) -> Mono.delay(java.time.Duration.ofMillis(500)).thenReturn(List.of(p("A"))) + .subscribeOn(reactor.core.scheduler.Schedulers.boundedElastic()); + RetrievePort alsoSlow = (query, topK) -> Mono.delay(java.time.Duration.ofMillis(500)).thenReturn(List.of(p("B"))); + RetrievePort hybrid = new HybridRetrievePort(List.of(slow, alsoSlow)); + + long start = System.nanoTime(); + List result = hybrid.retrieve("q", 2).block(); + long elapsedMillis = (System.nanoTime() - start) / 1_000_000; + + assertThat(result).extracting(Passage::id).containsExactlyInAnyOrder("A", "B"); + assertThat(elapsedMillis).isLessThan(900); + } + @Test void RRF得分使用常数60() { assertThat(HybridRetrievePort.RRF_K).isEqualTo(60); diff --git a/src/test/java/com/yoke/knowledge/VectorRetrievePortTests.java b/src/test/java/com/yoke/knowledge/VectorRetrievePortTests.java index 0c97633..9602e4f 100644 --- a/src/test/java/com/yoke/knowledge/VectorRetrievePortTests.java +++ b/src/test/java/com/yoke/knowledge/VectorRetrievePortTests.java @@ -44,6 +44,16 @@ public synchronized Mono> embed(List texts) { assertThat(port.retrieve("哪个工具", 2).block()).extracting(Passage::id).first().isEqualTo("b"); } + @Test + void 低于最低相似度的片段不返回_问无关问题时检索不到() { + VectorRetrievePort port = new VectorRetrievePort(PASSAGES, new KeywordEmbedding()); + + // 查询里没有任何已知关键字，与所有片段的相似度都接近 0 + assertThat(port.retrieve("今天吃什么", 3).block()).isEmpty(); + // 只有相关的片段留下，不会为了凑够 K 条把无关的也带上 + assertThat(port.retrieve("Redis", 3).block()).extracting(Passage::id).containsExactly("a"); + } + @Test void 片段向量只算一次_之后每次检索只多一次查询向量化() { KeywordEmbedding embedding = new KeywordEmbedding(); From 4fd89ea0f4e951373b440d9c1ba226fa647adc42 Mon Sep 17 00:00:00 2001 From: Kyle Date: Mon, 9 Nov 2026 10:25:08 +0800 Subject: [PATCH 04/19] fix: require retrieve tool when user explicitly asks to search the knowledge base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - retrieve 的工具说明加一句：用户明确要求查知识库时必须调用，不能没查就说没有 - 真实模型跑出来的现象：只问"请查一下知识库"时模型一次都没调 retrieve，直接说知识库是空的 - 加测试守着这句说明 --- src/main/java/com/yoke/tool/RetrieveTool.java | 2 +- src/test/java/com/yoke/tool/RetrieveToolTests.java | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/yoke/tool/RetrieveTool.java b/src/main/java/com/yoke/tool/RetrieveTool.java index 4bdc1b3..83740c9 100644 --- a/src/main/java/com/yoke/tool/RetrieveTool.java +++ b/src/main/java/com/yoke/tool/RetrieveTool.java @@ -42,7 +42,7 @@ public RetrieveTool(RetrievePort port) { @Tool(name = "retrieve", description = "当问题涉及 Yoke 自身的设计、配置、接口或使用方法，而你不确定答案时调用，先查资料再回答；" - + "闲聊、通用常识和计算不要调用。返回带 [1][2] 编号的资料片段，回答时请用相同的编号标注引用来源。" + + "用户明确要求查知识库或资料时必须调用，不能没查就说知识库里没有；闲聊、通用常识和计算不要调用。返回带 [1][2] 编号的资料片段，回答时请用相同的编号标注引用来源。" + "查不到时如实告诉用户，不要编造。") public Mono retrieve( @ToolParam(name = "query", description = "要查的问题或关键词，用完整的短句，例如「会话历史存在哪里」") String query, diff --git a/src/test/java/com/yoke/tool/RetrieveToolTests.java b/src/test/java/com/yoke/tool/RetrieveToolTests.java index 6801400..707b488 100644 --- a/src/test/java/com/yoke/tool/RetrieveToolTests.java +++ b/src/test/java/com/yoke/tool/RetrieveToolTests.java @@ -16,6 +16,14 @@ class RetrieveToolTests { new Passage("a", "会话与存储", "会话与存储 / 对话历史\n历史放在 Redis"), new Passage("b", "工具与权限", "工具与权限 / 模型能用哪些工具\n有 calculate")); + @Test + void 工具说明要求用户明确让查知识库时必须调用_不能没查就说没有() throws NoSuchMethodException { + String description = RetrieveTool.class.getMethod("retrieve", String.class, Integer.class) + .getAnnotation(io.agentscope.core.tool.Tool.class).description(); + + assertThat(description).contains("明确要求查知识库").contains("必须调用").contains("不能没查就说知识库里没有"); + } + @Test void 结果带编号和来源_方便模型按编号引用() { RetrieveTool tool = new RetrieveTool((query, topK) -> Mono.just(HITS)); From 9616dfde95a3e65838d72d3f3f99bbee8c6f333a Mon Sep 17 00:00:00 2001 From: Kyle Date: Mon, 9 Nov 2026 10:43:48 +0800 Subject: [PATCH 05/19] docs: refer to AgentScope instead of a pinned version in comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 代码注释、测试说明和一条启动报错信息里的「2.0.4」统一写成「AgentScope」 - 版本号只留在交代技术栈的地方，升级框架时不用满仓库改注释 --- src/main/java/com/yoke/api/ChatApi.java | 4 ++-- .../java/com/yoke/config/HarnessAgentFactory.java | 10 +++++----- src/main/java/com/yoke/config/SharedToolkit.java | 2 +- src/main/java/com/yoke/config/YokeSubagents.java | 14 +++++++------- .../com/yoke/knowledge/OpenAiEmbeddingClient.java | 2 +- src/main/java/com/yoke/knowledge/RetrievePort.java | 2 +- src/main/java/com/yoke/mcp/McpConnections.java | 4 ++-- src/main/java/com/yoke/tool/RetrieveTool.java | 2 +- src/test/java/com/yoke/ArchitectureTests.java | 2 +- src/test/java/com/yoke/ToolPermissionTests.java | 2 +- .../java/com/yoke/config/SharedToolkitTests.java | 2 +- src/test/java/com/yoke/config/SubagentTests.java | 10 +++++----- src/test/java/com/yoke/support/McpTestServer.java | 2 +- .../java/com/yoke/tool/ToolConversationTests.java | 2 +- 14 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/yoke/api/ChatApi.java b/src/main/java/com/yoke/api/ChatApi.java index 64aaa4b..aa19683 100644 --- a/src/main/java/com/yoke/api/ChatApi.java +++ b/src/main/java/com/yoke/api/ChatApi.java @@ -65,10 +65,10 @@ * *

子 Agent：{@code summarizer}（代码里声明，只做摘要，没有任何业务工具）、{@code kb-researcher} * （工作区文件 {@code workspace/subagents/kb-researcher.md} 声明，只能用 {@code retrieve}）和框架内置的 - * {@code general-purpose}（继承 Yoke 的业务工具）。2.0.4 里子 Agent 不继承父 Agent 的联网工具关闭，所以联网工具靠工具配置里的 + * {@code general-purpose}（继承 Yoke 的业务工具）。AgentScope 里子 Agent 不继承父 Agent 的联网工具关闭，所以联网工具靠工具配置里的 * deny 名单在所有层（主 Agent、内置子 Agent、声明的子 Agent）一并禁掉；子 Agent 同样没有文件读写和命令执行，也不能再派生子 Agent。 * - *

MCP 工具的授权：2.0.4 对 MCP 工具只放行服务端声明了 {@code readOnlyHint} 的，其余需要用户确认， + *

MCP 工具的授权：AgentScope 对 MCP 工具只放行服务端声明了 {@code readOnlyHint} 的，其余需要用户确认， * 而本接口没有人工确认通道，否则一轮对话会在工具调用处无声结束、调用方拿到空回复。 * 所以注册时就只放行声明了只读的工具，其余跳过（启动日志里记工具名），模型根本看不到它们， * 这对主 Agent 和子 Agent 都成立。MCP 服务配置的服务连不上时，应用启动失败。 diff --git a/src/main/java/com/yoke/config/HarnessAgentFactory.java b/src/main/java/com/yoke/config/HarnessAgentFactory.java index c9841fa..e6be536 100644 --- a/src/main/java/com/yoke/config/HarnessAgentFactory.java +++ b/src/main/java/com/yoke/config/HarnessAgentFactory.java @@ -15,7 +15,7 @@ /** * 每次调用 {@link #create()} 都新建一个 HarnessAgent，调用方用完负责 {@code close()}。 * - *

为什么不像 ReActAgent 那样共享一个 Builder：2.0.4 的 {@code HarnessAgent.Builder.build()} + *

为什么不像 ReActAgent 那样共享一个 Builder：AgentScope 的 {@code HarnessAgent.Builder.build()} * 会往内部持有的 ReActAgent.Builder 里追加一批装配期中间件，同一个 Builder 重复 build， * 中间件会越积越多（实测每次多 11 个），所以这里只共享不可变的部分（模型、存储、配置、中间件实例）， * Builder 每次现建。 @@ -46,7 +46,7 @@ public HarnessAgent create() { .distributedStore(distributedStore) .compaction(compactionConfig()) .middlewares(middlewares) - // 共享同一个模板是安全的：2.0.4 的 build() 先 toolkit.copy() 再注册内置工具，模板不会被污染（见 SharedToolkitTests） + // 共享同一个模板是安全的：AgentScope 的 build() 先 toolkit.copy() 再注册内置工具，模板不会被污染（见 SharedToolkitTests） .toolkit(toolkit) .subagents(subagents) // 权限取舍：对外服务的 Agent 不给读写文件和执行命令的能力 @@ -54,17 +54,17 @@ public HarnessAgent create() { .disableShellTool() // 联网工具两道防线并用：disableWebTools 管主 Agent（按开关，框架改名也有效）， // ToolsConfig.deny 管所有层——子 Agent 只继承文件和 shell 的关闭、不继承 web 的关闭 - // （2.0.4 的 HarnessAgentBuilderSupport 只传这两个），会自己注册 web_fetch / web_search，只能靠 deny 按名字移除， + // （AgentScope 的 HarnessAgentBuilderSupport 只传这两个），会自己注册 web_fetch / web_search，只能靠 deny 按名字移除， // 它会同时传给内置的 general-purpose 和所有声明的子 Agent。 // 代价：主 Agent 构建时 web 工具未注册，ToolFilter 对 deny 里的名字每次打 WARN（实测两条），在 application.yml 里调低了级别。 - // 显式传 toolsConfig 还让 2.0.4 不再去读工作区里的 tools.json（里面能声明 stdio MCP 服务） + // 显式传 toolsConfig 还让 AgentScope 不再去读工作区里的 tools.json（里面能声明 stdio MCP 服务） .disableWebTools() .toolsConfig(YokeSubagents.toolsConfig()) .build(); } /** - * 只按条数触发压缩。{@code triggerTokens} 在 2.0.4 里 0 表示「按模型上下文窗口动态计算」而不是关闭， + * 只按条数触发压缩。{@code triggerTokens} 在 AgentScope 里 0 表示「按模型上下文窗口动态计算」而不是关闭， * 所以用 {@link Integer#MAX_VALUE} 让 token 阈值实际上永远不会先触发； * {@code keepTokens} 为 0 才会改用 keepMessages（默认 -1 是动态）。 */ diff --git a/src/main/java/com/yoke/config/SharedToolkit.java b/src/main/java/com/yoke/config/SharedToolkit.java index a50db1d..99e4fc7 100644 --- a/src/main/java/com/yoke/config/SharedToolkit.java +++ b/src/main/java/com/yoke/config/SharedToolkit.java @@ -5,7 +5,7 @@ /** * 启动时装配好的 Toolkit 模板（Yoke 自己的工具加 MCP 工具），所有请求的 Agent 共用这一个。 * - *

能共用，依据是 2.0.4 的源码：{@code HarnessAgent.Builder.build()} 一上来就 {@code toolkit.copy()}， + *

能共用，依据是 AgentScope 的源码：{@code HarnessAgent.Builder.build()} 一上来就 {@code toolkit.copy()}， * Harness 自己的内置工具（记忆、会话、子 Agent 等）注册在副本上，传进去的模板不会被污染； * {@code Toolkit} 内部的注册表用的是 ConcurrentHashMap，只读的 {@code copy()} 可以并发。 * 前提是模板在启动之后不再被修改，并且里面的工具实例无状态、线程安全（副本按引用共享工具实例）。 diff --git a/src/main/java/com/yoke/config/YokeSubagents.java b/src/main/java/com/yoke/config/YokeSubagents.java index 17a36ce..6f4b9fd 100644 --- a/src/main/java/com/yoke/config/YokeSubagents.java +++ b/src/main/java/com/yoke/config/YokeSubagents.java @@ -22,10 +22,10 @@ *

    *
  • 编程式：{@link #programmatic()}，在代码里用 {@link SubagentDeclaration} 声明，例如 {@code summarizer}；
    • *
  • 工作区文件：{@code src/main/resources/workspace/subagents/*.md}，文件名即 agent id， - * 启动时拷贝到 {@code yoke.workspace/subagents/}，由 HarnessAgent 按 2.0.4 的约定从工作区加载。
    • + * 启动时拷贝到 {@code yoke.workspace/subagents/}，由 HarnessAgent 按 AgentScope 的约定从工作区加载。 *
* - *

同一个 id 同时出现在两处时，2.0.4 不会报错：默认的动态加载（DynamicSubagentsMiddleware）里 + *

同一个 id 同时出现在两处时，AgentScope 不会报错：默认的动态加载（DynamicSubagentsMiddleware）里 * 「动态声明胜出」，工作区文件静默覆盖代码里的声明；框架内置的 {@code general-purpose} 同理， * 一个叫 {@code general-purpose.md} 的文件就能把它换掉。配置错位却毫无提示很危险， * 所以 Yoke 在启动时自己检查，冲突直接启动失败，见 {@link #requireNoConflict}。 @@ -40,7 +40,7 @@ public final class YokeSubagents { public static final String SUMMARIZER = "summarizer"; /** - * 「一个工具都不要」的占位名。2.0.4 里声明的 {@code tools} 为空表示继承父 Agent 的全部工具， + * 「一个工具都不要」的占位名。AgentScope 里声明的 {@code tools} 为空表示继承父 Agent 的全部工具， * 没有办法直接表达「零个」；放一个不存在的名字，白名单非空，其余工具就都被过滤掉了。 */ static final String NO_TOOLS = "__no_tools__"; @@ -55,7 +55,7 @@ public final class YokeSubagents { static final List DENIED_TOOLS = List.of("web_fetch", "web_search", "memory_get", "memory_search", "session_history", "session_list", "session_search"); - /** 工作区里放子 Agent 声明文件的目录名，2.0.4 约定为 {@code /subagents/}。 */ + /** 工作区里放子 Agent 声明文件的目录名，AgentScope 约定为 {@code /subagents/}。 */ static final String WORKSPACE_DIR = "subagents"; private YokeSubagents() { @@ -75,7 +75,7 @@ public static List programmatic() { /** * 传给 {@code HarnessAgent.Builder.toolsConfig} 的工具配置：只含一份 deny 名单。 * - *

传入显式配置还有一个作用：2.0.4 在没有显式配置时会去读工作区里的 {@code tools.json}， + *

传入显式配置还有一个作用：AgentScope 在没有显式配置时会去读工作区里的 {@code tools.json}， * 其中可以声明要启动的 stdio MCP 服务。显式传入后不再读取那个文件，堵住这条绕过 MCP 传输限制的路。 */ public static io.agentscope.harness.agent.tools.ToolsConfig toolsConfig() { @@ -104,7 +104,7 @@ public static Set installWorkspaceFiles(Path workspace) { * 启动时的完整准备：安装随应用发布的声明文件，再扫描工作区里实际存在的全部声明文件做冲突检查。 * *

只检查 classpath 里的文件不够：{@code yoke.workspace} 是持久目录，里面手放的、或旧版本遗留的 - * {@code summarizer.md}、{@code general-purpose.md} 在 2.0.4 里「文件赢」，会静默替换代码声明和内置子 Agent。 + * {@code summarizer.md}、{@code general-purpose.md} 在 AgentScope 里「文件赢」，会静默替换代码声明和内置子 Agent。 * 所以以目录里实际的文件名为准；冲突直接启动失败。其余多出来的文件只记警告（它们会作为子 Agent 被加载）。 */ public static void prepareWorkspace(Path workspace, List declarations) { @@ -143,7 +143,7 @@ public static void requireNoConflict(List declarations, Set .collect(Collectors.toCollection(java.util.TreeSet::new)); if (!clashes.isEmpty()) { throw new IllegalStateException("子 Agent id 重复：" + clashes + " 同时被 workspace/subagents/*.md 和代码或框架内置声明占用。" - + "2.0.4 对此不报错，工作区文件会静默覆盖另一处，请改名或删掉其中一处"); + + "AgentScope 对此不报错，工作区文件会静默覆盖另一处，请改名或删掉其中一处"); } } diff --git a/src/main/java/com/yoke/knowledge/OpenAiEmbeddingClient.java b/src/main/java/com/yoke/knowledge/OpenAiEmbeddingClient.java index f56f290..c7fce1f 100644 --- a/src/main/java/com/yoke/knowledge/OpenAiEmbeddingClient.java +++ b/src/main/java/com/yoke/knowledge/OpenAiEmbeddingClient.java @@ -16,7 +16,7 @@ /** * 调用 OpenAI 兼容的 {@code POST {baseUrl}/embeddings} 接口。 * - *

为什么自己写而不用框架的 {@code OpenAITextEmbedding}：它在 rag-simple 模块里。2.0.4 里 agentscope-core 的 + *

为什么自己写而不用框架的 {@code OpenAITextEmbedding}：它在 rag-simple 模块里。AgentScope 里 agentscope-core 的 * {@code core/rag} 包（{@code Knowledge}、{@code Document}、{@code RetrieveConfig}）已整体标记为 * {@code @Deprecated(forRemoval = true, since = "2.0.0")}；rag-simple 的 pom 还把 Qdrant client、Milvus sdk、 * elasticsearch-java、postgresql、pgvector、pdfbox、poi-ooxml、tika-core、tika-parsers-standard-package、 diff --git a/src/main/java/com/yoke/knowledge/RetrievePort.java b/src/main/java/com/yoke/knowledge/RetrievePort.java index d9a5290..1120608 100644 --- a/src/main/java/com/yoke/knowledge/RetrievePort.java +++ b/src/main/java/com/yoke/knowledge/RetrievePort.java @@ -7,7 +7,7 @@ /** * 检索端口：给一个问题，返回最相关的若干片段。 * - *

这是 Yoke 自己的接口，不依赖 AgentScope：2.0.4 里框架的 rag 包已经整体标记为「将移除」 + *

这是 Yoke 自己的接口，不依赖 AgentScope：AgentScope 里框架的 rag 包已经整体标记为「将移除」 * （{@code @Deprecated(forRemoval = true)}，注释写明「在应用层集成检索」），所以检索放在应用层， * 以后换向量库、换检索算法都不碰 Agent 和工具的代码。 * diff --git a/src/main/java/com/yoke/mcp/McpConnections.java b/src/main/java/com/yoke/mcp/McpConnections.java index 5868a49..12cfa70 100644 --- a/src/main/java/com/yoke/mcp/McpConnections.java +++ b/src/main/java/com/yoke/mcp/McpConnections.java @@ -74,11 +74,11 @@ public static McpConnections connect(List servers, Cli /** * 把已连接服务的工具注册进 Toolkit，工具名就是服务端公布的原名。 * - *

只注册服务端声明了 {@code readOnlyHint} 的工具，其余跳过，模型根本看不到它们。原因：2.0.4 对 MCP 工具 + *

只注册服务端声明了 {@code readOnlyHint} 的工具，其余跳过，模型根本看不到它们。原因：AgentScope 对 MCP 工具 * 只放行只读的，其余都要用户确认；HTTP 接口没有人工确认通道，把它们暴露给模型只会让一轮对话在工具调用处无声结束， * 调用方拿到空回复。与其这样，不如一开始就不暴露。跳过时只记工具名到 INFO 日志，不记参数。 * 暴露的工具名若与 Toolkit 里已有的工具重名（例如内置的 {@code calculate}），启动失败，避免外部服务悄悄顶替内置工具。 - * 用 {@code disableTools} 而不是 {@code enableTools}：后者传空列表在 2.0.4 里表示「全部启用」。 + * 用 {@code disableTools} 而不是 {@code enableTools}：后者传空列表在 AgentScope 里表示「全部启用」。 */ public void registerInto(Toolkit toolkit) { for (McpClientWrapper client : clients) { diff --git a/src/main/java/com/yoke/tool/RetrieveTool.java b/src/main/java/com/yoke/tool/RetrieveTool.java index 83740c9..a941d2f 100644 --- a/src/main/java/com/yoke/tool/RetrieveTool.java +++ b/src/main/java/com/yoke/tool/RetrieveTool.java @@ -18,7 +18,7 @@ * *

只走这一条通道，不同时使用 HarnessAgent / ReActAgent 的 {@code knowledge()} 自动注入： * 两条通道同时开，同一个问题会被检索两次，两份资料挤进上下文还可能互相矛盾，引用编号也对不上； - * 而且 2.0.4 里框架的 rag 包已标记为将移除。工具化的另一个好处是闲聊不触发检索，省 token。 + * 而且 AgentScope 里框架的 rag 包已标记为将移除。工具化的另一个好处是闲聊不触发检索，省 token。 * *

返回值带 {@code [1][2]} 编号，模型回答时按编号引用，用户就能回头核对出处。 * 无状态，线程安全；{@code port} 为空表示没有配置知识库，工具返回提示语而不是抛异常。 diff --git a/src/test/java/com/yoke/ArchitectureTests.java b/src/test/java/com/yoke/ArchitectureTests.java index 8a3ccb6..f9c945e 100644 --- a/src/test/java/com/yoke/ArchitectureTests.java +++ b/src/test/java/com/yoke/ArchitectureTests.java @@ -75,7 +75,7 @@ public boolean test(JavaClass input) { .should().dependOnClassesThat().resideInAnyPackage("io.agentscope.core.message..", "io.agentscope.core.event.."); /** - * 检索端口是应用层自己的接口：不依赖 AgentScope（2.0.4 里框架的 rag 包已标记为将移除）， + * 检索端口是应用层自己的接口：不依赖 AgentScope（AgentScope 里框架的 rag 包已标记为将移除）， * 以后换向量库、换检索算法都不影响 Agent 和工具。向量化客户端、关键词检索、融合与降级都在这个包里。 */ @ArchTest diff --git a/src/test/java/com/yoke/ToolPermissionTests.java b/src/test/java/com/yoke/ToolPermissionTests.java index eaa7436..68e5733 100644 --- a/src/test/java/com/yoke/ToolPermissionTests.java +++ b/src/test/java/com/yoke/ToolPermissionTests.java @@ -60,7 +60,7 @@ private static Set toolsOfUnrestrictedAgent(Path workspace) { "memory_get", "memory_search", "session_history", "session_list", "session_search"); /** - * 保留下来的框架内置工具，逐个写明（2.0.4 实测）：写记忆、技能加载、异步结果、子 Agent 和后台任务。 + * 保留下来的框架内置工具，逐个写明（AgentScope 实测）：写记忆、技能加载、异步结果、子 Agent 和后台任务。 * 写成字面量而不是从参照 Agent 推导，是为了让框架升级新增了任何内置工具时这里失败，逼着人来决定放不放行。 */ private static final Set KEPT_BUILT_IN = Set.of( diff --git a/src/test/java/com/yoke/config/SharedToolkitTests.java b/src/test/java/com/yoke/config/SharedToolkitTests.java index 78c8e5e..88ddf8b 100644 --- a/src/test/java/com/yoke/config/SharedToolkitTests.java +++ b/src/test/java/com/yoke/config/SharedToolkitTests.java @@ -22,7 +22,7 @@ import static org.assertj.core.api.Assertions.assertThat; /** - * 守住「所有请求共用同一个 Toolkit 模板」这个前提：它成立靠的是 2.0.4 的 HarnessAgent.Builder.build() 会先拷贝 Toolkit。 + * 守住「所有请求共用同一个 Toolkit 模板」这个前提：它成立靠的是 AgentScope 的 HarnessAgent.Builder.build() 会先拷贝 Toolkit。 * 将来框架升级后如果不再拷贝，Harness 的内置工具会写进模板，这里会失败。 */ class SharedToolkitTests { diff --git a/src/test/java/com/yoke/config/SubagentTests.java b/src/test/java/com/yoke/config/SubagentTests.java index 620b56b..d43a7bc 100644 --- a/src/test/java/com/yoke/config/SubagentTests.java +++ b/src/test/java/com/yoke/config/SubagentTests.java @@ -23,7 +23,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; /** - * 子 Agent 的声明与权限。最关键的是安全回归：2.0.4 的子 Agent 不继承父 Agent 的 {@code disableWebTools}， + * 子 Agent 的声明与权限。最关键的是安全回归：AgentScope 的子 Agent 不继承父 Agent 的 {@code disableWebTools}， * 所以这里用「记录工具清单的假模型」真实驱动每一种子 Agent 跑一次，枚举它实际能看到的工具。 */ class SubagentTests { @@ -120,7 +120,7 @@ private static Set toolsSeenByParent(ScriptedModel model) { @Test void 声明文件忘了写tools白名单时也拿不到联网工具_靠的是父Agent的deny名单(@TempDir Path workspace) throws IOException { - // 2.0.4 里 tools 为空表示继承父 Agent 的全部工具；这是最容易写错的地方，所以要有第二道防线 + // AgentScope 里 tools 为空表示继承父 Agent 的全部工具；这是最容易写错的地方，所以要有第二道防线 TestAgents.factory(new ScriptedModel(c -> ScriptedModel.text("x")), workspace, template()); Files.writeString(workspace.resolve("subagents/open.md"), """ --- @@ -159,7 +159,7 @@ private static Set toolsSeenByParent(ScriptedModel model) { run(delegatingTo("summarizer"), workspace); - assertThat(marker).as("Yoke 显式传了 toolsConfig，2.0.4 就不再读 tools.json").doesNotExist(); + assertThat(marker).as("Yoke 显式传了 toolsConfig，AgentScope 就不再读 tools.json").doesNotExist(); } @Test @@ -176,7 +176,7 @@ private static Set toolsSeenByParent(ScriptedModel model) { // 握手失败无所谓，进程已经被拉起过 } - assertThat(marker).as("没有显式 toolsConfig 时，工作区里的 tools.json 能让 2.0.4 启动任意本地进程").exists(); + assertThat(marker).as("没有显式 toolsConfig 时，工作区里的 tools.json 能让 AgentScope 启动任意本地进程").exists(); } // ---- 声明与冲突 ---- @@ -258,7 +258,7 @@ private static Set toolsSeenByParent(ScriptedModel model) { @Test void 工作区里已有的同名声明文件会让启动失败_即使它不是随应用发布的(@TempDir Path workspace) throws IOException { - // yoke.workspace 是持久目录：手放的、旧版本遗留的 summarizer.md 在 2.0.4 里会静默替换代码声明 + // yoke.workspace 是持久目录：手放的、旧版本遗留的 summarizer.md 在 AgentScope 里会静默替换代码声明 Files.createDirectories(workspace.resolve("subagents")); Files.writeString(workspace.resolve("subagents/summarizer.md"), "---\ndescription: 遗留文件\n---\n旧提示词"); diff --git a/src/test/java/com/yoke/support/McpTestServer.java b/src/test/java/com/yoke/support/McpTestServer.java index 2c06b63..f1dcd27 100644 --- a/src/test/java/com/yoke/support/McpTestServer.java +++ b/src/test/java/com/yoke/support/McpTestServer.java @@ -117,7 +117,7 @@ private static McpServerFeatures.SyncToolSpecification echoTool() { .name("echo") .description("原样回显收到的文字，用来验证 MCP 连通") .inputSchema(schema) - // 声明只读：2.0.4 对只读的 MCP 工具自动放行，其余都要求显式授权 + // 声明只读：AgentScope 对只读的 MCP 工具自动放行，其余都要求显式授权 .annotations(new McpSchema.ToolAnnotations("echo", true, false, true, false, false)) .build(); return McpServerFeatures.SyncToolSpecification.builder() diff --git a/src/test/java/com/yoke/tool/ToolConversationTests.java b/src/test/java/com/yoke/tool/ToolConversationTests.java index 5c23752..42fd08e 100644 --- a/src/test/java/com/yoke/tool/ToolConversationTests.java +++ b/src/test/java/com/yoke/tool/ToolConversationTests.java @@ -135,7 +135,7 @@ private static ChatController controller(ScriptedModel model, Path workspace, To @Test void 未声明只读的MCP工具不会暴露给模型_主Agent和子Agent都看不到_即使模型硬调也不会执行(@TempDir Path workspace) { - // 2.0.4 只放行声明了 readOnlyHint 的 MCP 工具，其余要用户确认；HTTP 接口没有确认通道，所以注册时就不暴露 + // AgentScope 只放行声明了 readOnlyHint 的 MCP 工具，其余要用户确认；HTTP 接口没有确认通道，所以注册时就不暴露 try (McpTestServer server = new McpTestServer(); McpConnections mcp = McpConnections.connect(List.of(new YokeProperties.McpServer( "demo", "streamable-http", server.url(), Duration.ofSeconds(10), Map.of())))) { From ac1ee99bc27efc4e2c8d9c2df8249087f15e924f Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 12 Nov 2026 09:43:21 +0800 Subject: [PATCH 06/19] feat: add Volcano Ark multimodal embedding client and reorganize knowledge packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增火山方舟多模态向量客户端，并整理知识库相关包结构。 - 新增 ArkEmbeddingClient：方舟的向量模型走 /embeddings/multimodal， 请求体是内容对象数组、不支持批量、响应的 data 是单个对象，和 OpenAI 兼容的 /embeddings 完全不同，只改 base-url 会被方舟直接拒绝。一条文本一次请求， 并发上限 4，结果按输入顺序，并校验返回维度。 - 新增配置 yoke.knowledge.embedding.protocol（openai | ark-multimodal， 默认 openai）；local 环境改为 ark-multimodal，模型 doubao-embedding-vision-251215， 密钥复用 ARK_API_KEY。 - 包结构：检索实现移到 knowledge.retrieve，向量化客户端移到 knowledge.embed， Passage 和 UserContext 移到 model，SharedToolkit 移到 tool（model 包不允许依赖框架）。 - 修复 KnowledgeCorpus 重构引入的回归：片段正文在 chunk 还是空时就拼好， 导致所有片段只剩标题。 - SubagentTests 的 NEVER 补上 5 个读类记忆和会话工具，子 Agent 读不到 别的用户的数据也有测试守着。 - 测试随包搬迁；新增 ArkEmbeddingClientTests 和装配层用例，假服务只注册 多模态路径，误请求 /embeddings 会变红。README 中英文同步。 已用真实方舟验证：标准 /embeddings 会拒绝该模型，多模态接口返回 2048 维向量， 向量检索对改写后的问题仍能命中同一篇文档，对无关问题返回空。 --- README.md | 2 +- README_CN.md | 2 +- .../com/yoke/config/AgentConfiguration.java | 2 + .../yoke/config/KnowledgeConfiguration.java | 18 +- .../com/yoke/config/ToolConfiguration.java | 1 + .../java/com/yoke/config/YokeProperties.java | 18 +- .../com/yoke/controller/ChatController.java | 2 +- .../com/yoke/knowledge/KnowledgeCorpus.java | 10 +- .../java/com/yoke/knowledge/RetrievePort.java | 1 + .../knowledge/embed/ArkEmbeddingClient.java | 78 ++++++++ .../{ => embed}/OpenAiEmbeddingClient.java | 11 +- .../{ => retrieve}/HybridRetrievePort.java | 10 +- .../{ => retrieve}/KeywordRetrievePort.java | 4 +- .../{ => retrieve}/VectorRetrievePort.java | 11 +- .../yoke/{knowledge => model}/Passage.java | 2 +- .../com/yoke/{tool => model}/UserContext.java | 2 +- src/main/java/com/yoke/tool/ChatTools.java | 7 +- src/main/java/com/yoke/tool/RetrieveTool.java | 8 +- .../yoke/{config => tool}/SharedToolkit.java | 13 +- src/main/resources/application-local.yml | 8 +- src/main/resources/application.yml | 3 +- src/test/java/com/yoke/ProfilesTests.java | 4 +- .../AgentConfigurationConflictTests.java | 3 +- .../yoke/config/AgentConfigurationTests.java | 3 +- .../config/KnowledgeConfigurationTests.java | 39 +++- .../yoke/config/RedisConfigurationTests.java | 2 +- .../java/com/yoke/config/SubagentTests.java | 9 +- .../config/WorkspaceInitializerTests.java | 2 +- .../com/yoke/config/YokePropertiesTests.java | 9 +- .../yoke/knowledge/KnowledgeCorpusTests.java | 2 + .../embed/ArkEmbeddingClientTests.java | 166 ++++++++++++++++++ .../OpenAiEmbeddingClientTests.java | 4 +- .../HybridRetrievePortTests.java | 4 +- .../KeywordRetrievePortTests.java | 3 +- .../VectorRetrievePortTests.java | 4 +- .../java/com/yoke/mcp/McpStartupTests.java | 2 +- .../java/com/yoke/tool/ChatToolsTests.java | 1 + .../java/com/yoke/tool/RetrieveToolTests.java | 2 +- .../com/yoke/tool/ToolConversationTests.java | 3 +- 39 files changed, 393 insertions(+), 82 deletions(-) create mode 100644 src/main/java/com/yoke/knowledge/embed/ArkEmbeddingClient.java rename src/main/java/com/yoke/knowledge/{ => embed}/OpenAiEmbeddingClient.java (93%) rename src/main/java/com/yoke/knowledge/{ => retrieve}/HybridRetrievePort.java (95%) rename src/main/java/com/yoke/knowledge/{ => retrieve}/KeywordRetrievePort.java (98%) rename src/main/java/com/yoke/knowledge/{ => retrieve}/VectorRetrievePort.java (95%) rename src/main/java/com/yoke/{knowledge => model}/Passage.java (92%) rename src/main/java/com/yoke/{tool => model}/UserContext.java (95%) rename src/main/java/com/yoke/{config => tool}/SharedToolkit.java (81%) create mode 100644 src/test/java/com/yoke/knowledge/embed/ArkEmbeddingClientTests.java rename src/test/java/com/yoke/knowledge/{ => embed}/OpenAiEmbeddingClientTests.java (97%) rename src/test/java/com/yoke/knowledge/{ => retrieve}/HybridRetrievePortTests.java (98%) rename src/test/java/com/yoke/knowledge/{ => retrieve}/KeywordRetrievePortTests.java (96%) rename src/test/java/com/yoke/knowledge/{ => retrieve}/VectorRetrievePortTests.java (97%) diff --git a/README.md b/README.md index d48054b..fbdffbb 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ Yoke's own settings use the `yoke.*` prefix: | `yoke.compaction.trigger-messages` | Compact the conversation once it reaches this many messages, default 30 | | `yoke.compaction.keep-messages` | Messages kept verbatim after a compaction, default 10. Values `<= 0`, or a keep count not smaller than the trigger count, fall back to the defaults | | `yoke.workspace` | `HarnessAgent` workspace for local files (offloaded messages, memory), default `~/.yoke/workspace`; it stays on the machine and is not shared between replicas | -| `yoke.knowledge.embedding.model-name` / `api-key` / `base-url` / `dimensions` | Optional embedding service (OpenAI-compatible `/embeddings`) for vector retrieval. Vector retrieval is enabled only when both `model-name` and `api-key` (`YOKE_EMBEDDING_API_KEY`) are set; otherwise retrieval uses keywords only. If the service fails at runtime, retrieval falls back to keywords automatically | +| `yoke.knowledge.embedding.model-name` / `api-key` / `base-url` / `dimensions` / `protocol` | Optional embedding service for vector retrieval. `protocol` defaults to `openai` (OpenAI-compatible `/embeddings`, batchable); use `ark-multimodal` for Volcano Ark multimodal embedding models such as `doubao-embedding-vision-251215` (`/embeddings/multimodal`, one request per text; the `local` profile uses it and reuses `ARK_API_KEY`). Vector retrieval is enabled only when both `model-name` and `api-key` (`YOKE_EMBEDDING_API_KEY`) are set; otherwise retrieval uses keywords only. If the service fails at runtime, retrieval falls back to keywords automatically | | `yoke.mcp.servers[]` | External MCP servers, none by default. Each entry has `name`, `transport` (`sse` or `streamable-http` only), `url`, `timeout` (default 30s) and optional `headers`. `stdio` is rejected and the application fails at startup; so is a server that cannot be reached | The agent is built without file, shell or web tools. Reading the 2.0.4 source shows that a sub-agent inherits only the file and shell switches, not the web switch, so the web tools are also denied through a `ToolsConfig` deny list (on top of `disableWebTools()` for the main agent), which is applied to the main agent, the built-in `general-purpose` sub-agent and every declared sub-agent. Passing an explicit `ToolsConfig` also stops 2.0.4 from reading `tools.json` from the workspace, which could otherwise declare `stdio` MCP servers (processes started on the host). The read-side memory and session tools (`memory_get`, `memory_search`, `session_history`, `session_list`, `session_search`) are denied by the same list: `memory_get` only checks that a path stays inside the workspace root, not inside the current user's directory, and `session_history` puts model-supplied IDs straight into a path, so they would let the model read other users' memory and sessions. The model sees exactly these tools: Yoke's own (`get_weather`, `get_current_time`, `calculate`, `list_my_notes`, `retrieve`), the framework's memory-writing, skill-loading, async-result, sub-agent and background-task tools (`memory_save` only appends to the fixed `MEMORY.md` and `memory/.md` in the workspace, without a size limit), plus whatever the configured MCP servers publish. `ToolPermissionTests` asserts this list as an exact set, so a framework upgrade that adds a tool fails the build. A `MiddlewareBase` bean is attached to every new agent; `LoggingMiddleware` logs agent, model and tool timings without tool arguments or outputs. OpenTelemetry tracing is not attached by default: add an OpenTelemetry SDK and exporter and register an `OtelTracingMiddleware` bean when you need it. diff --git a/README_CN.md b/README_CN.md index c5e9824..009f8dd 100644 --- a/README_CN.md +++ b/README_CN.md @@ -165,7 +165,7 @@ Yoke 自己的配置使用 `yoke.*` 前缀： | `yoke.compaction.trigger-messages` | 对话消息达到这个条数就压缩，默认 30 | | `yoke.compaction.keep-messages` | 压缩后原样保留的最近条数，默认 10。小于等于 0，或保留条数不小于触发条数时，回落到默认值 | | `yoke.workspace` | `HarnessAgent` 的工作区，放本地文件（转存的消息、记忆），默认 `~/.yoke/workspace`；只在本机，副本之间不共享 | -| `yoke.knowledge.embedding.model-name` / `api-key` / `base-url` / `dimensions` | 可选的向量化服务（OpenAI 兼容的 `/embeddings`），用于向量检索。`model-name` 和 `api-key`（`YOKE_EMBEDDING_API_KEY`）都有值才启用向量检索，否则只用关键词检索；运行中向量化服务出错时自动退回关键词检索 | +| `yoke.knowledge.embedding.model-name` / `api-key` / `base-url` / `dimensions` / `protocol` | 可选的向量化服务，用于向量检索。`protocol` 默认 `openai`（OpenAI 兼容的 `/embeddings`，可批量）；火山方舟的多模态向量模型（如 `doubao-embedding-vision-251215`）要写 `ark-multimodal`（`/embeddings/multimodal`，一条文本一次请求，`local` 环境默认就是它，复用 `ARK_API_KEY`）。`model-name` 和 `api-key`（`YOKE_EMBEDDING_API_KEY`）都有值才启用向量检索，否则只用关键词检索；运行中向量化服务出错时自动退回关键词检索 | | `yoke.mcp.servers[]` | 外部 MCP 服务，默认一个也不接。每项有 `name`、`transport`（只能是 `sse` 或 `streamable-http`）、`url`、`timeout`（默认 30 秒）和可选的 `headers`。配置 `stdio` 会让应用启动失败；服务连不上同样启动失败 | Agent 创建时不开放文件、命令和联网工具。读 2.0.4 源码发现子 Agent 只继承文件和 shell 的关闭、不继承联网工具的关闭，所以联网工具除了主 Agent 的 `disableWebTools()`，还用 `ToolsConfig` 的 deny 名单禁掉，它同时作用于主 Agent、内置的 `general-purpose` 子 Agent 和所有声明的子 Agent。显式传入 `ToolsConfig` 还会让 2.0.4 不再去读工作区里的 `tools.json`（那个文件可以声明 `stdio` MCP 服务，也就是在宿主机上启动进程）。读类的记忆和会话工具（`memory_get`、`memory_search`、`session_history`、`session_list`、`session_search`）也被同一份 deny 名单拒绝：`memory_get` 只校验路径不越出工作区根目录、不校验是否在当前用户目录下，`session_history` 把模型给的 ID 直接拼进路径，开着就能让模型读到别的用户的记忆和会话。模型看到的工具恰好是：Yoke 自己的（`get_weather`、`get_current_time`、`calculate`、`list_my_notes`、`retrieve`），框架的写记忆、技能加载、异步结果、子 Agent 和后台任务类工具（`memory_save` 只往工作区固定的 `MEMORY.md` 与 `memory/<日期>.md` 追加，内容没有大小限制），再加上所配置的 MCP 服务公布的工具。`ToolPermissionTests` 用精确集合断言这份清单，框架升级新增了工具会让构建失败。容器里的 `MiddlewareBase` Bean 会挂到每个新建的 Agent 上；`LoggingMiddleware` 记录 Agent、模型、工具的耗时，不记录工具的参数和输出。OpenTelemetry 追踪默认不挂载：需要时自行引入 OpenTelemetry SDK 和导出器，再注册一个 `OtelTracingMiddleware` Bean。 diff --git a/src/main/java/com/yoke/config/AgentConfiguration.java b/src/main/java/com/yoke/config/AgentConfiguration.java index 43672b7..618ca87 100644 --- a/src/main/java/com/yoke/config/AgentConfiguration.java +++ b/src/main/java/com/yoke/config/AgentConfiguration.java @@ -2,6 +2,8 @@ import java.nio.file.Path; import java.util.List; + +import com.yoke.tool.SharedToolkit; import io.agentscope.core.middleware.MiddlewareBase; import io.agentscope.core.model.Model; import io.agentscope.harness.agent.DistributedStore; diff --git a/src/main/java/com/yoke/config/KnowledgeConfiguration.java b/src/main/java/com/yoke/config/KnowledgeConfiguration.java index 3b0851c..425e284 100644 --- a/src/main/java/com/yoke/config/KnowledgeConfiguration.java +++ b/src/main/java/com/yoke/config/KnowledgeConfiguration.java @@ -5,13 +5,14 @@ import java.util.List; import com.yoke.knowledge.EmbeddingClient; -import com.yoke.knowledge.HybridRetrievePort; -import com.yoke.knowledge.KeywordRetrievePort; +import com.yoke.knowledge.retrieve.HybridRetrievePort; +import com.yoke.knowledge.retrieve.KeywordRetrievePort; import com.yoke.knowledge.KnowledgeCorpus; -import com.yoke.knowledge.OpenAiEmbeddingClient; -import com.yoke.knowledge.Passage; +import com.yoke.knowledge.embed.ArkEmbeddingClient; +import com.yoke.knowledge.embed.OpenAiEmbeddingClient; +import com.yoke.model.Passage; import com.yoke.knowledge.RetrievePort; -import com.yoke.knowledge.VectorRetrievePort; +import com.yoke.knowledge.retrieve.VectorRetrievePort; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Bean; @@ -55,8 +56,11 @@ public RetrievePort retrievePort(YokeProperties yoke) { .defaultHeader("Authorization", "Bearer " + embedding.apiKey()) .requestFactory(requestFactory) .build(); - EmbeddingClient embeddings = new OpenAiEmbeddingClient(client, embedding.modelName(), embedding.dimensions()); - log.info("检索使用向量加关键词双路融合，向量模型：{}", embedding.modelName()); + EmbeddingClient embeddings = switch (embedding.protocol()) { + case ARK_MULTIMODAL -> new ArkEmbeddingClient(client, embedding.modelName(), embedding.dimensions()); + case OPENAI -> new OpenAiEmbeddingClient(client, embedding.modelName(), embedding.dimensions()); + }; + log.info("检索使用向量加关键词双路融合，向量模型：{}，协议：{}", embedding.modelName(), embedding.protocol()); return new HybridRetrievePort(List.of(new VectorRetrievePort(passages, embeddings), keyword)); } } diff --git a/src/main/java/com/yoke/config/ToolConfiguration.java b/src/main/java/com/yoke/config/ToolConfiguration.java index 5b7a73d..0000bae 100644 --- a/src/main/java/com/yoke/config/ToolConfiguration.java +++ b/src/main/java/com/yoke/config/ToolConfiguration.java @@ -2,6 +2,7 @@ import com.yoke.knowledge.RetrievePort; import com.yoke.mcp.McpConnections; +import com.yoke.tool.SharedToolkit; import com.yoke.tool.ChatTools; import com.yoke.tool.RetrieveTool; import com.yoke.tool.WeatherTool; diff --git a/src/main/java/com/yoke/config/YokeProperties.java b/src/main/java/com/yoke/config/YokeProperties.java index b6e1768..cf66a31 100644 --- a/src/main/java/com/yoke/config/YokeProperties.java +++ b/src/main/java/com/yoke/config/YokeProperties.java @@ -91,18 +91,28 @@ public record Compaction( public record Knowledge(@DefaultValue Embedding embedding) { /** - * 向量化服务（OpenAI 兼容的 {@code /embeddings} 接口），{@code modelName} 和 {@code apiKey} 都有值才启用。 + * 向量化服务，{@code modelName} 和 {@code apiKey} 都有值才启用。 * * @param modelName 向量模型名，例如 text-embedding-3-small；为空表示不启用向量检索 * @param baseUrl 接口根地址，默认 OpenAI 官方；用 DashScope 等兼容服务时改它 * @param apiKey 密钥，只从环境变量或外部配置提供，不要写进仓库 - * @param dimensions 向量维度，可选；不传用模型默认值 + * @param dimensions 向量维度，可选；不传用模型默认值。方舟多模态协议会校验返回的维度与它一致 + * @param protocol 接口协议，默认 OpenAI 兼容的 {@code /embeddings}；火山方舟的多模态向量模型要选 ark-multimodal */ public record Embedding( String modelName, @DefaultValue("https://api.openai.com/v1") String baseUrl, String apiKey, - Integer dimensions) { + Integer dimensions, + @DefaultValue("openai") Protocol protocol) { + + /** 向量化接口的协议。 */ + public enum Protocol { + /** OpenAI 兼容的 {@code POST /embeddings}，支持批量。 */ + OPENAI, + /** 火山方舟的 {@code POST /embeddings/multimodal}，一条文本一次请求。 */ + ARK_MULTIMODAL + } public boolean enabled() { return StrUtil.isNotBlank(modelName) && StrUtil.isNotBlank(apiKey); @@ -112,7 +122,7 @@ public boolean enabled() { @Override public String toString() { return "Embedding[modelName=" + modelName + ", baseUrl=" + baseUrl + ", apiKey=" - + (StrUtil.isNotBlank(apiKey) ? "***" : null) + ", dimensions=" + dimensions + "]"; + + (StrUtil.isNotBlank(apiKey) ? "***" : null) + ", dimensions=" + dimensions + ", protocol=" + protocol + "]"; } } } diff --git a/src/main/java/com/yoke/controller/ChatController.java b/src/main/java/com/yoke/controller/ChatController.java index 286bfc8..dfcbc8d 100644 --- a/src/main/java/com/yoke/controller/ChatController.java +++ b/src/main/java/com/yoke/controller/ChatController.java @@ -11,7 +11,7 @@ import com.yoke.config.HarnessAgentFactory; import com.yoke.model.ChatRequest; import com.yoke.model.ChatResponse; -import com.yoke.tool.UserContext; +import com.yoke.model.UserContext; import io.agentscope.core.agent.RuntimeContext; import io.agentscope.harness.agent.HarnessAgent; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java b/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java index 5024fb0..e00146b 100644 --- a/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java +++ b/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java @@ -9,6 +9,8 @@ import java.util.List; import java.util.Optional; +import com.yoke.model.Passage; +import lombok.NoArgsConstructor; import org.springframework.core.io.Resource; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; @@ -19,13 +21,11 @@ * 片段正文超过 {@link #MAX_CHUNK_LENGTH} 个字符时按段落再切。每个片段正文前面带上「文档标题 / 小节标题」， * 这样单看一个片段也知道它在讲什么。 */ +@NoArgsConstructor(access = lombok.AccessLevel.PRIVATE) public final class KnowledgeCorpus { static final int MAX_CHUNK_LENGTH = 500; - private KnowledgeCorpus() { - } - /** 读取 classpath 下的全部知识文档并切片；没有文档时返回空列表。 */ public static List loadFromClasspath(String pattern) { try { @@ -78,13 +78,13 @@ private static void flush(List passages, String docId, String title, St String heading = Optional.ofNullable(section).map(name -> title + " / " + name).orElse(title); StringBuilder chunk = new StringBuilder(); for (String paragraph : text.split("\\n\\s*\\n")) { - if (chunk.length() > 0 && chunk.length() + paragraph.length() > MAX_CHUNK_LENGTH) { + if (!chunk.isEmpty() && chunk.length() + paragraph.length() > MAX_CHUNK_LENGTH) { passages.add(new Passage(docId + "#" + passages.size(), title, heading + "\n" + chunk.toString().strip())); chunk.setLength(0); } chunk.append(paragraph.strip()).append("\n\n"); } - if (chunk.length() > 0) { + if (!chunk.isEmpty()) { passages.add(new Passage(docId + "#" + passages.size(), title, heading + "\n" + chunk.toString().strip())); } } diff --git a/src/main/java/com/yoke/knowledge/RetrievePort.java b/src/main/java/com/yoke/knowledge/RetrievePort.java index 1120608..5c6c10a 100644 --- a/src/main/java/com/yoke/knowledge/RetrievePort.java +++ b/src/main/java/com/yoke/knowledge/RetrievePort.java @@ -2,6 +2,7 @@ import java.util.List; +import com.yoke.model.Passage; import reactor.core.publisher.Mono; /** diff --git a/src/main/java/com/yoke/knowledge/embed/ArkEmbeddingClient.java b/src/main/java/com/yoke/knowledge/embed/ArkEmbeddingClient.java new file mode 100644 index 0000000..3ffc3bb --- /dev/null +++ b/src/main/java/com/yoke/knowledge/embed/ArkEmbeddingClient.java @@ -0,0 +1,78 @@ +package com.yoke.knowledge.embed; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import com.yoke.knowledge.EmbeddingClient; +import lombok.RequiredArgsConstructor; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import org.springframework.http.MediaType; +import org.springframework.web.client.RestClient; + +/** + * 火山方舟多模态向量模型（例如 doubao-embedding-vision）的客户端，调用 {@code POST {baseUrl}/embeddings/multimodal}。 + * + *

它和 OpenAI 兼容的 {@code /embeddings} 有三处不同，所以不能复用 {@link OpenAiEmbeddingClient}： + *

    + *
  • 路径是 {@code /embeddings/multimodal}；
    • + *
  • 请求体的 {@code input} 是内容对象的数组（{@code {"type":"text","text":"……"}}），不是字符串数组；
    • + *
  • 响应的 {@code data} 是单个对象（也兼容数组），没有 {@code index}； + * 而且不支持批量：一次传多个 input 会被融合成一个向量，所以一个文本必须一次请求。
    • + *
+ * + *

一批文本拆成逐条请求，并发度受 {@link #CONCURRENCY} 限制，结果按输入顺序返回。 + * 请求和响应格式参考 Peliplat 的 {@code ArkEmbeddingModel}；它那套退避重试和全局背压是给批量入库用的， + * 这里语料只有几篇小文档，向量化失败直接报错，由上层降级到关键词检索，不重试。 + * + *

隐私：用户的检索问题和知识库语料会发送给方舟。RestClient 是阻塞的，所以放到 boundedElastic 上执行。 + */ +@RequiredArgsConstructor +public final class ArkEmbeddingClient implements EmbeddingClient { + + /** 同时在途的请求数：不支持批量，片段多时不能一次把方舟打满。 */ + static final int CONCURRENCY = 4; + + private final RestClient client; + private final String modelName; + private final Integer dimensions; + + @Override + public Mono> embed(List texts) { + return Flux.fromIterable(texts) + .flatMapSequential(text -> Mono.fromCallable(() -> embedOne(text)).subscribeOn(Schedulers.boundedElastic()), + CONCURRENCY) + .collectList(); + } + + @SuppressWarnings("unchecked") + private double[] embedOne(String text) { + Map body = new HashMap<>(); + body.put("model", modelName); + body.put("input", List.of(Map.of("type", "text", "text", text))); + if (Objects.nonNull(dimensions)) { + body.put("dimensions", dimensions); + } + Map response = client.post() + .uri("/embeddings/multimodal") + .contentType(MediaType.APPLICATION_JSON) + .body(body) + .retrieve() + .body(Map.class); + Object data = Objects.isNull(response) ? null : response.get("data"); + // 实测 data 是单个对象；兼容它改成数组的情况 + Object item = data instanceof List list && !list.isEmpty() ? list.getFirst() : data; + if (!(item instanceof Map map) || !(map.get("embedding") instanceof List values) || values.isEmpty()) { + throw new IllegalStateException("向量化接口的响应里没有 embedding"); + } + double[] vector = ((List) values).stream().mapToDouble(Number::doubleValue).toArray(); + if (Objects.nonNull(dimensions) && vector.length != dimensions) { + throw new IllegalStateException("向量化接口返回的维度 " + vector.length + " 与配置的 " + dimensions + " 不一致"); + } + return vector; + } +} diff --git a/src/main/java/com/yoke/knowledge/OpenAiEmbeddingClient.java b/src/main/java/com/yoke/knowledge/embed/OpenAiEmbeddingClient.java similarity index 93% rename from src/main/java/com/yoke/knowledge/OpenAiEmbeddingClient.java rename to src/main/java/com/yoke/knowledge/embed/OpenAiEmbeddingClient.java index c7fce1f..8289b68 100644 --- a/src/main/java/com/yoke/knowledge/OpenAiEmbeddingClient.java +++ b/src/main/java/com/yoke/knowledge/embed/OpenAiEmbeddingClient.java @@ -1,4 +1,4 @@ -package com.yoke.knowledge; +package com.yoke.knowledge.embed; import java.util.ArrayList; import java.util.Comparator; @@ -7,6 +7,8 @@ import java.util.Map; import java.util.Objects; +import com.yoke.knowledge.EmbeddingClient; +import lombok.RequiredArgsConstructor; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @@ -26,18 +28,13 @@ *

隐私：用户的检索问题和知识库语料会发送给配置的 embedding 服务提供方。 * RestClient 是阻塞的，所以放到 boundedElastic 上执行，不占用响应式线程。 */ +@RequiredArgsConstructor public final class OpenAiEmbeddingClient implements EmbeddingClient { private final RestClient client; private final String modelName; private final Integer dimensions; - public OpenAiEmbeddingClient(RestClient client, String modelName, Integer dimensions) { - this.client = client; - this.modelName = modelName; - this.dimensions = dimensions; - } - @Override public Mono> embed(List texts) { return Mono.fromCallable(() -> request(texts)).subscribeOn(Schedulers.boundedElastic()); diff --git a/src/main/java/com/yoke/knowledge/HybridRetrievePort.java b/src/main/java/com/yoke/knowledge/retrieve/HybridRetrievePort.java similarity index 95% rename from src/main/java/com/yoke/knowledge/HybridRetrievePort.java rename to src/main/java/com/yoke/knowledge/retrieve/HybridRetrievePort.java index e6e3269..6a5799d 100644 --- a/src/main/java/com/yoke/knowledge/HybridRetrievePort.java +++ b/src/main/java/com/yoke/knowledge/retrieve/HybridRetrievePort.java @@ -1,4 +1,4 @@ -package com.yoke.knowledge; +package com.yoke.knowledge.retrieve; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -6,8 +6,9 @@ import java.util.Map; import java.util.Objects; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import com.yoke.model.Passage; +import com.yoke.knowledge.RetrievePort; +import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -25,10 +26,9 @@ * * 融合只发生在成功的路之间，所以向量路挂掉时结果自然退化成纯关键词检索。 */ +@Slf4j public final class HybridRetrievePort implements RetrievePort { - private static final Logger log = LoggerFactory.getLogger(HybridRetrievePort.class); - /** RRF 常数。 */ static final int RRF_K = 60; diff --git a/src/main/java/com/yoke/knowledge/KeywordRetrievePort.java b/src/main/java/com/yoke/knowledge/retrieve/KeywordRetrievePort.java similarity index 98% rename from src/main/java/com/yoke/knowledge/KeywordRetrievePort.java rename to src/main/java/com/yoke/knowledge/retrieve/KeywordRetrievePort.java index e788157..02ee7d3 100644 --- a/src/main/java/com/yoke/knowledge/KeywordRetrievePort.java +++ b/src/main/java/com/yoke/knowledge/retrieve/KeywordRetrievePort.java @@ -1,4 +1,4 @@ -package com.yoke.knowledge; +package com.yoke.knowledge.retrieve; import java.util.ArrayList; import java.util.Comparator; @@ -11,6 +11,8 @@ import java.util.Set; import cn.hutool.core.util.StrUtil; +import com.yoke.model.Passage; +import com.yoke.knowledge.RetrievePort; import reactor.core.publisher.Mono; /** diff --git a/src/main/java/com/yoke/knowledge/VectorRetrievePort.java b/src/main/java/com/yoke/knowledge/retrieve/VectorRetrievePort.java similarity index 95% rename from src/main/java/com/yoke/knowledge/VectorRetrievePort.java rename to src/main/java/com/yoke/knowledge/retrieve/VectorRetrievePort.java index 50da189..dcae5a0 100644 --- a/src/main/java/com/yoke/knowledge/VectorRetrievePort.java +++ b/src/main/java/com/yoke/knowledge/retrieve/VectorRetrievePort.java @@ -1,4 +1,4 @@ -package com.yoke.knowledge; +package com.yoke.knowledge.retrieve; import java.util.ArrayList; import java.util.Comparator; @@ -6,8 +6,10 @@ import java.util.Objects; import cn.hutool.core.util.StrUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import com.yoke.knowledge.EmbeddingClient; +import com.yoke.model.Passage; +import com.yoke.knowledge.RetrievePort; +import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -17,10 +19,9 @@ *

片段向量在第一次检索时才算（懒索引），启动不依赖向量化服务；索引失败时这次检索报错， * 下一次检索会重试，不会把失败结果缓存下来。索引成功后只读，线程安全。 */ +@Slf4j public final class VectorRetrievePort implements RetrievePort { - private static final Logger log = LoggerFactory.getLogger(VectorRetrievePort.class); - /** 一次向量化请求最多带多少个片段。 */ static final int BATCH_SIZE = 16; diff --git a/src/main/java/com/yoke/knowledge/Passage.java b/src/main/java/com/yoke/model/Passage.java similarity index 92% rename from src/main/java/com/yoke/knowledge/Passage.java rename to src/main/java/com/yoke/model/Passage.java index cb8af03..30bee87 100644 --- a/src/main/java/com/yoke/knowledge/Passage.java +++ b/src/main/java/com/yoke/model/Passage.java @@ -1,4 +1,4 @@ -package com.yoke.knowledge; +package com.yoke.model; /** * 知识库里的一个片段，也是检索的返回单位。 diff --git a/src/main/java/com/yoke/tool/UserContext.java b/src/main/java/com/yoke/model/UserContext.java similarity index 95% rename from src/main/java/com/yoke/tool/UserContext.java rename to src/main/java/com/yoke/model/UserContext.java index acd9483..bcc8f17 100644 --- a/src/main/java/com/yoke/tool/UserContext.java +++ b/src/main/java/com/yoke/model/UserContext.java @@ -1,4 +1,4 @@ -package com.yoke.tool; +package com.yoke.model; /** * 当前请求的用户上下文，由控制器放进 {@code RuntimeContext}，工具方法里把它声明成不带 @ToolParam 的参数， diff --git a/src/main/java/com/yoke/tool/ChatTools.java b/src/main/java/com/yoke/tool/ChatTools.java index cfcf4e9..e34020d 100644 --- a/src/main/java/com/yoke/tool/ChatTools.java +++ b/src/main/java/com/yoke/tool/ChatTools.java @@ -12,10 +12,10 @@ import java.util.Optional; import cn.hutool.core.util.StrUtil; +import com.yoke.model.UserContext; import io.agentscope.core.tool.Tool; import io.agentscope.core.tool.ToolParam; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; /** * Yoke 自带的几个通用工具。 @@ -27,10 +27,9 @@ *

description 写的是「什么时候该用它」而不是功能罗列：模型靠它决定要不要调用。 * 参数名用 {@code @ToolParam(name = ...)} 显式声明，不依赖编译参数。 */ +@Slf4j public class ChatTools { - private static final Logger log = LoggerFactory.getLogger(ChatTools.class); - private static final DateTimeFormatter TIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss xxx"); /** 笔记工具单次最多返回的条数。 */ diff --git a/src/main/java/com/yoke/tool/RetrieveTool.java b/src/main/java/com/yoke/tool/RetrieveTool.java index a941d2f..ffbeb8e 100644 --- a/src/main/java/com/yoke/tool/RetrieveTool.java +++ b/src/main/java/com/yoke/tool/RetrieveTool.java @@ -5,12 +5,11 @@ import java.util.Optional; import cn.hutool.core.util.StrUtil; -import com.yoke.knowledge.Passage; +import com.yoke.model.Passage; import com.yoke.knowledge.RetrievePort; import io.agentscope.core.tool.Tool; import io.agentscope.core.tool.ToolParam; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Mono; /** @@ -23,10 +22,9 @@ *

返回值带 {@code [1][2]} 编号，模型回答时按编号引用，用户就能回头核对出处。 * 无状态，线程安全；{@code port} 为空表示没有配置知识库，工具返回提示语而不是抛异常。 */ +@Slf4j public class RetrieveTool { - private static final Logger log = LoggerFactory.getLogger(RetrieveTool.class); - static final int DEFAULT_TOP_K = 3; /** 单次最多返回几条：模型传多大都不会超过，避免一次把上下文塞满。 */ diff --git a/src/main/java/com/yoke/config/SharedToolkit.java b/src/main/java/com/yoke/tool/SharedToolkit.java similarity index 81% rename from src/main/java/com/yoke/config/SharedToolkit.java rename to src/main/java/com/yoke/tool/SharedToolkit.java index 99e4fc7..8780bc2 100644 --- a/src/main/java/com/yoke/config/SharedToolkit.java +++ b/src/main/java/com/yoke/tool/SharedToolkit.java @@ -1,4 +1,4 @@ -package com.yoke.config; +package com.yoke.tool; import io.agentscope.core.tool.Toolkit; @@ -14,15 +14,6 @@ *

单独包一层类型，是为了不和官方 starter 注册的原型 {@code Toolkit} Bean 在按类型注入时撞车。 * MCP 连接由 {@code McpConnections} 持有，模板和副本都不拥有连接。 */ -public final class SharedToolkit { +public record SharedToolkit(Toolkit toolkit) { - private final Toolkit toolkit; - - public SharedToolkit(Toolkit toolkit) { - this.toolkit = toolkit; - } - - public Toolkit toolkit() { - return toolkit; - } } diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index d548016..2bb09a5 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -9,8 +9,12 @@ agentscope: yoke: knowledge: embedding: - # 没有设置环境变量时为空，检索只用关键词 - api-key: ${YOKE_EMBEDDING_API_KEY:} + # 火山方舟的多模态向量模型，和上面的对话模型共用同一个 ARK_API_KEY；没有设置环境变量时密钥为空，检索只用关键词 + protocol: ark-multimodal + model-name: doubao-embedding-vision-251215 + base-url: https://ark.cn-beijing.volces.com/api/v3 + api-key: ${ARK_API_KEY:} + dimensions: 2048 redis: # 对应 docker-compose.yml 里的 Redis，本地研发先 docker compose up -d uri: ${YOKE_REDIS_URI:redis://127.0.0.1:6379} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 5d47ef7..bbf3f10 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -38,7 +38,8 @@ yoke: knowledge: # 知识库检索（retrieve 工具）：语料是 classpath:knowledge/*.md。 # embedding 的 model-name 和 api-key 都有值才启用向量检索（与关键词检索融合）；任意一个为空就只用关键词检索， - # 向量化服务出错时也会自动退回关键词。密钥见各环境文件，base-url 可换成其他 OpenAI 兼容服务 + # 向量化服务出错时也会自动退回关键词。密钥见各环境文件，base-url 可换成其他 OpenAI 兼容服务。 + # protocol 默认 openai（OpenAI 兼容的 /embeddings）；火山方舟的多模态向量模型要写 ark-multimodal embedding: model-name: ${YOKE_EMBEDDING_MODEL:} base-url: ${YOKE_EMBEDDING_BASE_URL:https://api.openai.com/v1} diff --git a/src/test/java/com/yoke/ProfilesTests.java b/src/test/java/com/yoke/ProfilesTests.java index 544f1b1..7770917 100644 --- a/src/test/java/com/yoke/ProfilesTests.java +++ b/src/test/java/com/yoke/ProfilesTests.java @@ -49,7 +49,9 @@ private static Object yamlValue(String file, String key) throws IOException { @Test void 向量化密钥在三个环境里的写法一致_都不会让缺失变成启动失败() throws IOException { - assertThat(yamlValue("application-local.yml", "yoke.knowledge.embedding.api-key")).isEqualTo("${YOKE_EMBEDDING_API_KEY:}"); + // local 接火山方舟，向量化和对话模型共用 ARK_API_KEY + assertThat(yamlValue("application-local.yml", "yoke.knowledge.embedding.api-key")).isEqualTo("${ARK_API_KEY:}"); + assertThat(yamlValue("application-local.yml", "yoke.knowledge.embedding.protocol")).isEqualTo("ark-multimodal"); assertThat(yamlValue("application-prod.yml", "yoke.knowledge.embedding.api-key")).isEqualTo("${YOKE_EMBEDDING_API_KEY:}"); // test 环境故意留空，保证测试不访问向量化服务 assertThat(yamlValue("application-test.yml", "yoke.knowledge.embedding.api-key")).isIn("", null); diff --git a/src/test/java/com/yoke/config/AgentConfigurationConflictTests.java b/src/test/java/com/yoke/config/AgentConfigurationConflictTests.java index 158d61d..ea9b5e5 100644 --- a/src/test/java/com/yoke/config/AgentConfigurationConflictTests.java +++ b/src/test/java/com/yoke/config/AgentConfigurationConflictTests.java @@ -4,6 +4,7 @@ import java.nio.file.Path; import java.util.List; +import com.yoke.tool.SharedToolkit; import io.agentscope.core.model.Model; import io.agentscope.core.tool.Toolkit; import io.agentscope.harness.agent.DistributedStore; @@ -27,7 +28,7 @@ private ApplicationContextRunner runner(Path workspace) { .withBean(SharedToolkit.class, () -> new SharedToolkit(new Toolkit())) .withBean(YokeProperties.class, () -> new YokeProperties(new YokeProperties.Redis("redis://127.0.0.1:6379", "yoke:"), new YokeProperties.Compaction(30, 10), workspace.toString(), - new YokeProperties.Knowledge(new YokeProperties.Knowledge.Embedding(null, null, null, null)), + new YokeProperties.Knowledge(new YokeProperties.Knowledge.Embedding(null, null, null, null, null)), new YokeProperties.Mcp(List.of()))) .withUserConfiguration(AgentConfiguration.class); } diff --git a/src/test/java/com/yoke/config/AgentConfigurationTests.java b/src/test/java/com/yoke/config/AgentConfigurationTests.java index 65f16cd..a74ef59 100644 --- a/src/test/java/com/yoke/config/AgentConfigurationTests.java +++ b/src/test/java/com/yoke/config/AgentConfigurationTests.java @@ -2,6 +2,7 @@ import java.util.List; +import com.yoke.tool.SharedToolkit; import io.agentscope.core.middleware.MiddlewareBase; import io.agentscope.core.model.Model; import io.agentscope.core.tool.Toolkit; @@ -42,7 +43,7 @@ AgentscopeProperties agentscopeProperties() { YokeProperties yokeProperties() { return new YokeProperties(new YokeProperties.Redis("redis://127.0.0.1:6379", "yoke:"), new YokeProperties.Compaction(30, 10), "/tmp/yoke-test", - new YokeProperties.Knowledge(new YokeProperties.Knowledge.Embedding(null, null, null, null)), + new YokeProperties.Knowledge(new YokeProperties.Knowledge.Embedding(null, null, null, null, null)), new YokeProperties.Mcp(List.of())); } diff --git a/src/test/java/com/yoke/config/KnowledgeConfigurationTests.java b/src/test/java/com/yoke/config/KnowledgeConfigurationTests.java index a97c40a..10d7fdf 100644 --- a/src/test/java/com/yoke/config/KnowledgeConfigurationTests.java +++ b/src/test/java/com/yoke/config/KnowledgeConfigurationTests.java @@ -3,11 +3,12 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicInteger; import com.sun.net.httpserver.HttpServer; -import com.yoke.knowledge.HybridRetrievePort; -import com.yoke.knowledge.KeywordRetrievePort; -import com.yoke.knowledge.Passage; +import com.yoke.knowledge.retrieve.HybridRetrievePort; +import com.yoke.knowledge.retrieve.KeywordRetrievePort; +import com.yoke.model.Passage; import com.yoke.knowledge.RetrievePort; import org.junit.jupiter.api.Test; @@ -53,6 +54,38 @@ static class Context { .run(context -> assertThat(context.getBean(RetrievePort.class)).isInstanceOf(HybridRetrievePort.class)); } + @Test + void 方舟协议的向量化走多模态接口且向量路真的生效() throws IOException { + // 只在 /embeddings/multimodal 上应答；含「Redis」的片段与问题向量相同，必须被向量路排到前面 + AtomicInteger calls = new AtomicInteger(); + HttpServer ark = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + ark.createContext("/api/v3/embeddings/multimodal", exchange -> { + String request = new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8); + calls.incrementAndGet(); + String vector = request.contains("Redis") || request.contains("历史") ? "[1.0,0.0]" : "[0.0,1.0]"; + byte[] body = ("{\"data\":{\"embedding\":" + vector + "}}").getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + ark.start(); + try { + runner.withPropertyValues("yoke.knowledge.embedding.protocol=ark-multimodal", + "yoke.knowledge.embedding.model-name=doubao-embedding-vision-251215", + "yoke.knowledge.embedding.api-key=ark-test", + "yoke.knowledge.embedding.dimensions=2", + "yoke.knowledge.embedding.base-url=http://127.0.0.1:" + ark.getAddress().getPort() + "/api/v3") + .run(context -> { + RetrievePort port = context.getBean(RetrievePort.class); + assertThat(port).isInstanceOf(HybridRetrievePort.class); + assertThat(port.retrieve("对话历史存在哪里", 3).block()).isNotEmpty(); + assertThat(calls.get()).as("向量路确实请求了方舟的多模态接口").isPositive(); + }); + } finally { + ark.stop(0); + } + } + @Test void 向量化服务不可用时检索自动退回关键词() throws IOException { // 本地起一个永远返回 500 的向量化服务：向量路失败，关键词路必须兜住 diff --git a/src/test/java/com/yoke/config/RedisConfigurationTests.java b/src/test/java/com/yoke/config/RedisConfigurationTests.java index 8824997..3af1b4e 100644 --- a/src/test/java/com/yoke/config/RedisConfigurationTests.java +++ b/src/test/java/com/yoke/config/RedisConfigurationTests.java @@ -11,7 +11,7 @@ class RedisConfigurationTests { private static YokeProperties withUri(String uri) { return new YokeProperties(new YokeProperties.Redis(uri, "yoke:"), new YokeProperties.Compaction(30, 10), "/tmp/yoke", - new YokeProperties.Knowledge(new YokeProperties.Knowledge.Embedding(null, null, null, null)), + new YokeProperties.Knowledge(new YokeProperties.Knowledge.Embedding(null, null, null, null, null)), new YokeProperties.Mcp(java.util.List.of())); } diff --git a/src/test/java/com/yoke/config/SubagentTests.java b/src/test/java/com/yoke/config/SubagentTests.java index d43a7bc..55113b5 100644 --- a/src/test/java/com/yoke/config/SubagentTests.java +++ b/src/test/java/com/yoke/config/SubagentTests.java @@ -28,11 +28,16 @@ */ class SubagentTests { - /** 对外服务的 Agent 任何一层都不该看到的工具：联网、文件读写、命令执行。 */ + /** + * 对外服务的 Agent 任何一层都不该看到的工具：联网、文件读写、命令执行， + * 以及能读到别的用户记忆和会话的读类工具（路径由模型给的参数决定，@SafeId 管不到）。 + * 只保留 memory_save，它只往当前用户固定的记忆文件里追加。 + */ private static final Set NEVER = Set.of( "web_fetch", "web_search", "read_file", "write_file", "edit_file", "list_files", "glob_files", "grep_files", - "execute"); + "execute", + "memory_get", "memory_search", "session_history", "session_list", "session_search"); private static final String PARENT_PROMPT = "你是测试助手"; diff --git a/src/test/java/com/yoke/config/WorkspaceInitializerTests.java b/src/test/java/com/yoke/config/WorkspaceInitializerTests.java index b9473e4..9b487d5 100644 --- a/src/test/java/com/yoke/config/WorkspaceInitializerTests.java +++ b/src/test/java/com/yoke/config/WorkspaceInitializerTests.java @@ -17,7 +17,7 @@ class WorkspaceInitializerTests { private static YokeProperties props(Path workspace) { return new YokeProperties(new YokeProperties.Redis("redis://127.0.0.1:6379", "yoke:"), new YokeProperties.Compaction(30, 10), workspace.toString(), - new YokeProperties.Knowledge(new YokeProperties.Knowledge.Embedding(null, null, null, null)), + new YokeProperties.Knowledge(new YokeProperties.Knowledge.Embedding(null, null, null, null, null)), new YokeProperties.Mcp(List.of())); } diff --git a/src/test/java/com/yoke/config/YokePropertiesTests.java b/src/test/java/com/yoke/config/YokePropertiesTests.java index d2a3332..d14dd9e 100644 --- a/src/test/java/com/yoke/config/YokePropertiesTests.java +++ b/src/test/java/com/yoke/config/YokePropertiesTests.java @@ -107,6 +107,7 @@ private static Map required() { assertThat(properties.mcp().servers()).isEmpty(); assertThat(properties.knowledge().embedding().enabled()).isFalse(); assertThat(properties.knowledge().embedding().baseUrl()).isEqualTo("https://api.openai.com/v1"); + assertThat(properties.knowledge().embedding().protocol()).isEqualTo(YokeProperties.Knowledge.Embedding.Protocol.OPENAI); } @Test @@ -183,11 +184,13 @@ private static Map required() { @Test void 向量化配置的toString不泄漏密钥() { YokeProperties.Knowledge.Embedding embedding = new YokeProperties.Knowledge.Embedding( - "text-embedding-3-small", "https://api.openai.com/v1", "sk-very-secret", 512); + "text-embedding-3-small", "https://api.openai.com/v1", "sk-very-secret", 512, + YokeProperties.Knowledge.Embedding.Protocol.OPENAI); assertThat(embedding.toString()).contains("text-embedding-3-small").doesNotContain("sk-very-secret"); assertThat(embedding.enabled()).isTrue(); - assertThat(new YokeProperties.Knowledge.Embedding("m", "u", " ", null).enabled()).isFalse(); - assertThat(new YokeProperties.Knowledge.Embedding(null, "u", "k", null).enabled()).isFalse(); + assertThat(embedding.toString()).contains("protocol=OPENAI"); + assertThat(new YokeProperties.Knowledge.Embedding("m", "u", " ", null, null).enabled()).isFalse(); + assertThat(new YokeProperties.Knowledge.Embedding(null, "u", "k", null, null).enabled()).isFalse(); } } diff --git a/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java b/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java index 3e98262..1ada2d0 100644 --- a/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java +++ b/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java @@ -2,6 +2,8 @@ import java.util.List; +import com.yoke.knowledge.retrieve.KeywordRetrievePort; +import com.yoke.model.Passage; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; diff --git a/src/test/java/com/yoke/knowledge/embed/ArkEmbeddingClientTests.java b/src/test/java/com/yoke/knowledge/embed/ArkEmbeddingClientTests.java new file mode 100644 index 0000000..b711cc6 --- /dev/null +++ b/src/test/java/com/yoke/knowledge/embed/ArkEmbeddingClientTests.java @@ -0,0 +1,166 @@ +package com.yoke.knowledge.embed; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; + +import com.sun.net.httpserver.HttpServer; +import com.yoke.knowledge.retrieve.VectorRetrievePort; +import com.yoke.model.Passage; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import org.springframework.web.client.RestClient; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** 用 JDK 自带的 HttpServer 模拟火山方舟的 /embeddings/multimodal 接口，不访问外网。 */ +class ArkEmbeddingClientTests { + + private HttpServer server; + private final List requestBodies = new CopyOnWriteArrayList<>(); + private final List authHeaders = new CopyOnWriteArrayList<>(); + private final AtomicInteger inFlight = new AtomicInteger(); + private final AtomicInteger maxInFlight = new AtomicInteger(); + private volatile int status = 200; + private volatile String responseBody; + private volatile boolean computeFromRequest; + + @BeforeEach + void start() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + // 只注册多模态路径：客户端如果误请求标准的 /embeddings，服务端返回 404，测试会红 + server.createContext("/api/v3/embeddings/multimodal", exchange -> { + String request = new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8); + requestBodies.add(request); + authHeaders.add(exchange.getRequestHeaders().getFirst("Authorization")); + maxInFlight.accumulateAndGet(inFlight.incrementAndGet(), Math::max); + try { + Thread.sleep(50); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + byte[] body = (computeFromRequest ? vectorFor(request) : responseBody).getBytes(StandardCharsets.UTF_8); + inFlight.decrementAndGet(); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + server.start(); + } + + @AfterEach + void stop() { + server.stop(0); + } + + private ArkEmbeddingClient client(Integer dimensions) { + RestClient rest = RestClient.builder() + .baseUrl("http://127.0.0.1:" + server.getAddress().getPort() + "/api/v3") + .defaultHeader("Authorization", "Bearer ark-test") + .build(); + return new ArkEmbeddingClient(rest, "doubao-embedding-vision-251215", dimensions); + } + + @Test + void 请求体按多模态协议组装_data是单个对象() { + responseBody = """ + {"data":{"embedding":[0.25,0.75],"object":"embedding"},"model":"doubao-embedding-vision-251215"}"""; + + List vectors = client(2).embed(List.of("你好")).block(); + + assertThat(vectors).hasSize(1); + assertThat(vectors.getFirst()).containsExactly(0.25, 0.75); + assertThat(requestBodies.getFirst()).contains("\"model\":\"doubao-embedding-vision-251215\"") + .contains("\"input\":[{").contains("\"type\":\"text\"").contains("\"text\":\"你好\"") + .contains("\"dimensions\":2"); + assertThat(authHeaders.getFirst()).isEqualTo("Bearer ark-test"); + } + + @Test + void 兼容data是数组的响应() { + responseBody = """ + {"data":[{"index":0,"embedding":[1.0,0.0]}]}"""; + + assertThat(client(null).embed(List.of("x")).block().getFirst()).containsExactly(1.0, 0.0); + } + + @Test + void 不指定维度时请求里不带dimensions() { + responseBody = """ + {"data":{"embedding":[1.0]}}"""; + + client(null).embed(List.of("x")).block(); + + assertThat(requestBodies.getFirst()).doesNotContain("dimensions"); + } + + @Test + void 不支持批量_多条文本拆成逐条请求且结果顺序与输入一致() { + // 含「甲」的是 [1,0]，其余是 [0,1]；每条请求的 input 里只有一条文本 + computeFromRequest = true; + + List vectors = client(null).embed(List.of("乙1", "甲2", "乙3", "甲4", "乙5")).block(); + + assertThat(requestBodies).hasSize(5).allSatisfy(body -> assertThat(body).containsOnlyOnce("\"type\":\"text\"")); + assertThat(vectors).extracting(vector -> vector[0]).containsExactly(0.0, 1.0, 0.0, 1.0, 0.0); + } + + @Test + void 并发请求数不超过上限() { + computeFromRequest = true; + + client(null).embed(List.of("a", "b", "c", "d", "e", "f", "g", "h", "i", "j")).block(); + + assertThat(maxInFlight.get()).isLessThanOrEqualTo(ArkEmbeddingClient.CONCURRENCY); + } + + @Test + void 返回的维度与配置不一致时报错() { + responseBody = """ + {"data":{"embedding":[1.0,2.0,3.0]}}"""; + + assertThatThrownBy(() -> client(2).embed(List.of("x")).block()).hasMessageContaining("不一致"); + } + + @Test + void 响应里没有embedding时报错() { + responseBody = """ + {"data":{}}"""; + + assertThatThrownBy(() -> client(null).embed(List.of("x")).block()).hasMessageContaining("没有 embedding"); + } + + @Test + void 上游返回错误状态码时报错且不泄漏密钥() { + status = 429; + responseBody = "{}"; + + assertThatThrownBy(() -> client(null).embed(List.of("a")).block()) + .isInstanceOf(RuntimeException.class) + .satisfies(error -> assertThat(String.valueOf(error.getMessage())).doesNotContain("ark-test")); + } + + @Test + void 向量检索端到端_经过真实HTTP调用的向量化() { + computeFromRequest = true; + List passages = List.of(new Passage("a", "A", "甲方"), new Passage("b", "B", "乙方")); + VectorRetrievePort port = new VectorRetrievePort(passages, client(null)); + + assertThat(port.retrieve("甲", 1).block()).extracting(Passage::id).containsExactly("a"); + assertThat(port.retrieve("乙", 1).block()).extracting(Passage::id).containsExactly("b"); + // 建索引 2 条片段 + 两次查询各 1 条，不支持批量所以是 4 次请求 + assertThat(requestBodies).hasSize(4); + } + + private static String vectorFor(String requestBody) { + String vector = requestBody.contains("甲") ? "[1.0,0.0]" : "[0.0,1.0]"; + return "{\"data\":{\"embedding\":" + vector + "}}"; + } +} diff --git a/src/test/java/com/yoke/knowledge/OpenAiEmbeddingClientTests.java b/src/test/java/com/yoke/knowledge/embed/OpenAiEmbeddingClientTests.java similarity index 97% rename from src/test/java/com/yoke/knowledge/OpenAiEmbeddingClientTests.java rename to src/test/java/com/yoke/knowledge/embed/OpenAiEmbeddingClientTests.java index 6adfcb4..bc59f46 100644 --- a/src/test/java/com/yoke/knowledge/OpenAiEmbeddingClientTests.java +++ b/src/test/java/com/yoke/knowledge/embed/OpenAiEmbeddingClientTests.java @@ -1,4 +1,4 @@ -package com.yoke.knowledge; +package com.yoke.knowledge.embed; import java.io.IOException; import java.net.InetSocketAddress; @@ -7,6 +7,8 @@ import java.util.concurrent.CopyOnWriteArrayList; import com.sun.net.httpserver.HttpServer; +import com.yoke.knowledge.retrieve.VectorRetrievePort; +import com.yoke.model.Passage; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/src/test/java/com/yoke/knowledge/HybridRetrievePortTests.java b/src/test/java/com/yoke/knowledge/retrieve/HybridRetrievePortTests.java similarity index 98% rename from src/test/java/com/yoke/knowledge/HybridRetrievePortTests.java rename to src/test/java/com/yoke/knowledge/retrieve/HybridRetrievePortTests.java index 3912a0a..7ef9a4a 100644 --- a/src/test/java/com/yoke/knowledge/HybridRetrievePortTests.java +++ b/src/test/java/com/yoke/knowledge/retrieve/HybridRetrievePortTests.java @@ -1,8 +1,10 @@ -package com.yoke.knowledge; +package com.yoke.knowledge.retrieve; +import com.yoke.knowledge.RetrievePort; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import com.yoke.model.Passage; import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; diff --git a/src/test/java/com/yoke/knowledge/KeywordRetrievePortTests.java b/src/test/java/com/yoke/knowledge/retrieve/KeywordRetrievePortTests.java similarity index 96% rename from src/test/java/com/yoke/knowledge/KeywordRetrievePortTests.java rename to src/test/java/com/yoke/knowledge/retrieve/KeywordRetrievePortTests.java index cdc8612..cffd5dd 100644 --- a/src/test/java/com/yoke/knowledge/KeywordRetrievePortTests.java +++ b/src/test/java/com/yoke/knowledge/retrieve/KeywordRetrievePortTests.java @@ -1,7 +1,8 @@ -package com.yoke.knowledge; +package com.yoke.knowledge.retrieve; import java.util.List; +import com.yoke.model.Passage; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; diff --git a/src/test/java/com/yoke/knowledge/VectorRetrievePortTests.java b/src/test/java/com/yoke/knowledge/retrieve/VectorRetrievePortTests.java similarity index 97% rename from src/test/java/com/yoke/knowledge/VectorRetrievePortTests.java rename to src/test/java/com/yoke/knowledge/retrieve/VectorRetrievePortTests.java index 9602e4f..977d784 100644 --- a/src/test/java/com/yoke/knowledge/VectorRetrievePortTests.java +++ b/src/test/java/com/yoke/knowledge/retrieve/VectorRetrievePortTests.java @@ -1,9 +1,11 @@ -package com.yoke.knowledge; +package com.yoke.knowledge.retrieve; +import com.yoke.knowledge.EmbeddingClient; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import com.yoke.model.Passage; import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; diff --git a/src/test/java/com/yoke/mcp/McpStartupTests.java b/src/test/java/com/yoke/mcp/McpStartupTests.java index bd4d2bd..f91e96c 100644 --- a/src/test/java/com/yoke/mcp/McpStartupTests.java +++ b/src/test/java/com/yoke/mcp/McpStartupTests.java @@ -2,7 +2,7 @@ import com.yoke.config.KnowledgeConfiguration; import com.yoke.config.ToolConfiguration; -import com.yoke.config.SharedToolkit; +import com.yoke.tool.SharedToolkit; import com.yoke.config.YokeProperties; import com.yoke.support.McpTestServer; import com.yoke.tool.WeatherTool; diff --git a/src/test/java/com/yoke/tool/ChatToolsTests.java b/src/test/java/com/yoke/tool/ChatToolsTests.java index 719a056..8674d19 100644 --- a/src/test/java/com/yoke/tool/ChatToolsTests.java +++ b/src/test/java/com/yoke/tool/ChatToolsTests.java @@ -11,6 +11,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yoke.model.UserContext; import io.agentscope.core.agent.RuntimeContext; import io.agentscope.core.message.ToolResultBlock; import io.agentscope.core.message.TextBlock; diff --git a/src/test/java/com/yoke/tool/RetrieveToolTests.java b/src/test/java/com/yoke/tool/RetrieveToolTests.java index 707b488..733fc6f 100644 --- a/src/test/java/com/yoke/tool/RetrieveToolTests.java +++ b/src/test/java/com/yoke/tool/RetrieveToolTests.java @@ -3,7 +3,7 @@ import java.util.ArrayList; import java.util.List; -import com.yoke.knowledge.Passage; +import com.yoke.model.Passage; import com.yoke.knowledge.RetrievePort; import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; diff --git a/src/test/java/com/yoke/tool/ToolConversationTests.java b/src/test/java/com/yoke/tool/ToolConversationTests.java index 42fd08e..0722f2d 100644 --- a/src/test/java/com/yoke/tool/ToolConversationTests.java +++ b/src/test/java/com/yoke/tool/ToolConversationTests.java @@ -6,6 +6,7 @@ import java.util.Map; import com.yoke.controller.ChatController; +import com.yoke.knowledge.retrieve.KeywordRetrievePort; import com.yoke.model.ChatRequest; import com.yoke.model.ChatResponse; import com.yoke.support.AguiTestSupport; @@ -102,7 +103,7 @@ private static ChatController controller(ScriptedModel model, Path workspace, To @Test void 带编号的检索结果经过完整一轮对话回到模型手里(@TempDir Path workspace) { - var port = new com.yoke.knowledge.KeywordRetrievePort( + var port = new KeywordRetrievePort( com.yoke.knowledge.KnowledgeCorpus.loadFromClasspath("classpath*:knowledge/*.md")); Toolkit toolkit = new Toolkit(); toolkit.registerTool(new RetrieveTool(port)); From 21ed58b03265c2bde2a128b75c305842749d9dbf Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 12 Nov 2026 18:58:06 +0800 Subject: [PATCH 07/19] feat: cite retrieved passages by passage id instead of per-call numbers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit retrieve 返回的每条资料改用片段自身的标识（形如 [api-usage.md#2]）做引用编号， 不再每次检索都从 [1] 开始编号。同一轮对话里模型换个说法再查一次时，两次结果里的 [1] 会指向不同片段，引用就对不上了。工具说明、kb-researcher 提示词、知识文档、ChatApi 和两份 README 同步更新。 --- README.md | 4 +-- README_CN.md | 4 +-- src/main/java/com/yoke/api/ChatApi.java | 4 +-- src/main/java/com/yoke/model/Passage.java | 3 +- src/main/java/com/yoke/tool/RetrieveTool.java | 14 ++++---- .../knowledge/tools-and-permissions.md | 2 +- .../workspace/subagents/kb-researcher.md | 4 +-- .../java/com/yoke/ToolsEndToEndTests.java | 4 +-- .../java/com/yoke/tool/RetrieveToolTests.java | 33 ++++++++++++++++--- .../com/yoke/tool/ToolConversationTests.java | 4 +-- 10 files changed, 51 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index fbdffbb..5ff02a8 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Yoke is an open-source project of 友客AI (YouKe AI). We read the AgentScope Ja - One endpoint for multi-turn conversations: `POST /api/chat`, plus `POST /api/chat/stream` that streams the reasoning process over SSE using the AG-UI protocol. - A built-in `get_weather` tool looks up current weather through the open-source [Open-Meteo](https://open-meteo.com) API, no key needed (the free tier is for non-commercial use only). - Every request builds a temporary `HarnessAgent` and closes it afterwards. Conversation history lives in Redis, so it survives restarts and is shared by all replicas, isolated per `(userId, sessionId)`, and compacted automatically when a conversation gets long. -- Tools, knowledge and sub-agents: calculator, clock and per-user notes tools, a `retrieve` tool over a built-in knowledge base that returns `[1][2]`-numbered citations, external MCP servers over HTTP, and two sub-agents (one declared in code, one declared as a workspace Markdown file). The agent still has no file, shell or web tools, and a logging middleware records a timing trace for every call. +- Tools, knowledge and sub-agents: calculator, clock and per-user notes tools, a `retrieve` tool over a built-in knowledge base that returns citations keyed by stable passage ids such as `[api-usage.md#2]`, external MCP servers over HTTP, and two sub-agents (one declared in code, one declared as a workspace Markdown file). The agent still has no file, shell or web tools, and a logging middleware records a timing trace for every call. - Models are plugged in through the official AgentScope starters: OpenAI-compatible APIs (works with DashScope, DeepSeek and others), Anthropic, Gemini and Ollama. Switching is a dependency and configuration change. - `local` / `test` / `prod` profiles. In production, a missing API key fails the startup instead of shipping with an empty key. - Virtual threads are enabled by default, for both Tomcat and Spring's default thread pool. @@ -173,7 +173,7 @@ The agent is built without file, shell or web tools. Reading the 2.0.4 source sh ## Tools, Knowledge and Sub-agents - **Tools** (`com.yoke.tool`): plain classes with `@Tool` methods, registered once into a shared `Toolkit`. `calculate` uses a hand-written recursive-descent parser (no script engine, input length and nesting limits, division by zero reported as text). `list_my_notes` takes a `UserContext` parameter without `@ToolParam`: the framework injects it by type from the request's `RuntimeContext`, so the model never sees or forges the user id. -- **Retrieval**: the `retrieve` tool searches `src/main/resources/knowledge/*.md` and returns passages numbered `[1]`, `[2]` with their source, which the model is asked to cite. It is the only retrieval channel; the framework's `knowledge()` auto-injection is deliberately not used. Retrieval is layered behind the application's own `RetrievePort` (no framework types): BM25 keyword search is always available, and with an embedding service configured a vector channel is added and the two are fused with RRF (constant 60); if one channel fails the other is used. Note that with embeddings enabled, user queries and the knowledge-base text are sent to the embedding provider. The framework's own `rag-simple` module is not used: the `core/rag` package is `@Deprecated(forRemoval = true)` in 2.0.4 and the module drags in Qdrant, Milvus, Elasticsearch, PostgreSQL, PDFBox, POI, Tika and more. +- **Retrieval**: the `retrieve` tool searches `src/main/resources/knowledge/*.md` and returns passages labelled with a stable id such as `[api-usage.md#2]` plus their source, which the model is asked to cite verbatim (ids stay unique across repeated retrievals in one conversation, unlike per-call `[1]`, `[2]` counters). It is the only retrieval channel; the framework's `knowledge()` auto-injection is deliberately not used. Retrieval is layered behind the application's own `RetrievePort` (no framework types): BM25 keyword search is always available, and with an embedding service configured a vector channel is added and the two are fused with RRF (constant 60); if one channel fails the other is used. Note that with embeddings enabled, user queries and the knowledge-base text are sent to the embedding provider. The framework's own `rag-simple` module is not used: the `core/rag` package is `@Deprecated(forRemoval = true)` in 2.0.4 and the module drags in Qdrant, Milvus, Elasticsearch, PostgreSQL, PDFBox, POI, Tika and more. - **MCP**: only HTTP transports are accepted. Example: ```yaml diff --git a/README_CN.md b/README_CN.md index 009f8dd..34945ec 100644 --- a/README_CN.md +++ b/README_CN.md @@ -29,7 +29,7 @@ Yoke 是友客AI的开源项目。大家一起读 AgentScope Java V2 和 Spring - 一个接口完成多轮对话：`POST /api/chat`；另有 `POST /api/chat/stream`，按 AG-UI 协议通过 SSE 边推理边推送。 - 内置 `get_weather` 工具，通过开源的 [Open-Meteo](https://open-meteo.com) 查当前天气，免密钥（免费版仅限非商业使用）。 - 每次请求新建一个临时 `HarnessAgent`，用完即关闭。对话历史存在 Redis 里，重启不丢，多个副本共享，按 `(userId, sessionId)` 隔离，对话过长时自动压缩。 -- 工具、知识库与子 Agent：计算器、时钟、按用户隔离的笔记工具，基于内置知识库的 `retrieve` 工具（返回带 `[1][2]` 编号的引用），通过 HTTP 接入外部 MCP 服务，以及两个子 Agent（一个在代码里声明，一个由工作区 Markdown 文件声明）。Agent 仍不开放文件、命令和联网工具，日志中间件为每次调用记录耗时轨迹。 +- 工具、知识库与子 Agent：计算器、时钟、按用户隔离的笔记工具，基于内置知识库的 `retrieve` 工具（返回带稳定片段编号的引用，形如 `[api-usage.md#2]`），通过 HTTP 接入外部 MCP 服务，以及两个子 Agent（一个在代码里声明，一个由工作区 Markdown 文件声明）。Agent 仍不开放文件、命令和联网工具，日志中间件为每次调用记录耗时轨迹。 - 通过 AgentScope 官方 starter 接入各种模型：OpenAI 兼容协议（可对接 DashScope、DeepSeek 等）、Anthropic、Gemini、Ollama，换依赖和配置即可。 - 有 local / test / prod 多环境配置；生产环境缺少密钥时启动即失败，避免带着空密钥上线。 - 默认开启虚拟线程，Tomcat 和 Spring 默认线程池都使用虚拟线程。 @@ -173,7 +173,7 @@ Agent 创建时不开放文件、命令和联网工具。读 2.0.4 源码发现 ## 工具、知识库与子 Agent - **工具**（`com.yoke.tool`）：带 `@Tool` 方法的普通类，启动时一次性注册进共享的 `Toolkit`。`calculate` 用手写的递归下降解析器（不用脚本引擎，输入长度和嵌套深度有上限，除零以文本形式报告）。`list_my_notes` 的 `UserContext` 参数不带 `@ToolParam`：框架按类型从本次请求的 `RuntimeContext` 里注入，模型既看不到也伪造不了用户标识。 -- **检索**：`retrieve` 工具检索 `src/main/resources/knowledge/*.md`，返回带 `[1]`、`[2]` 编号和来源的片段，并要求模型据此引用。它是唯一的检索通道，有意不同时使用框架的 `knowledge()` 自动注入。检索分层放在应用自己的 `RetrievePort` 后面（不含框架类型）：BM25 关键词检索永远可用；配置了向量化服务就再加一路向量检索，两路用 RRF（常数 60）融合，任何一路失败就用另一路。注意开启向量化后，用户的检索问题和知识库语料会发送给 embedding 服务提供方。没有使用框架自带的 `rag-simple`：`core/rag` 包在 2.0.4 里已 `@Deprecated(forRemoval = true)`，而且该模块会拖进 Qdrant、Milvus、Elasticsearch、PostgreSQL、PDFBox、POI、Tika 等一大批依赖。 +- **检索**：`retrieve` 工具检索 `src/main/resources/knowledge/*.md`，返回带稳定编号（形如 `[api-usage.md#2]`，文档名加序号）和来源的片段，并要求模型原样照抄编号引用；同一轮对话里多次检索编号也不会重复，不像每次从 `[1]` 开始的序号会冲突。它是唯一的检索通道，有意不同时使用框架的 `knowledge()` 自动注入。检索分层放在应用自己的 `RetrievePort` 后面（不含框架类型）：BM25 关键词检索永远可用；配置了向量化服务就再加一路向量检索，两路用 RRF（常数 60）融合，任何一路失败就用另一路。注意开启向量化后，用户的检索问题和知识库语料会发送给 embedding 服务提供方。没有使用框架自带的 `rag-simple`：`core/rag` 包在 2.0.4 里已 `@Deprecated(forRemoval = true)`，而且该模块会拖进 Qdrant、Milvus、Elasticsearch、PostgreSQL、PDFBox、POI、Tika 等一大批依赖。 - **MCP**：只接受 HTTP 传输。示例： ```yaml diff --git a/src/main/java/com/yoke/api/ChatApi.java b/src/main/java/com/yoke/api/ChatApi.java index aa19683..01a69c2 100644 --- a/src/main/java/com/yoke/api/ChatApi.java +++ b/src/main/java/com/yoke/api/ChatApi.java @@ -59,8 +59,8 @@ *

业务上下文不经过模型：{@code list_my_notes} 要知道「当前用户是谁」，userId 由控制器放进 RuntimeContext， * 工具按类型取用，不是模型传的参数，模型看不到也伪造不了。这个 userId 本身仍是调用方自报的（见上面的实现说明）。 * - *

引用格式：{@code retrieve} 返回的资料带 {@code [1]}、{@code [2]} 这样的编号和来源文档标题， - * 模型回答时按相同编号标注出处，调用方可以据此回溯。检索是工具化的，只有模型认为需要时才会调用， + *

引用格式：{@code retrieve} 返回的资料带 {@code [api-usage.md#2]} 这样的编号和来源文档标题，编号是片段的稳定标识 + * （文档名加序号），同一轮对话里多次检索也不会重复；模型回答时原样照抄编号标注出处，调用方可以据此回溯。检索是工具化的，只有模型认为需要时才会调用， * 没有同时开启框架的 knowledge 自动注入。没有配置向量化服务时只用关键词检索，向量化服务出错时自动退回关键词检索。 * *

子 Agent：{@code summarizer}（代码里声明，只做摘要，没有任何业务工具）、{@code kb-researcher} diff --git a/src/main/java/com/yoke/model/Passage.java b/src/main/java/com/yoke/model/Passage.java index 30bee87..95d9c2a 100644 --- a/src/main/java/com/yoke/model/Passage.java +++ b/src/main/java/com/yoke/model/Passage.java @@ -3,7 +3,8 @@ /** * 知识库里的一个片段，也是检索的返回单位。 * - * @param id 片段的稳定标识（文档名加序号），多路检索融合时靠它判断是不是同一个片段 + * @param id 片段的稳定标识（文档名加序号，如 api-usage.md#2）：多路检索融合时靠它判断是不是同一个片段， + * {@code retrieve} 也把它当作引用编号返回给模型，所以它在整个对话里必须固定指向同一个片段 * @param source 片段所属文档的标题，引用时展示给用户 * @param text 片段正文 */ diff --git a/src/main/java/com/yoke/tool/RetrieveTool.java b/src/main/java/com/yoke/tool/RetrieveTool.java index ffbeb8e..dc854c8 100644 --- a/src/main/java/com/yoke/tool/RetrieveTool.java +++ b/src/main/java/com/yoke/tool/RetrieveTool.java @@ -19,7 +19,9 @@ * 两条通道同时开，同一个问题会被检索两次，两份资料挤进上下文还可能互相矛盾，引用编号也对不上； * 而且 AgentScope 里框架的 rag 包已标记为将移除。工具化的另一个好处是闲聊不触发检索，省 token。 * - *

返回值带 {@code [1][2]} 编号，模型回答时按编号引用，用户就能回头核对出处。 + *

返回的每条资料前带方括号编号，模型回答时原样照抄编号来引用，用户就能回头核对出处。编号用片段自己的稳定标识 + * （文档名加序号，形如 {@code [api-usage.md#2]}），而不是每次检索都从 1 开始的序号：同一轮对话里模型常常换个说法再查一次， + * 两次结果里的 {@code [1]} 会指向不同的片段，引用就对不上了。稳定标识在整个对话里固定指向同一个片段。 * 无状态，线程安全；{@code port} 为空表示没有配置知识库，工具返回提示语而不是抛异常。 */ @Slf4j @@ -40,7 +42,7 @@ public RetrieveTool(RetrievePort port) { @Tool(name = "retrieve", description = "当问题涉及 Yoke 自身的设计、配置、接口或使用方法，而你不确定答案时调用，先查资料再回答；" - + "用户明确要求查知识库或资料时必须调用，不能没查就说知识库里没有；闲聊、通用常识和计算不要调用。返回带 [1][2] 编号的资料片段，回答时请用相同的编号标注引用来源。" + + "用户明确要求查知识库或资料时必须调用，不能没查就说知识库里没有；闲聊、通用常识和计算不要调用。返回的每条资料前有方括号编号（形如 [api-usage.md#2]），回答时请原样照抄编号标注引用来源。" + "查不到时如实告诉用户，不要编造。") public Mono retrieve( @ToolParam(name = "query", description = "要查的问题或关键词，用完整的短句，例如「会话历史存在哪里」") String query, @@ -66,10 +68,10 @@ static String format(List passages) { if (passages.isEmpty()) { return "没有检索到相关资料。请如实告诉用户知识库里没有查到，不要编造。"; } - StringBuilder out = new StringBuilder("检索到 ").append(passages.size()).append(" 条相关资料，回答时用 [编号] 标注引用："); - for (int i = 0; i < passages.size(); i++) { - Passage passage = passages.get(i); - out.append("\n\n[").append(i + 1).append("] 来源：").append(passage.source()).append('\n').append(passage.text()); + StringBuilder out = new StringBuilder("检索到 ").append(passages.size()) + .append(" 条相关资料，回答时原样照抄方括号里的编号标注引用（编号在整个对话里固定指向同一个片段，多次检索也不会重复）："); + for (Passage passage : passages) { + out.append("\n\n[").append(passage.id()).append("] 来源：").append(passage.source()).append('\n').append(passage.text()); } return out.toString(); } diff --git a/src/main/resources/knowledge/tools-and-permissions.md b/src/main/resources/knowledge/tools-and-permissions.md index fb9c1aa..5780ce5 100644 --- a/src/main/resources/knowledge/tools-and-permissions.md +++ b/src/main/resources/knowledge/tools-and-permissions.md @@ -10,7 +10,7 @@ Yoke 自带的工具有 get_current_time、calculate、list_my_notes、get_weath list_my_notes 需要知道当前用户是谁，但 userId 不是模型传的参数：它由控制器放进 RuntimeContext，工具方法里不带 @ToolParam 的 UserContext 参数由框架按类型注入。模型既看不到这个参数，也没法伪造。 ## 检索如何降级 -retrieve 工具背后是检索端口。配置了 embedding 模型时，向量检索和关键词检索并行，用 RRF 融合排名；没有配置或向量检索失败时，自动退回纯关键词检索。返回的资料带 [1][2] 编号，回答时应按编号标注引用。 +retrieve 工具背后是检索端口。配置了 embedding 模型时，向量检索和关键词检索并行，用 RRF 融合排名；没有配置或向量检索失败时，自动退回纯关键词检索。返回的每条资料前带方括号编号（形如 [api-usage.md#2]，是片段的稳定标识，多次检索也不会重复），回答时应原样照抄编号标注引用。 ## 接入 MCP 服务 通过 yoke.mcp.servers 可以接入外部 MCP 服务，传输方式只允许 sse 和 streamable-http 两种 HTTP 传输。配置 stdio 会在启动时直接失败，因为 stdio 会让服务端按配置去启动本地进程。 diff --git a/src/main/resources/workspace/subagents/kb-researcher.md b/src/main/resources/workspace/subagents/kb-researcher.md index 63190d1..32face0 100644 --- a/src/main/resources/workspace/subagents/kb-researcher.md +++ b/src/main/resources/workspace/subagents/kb-researcher.md @@ -1,5 +1,5 @@ --- -description: 专门查证 Yoke 自身设计、配置和接口细节的研究员。需要核对「Yoke 怎么做某件事」又不想占用主对话的上下文时，把问题委派给它；它只会查知识库，并返回带 [编号] 引用的结论。 +description: 专门查证 Yoke 自身设计、配置和接口细节的研究员。需要核对「Yoke 怎么做某件事」又不想占用主对话的上下文时，把问题委派给它；它只会查知识库，并返回带方括号编号引用的结论。 maxIters: 4 tools: [retrieve] --- @@ -9,4 +9,4 @@ tools: [retrieve] 工作方式： 1. 先用 retrieve 工具查资料，必要时换个说法再查一次，最多查两次。 2. 只根据检索到的资料作答，资料里没有的内容就明确说「知识库里没有查到」，不要凭印象补充。 -3. 回答要简短，每个结论后面用 [1]、[2] 这样的编号标注出处，编号与检索结果里的编号一致。 +3. 回答要简短，每个结论后面用方括号编号标注出处（形如 [api-usage.md#2]），编号必须原样照抄检索结果里的，不要自己改成 [1]、[2]。多次检索的结果里编号不会重复，直接引用即可。 diff --git a/src/test/java/com/yoke/ToolsEndToEndTests.java b/src/test/java/com/yoke/ToolsEndToEndTests.java index 4b3a462..fb11ab6 100644 --- a/src/test/java/com/yoke/ToolsEndToEndTests.java +++ b/src/test/java/com/yoke/ToolsEndToEndTests.java @@ -81,7 +81,7 @@ private List> stream(String threadId, String text) { } @Test - void 检索工具在真实装配下连着知识库_返回带编号的资料() { + void 检索工具在真实装配下连着知识库_返回带稳定编号的资料() { List> events = stream("e2e-r-" + System.nanoTime(), "对话历史存在哪里"); Map call = AguiTestSupport.first(events, "TOOL_CALL_START"); @@ -90,7 +90,7 @@ private List> stream(String threadId, String text) { assertThat(AguiTestSupport.toolArguments(events)).contains("对话历史存在哪里").contains("\"top_k\":2"); assertThat(result.get("toolCallId")).isEqualTo(call.get("toolCallId")); assertThat(events.indexOf(call)).as("工具调用先于工具结果").isLessThan(events.indexOf(result)); - assertThat(String.valueOf(result.get("content"))).contains("[1] 来源：").contains("Redis").doesNotContain("尚未配置"); + assertThat(String.valueOf(result.get("content"))).containsPattern("\\[[\\w-]+\\.md#\\d+\\] 来源：").contains("Redis").doesNotContain("尚未配置"); assertThat(events.getLast().get("type")).isEqualTo("RUN_FINISHED"); } diff --git a/src/test/java/com/yoke/tool/RetrieveToolTests.java b/src/test/java/com/yoke/tool/RetrieveToolTests.java index 733fc6f..c802ded 100644 --- a/src/test/java/com/yoke/tool/RetrieveToolTests.java +++ b/src/test/java/com/yoke/tool/RetrieveToolTests.java @@ -13,8 +13,8 @@ class RetrieveToolTests { private static final List HITS = List.of( - new Passage("a", "会话与存储", "会话与存储 / 对话历史\n历史放在 Redis"), - new Passage("b", "工具与权限", "工具与权限 / 模型能用哪些工具\n有 calculate")); + new Passage("sessions.md#0", "会话与存储", "会话与存储 / 对话历史\n历史放在 Redis"), + new Passage("tools.md#3", "工具与权限", "工具与权限 / 模型能用哪些工具\n有 calculate")); @Test void 工具说明要求用户明确让查知识库时必须调用_不能没查就说没有() throws NoSuchMethodException { @@ -31,9 +31,32 @@ class RetrieveToolTests { String result = tool.retrieve("历史", 3).block(); assertThat(result).contains("检索到 2 条相关资料") - .contains("[1] 来源：会话与存储").contains("历史放在 Redis") - .contains("[2] 来源：工具与权限").contains("有 calculate"); - assertThat(result.indexOf("[1]")).isLessThan(result.indexOf("[2]")); + .contains("[sessions.md#0] 来源：会话与存储").contains("历史放在 Redis") + .contains("[tools.md#3] 来源：工具与权限").contains("有 calculate"); + assertThat(result.indexOf("[sessions.md#0]")).isLessThan(result.indexOf("[tools.md#3]")); + } + + @Test + void 多次检索的编号不冲突_同一个片段始终是同一个编号() { + // 模型常常换个说法再查一次。每次从 [1] 开始编号的话，两次结果里的 [1] 指向不同的片段，引用就对不上 + Passage shared = new Passage("api.md#2", "接口使用", "接口使用 / 普通对话\n发送 JSON"); + Passage onlyFirst = new Passage("mcp.md#1", "工具与权限", "工具与权限 / 接入 MCP\n只允许 HTTP"); + Passage onlySecond = new Passage("redis.md#0", "会话与存储", "会话与存储 / Redis\n历史放在 Redis"); + List> answers = List.of(List.of(onlyFirst, shared), List.of(onlySecond, shared)); + java.util.concurrent.atomic.AtomicInteger call = new java.util.concurrent.atomic.AtomicInteger(); + RetrieveTool tool = new RetrieveTool((query, topK) -> Mono.just(answers.get(call.getAndIncrement()))); + + String first = tool.retrieve("q1", 3).block(); + String second = tool.retrieve("q2", 3).block(); + + assertThat(first).contains("[mcp.md#1] 来源：").contains("[api.md#2] 来源："); + assertThat(second).contains("[redis.md#0] 来源：").contains("[api.md#2] 来源："); + // 不再出现每次都从 1 开始的序号 + assertThat(first).doesNotContain("[1]").doesNotContain("[2]"); + assertThat(second).doesNotContain("[1]").doesNotContain("[2]"); + // 同一个编号在两次结果里对应同一段正文，不同片段的编号互不相同 + assertThat(first).contains("[api.md#2] 来源：接口使用\n接口使用 / 普通对话"); + assertThat(second).contains("[api.md#2] 来源：接口使用\n接口使用 / 普通对话"); } @Test diff --git a/src/test/java/com/yoke/tool/ToolConversationTests.java b/src/test/java/com/yoke/tool/ToolConversationTests.java index 0722f2d..66f439a 100644 --- a/src/test/java/com/yoke/tool/ToolConversationTests.java +++ b/src/test/java/com/yoke/tool/ToolConversationTests.java @@ -102,7 +102,7 @@ private static ChatController controller(ScriptedModel model, Path workspace, To } @Test - void 带编号的检索结果经过完整一轮对话回到模型手里(@TempDir Path workspace) { + void 带稳定编号的检索结果经过完整一轮对话回到模型手里(@TempDir Path workspace) { var port = new KeywordRetrievePort( com.yoke.knowledge.KnowledgeCorpus.loadFromClasspath("classpath*:knowledge/*.md")); Toolkit toolkit = new Toolkit(); @@ -113,7 +113,7 @@ private static ChatController controller(ScriptedModel model, Path workspace, To List> events = stream(controller, "s-r", "历史存哪"); String output = resultOf(events); - assertThat(output).contains("[1] 来源：").contains("[2] 来源：").contains("Redis").doesNotContain("[3]"); + assertThat(output).containsPattern("\\[[\\w-]+\\.md#\\d+\\] 来源：").contains("Redis").doesNotContain("[1]").doesNotContain("[2]"); } @Test From 14b2c8febd7e7c35ac879cbdf41189d5165924cc Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 12 Nov 2026 19:00:14 +0800 Subject: [PATCH 08/19] fix: derive passage ids from passage text instead of position MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 片段 id 原来是「文档名#片段序号」，现在它是写进 Redis 对话历史的引用编号，跨重启、 跨部署都还在。文档前面插一节再部署，后面所有片段的 id 就换成了别的正文，旧引用会指错。 改成「文档名#正文 SHA-256 前 8 位」：同一个 id 一定是同一段正文，正文改了 id 跟着变， 旧引用只会找不到，不会指到别处。补了插入小节、修改正文两个测试。 --- README.md | 2 +- README_CN.md | 2 +- src/main/java/com/yoke/api/ChatApi.java | 2 +- .../com/yoke/knowledge/KnowledgeCorpus.java | 30 +++++++++++++++++-- src/main/java/com/yoke/model/Passage.java | 4 +-- src/main/java/com/yoke/tool/RetrieveTool.java | 2 +- .../java/com/yoke/ToolsEndToEndTests.java | 2 +- .../yoke/knowledge/KnowledgeCorpusTests.java | 22 ++++++++++++++ .../com/yoke/tool/ToolConversationTests.java | 2 +- 9 files changed, 57 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 5ff02a8..44518fb 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ The agent is built without file, shell or web tools. Reading the 2.0.4 source sh ## Tools, Knowledge and Sub-agents - **Tools** (`com.yoke.tool`): plain classes with `@Tool` methods, registered once into a shared `Toolkit`. `calculate` uses a hand-written recursive-descent parser (no script engine, input length and nesting limits, division by zero reported as text). `list_my_notes` takes a `UserContext` parameter without `@ToolParam`: the framework injects it by type from the request's `RuntimeContext`, so the model never sees or forges the user id. -- **Retrieval**: the `retrieve` tool searches `src/main/resources/knowledge/*.md` and returns passages labelled with a stable id such as `[api-usage.md#2]` plus their source, which the model is asked to cite verbatim (ids stay unique across repeated retrievals in one conversation, unlike per-call `[1]`, `[2]` counters). It is the only retrieval channel; the framework's `knowledge()` auto-injection is deliberately not used. Retrieval is layered behind the application's own `RetrievePort` (no framework types): BM25 keyword search is always available, and with an embedding service configured a vector channel is added and the two are fused with RRF (constant 60); if one channel fails the other is used. Note that with embeddings enabled, user queries and the knowledge-base text are sent to the embedding provider. The framework's own `rag-simple` module is not used: the `core/rag` package is `@Deprecated(forRemoval = true)` in 2.0.4 and the module drags in Qdrant, Milvus, Elasticsearch, PostgreSQL, PDFBox, POI, Tika and more. +- **Retrieval**: the `retrieve` tool searches `src/main/resources/knowledge/*.md` and returns passages labelled with a stable id such as `[api-usage.md#2]` (document name plus a digest of the passage text, so editing a passage changes its id and old citations never point at different text) plus their source, which the model is asked to cite verbatim (ids stay unique across repeated retrievals in one conversation, unlike per-call `[1]`, `[2]` counters). It is the only retrieval channel; the framework's `knowledge()` auto-injection is deliberately not used. Retrieval is layered behind the application's own `RetrievePort` (no framework types): BM25 keyword search is always available, and with an embedding service configured a vector channel is added and the two are fused with RRF (constant 60); if one channel fails the other is used. Note that with embeddings enabled, user queries and the knowledge-base text are sent to the embedding provider. The framework's own `rag-simple` module is not used: the `core/rag` package is `@Deprecated(forRemoval = true)` in 2.0.4 and the module drags in Qdrant, Milvus, Elasticsearch, PostgreSQL, PDFBox, POI, Tika and more. - **MCP**: only HTTP transports are accepted. Example: ```yaml diff --git a/README_CN.md b/README_CN.md index 34945ec..9285456 100644 --- a/README_CN.md +++ b/README_CN.md @@ -173,7 +173,7 @@ Agent 创建时不开放文件、命令和联网工具。读 2.0.4 源码发现 ## 工具、知识库与子 Agent - **工具**（`com.yoke.tool`）：带 `@Tool` 方法的普通类，启动时一次性注册进共享的 `Toolkit`。`calculate` 用手写的递归下降解析器（不用脚本引擎，输入长度和嵌套深度有上限，除零以文本形式报告）。`list_my_notes` 的 `UserContext` 参数不带 `@ToolParam`：框架按类型从本次请求的 `RuntimeContext` 里注入，模型既看不到也伪造不了用户标识。 -- **检索**：`retrieve` 工具检索 `src/main/resources/knowledge/*.md`，返回带稳定编号（形如 `[api-usage.md#2]`，文档名加序号）和来源的片段，并要求模型原样照抄编号引用；同一轮对话里多次检索编号也不会重复，不像每次从 `[1]` 开始的序号会冲突。它是唯一的检索通道，有意不同时使用框架的 `knowledge()` 自动注入。检索分层放在应用自己的 `RetrievePort` 后面（不含框架类型）：BM25 关键词检索永远可用；配置了向量化服务就再加一路向量检索，两路用 RRF（常数 60）融合，任何一路失败就用另一路。注意开启向量化后，用户的检索问题和知识库语料会发送给 embedding 服务提供方。没有使用框架自带的 `rag-simple`：`core/rag` 包在 2.0.4 里已 `@Deprecated(forRemoval = true)`，而且该模块会拖进 Qdrant、Milvus、Elasticsearch、PostgreSQL、PDFBox、POI、Tika 等一大批依赖。 +- **检索**：`retrieve` 工具检索 `src/main/resources/knowledge/*.md`，返回带稳定编号（形如 `[api-usage.md#2]`，文档名加正文摘要，正文改了编号也跟着变，旧引用不会指到别的内容上）和来源的片段，并要求模型原样照抄编号引用；同一轮对话里多次检索编号也不会重复，不像每次从 `[1]` 开始的序号会冲突。它是唯一的检索通道，有意不同时使用框架的 `knowledge()` 自动注入。检索分层放在应用自己的 `RetrievePort` 后面（不含框架类型）：BM25 关键词检索永远可用；配置了向量化服务就再加一路向量检索，两路用 RRF（常数 60）融合，任何一路失败就用另一路。注意开启向量化后，用户的检索问题和知识库语料会发送给 embedding 服务提供方。没有使用框架自带的 `rag-simple`：`core/rag` 包在 2.0.4 里已 `@Deprecated(forRemoval = true)`，而且该模块会拖进 Qdrant、Milvus、Elasticsearch、PostgreSQL、PDFBox、POI、Tika 等一大批依赖。 - **MCP**：只接受 HTTP 传输。示例： ```yaml diff --git a/src/main/java/com/yoke/api/ChatApi.java b/src/main/java/com/yoke/api/ChatApi.java index 01a69c2..01e74c8 100644 --- a/src/main/java/com/yoke/api/ChatApi.java +++ b/src/main/java/com/yoke/api/ChatApi.java @@ -60,7 +60,7 @@ * 工具按类型取用，不是模型传的参数，模型看不到也伪造不了。这个 userId 本身仍是调用方自报的（见上面的实现说明）。 * *

引用格式：{@code retrieve} 返回的资料带 {@code [api-usage.md#2]} 这样的编号和来源文档标题，编号是片段的稳定标识 - * （文档名加序号），同一轮对话里多次检索也不会重复；模型回答时原样照抄编号标注出处，调用方可以据此回溯。检索是工具化的，只有模型认为需要时才会调用， + * （文档名加正文摘要，正文改了编号也跟着变），同一轮对话里多次检索也不会重复；模型回答时原样照抄编号标注出处，调用方可以据此回溯。检索是工具化的，只有模型认为需要时才会调用， * 没有同时开启框架的 knowledge 自动注入。没有配置向量化服务时只用关键词检索，向量化服务出错时自动退回关键词检索。 * *

子 Agent：{@code summarizer}（代码里声明，只做摘要，没有任何业务工具）、{@code kb-researcher} diff --git a/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java b/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java index e00146b..9bc2b1c 100644 --- a/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java +++ b/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java @@ -3,9 +3,12 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; +import java.util.HexFormat; import java.util.List; import java.util.Optional; @@ -19,13 +22,15 @@ * *

切法：文档第一个一级标题（{@code # }）当作文档标题，之后每个二级标题（{@code ## }）起一个片段， * 片段正文超过 {@link #MAX_CHUNK_LENGTH} 个字符时按段落再切。每个片段正文前面带上「文档标题 / 小节标题」， - * 这样单看一个片段也知道它在讲什么。 + * 这样单看一个片段也知道它在讲什么。片段 id 由文档名和正文摘要组成，见 {@link #passage}。 */ @NoArgsConstructor(access = lombok.AccessLevel.PRIVATE) public final class KnowledgeCorpus { static final int MAX_CHUNK_LENGTH = 500; + private static final int DIGEST_BYTES = 4; + /** 读取 classpath 下的全部知识文档并切片；没有文档时返回空列表。 */ public static List loadFromClasspath(String pattern) { try { @@ -79,13 +84,32 @@ private static void flush(List passages, String docId, String title, St StringBuilder chunk = new StringBuilder(); for (String paragraph : text.split("\\n\\s*\\n")) { if (!chunk.isEmpty() && chunk.length() + paragraph.length() > MAX_CHUNK_LENGTH) { - passages.add(new Passage(docId + "#" + passages.size(), title, heading + "\n" + chunk.toString().strip())); + passages.add(passage(docId, title, heading + "\n" + chunk.toString().strip())); chunk.setLength(0); } chunk.append(paragraph.strip()).append("\n\n"); } if (!chunk.isEmpty()) { - passages.add(new Passage(docId + "#" + passages.size(), title, heading + "\n" + chunk.toString().strip())); + passages.add(passage(docId, title, heading + "\n" + chunk.toString().strip())); + } + } + + /** + * 片段 id 是「文档名#正文摘要」，不用片段在文档里的位置：id 会作为引用编号写进 Redis 里的对话历史， + * 跨重启、跨部署都还在。按位置编号的话，文档前面插一节，后面所有片段的 id 就都换成了别的正文； + * 按内容算，同一个 id 一定是同一段正文，正文改了 id 也跟着变，旧引用只会找不到，不会指错。 + */ + private static Passage passage(String docId, String title, String text) { + return new Passage(docId + "#" + digest(text), title, text); + } + + /** SHA-256 的前 4 个字节（8 位十六进制）：知识库只有几十个片段，足够区分，又不会让引用编号太长。 */ + private static String digest(String text) { + try { + byte[] hash = MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.UTF_8)); + return HexFormat.of().formatHex(hash, 0, DIGEST_BYTES); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("JDK 缺少 SHA-256", e); } } } diff --git a/src/main/java/com/yoke/model/Passage.java b/src/main/java/com/yoke/model/Passage.java index 95d9c2a..7a62b0c 100644 --- a/src/main/java/com/yoke/model/Passage.java +++ b/src/main/java/com/yoke/model/Passage.java @@ -3,8 +3,8 @@ /** * 知识库里的一个片段，也是检索的返回单位。 * - * @param id 片段的稳定标识（文档名加序号，如 api-usage.md#2）：多路检索融合时靠它判断是不是同一个片段， - * {@code retrieve} 也把它当作引用编号返回给模型，所以它在整个对话里必须固定指向同一个片段 + * @param id 片段的稳定标识（文档名加正文摘要，形如「文档名.md#8 位十六进制」）：多路检索融合时靠它判断是不是同一个片段， + * {@code retrieve} 也把它当作引用编号返回给模型，引用会留在对话历史里跨部署存在，所以同一个 id 只能对应同一段正文 * @param source 片段所属文档的标题，引用时展示给用户 * @param text 片段正文 */ diff --git a/src/main/java/com/yoke/tool/RetrieveTool.java b/src/main/java/com/yoke/tool/RetrieveTool.java index dc854c8..7b5ea1b 100644 --- a/src/main/java/com/yoke/tool/RetrieveTool.java +++ b/src/main/java/com/yoke/tool/RetrieveTool.java @@ -20,7 +20,7 @@ * 而且 AgentScope 里框架的 rag 包已标记为将移除。工具化的另一个好处是闲聊不触发检索，省 token。 * *

返回的每条资料前带方括号编号，模型回答时原样照抄编号来引用，用户就能回头核对出处。编号用片段自己的稳定标识 - * （文档名加序号，形如 {@code [api-usage.md#2]}），而不是每次检索都从 1 开始的序号：同一轮对话里模型常常换个说法再查一次， + * （文档名加正文摘要，形如 {@code [api-usage.md#2]}），而不是每次检索都从 1 开始的序号：同一轮对话里模型常常换个说法再查一次， * 两次结果里的 {@code [1]} 会指向不同的片段，引用就对不上了。稳定标识在整个对话里固定指向同一个片段。 * 无状态，线程安全；{@code port} 为空表示没有配置知识库，工具返回提示语而不是抛异常。 */ diff --git a/src/test/java/com/yoke/ToolsEndToEndTests.java b/src/test/java/com/yoke/ToolsEndToEndTests.java index fb11ab6..41a914e 100644 --- a/src/test/java/com/yoke/ToolsEndToEndTests.java +++ b/src/test/java/com/yoke/ToolsEndToEndTests.java @@ -90,7 +90,7 @@ private List> stream(String threadId, String text) { assertThat(AguiTestSupport.toolArguments(events)).contains("对话历史存在哪里").contains("\"top_k\":2"); assertThat(result.get("toolCallId")).isEqualTo(call.get("toolCallId")); assertThat(events.indexOf(call)).as("工具调用先于工具结果").isLessThan(events.indexOf(result)); - assertThat(String.valueOf(result.get("content"))).containsPattern("\\[[\\w-]+\\.md#\\d+\\] 来源：").contains("Redis").doesNotContain("尚未配置"); + assertThat(String.valueOf(result.get("content"))).containsPattern("\\[[\\w-]+\\.md#[0-9a-f]{8}\\] 来源：").contains("Redis").doesNotContain("尚未配置"); assertThat(events.getLast().get("type")).isEqualTo("RUN_FINISHED"); } diff --git a/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java b/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java index 1ada2d0..7bbe1cf 100644 --- a/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java +++ b/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java @@ -21,6 +21,28 @@ class KnowledgeCorpusTests { assertThat(passages).extracting(Passage::id).doesNotHaveDuplicates(); } + @Test + void 片段id跟着正文走_前面插入新小节不改变后面片段的id() { + // id 会作为引用编号留在 Redis 的对话历史里，按位置编号的话，插一节之后旧引用就指到了别的正文上 + String before = "# 标题\n## 第一节\n内容一\n## 第二节\n内容二\n"; + String inserted = "# 标题\n## 新加的一节\n新内容\n## 第一节\n内容一\n## 第二节\n内容二\n"; + + List original = KnowledgeCorpus.split("demo.md", before); + List afterInsert = KnowledgeCorpus.split("demo.md", inserted); + + assertThat(original).extracting(Passage::id).allMatch(id -> id.matches("demo\\.md#[0-9a-f]{8}")); + assertThat(afterInsert.subList(1, 3)).extracting(Passage::id) + .containsExactlyElementsOf(original.stream().map(Passage::id).toList()); + } + + @Test + void 正文改了片段id也跟着变() { + String id = KnowledgeCorpus.split("demo.md", "# 标题\n## 一节\n旧内容").getFirst().id(); + String changed = KnowledgeCorpus.split("demo.md", "# 标题\n## 一节\n新内容").getFirst().id(); + + assertThat(changed).isNotEqualTo(id); + } + @Test void 过长的小节按段落再切() { String paragraph = "字".repeat(KnowledgeCorpus.MAX_CHUNK_LENGTH - 50); diff --git a/src/test/java/com/yoke/tool/ToolConversationTests.java b/src/test/java/com/yoke/tool/ToolConversationTests.java index 66f439a..7182b52 100644 --- a/src/test/java/com/yoke/tool/ToolConversationTests.java +++ b/src/test/java/com/yoke/tool/ToolConversationTests.java @@ -113,7 +113,7 @@ private static ChatController controller(ScriptedModel model, Path workspace, To List> events = stream(controller, "s-r", "历史存哪"); String output = resultOf(events); - assertThat(output).containsPattern("\\[[\\w-]+\\.md#\\d+\\] 来源：").contains("Redis").doesNotContain("[1]").doesNotContain("[2]"); + assertThat(output).containsPattern("\\[[\\w-]+\\.md#[0-9a-f]{8}\\] 来源：").contains("Redis").doesNotContain("[1]").doesNotContain("[2]"); } @Test From 415e9b490b8b8bd0f550cac4bee1733d77a17f91 Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 12 Nov 2026 19:01:34 +0800 Subject: [PATCH 09/19] fix: fail fast on duplicate knowledge document file names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 片段 id 以文件名开头，而 classpath*:knowledge/*.md 会把每个 classpath 根（依赖 jar、 测试资源）里的同名文件都找出来。两篇同名文档的片段会共用 id 前缀，融合检索按 id 去重时 一篇会悄悄覆盖另一篇，引用也分不清出自哪篇。现在加载时发现文件名重复就直接失败并列出 重复的文件名。加载逻辑拆出 load(List) 以便测试。 --- README.md | 2 +- README_CN.md | 2 +- .../com/yoke/knowledge/KnowledgeCorpus.java | 46 +++++++++++++------ .../yoke/knowledge/KnowledgeCorpusTests.java | 28 +++++++++++ 4 files changed, 63 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 44518fb..faf7ac9 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ The agent is built without file, shell or web tools. Reading the 2.0.4 source sh ## Tools, Knowledge and Sub-agents - **Tools** (`com.yoke.tool`): plain classes with `@Tool` methods, registered once into a shared `Toolkit`. `calculate` uses a hand-written recursive-descent parser (no script engine, input length and nesting limits, division by zero reported as text). `list_my_notes` takes a `UserContext` parameter without `@ToolParam`: the framework injects it by type from the request's `RuntimeContext`, so the model never sees or forges the user id. -- **Retrieval**: the `retrieve` tool searches `src/main/resources/knowledge/*.md` and returns passages labelled with a stable id such as `[api-usage.md#2]` (document name plus a digest of the passage text, so editing a passage changes its id and old citations never point at different text) plus their source, which the model is asked to cite verbatim (ids stay unique across repeated retrievals in one conversation, unlike per-call `[1]`, `[2]` counters). It is the only retrieval channel; the framework's `knowledge()` auto-injection is deliberately not used. Retrieval is layered behind the application's own `RetrievePort` (no framework types): BM25 keyword search is always available, and with an embedding service configured a vector channel is added and the two are fused with RRF (constant 60); if one channel fails the other is used. Note that with embeddings enabled, user queries and the knowledge-base text are sent to the embedding provider. The framework's own `rag-simple` module is not used: the `core/rag` package is `@Deprecated(forRemoval = true)` in 2.0.4 and the module drags in Qdrant, Milvus, Elasticsearch, PostgreSQL, PDFBox, POI, Tika and more. +- **Retrieval**: the `retrieve` tool searches `src/main/resources/knowledge/*.md` and returns passages labelled with a stable id such as `[api-usage.md#2]` (document name plus a digest of the passage text, so editing a passage changes its id and old citations never point at different text; two knowledge documents with the same file name in different classpath roots fail the startup) plus their source, which the model is asked to cite verbatim (ids stay unique across repeated retrievals in one conversation, unlike per-call `[1]`, `[2]` counters). It is the only retrieval channel; the framework's `knowledge()` auto-injection is deliberately not used. Retrieval is layered behind the application's own `RetrievePort` (no framework types): BM25 keyword search is always available, and with an embedding service configured a vector channel is added and the two are fused with RRF (constant 60); if one channel fails the other is used. Note that with embeddings enabled, user queries and the knowledge-base text are sent to the embedding provider. The framework's own `rag-simple` module is not used: the `core/rag` package is `@Deprecated(forRemoval = true)` in 2.0.4 and the module drags in Qdrant, Milvus, Elasticsearch, PostgreSQL, PDFBox, POI, Tika and more. - **MCP**: only HTTP transports are accepted. Example: ```yaml diff --git a/README_CN.md b/README_CN.md index 9285456..a776ad1 100644 --- a/README_CN.md +++ b/README_CN.md @@ -173,7 +173,7 @@ Agent 创建时不开放文件、命令和联网工具。读 2.0.4 源码发现 ## 工具、知识库与子 Agent - **工具**（`com.yoke.tool`）：带 `@Tool` 方法的普通类，启动时一次性注册进共享的 `Toolkit`。`calculate` 用手写的递归下降解析器（不用脚本引擎，输入长度和嵌套深度有上限，除零以文本形式报告）。`list_my_notes` 的 `UserContext` 参数不带 `@ToolParam`：框架按类型从本次请求的 `RuntimeContext` 里注入，模型既看不到也伪造不了用户标识。 -- **检索**：`retrieve` 工具检索 `src/main/resources/knowledge/*.md`，返回带稳定编号（形如 `[api-usage.md#2]`，文档名加正文摘要，正文改了编号也跟着变，旧引用不会指到别的内容上）和来源的片段，并要求模型原样照抄编号引用；同一轮对话里多次检索编号也不会重复，不像每次从 `[1]` 开始的序号会冲突。它是唯一的检索通道，有意不同时使用框架的 `knowledge()` 自动注入。检索分层放在应用自己的 `RetrievePort` 后面（不含框架类型）：BM25 关键词检索永远可用；配置了向量化服务就再加一路向量检索，两路用 RRF（常数 60）融合，任何一路失败就用另一路。注意开启向量化后，用户的检索问题和知识库语料会发送给 embedding 服务提供方。没有使用框架自带的 `rag-simple`：`core/rag` 包在 2.0.4 里已 `@Deprecated(forRemoval = true)`，而且该模块会拖进 Qdrant、Milvus、Elasticsearch、PostgreSQL、PDFBox、POI、Tika 等一大批依赖。 +- **检索**：`retrieve` 工具检索 `src/main/resources/knowledge/*.md`，返回带稳定编号（形如 `[api-usage.md#2]`，文档名加正文摘要，正文改了编号也跟着变，旧引用不会指到别的内容上；不同 classpath 根里有同名知识文档时启动即失败）和来源的片段，并要求模型原样照抄编号引用；同一轮对话里多次检索编号也不会重复，不像每次从 `[1]` 开始的序号会冲突。它是唯一的检索通道，有意不同时使用框架的 `knowledge()` 自动注入。检索分层放在应用自己的 `RetrievePort` 后面（不含框架类型）：BM25 关键词检索永远可用；配置了向量化服务就再加一路向量检索，两路用 RRF（常数 60）融合，任何一路失败就用另一路。注意开启向量化后，用户的检索问题和知识库语料会发送给 embedding 服务提供方。没有使用框架自带的 `rag-simple`：`core/rag` 包在 2.0.4 里已 `@Deprecated(forRemoval = true)`，而且该模块会拖进 Qdrant、Milvus、Elasticsearch、PostgreSQL、PDFBox、POI、Tika 等一大批依赖。 - **MCP**：只接受 HTTP 传输。示例： ```yaml diff --git a/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java b/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java index 9bc2b1c..8c62e26 100644 --- a/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java +++ b/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java @@ -10,7 +10,10 @@ import java.util.Comparator; import java.util.HexFormat; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.TreeMap; +import java.util.stream.Collectors; import com.yoke.model.Passage; import lombok.NoArgsConstructor; @@ -34,24 +37,41 @@ public final class KnowledgeCorpus { /** 读取 classpath 下的全部知识文档并切片；没有文档时返回空列表。 */ public static List loadFromClasspath(String pattern) { try { - Resource[] resources = new PathMatchingResourcePatternResolver().getResources(pattern); - List passages = new ArrayList<>(); - Arrays.stream(resources) - .sorted(Comparator.comparing(Resource::getFilename, Comparator.nullsLast(Comparator.naturalOrder()))) - .forEach(resource -> { - try { - String markdown = new String(resource.getInputStream().readAllBytes(), StandardCharsets.UTF_8); - passages.addAll(split(resource.getFilename(), markdown)); - } catch (IOException e) { - throw new UncheckedIOException("读取知识文档失败：" + resource.getFilename(), e); - } - }); - return passages; + return load(Arrays.asList(new PathMatchingResourcePatternResolver().getResources(pattern))); } catch (IOException e) { throw new UncheckedIOException("扫描知识文档失败：" + pattern, e); } } + /** + * 片段 id 以文件名开头，{@code classpath*:} 又会把每个 classpath 根（依赖 jar、测试资源）里的同名文件都找出来， + * 两篇同名文档的片段会被当成同一个来源，融合检索时还会互相覆盖，所以文件名重复直接失败，而不是悄悄丢掉一篇。 + */ + static List load(List resources) { + List duplicates = resources.stream() + .collect(Collectors.groupingBy(resource -> Optional.ofNullable(resource.getFilename()).orElse(""), + TreeMap::new, Collectors.counting())) + .entrySet().stream() + .filter(entry -> entry.getValue() > 1) + .map(Map.Entry::getKey) + .toList(); + if (!duplicates.isEmpty()) { + throw new IllegalStateException("知识文档文件名重复，片段 id 会冲突：" + duplicates); + } + List passages = new ArrayList<>(); + resources.stream() + .sorted(Comparator.comparing(Resource::getFilename, Comparator.nullsLast(Comparator.naturalOrder()))) + .forEach(resource -> { + try { + String markdown = new String(resource.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + passages.addAll(split(resource.getFilename(), markdown)); + } catch (IOException e) { + throw new UncheckedIOException("读取知识文档失败：" + resource.getFilename(), e); + } + }); + return passages; + } + /** 把一篇 Markdown 切成片段；{@code fileName} 只在文档没有一级标题时充当标题。 */ static List split(String fileName, String markdown) { String title = Optional.ofNullable(fileName).orElse("未命名文档"); diff --git a/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java b/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java index 7bbe1cf..aef98cf 100644 --- a/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java +++ b/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java @@ -1,12 +1,17 @@ package com.yoke.knowledge; +import java.nio.charset.StandardCharsets; import java.util.List; import com.yoke.knowledge.retrieve.KeywordRetrievePort; import com.yoke.model.Passage; import org.junit.jupiter.api.Test; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.Resource; + import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; class KnowledgeCorpusTests { @@ -70,6 +75,29 @@ class KnowledgeCorpusTests { assertThat(passages).extracting(Passage::id).doesNotHaveDuplicates(); } + @Test + void 不同classpath根里有同名文档时直接失败() { + // classpath*: 会把依赖 jar 里的同名文件也找出来，两篇文档的片段 id 都以 api-usage.md 开头 + List resources = List.of( + named("api-usage.md", "# 接口使用\n## 一节\n内容一"), + named("api-usage.md", "# 别的接口\n## 一节\n内容二"), + named("yoke-overview.md", "# 概览\n## 一节\n内容三")); + + assertThatThrownBy(() -> KnowledgeCorpus.load(resources)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("api-usage.md") + .hasMessageNotContaining("yoke-overview.md"); + } + + private static Resource named(String fileName, String markdown) { + return new ByteArrayResource(markdown.getBytes(StandardCharsets.UTF_8)) { + @Override + public String getFilename() { + return fileName; + } + }; + } + @Test void 没有匹配的文档时返回空列表() { assertThat(KnowledgeCorpus.loadFromClasspath("classpath*:no-such-dir/*.md")).isEmpty(); From 3e08e9182019ddb7124086f851aff9a999404cf0 Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 12 Nov 2026 19:03:09 +0800 Subject: [PATCH 10/19] fix: use a placeholder instead of a real passage id in citation examples MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit retrieve 工具说明、kb-researcher 提示词和知识文档里举例用的 [api-usage.md#2] 原本 正好是一个真实片段的编号。模型被要求原样照抄方括号里的编号，可能把这个没检索到的片段 也当作引用写出来。示例统一改成 [文档名.md#片段编号]（README 英文版用 [doc-name.md#passage-id]），并加测试确认这些给模型看的文本里不出现任何真实片段编号。 --- README.md | 4 ++-- README_CN.md | 4 ++-- src/main/java/com/yoke/api/ChatApi.java | 2 +- src/main/java/com/yoke/tool/RetrieveTool.java | 4 ++-- .../knowledge/tools-and-permissions.md | 2 +- .../workspace/subagents/kb-researcher.md | 2 +- .../java/com/yoke/tool/RetrieveToolTests.java | 21 +++++++++++++++++++ 7 files changed, 30 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index faf7ac9..13696a6 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Yoke is an open-source project of 友客AI (YouKe AI). We read the AgentScope Ja - One endpoint for multi-turn conversations: `POST /api/chat`, plus `POST /api/chat/stream` that streams the reasoning process over SSE using the AG-UI protocol. - A built-in `get_weather` tool looks up current weather through the open-source [Open-Meteo](https://open-meteo.com) API, no key needed (the free tier is for non-commercial use only). - Every request builds a temporary `HarnessAgent` and closes it afterwards. Conversation history lives in Redis, so it survives restarts and is shared by all replicas, isolated per `(userId, sessionId)`, and compacted automatically when a conversation gets long. -- Tools, knowledge and sub-agents: calculator, clock and per-user notes tools, a `retrieve` tool over a built-in knowledge base that returns citations keyed by stable passage ids such as `[api-usage.md#2]`, external MCP servers over HTTP, and two sub-agents (one declared in code, one declared as a workspace Markdown file). The agent still has no file, shell or web tools, and a logging middleware records a timing trace for every call. +- Tools, knowledge and sub-agents: calculator, clock and per-user notes tools, a `retrieve` tool over a built-in knowledge base that returns citations keyed by stable passage ids such as `[doc-name.md#passage-id]`, external MCP servers over HTTP, and two sub-agents (one declared in code, one declared as a workspace Markdown file). The agent still has no file, shell or web tools, and a logging middleware records a timing trace for every call. - Models are plugged in through the official AgentScope starters: OpenAI-compatible APIs (works with DashScope, DeepSeek and others), Anthropic, Gemini and Ollama. Switching is a dependency and configuration change. - `local` / `test` / `prod` profiles. In production, a missing API key fails the startup instead of shipping with an empty key. - Virtual threads are enabled by default, for both Tomcat and Spring's default thread pool. @@ -173,7 +173,7 @@ The agent is built without file, shell or web tools. Reading the 2.0.4 source sh ## Tools, Knowledge and Sub-agents - **Tools** (`com.yoke.tool`): plain classes with `@Tool` methods, registered once into a shared `Toolkit`. `calculate` uses a hand-written recursive-descent parser (no script engine, input length and nesting limits, division by zero reported as text). `list_my_notes` takes a `UserContext` parameter without `@ToolParam`: the framework injects it by type from the request's `RuntimeContext`, so the model never sees or forges the user id. -- **Retrieval**: the `retrieve` tool searches `src/main/resources/knowledge/*.md` and returns passages labelled with a stable id such as `[api-usage.md#2]` (document name plus a digest of the passage text, so editing a passage changes its id and old citations never point at different text; two knowledge documents with the same file name in different classpath roots fail the startup) plus their source, which the model is asked to cite verbatim (ids stay unique across repeated retrievals in one conversation, unlike per-call `[1]`, `[2]` counters). It is the only retrieval channel; the framework's `knowledge()` auto-injection is deliberately not used. Retrieval is layered behind the application's own `RetrievePort` (no framework types): BM25 keyword search is always available, and with an embedding service configured a vector channel is added and the two are fused with RRF (constant 60); if one channel fails the other is used. Note that with embeddings enabled, user queries and the knowledge-base text are sent to the embedding provider. The framework's own `rag-simple` module is not used: the `core/rag` package is `@Deprecated(forRemoval = true)` in 2.0.4 and the module drags in Qdrant, Milvus, Elasticsearch, PostgreSQL, PDFBox, POI, Tika and more. +- **Retrieval**: the `retrieve` tool searches `src/main/resources/knowledge/*.md` and returns passages labelled with a stable id such as `[doc-name.md#passage-id]` (document name plus a digest of the passage text, so editing a passage changes its id and old citations never point at different text; two knowledge documents with the same file name in different classpath roots fail the startup) plus their source, which the model is asked to cite verbatim (ids stay unique across repeated retrievals in one conversation, unlike per-call `[1]`, `[2]` counters). It is the only retrieval channel; the framework's `knowledge()` auto-injection is deliberately not used. Retrieval is layered behind the application's own `RetrievePort` (no framework types): BM25 keyword search is always available, and with an embedding service configured a vector channel is added and the two are fused with RRF (constant 60); if one channel fails the other is used. Note that with embeddings enabled, user queries and the knowledge-base text are sent to the embedding provider. The framework's own `rag-simple` module is not used: the `core/rag` package is `@Deprecated(forRemoval = true)` in 2.0.4 and the module drags in Qdrant, Milvus, Elasticsearch, PostgreSQL, PDFBox, POI, Tika and more. - **MCP**: only HTTP transports are accepted. Example: ```yaml diff --git a/README_CN.md b/README_CN.md index a776ad1..da8b178 100644 --- a/README_CN.md +++ b/README_CN.md @@ -29,7 +29,7 @@ Yoke 是友客AI的开源项目。大家一起读 AgentScope Java V2 和 Spring - 一个接口完成多轮对话：`POST /api/chat`；另有 `POST /api/chat/stream`，按 AG-UI 协议通过 SSE 边推理边推送。 - 内置 `get_weather` 工具，通过开源的 [Open-Meteo](https://open-meteo.com) 查当前天气，免密钥（免费版仅限非商业使用）。 - 每次请求新建一个临时 `HarnessAgent`，用完即关闭。对话历史存在 Redis 里，重启不丢，多个副本共享，按 `(userId, sessionId)` 隔离，对话过长时自动压缩。 -- 工具、知识库与子 Agent：计算器、时钟、按用户隔离的笔记工具，基于内置知识库的 `retrieve` 工具（返回带稳定片段编号的引用，形如 `[api-usage.md#2]`），通过 HTTP 接入外部 MCP 服务，以及两个子 Agent（一个在代码里声明，一个由工作区 Markdown 文件声明）。Agent 仍不开放文件、命令和联网工具，日志中间件为每次调用记录耗时轨迹。 +- 工具、知识库与子 Agent：计算器、时钟、按用户隔离的笔记工具，基于内置知识库的 `retrieve` 工具（返回带稳定片段编号的引用，形如 `[文档名.md#片段编号]`），通过 HTTP 接入外部 MCP 服务，以及两个子 Agent（一个在代码里声明，一个由工作区 Markdown 文件声明）。Agent 仍不开放文件、命令和联网工具，日志中间件为每次调用记录耗时轨迹。 - 通过 AgentScope 官方 starter 接入各种模型：OpenAI 兼容协议（可对接 DashScope、DeepSeek 等）、Anthropic、Gemini、Ollama，换依赖和配置即可。 - 有 local / test / prod 多环境配置；生产环境缺少密钥时启动即失败，避免带着空密钥上线。 - 默认开启虚拟线程，Tomcat 和 Spring 默认线程池都使用虚拟线程。 @@ -173,7 +173,7 @@ Agent 创建时不开放文件、命令和联网工具。读 2.0.4 源码发现 ## 工具、知识库与子 Agent - **工具**（`com.yoke.tool`）：带 `@Tool` 方法的普通类，启动时一次性注册进共享的 `Toolkit`。`calculate` 用手写的递归下降解析器（不用脚本引擎，输入长度和嵌套深度有上限，除零以文本形式报告）。`list_my_notes` 的 `UserContext` 参数不带 `@ToolParam`：框架按类型从本次请求的 `RuntimeContext` 里注入，模型既看不到也伪造不了用户标识。 -- **检索**：`retrieve` 工具检索 `src/main/resources/knowledge/*.md`，返回带稳定编号（形如 `[api-usage.md#2]`，文档名加正文摘要，正文改了编号也跟着变，旧引用不会指到别的内容上；不同 classpath 根里有同名知识文档时启动即失败）和来源的片段，并要求模型原样照抄编号引用；同一轮对话里多次检索编号也不会重复，不像每次从 `[1]` 开始的序号会冲突。它是唯一的检索通道，有意不同时使用框架的 `knowledge()` 自动注入。检索分层放在应用自己的 `RetrievePort` 后面（不含框架类型）：BM25 关键词检索永远可用；配置了向量化服务就再加一路向量检索，两路用 RRF（常数 60）融合，任何一路失败就用另一路。注意开启向量化后，用户的检索问题和知识库语料会发送给 embedding 服务提供方。没有使用框架自带的 `rag-simple`：`core/rag` 包在 2.0.4 里已 `@Deprecated(forRemoval = true)`，而且该模块会拖进 Qdrant、Milvus、Elasticsearch、PostgreSQL、PDFBox、POI、Tika 等一大批依赖。 +- **检索**：`retrieve` 工具检索 `src/main/resources/knowledge/*.md`，返回带稳定编号（形如 `[文档名.md#片段编号]`，文档名加正文摘要，正文改了编号也跟着变，旧引用不会指到别的内容上；不同 classpath 根里有同名知识文档时启动即失败）和来源的片段，并要求模型原样照抄编号引用；同一轮对话里多次检索编号也不会重复，不像每次从 `[1]` 开始的序号会冲突。它是唯一的检索通道，有意不同时使用框架的 `knowledge()` 自动注入。检索分层放在应用自己的 `RetrievePort` 后面（不含框架类型）：BM25 关键词检索永远可用；配置了向量化服务就再加一路向量检索，两路用 RRF（常数 60）融合，任何一路失败就用另一路。注意开启向量化后，用户的检索问题和知识库语料会发送给 embedding 服务提供方。没有使用框架自带的 `rag-simple`：`core/rag` 包在 2.0.4 里已 `@Deprecated(forRemoval = true)`，而且该模块会拖进 Qdrant、Milvus、Elasticsearch、PostgreSQL、PDFBox、POI、Tika 等一大批依赖。 - **MCP**：只接受 HTTP 传输。示例： ```yaml diff --git a/src/main/java/com/yoke/api/ChatApi.java b/src/main/java/com/yoke/api/ChatApi.java index 01e74c8..f03349c 100644 --- a/src/main/java/com/yoke/api/ChatApi.java +++ b/src/main/java/com/yoke/api/ChatApi.java @@ -59,7 +59,7 @@ *

业务上下文不经过模型：{@code list_my_notes} 要知道「当前用户是谁」，userId 由控制器放进 RuntimeContext， * 工具按类型取用，不是模型传的参数，模型看不到也伪造不了。这个 userId 本身仍是调用方自报的（见上面的实现说明）。 * - *

引用格式：{@code retrieve} 返回的资料带 {@code [api-usage.md#2]} 这样的编号和来源文档标题，编号是片段的稳定标识 + *

引用格式：{@code retrieve} 返回的资料带 {@code [文档名.md#片段编号]} 这样的编号和来源文档标题，编号是片段的稳定标识 * （文档名加正文摘要，正文改了编号也跟着变），同一轮对话里多次检索也不会重复；模型回答时原样照抄编号标注出处，调用方可以据此回溯。检索是工具化的，只有模型认为需要时才会调用， * 没有同时开启框架的 knowledge 自动注入。没有配置向量化服务时只用关键词检索，向量化服务出错时自动退回关键词检索。 * diff --git a/src/main/java/com/yoke/tool/RetrieveTool.java b/src/main/java/com/yoke/tool/RetrieveTool.java index 7b5ea1b..993687b 100644 --- a/src/main/java/com/yoke/tool/RetrieveTool.java +++ b/src/main/java/com/yoke/tool/RetrieveTool.java @@ -20,7 +20,7 @@ * 而且 AgentScope 里框架的 rag 包已标记为将移除。工具化的另一个好处是闲聊不触发检索，省 token。 * *

返回的每条资料前带方括号编号，模型回答时原样照抄编号来引用，用户就能回头核对出处。编号用片段自己的稳定标识 - * （文档名加正文摘要，形如 {@code [api-usage.md#2]}），而不是每次检索都从 1 开始的序号：同一轮对话里模型常常换个说法再查一次， + * （文档名加正文摘要，形如 {@code [文档名.md#片段编号]}），而不是每次检索都从 1 开始的序号：同一轮对话里模型常常换个说法再查一次， * 两次结果里的 {@code [1]} 会指向不同的片段，引用就对不上了。稳定标识在整个对话里固定指向同一个片段。 * 无状态，线程安全；{@code port} 为空表示没有配置知识库，工具返回提示语而不是抛异常。 */ @@ -42,7 +42,7 @@ public RetrieveTool(RetrievePort port) { @Tool(name = "retrieve", description = "当问题涉及 Yoke 自身的设计、配置、接口或使用方法，而你不确定答案时调用，先查资料再回答；" - + "用户明确要求查知识库或资料时必须调用，不能没查就说知识库里没有；闲聊、通用常识和计算不要调用。返回的每条资料前有方括号编号（形如 [api-usage.md#2]），回答时请原样照抄编号标注引用来源。" + + "用户明确要求查知识库或资料时必须调用，不能没查就说知识库里没有；闲聊、通用常识和计算不要调用。返回的每条资料前有方括号编号（形如 [文档名.md#片段编号]），回答时请原样照抄编号标注引用来源。" + "查不到时如实告诉用户，不要编造。") public Mono retrieve( @ToolParam(name = "query", description = "要查的问题或关键词，用完整的短句，例如「会话历史存在哪里」") String query, diff --git a/src/main/resources/knowledge/tools-and-permissions.md b/src/main/resources/knowledge/tools-and-permissions.md index 5780ce5..093cee7 100644 --- a/src/main/resources/knowledge/tools-and-permissions.md +++ b/src/main/resources/knowledge/tools-and-permissions.md @@ -10,7 +10,7 @@ Yoke 自带的工具有 get_current_time、calculate、list_my_notes、get_weath list_my_notes 需要知道当前用户是谁，但 userId 不是模型传的参数：它由控制器放进 RuntimeContext，工具方法里不带 @ToolParam 的 UserContext 参数由框架按类型注入。模型既看不到这个参数，也没法伪造。 ## 检索如何降级 -retrieve 工具背后是检索端口。配置了 embedding 模型时，向量检索和关键词检索并行，用 RRF 融合排名；没有配置或向量检索失败时，自动退回纯关键词检索。返回的每条资料前带方括号编号（形如 [api-usage.md#2]，是片段的稳定标识，多次检索也不会重复），回答时应原样照抄编号标注引用。 +retrieve 工具背后是检索端口。配置了 embedding 模型时，向量检索和关键词检索并行，用 RRF 融合排名；没有配置或向量检索失败时，自动退回纯关键词检索。返回的每条资料前带方括号编号（形如 [文档名.md#片段编号]，是片段的稳定标识，多次检索也不会重复），回答时应原样照抄编号标注引用。 ## 接入 MCP 服务 通过 yoke.mcp.servers 可以接入外部 MCP 服务，传输方式只允许 sse 和 streamable-http 两种 HTTP 传输。配置 stdio 会在启动时直接失败，因为 stdio 会让服务端按配置去启动本地进程。 diff --git a/src/main/resources/workspace/subagents/kb-researcher.md b/src/main/resources/workspace/subagents/kb-researcher.md index 32face0..9ac5f73 100644 --- a/src/main/resources/workspace/subagents/kb-researcher.md +++ b/src/main/resources/workspace/subagents/kb-researcher.md @@ -9,4 +9,4 @@ tools: [retrieve] 工作方式： 1. 先用 retrieve 工具查资料，必要时换个说法再查一次，最多查两次。 2. 只根据检索到的资料作答，资料里没有的内容就明确说「知识库里没有查到」，不要凭印象补充。 -3. 回答要简短，每个结论后面用方括号编号标注出处（形如 [api-usage.md#2]），编号必须原样照抄检索结果里的，不要自己改成 [1]、[2]。多次检索的结果里编号不会重复，直接引用即可。 +3. 回答要简短，每个结论后面用方括号编号标注出处（形如 [文档名.md#片段编号]），编号必须原样照抄检索结果里的，不要自己改成 [1]、[2]。多次检索的结果里编号不会重复，直接引用即可。 diff --git a/src/test/java/com/yoke/tool/RetrieveToolTests.java b/src/test/java/com/yoke/tool/RetrieveToolTests.java index c802ded..a5c5665 100644 --- a/src/test/java/com/yoke/tool/RetrieveToolTests.java +++ b/src/test/java/com/yoke/tool/RetrieveToolTests.java @@ -1,13 +1,17 @@ package com.yoke.tool; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import com.yoke.knowledge.KnowledgeCorpus; import com.yoke.model.Passage; import com.yoke.knowledge.RetrievePort; import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; +import org.springframework.core.io.ClassPathResource; + import static org.assertj.core.api.Assertions.assertThat; class RetrieveToolTests { @@ -24,6 +28,23 @@ class RetrieveToolTests { assertThat(description).contains("明确要求查知识库").contains("必须调用").contains("不能没查就说知识库里没有"); } + @Test + void 给模型看的编号示例不是真实存在的片段编号() throws Exception { + // 模型被要求原样照抄方括号里的编号，示例要是真实编号，模型可能把没检索到的片段也引出来 + String description = RetrieveTool.class.getMethod("retrieve", String.class, Integer.class) + .getAnnotation(io.agentscope.core.tool.Tool.class).description(); + String researcher = new ClassPathResource("workspace/subagents/kb-researcher.md") + .getContentAsString(StandardCharsets.UTF_8); + List corpus = KnowledgeCorpus.loadFromClasspath("classpath*:knowledge/*.md"); + List modelFacing = new ArrayList<>(List.of(description, researcher)); + corpus.forEach(passage -> modelFacing.add(passage.text())); + + assertThat(description).contains("[文档名.md#片段编号]"); + for (Passage passage : corpus) { + assertThat(modelFacing).noneMatch(text -> text.contains("[" + passage.id() + "]")); + } + } + @Test void 结果带编号和来源_方便模型按编号引用() { RetrieveTool tool = new RetrieveTool((query, topK) -> Mono.just(HITS)); From 265f707d945404d81bf95d832387469b3c7829ed Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 12 Nov 2026 19:04:11 +0800 Subject: [PATCH 11/19] test: assert retrieval returns exactly top_k cited passages again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 改成片段 id 引用时，断言从 contains("[2] 来源：").doesNotContain("[3]") 换成了只匹配 一条编号的正则，不再检查条数：RetrieveTool 不把 top_k 传给检索端口时测试照样通过。 现在数一遍结果里带编号的资料，要求正好两条（脚本里模型要的 top_k=2）。 --- src/test/java/com/yoke/tool/ToolConversationTests.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/yoke/tool/ToolConversationTests.java b/src/test/java/com/yoke/tool/ToolConversationTests.java index 7182b52..683e7ca 100644 --- a/src/test/java/com/yoke/tool/ToolConversationTests.java +++ b/src/test/java/com/yoke/tool/ToolConversationTests.java @@ -4,6 +4,7 @@ import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; import com.yoke.controller.ChatController; import com.yoke.knowledge.retrieve.KeywordRetrievePort; @@ -113,7 +114,9 @@ private static ChatController controller(ScriptedModel model, Path workspace, To List> events = stream(controller, "s-r", "历史存哪"); String output = resultOf(events); - assertThat(output).containsPattern("\\[[\\w-]+\\.md#[0-9a-f]{8}\\] 来源：").contains("Redis").doesNotContain("[1]").doesNotContain("[2]"); + // 模型要的是 top_k=2，结果里正好两条带编号的资料，少了多了都说明条数没有传到检索端口 + assertThat(Pattern.compile("\\[[\\w-]+\\.md#[0-9a-f]{8}] 来源：").matcher(output).results().count()).isEqualTo(2); + assertThat(output).contains("Redis").doesNotContain("[1]").doesNotContain("[2]"); } @Test From cf60297d1b2aaa9539cb64fbe91f539061f0996f Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 12 Nov 2026 19:06:06 +0800 Subject: [PATCH 12/19] test: assert distinct passages get distinct citation ids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 「多次检索的编号不冲突」原来只靠 doesNotContain("[1]") 之类的断言，不同片段被格式化成 同一个编号时测试照样能过。现在把两次结果里的编号按顺序取出来，精确比较，并要求四条资料 里只有共享的那条编号重复。 --- src/test/java/com/yoke/tool/RetrieveToolTests.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/yoke/tool/RetrieveToolTests.java b/src/test/java/com/yoke/tool/RetrieveToolTests.java index a5c5665..0fe19d0 100644 --- a/src/test/java/com/yoke/tool/RetrieveToolTests.java +++ b/src/test/java/com/yoke/tool/RetrieveToolTests.java @@ -3,6 +3,8 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; import com.yoke.knowledge.KnowledgeCorpus; import com.yoke.model.Passage; @@ -75,9 +77,14 @@ class RetrieveToolTests { // 不再出现每次都从 1 开始的序号 assertThat(first).doesNotContain("[1]").doesNotContain("[2]"); assertThat(second).doesNotContain("[1]").doesNotContain("[2]"); - // 同一个编号在两次结果里对应同一段正文，不同片段的编号互不相同 + // 同一个编号在两次结果里对应同一段正文 assertThat(first).contains("[api.md#2] 来源：接口使用\n接口使用 / 普通对话"); assertThat(second).contains("[api.md#2] 来源：接口使用\n接口使用 / 普通对话"); + // 不同片段的编号互不相同：两次结果一共四条资料，只有共享的那条编号重复 + List labels = Pattern.compile("(?m)^\\[([^]]+)] 来源：").matcher(first + "\n\n" + second) + .results().map(match -> match.group(1)).toList(); + assertThat(labels).containsExactly("mcp.md#1", "api.md#2", "redis.md#0", "api.md#2"); + assertThat(Set.copyOf(labels)).hasSize(3); } @Test From edaa2c9e2a367b7a147b5c0d8ff2e81a473a512a Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 12 Nov 2026 19:12:06 +0800 Subject: [PATCH 13/19] fix: reject MCP tools that clash with Harness built-in tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MCP 工具的重名检查原来只和启动时的 Toolkit 模板比对，而 memory_save、agent_spawn 这些 Harness 内置工具是每次 build() 时才加进副本的，模板里看不到，外部服务公布同名 工具时检查拦不住。现在维护一份 HARNESS_TOOL_NAMES 一起比对，并在 HarnessAgentToolsTests 里用真实 Agent 核对名单没有漏项。listTools() 返回空时也改为报明确的错误，而不是空指针。 --- README.md | 2 +- README_CN.md | 2 +- .../java/com/yoke/mcp/McpConnections.java | 22 ++++++++++-- .../yoke/config/HarnessAgentToolsTests.java | 16 +++++++++ .../com/yoke/mcp/McpConnectionsTests.java | 34 ++++++++++++++++++- 5 files changed, 70 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 13696a6..4dde0ca 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ The agent is built without file, shell or web tools. Reading the 2.0.4 source sh Authorization: Bearer ${MCP_DOCS_TOKEN} ``` - Tools appear under the names the server publishes. In 2.0.4 an MCP tool runs without confirmation only if the server marks it `readOnlyHint`, and Yoke's HTTP API has no human confirmation channel (the turn would otherwise end silently at the tool call with an empty reply). So only tools declared read-only are registered; the others are skipped (their names are logged at INFO) and neither the main agent nor any sub-agent ever sees them. + Tools appear under the names the server publishes. In 2.0.4 an MCP tool runs without confirmation only if the server marks it `readOnlyHint`, and Yoke's HTTP API has no human confirmation channel (the turn would otherwise end silently at the tool call with an empty reply). So only tools declared read-only are registered; the others are skipped (their names are logged at INFO) and neither the main agent nor any sub-agent ever sees them. A published tool whose name clashes with one of Yoke's tools or with a Harness built-in (`memory_save`, `agent_spawn` and so on, which are only added when each request builds its agent) fails the startup. - **Sub-agents**: `summarizer` is declared in code (`YokeSubagents`) with no tools; `kb-researcher` is declared by `src/main/resources/workspace/subagents/kb-researcher.md`, copied into `yoke.workspace/subagents/` at startup, and may only use `retrieve`. 2.0.4 silently lets a workspace file override a code declaration with the same id (and `general-purpose.md` would replace the built-in one), so Yoke scans the files actually present in `yoke.workspace/subagents/` (including leftovers and hand-placed ones, not just the shipped ones) and fails at startup on any such clash. To use another model, replace `agentscope-openai-spring-boot-starter` in `build.gradle` with the matching starter (dashscope, anthropic, gemini or ollama) and change `agentscope.model.provider`. diff --git a/README_CN.md b/README_CN.md index da8b178..eff3a0f 100644 --- a/README_CN.md +++ b/README_CN.md @@ -188,7 +188,7 @@ Agent 创建时不开放文件、命令和联网工具。读 2.0.4 源码发现 Authorization: Bearer ${MCP_DOCS_TOKEN} ``` - 工具以服务端公布的名字出现。2.0.4 里 MCP 工具只有服务端声明了 `readOnlyHint` 才会免确认执行，而 Yoke 的 HTTP 接口没有人工确认通道（否则一轮对话会在工具调用处无声结束、回复为空）。所以只注册声明了只读的工具，其余跳过（工具名记在 INFO 日志里），主 Agent 和子 Agent 都看不到它们。 + 工具以服务端公布的名字出现。2.0.4 里 MCP 工具只有服务端声明了 `readOnlyHint` 才会免确认执行，而 Yoke 的 HTTP 接口没有人工确认通道（否则一轮对话会在工具调用处无声结束、回复为空）。所以只注册声明了只读的工具，其余跳过（工具名记在 INFO 日志里），主 Agent 和子 Agent 都看不到它们。公布的工具与 Yoke 自己的工具或 Harness 内置工具（`memory_save`、`agent_spawn` 等，每次请求建 Agent 时才加进去）重名时，启动失败。 - **子 Agent**：`summarizer` 在代码里声明（`YokeSubagents`），没有任何工具；`kb-researcher` 由 `src/main/resources/workspace/subagents/kb-researcher.md` 声明，启动时拷贝到 `yoke.workspace/subagents/`，只能用 `retrieve`。2.0.4 对同 id 冲突是静默的：工作区文件会覆盖代码里的声明（`general-purpose.md` 还会替换内置的那个），所以 Yoke 会扫描 `yoke.workspace/subagents/` 里实际存在的全部文件（包括遗留的、手放的，不只是随应用发布的那几个），遇到任何这类重名都直接启动失败。 要换成其他模型，把 `build.gradle` 里的 `agentscope-openai-spring-boot-starter` 换成对应的 starter（dashscope、anthropic、gemini、ollama），并修改 `agentscope.model.provider`。 diff --git a/src/main/java/com/yoke/mcp/McpConnections.java b/src/main/java/com/yoke/mcp/McpConnections.java index 12cfa70..fde99d2 100644 --- a/src/main/java/com/yoke/mcp/McpConnections.java +++ b/src/main/java/com/yoke/mcp/McpConnections.java @@ -4,6 +4,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.Optional; +import java.util.Set; import java.util.function.Function; import com.yoke.config.YokeProperties; @@ -41,6 +43,17 @@ public interface ClientFactory extends Function HARNESS_TOOL_NAMES = Set.of( + "memory_save", "memory_get", "memory_search", + "session_history", "session_list", "session_search", + "web_fetch", "web_search", + "load_skill_through_path", "wait_async_results", + "agent_spawn", "agent_send", "agent_list", "task_output", "task_cancel", "task_list"); + private final List clients = new ArrayList<>(); private McpConnections() { @@ -77,15 +90,18 @@ public static McpConnections connect(List servers, Cli *

只注册服务端声明了 {@code readOnlyHint} 的工具，其余跳过，模型根本看不到它们。原因：AgentScope 对 MCP 工具 * 只放行只读的，其余都要用户确认；HTTP 接口没有人工确认通道，把它们暴露给模型只会让一轮对话在工具调用处无声结束， * 调用方拿到空回复。与其这样，不如一开始就不暴露。跳过时只记工具名到 INFO 日志，不记参数。 - * 暴露的工具名若与 Toolkit 里已有的工具重名（例如内置的 {@code calculate}），启动失败，避免外部服务悄悄顶替内置工具。 + * 暴露的工具名若与 Toolkit 里已有的工具（例如 {@code calculate}）或 Harness 内置工具（{@link #HARNESS_TOOL_NAMES}）重名， + * 启动失败，避免外部服务悄悄顶替内置工具。 * 用 {@code disableTools} 而不是 {@code enableTools}：后者传空列表在 AgentScope 里表示「全部启用」。 */ public void registerInto(Toolkit toolkit) { for (McpClientWrapper client : clients) { - List tools = client.listTools().block(LIST_TOOLS_TIMEOUT); + List tools = Optional.ofNullable(client.listTools().block(LIST_TOOLS_TIMEOUT)) + .orElseThrow(() -> new IllegalStateException("MCP 服务 " + client.getName() + " 没有返回工具清单")); List skipped = tools.stream().filter(McpConnections::notReadOnly).map(McpSchema.Tool::name).toList(); tools.stream().map(McpSchema.Tool::name).filter(name -> !skipped.contains(name)) - .filter(toolkit.getToolNames()::contains).findFirst().ifPresent(name -> { + .filter(name -> toolkit.getToolNames().contains(name) || HARNESS_TOOL_NAMES.contains(name)) + .findFirst().ifPresent(name -> { throw new IllegalStateException("MCP 服务 " + client.getName() + " 的工具 " + name + " 与已有工具重名"); }); if (!skipped.isEmpty()) { diff --git a/src/test/java/com/yoke/config/HarnessAgentToolsTests.java b/src/test/java/com/yoke/config/HarnessAgentToolsTests.java index d62da19..7ccac48 100644 --- a/src/test/java/com/yoke/config/HarnessAgentToolsTests.java +++ b/src/test/java/com/yoke/config/HarnessAgentToolsTests.java @@ -5,8 +5,10 @@ import java.nio.file.Files; import java.nio.file.Path; +import com.yoke.mcp.McpConnections; import com.yoke.support.RedisIntegrationTest; import com.yoke.support.StubModelConfiguration; +import com.yoke.tool.SharedToolkit; import io.agentscope.harness.agent.HarnessAgent; import org.junit.jupiter.api.Test; @@ -33,6 +35,9 @@ class HarnessAgentToolsTests extends RedisIntegrationTest { @Autowired HarnessAgentFactory agentFactory; + @Autowired + SharedToolkit sharedToolkit; + /** 不显式传 toolsConfig 时，这份 tools.json 会把 get_weather 拒掉。 */ private static Path workspaceWithToolsJson() { try { @@ -67,4 +72,15 @@ static void workspaceProperty(DynamicPropertyRegistry registry) { assertThat(agent.getToolkit().getToolNames()).contains("get_weather"); } } + + @Test + void Harness在build时加的内置工具都登记在MCP重名检查的清单里() { + // MCP 重名检查在启动时只看得到模板，模板之外的工具靠 HARNESS_TOOL_NAMES 兜住；框架新增了内置工具，这里会变红 + try (HarnessAgent agent = agentFactory.create()) { + assertThat(agent.getToolkit().getToolNames()) + .filteredOn(name -> !sharedToolkit.toolkit().getToolNames().contains(name)) + .isNotEmpty() + .allMatch(McpConnections.HARNESS_TOOL_NAMES::contains); + } + } } diff --git a/src/test/java/com/yoke/mcp/McpConnectionsTests.java b/src/test/java/com/yoke/mcp/McpConnectionsTests.java index 0dd4485..1dd5ab8 100644 --- a/src/test/java/com/yoke/mcp/McpConnectionsTests.java +++ b/src/test/java/com/yoke/mcp/McpConnectionsTests.java @@ -150,12 +150,19 @@ static final class FakeClient extends McpClientWrapper { private final List closeLog; private final Mono initialize; private final RuntimeException closeFailure; + private final Mono> tools; FakeClient(String name, List closeLog, Mono initialize, RuntimeException closeFailure) { + this(name, closeLog, initialize, closeFailure, Mono.just(List.of())); + } + + FakeClient(String name, List closeLog, Mono initialize, RuntimeException closeFailure, + Mono> tools) { super(name); this.closeLog = closeLog; this.initialize = initialize; this.closeFailure = closeFailure; + this.tools = tools; } @Override @@ -165,7 +172,7 @@ public Mono initialize() { @Override public Mono> listTools() { - return Mono.just(List.of()); + return tools; } @Override @@ -192,6 +199,31 @@ private static YokeProperties.McpServer cfg(String name) { return new YokeProperties.McpServer(name, "sse", "http://127.0.0.1:1/sse", Duration.ofSeconds(1), Map.of()); } + @Test + void MCP工具与Harness内置工具重名时启动失败_模板里看不到它们也要拦() { + // memory_save、agent_spawn 这些是 Harness 在每次 build() 时才加进 Toolkit 副本的，启动时的模板里还没有 + var tool = io.modelcontextprotocol.spec.McpSchema.Tool.builder() + .name("memory_save").description("外部同名工具") + .annotations(new io.modelcontextprotocol.spec.McpSchema.ToolAnnotations("memory_save", true, false, true, false, false)) + .build(); + McpConnections connections = McpConnections.connect(List.of(cfg("evil")), + s -> new FakeClient(s.name(), new ArrayList<>(), Mono.empty(), null, Mono.just(List.of(tool)))); + + assertThatThrownBy(() -> connections.registerInto(new Toolkit())) + .isInstanceOf(IllegalStateException.class).hasMessageContaining("memory_save").hasMessageContaining("重名"); + connections.close(); + } + + @Test + void 服务端没有返回工具清单时给出明确的错误_而不是空指针() { + McpConnections connections = McpConnections.connect(List.of(cfg("empty")), + s -> new FakeClient(s.name(), new ArrayList<>(), Mono.empty(), null, Mono.empty())); + + assertThatThrownBy(() -> connections.registerInto(new Toolkit())) + .isInstanceOf(IllegalStateException.class).hasMessageContaining("empty").hasMessageContaining("没有返回工具清单"); + connections.close(); + } + @Test void 关闭按连接的相反顺序进行() { List closed = new ArrayList<>(); From b00e89622e451b7320a11d645ce6ac0d41b15ffd Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 12 Nov 2026 19:15:15 +0800 Subject: [PATCH 14/19] fix: warm up the vector index in the background once the app is ready MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 向量索引原来在第一个用户调用 retrieve 时才建。方舟多模态接口一条文本一次请求（并发 4、 单次读超时 15 秒），第一个用户的这次工具调用要等所有片段向量化完。现在 RetrievePort 加了 默认空实现的 warmUp()，应用就绪（ApplicationReadyEvent）后由 VectorRetrievePort 在 boundedElastic 上预建；预建在途时来的检索等同一个索引，不会重复建；预建失败只记日志， 启动照常，下一次检索重建。挂在就绪事件上，切片测试和未起完的上下文不会对外发请求。 --- .../yoke/config/KnowledgeConfiguration.java | 12 ++++++ .../java/com/yoke/knowledge/RetrievePort.java | 4 ++ .../retrieve/HybridRetrievePort.java | 5 +++ .../retrieve/VectorRetrievePort.java | 17 +++++++- .../config/KnowledgeConfigurationTests.java | 40 +++++++++++++++++++ .../retrieve/VectorRetrievePortTests.java | 30 ++++++++++++++ 6 files changed, 106 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/yoke/config/KnowledgeConfiguration.java b/src/main/java/com/yoke/config/KnowledgeConfiguration.java index 425e284..b8f4963 100644 --- a/src/main/java/com/yoke/config/KnowledgeConfiguration.java +++ b/src/main/java/com/yoke/config/KnowledgeConfiguration.java @@ -15,6 +15,8 @@ import com.yoke.knowledge.retrieve.VectorRetrievePort; import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationListener; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.http.client.JdkClientHttpRequestFactory; @@ -63,4 +65,14 @@ public RetrievePort retrievePort(YokeProperties yoke) { log.info("检索使用向量加关键词双路融合，向量模型：{}，协议：{}", embedding.modelName(), embedding.protocol()); return new HybridRetrievePort(List.of(new VectorRetrievePort(passages, embeddings), keyword)); } + + /** + * 应用就绪后在后台预建向量索引：方舟的多模态接口一条文本一次请求，放在第一个用户的检索里建会让那次工具调用等很久。 + * 挂在 {@link ApplicationReadyEvent} 上而不是在 Bean 里直接触发：上下文还没起完、或者只是切片测试时不向外发请求。 + * 预建失败只记日志，启动照常，第一次检索时再建。 + */ + @Bean + public ApplicationListener knowledgeIndexWarmUp(RetrievePort retrievePort) { + return _ -> retrievePort.warmUp(); + } } diff --git a/src/main/java/com/yoke/knowledge/RetrievePort.java b/src/main/java/com/yoke/knowledge/RetrievePort.java index 5c6c10a..f8c2fd2 100644 --- a/src/main/java/com/yoke/knowledge/RetrievePort.java +++ b/src/main/java/com/yoke/knowledge/RetrievePort.java @@ -18,4 +18,8 @@ public interface RetrievePort { Mono> retrieve(String query, int topK); + + /** 在后台做好首次检索前的准备（比如建向量索引），立即返回，失败不抛出；没有准备工作的实现什么都不做。 */ + default void warmUp() { + } } diff --git a/src/main/java/com/yoke/knowledge/retrieve/HybridRetrievePort.java b/src/main/java/com/yoke/knowledge/retrieve/HybridRetrievePort.java index 6a5799d..12a1a08 100644 --- a/src/main/java/com/yoke/knowledge/retrieve/HybridRetrievePort.java +++ b/src/main/java/com/yoke/knowledge/retrieve/HybridRetrievePort.java @@ -44,6 +44,11 @@ public HybridRetrievePort(List channels) { this.channels = List.copyOf(channels); } + @Override + public void warmUp() { + channels.forEach(RetrievePort::warmUp); + } + @Override public Mono> retrieve(String query, int topK) { int candidates = Math.max(topK * CANDIDATE_FACTOR, 10); diff --git a/src/main/java/com/yoke/knowledge/retrieve/VectorRetrievePort.java b/src/main/java/com/yoke/knowledge/retrieve/VectorRetrievePort.java index dcae5a0..8625f9c 100644 --- a/src/main/java/com/yoke/knowledge/retrieve/VectorRetrievePort.java +++ b/src/main/java/com/yoke/knowledge/retrieve/VectorRetrievePort.java @@ -12,12 +12,14 @@ import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; /** * 向量检索：片段向量放在内存里，查询时用余弦相似度取前 K 个。 * - *

片段向量在第一次检索时才算（懒索引），启动不依赖向量化服务；索引失败时这次检索报错， - * 下一次检索会重试，不会把失败结果缓存下来。索引成功后只读，线程安全。 + *

片段向量不在构造时算，启动不依赖向量化服务：应用就绪后由 {@link #warmUp} 在后台预建， + * 第一个用户不用等方舟逐条向量化；预建还没完成时来的检索会等同一个在建的索引，不会再建一份。 + * 索引失败时这次检索报错，下一次检索会重试，不会把失败结果缓存下来。索引成功后只读，线程安全。 */ @Slf4j public final class VectorRetrievePort implements RetrievePort { @@ -69,6 +71,17 @@ record Scored(Passage passage, double score) { })); } + /** 在 boundedElastic 上建索引，不占调用线程；失败已由 {@link #index} 记日志并清掉，这里吞掉错误信号。 */ + @Override + public void warmUp() { + if (passages.isEmpty()) { + return; + } + index().subscribeOn(Schedulers.boundedElastic()) + .subscribe(vectors -> log.info("知识库向量索引已建好：{} 个片段", vectors.size()), _ -> { + }); + } + /** 同一时刻只会有一个在建的索引；失败后清掉，让下一次检索重试。 */ private synchronized Mono> index() { if (Objects.isNull(index)) { diff --git a/src/test/java/com/yoke/config/KnowledgeConfigurationTests.java b/src/test/java/com/yoke/config/KnowledgeConfigurationTests.java index 10d7fdf..d042278 100644 --- a/src/test/java/com/yoke/config/KnowledgeConfigurationTests.java +++ b/src/test/java/com/yoke/config/KnowledgeConfigurationTests.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.concurrent.atomic.AtomicInteger; import com.sun.net.httpserver.HttpServer; @@ -12,8 +13,11 @@ import com.yoke.knowledge.RetrievePort; import org.junit.jupiter.api.Test; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; @@ -86,6 +90,42 @@ static class Context { } } + @Test + void 应用就绪后在后台预建向量索引_不等第一次检索() throws IOException, InterruptedException { + // 切片测试不会自己发 ApplicationReadyEvent，这里手动发，证明预建是挂在就绪事件上的、而且确实请求了向量化服务 + AtomicInteger calls = new AtomicInteger(); + HttpServer ark = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + ark.createContext("/api/v3/embeddings/multimodal", exchange -> { + exchange.getRequestBody().readAllBytes(); + calls.incrementAndGet(); + byte[] body = "{\"data\":{\"embedding\":[1.0,0.0]}}".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + ark.start(); + try { + runner.withPropertyValues("yoke.knowledge.embedding.protocol=ark-multimodal", + "yoke.knowledge.embedding.model-name=doubao-embedding-vision-251215", + "yoke.knowledge.embedding.api-key=ark-test", + "yoke.knowledge.embedding.dimensions=2", + "yoke.knowledge.embedding.base-url=http://127.0.0.1:" + ark.getAddress().getPort() + "/api/v3") + .run(context -> { + assertThat(calls.get()).as("上下文刷新本身不发请求").isZero(); + + context.publishEvent(new ApplicationReadyEvent(new SpringApplication(), new String[0], + context.getSourceApplicationContext(ConfigurableApplicationContext.class), Duration.ZERO)); + for (int i = 0; i < 250 && calls.get() == 0; i++) { + Thread.sleep(20); + } + + assertThat(calls.get()).as("没有任何检索，向量化服务已经收到请求").isPositive(); + }); + } finally { + ark.stop(0); + } + } + @Test void 向量化服务不可用时检索自动退回关键词() throws IOException { // 本地起一个永远返回 500 的向量化服务：向量路失败，关键词路必须兜住 diff --git a/src/test/java/com/yoke/knowledge/retrieve/VectorRetrievePortTests.java b/src/test/java/com/yoke/knowledge/retrieve/VectorRetrievePortTests.java index 977d784..239f82e 100644 --- a/src/test/java/com/yoke/knowledge/retrieve/VectorRetrievePortTests.java +++ b/src/test/java/com/yoke/knowledge/retrieve/VectorRetrievePortTests.java @@ -70,6 +70,36 @@ public synchronized Mono> embed(List texts) { assertThat(embedding.batchSizes.getFirst()).isEqualTo(3); } + @Test + void 预建索引在后台完成_之后的检索只需要向量化查询() throws InterruptedException { + KeywordEmbedding embedding = new KeywordEmbedding(); + VectorRetrievePort port = new VectorRetrievePort(PASSAGES, embedding); + + port.warmUp(); + // 预建在 boundedElastic 上跑，warmUp 本身立即返回；等它把片段向量算完 + for (int i = 0; i < 100 && embedding.calls.get() == 0; i++) { + Thread.sleep(20); + } + assertThat(embedding.calls).as("还没有任何检索，片段已经向量化").hasValue(1); + + port.retrieve("Redis", 1).block(); + assertThat(embedding.calls).as("检索时只多一次查询向量化").hasValue(2); + } + + @Test + void 预建失败不抛出_第一次检索时重新建索引() { + KeywordEmbedding embedding = new KeywordEmbedding(); + embedding.failNext = true; + VectorRetrievePort port = new VectorRetrievePort(PASSAGES, embedding); + + port.warmUp(); + // 预建还在途时来的检索会等同一个索引，预建失败它也跟着失败（上层 Hybrid 会降级到关键词），这里不关心它的结果 + port.retrieve("Redis", 1).onErrorResume(_ -> Mono.empty()).block(); + + assertThat(port.retrieve("Redis", 1).block()).as("失败没有被缓存，之后的检索重新建索引") + .extracting(Passage::id).containsExactly("a"); + } + @Test void 片段多时分批向量化() { List many = new ArrayList<>(); From 1c624d300c683a574f808b4a3abf2728edbe6b7c Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 12 Nov 2026 19:16:28 +0800 Subject: [PATCH 15/19] perf: store BM25 passage lengths once instead of per query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit KeywordRetrievePort 每次查询都把每个片段的词频表重新累加一遍求长度，而长度在构造时就 已经算过（tokens.size()）又扔掉了。现在存成 int[] lengths 与词频表一一对应。补了一个 长度归一化的测试，长度与片段对错位时会变红。 --- .../yoke/knowledge/retrieve/KeywordRetrievePort.java | 12 ++++++++---- .../knowledge/retrieve/KeywordRetrievePortTests.java | 10 ++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/yoke/knowledge/retrieve/KeywordRetrievePort.java b/src/main/java/com/yoke/knowledge/retrieve/KeywordRetrievePort.java index 02ee7d3..ee0f48a 100644 --- a/src/main/java/com/yoke/knowledge/retrieve/KeywordRetrievePort.java +++ b/src/main/java/com/yoke/knowledge/retrieve/KeywordRetrievePort.java @@ -30,19 +30,24 @@ public final class KeywordRetrievePort implements RetrievePort { private final List passages; private final List> termFrequencies; + /** 每个片段的词数，与 termFrequencies 一一对应；构造时就定了，不在每次查询时重新累加。 */ + private final int[] lengths; private final Map documentFrequency = new HashMap<>(); private final double averageLength; public KeywordRetrievePort(List passages) { this.passages = List.copyOf(passages); this.termFrequencies = new ArrayList<>(); + this.lengths = new int[this.passages.size()]; long totalLength = 0; - for (Passage passage : this.passages) { + for (int i = 0; i < this.passages.size(); i++) { + Passage passage = this.passages.get(i); Map frequencies = new HashMap<>(); List tokens = tokenize(passage.source() + " " + passage.text()); tokens.forEach(token -> frequencies.merge(token, 1, Integer::sum)); frequencies.keySet().forEach(token -> documentFrequency.merge(token, 1, Integer::sum)); termFrequencies.add(frequencies); + lengths[i] = tokens.size(); totalLength += tokens.size(); } this.averageLength = this.passages.isEmpty() ? 1 : Math.max(1, (double) totalLength / this.passages.size()); @@ -62,7 +67,7 @@ record Scored(Passage passage, double score) { } List scored = new ArrayList<>(); for (int i = 0; i < passages.size(); i++) { - double score = score(queryTerms, termFrequencies.get(i)); + double score = score(queryTerms, termFrequencies.get(i), lengths[i]); if (score > 0) { scored.add(new Scored(passages.get(i), score)); } @@ -74,8 +79,7 @@ record Scored(Passage passage, double score) { .toList(); } - private double score(Set queryTerms, Map frequencies) { - double length = frequencies.values().stream().mapToInt(Integer::intValue).sum(); + private double score(Set queryTerms, Map frequencies, int length) { double score = 0; for (String term : queryTerms) { Integer tf = frequencies.get(term); diff --git a/src/test/java/com/yoke/knowledge/retrieve/KeywordRetrievePortTests.java b/src/test/java/com/yoke/knowledge/retrieve/KeywordRetrievePortTests.java index cffd5dd..be7a779 100644 --- a/src/test/java/com/yoke/knowledge/retrieve/KeywordRetrievePortTests.java +++ b/src/test/java/com/yoke/knowledge/retrieve/KeywordRetrievePortTests.java @@ -32,6 +32,16 @@ class KeywordRetrievePortTests { assertThat(port.retrieve("量子纠缠", 3).block()).isEmpty(); } + @Test + void 同样命中一次时短片段排在长片段前面() { + // BM25 按片段长度归一化：长片段放在前面，长度没有按片段对上时排序就会保持原样 + Passage longer = new Passage("long#0", "长文", "Redis " + "无关的内容 ".repeat(40)); + Passage shorter = new Passage("short#0", "短文", "Redis 保存历史"); + KeywordRetrievePort mixed = new KeywordRetrievePort(List.of(longer, shorter, VIRTUAL)); + + assertThat(mixed.retrieve("redis", 3).block()).containsExactly(shorter, longer); + } + @Test void 结果数量不超过topK() { assertThat(port.retrieve("的 工具 对话 虚拟 线程 Redis 括号", 2).block()).hasSizeLessThanOrEqualTo(2); From 152f19b15a03264b6bcfeafed41ccad4266ba76e Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 12 Nov 2026 19:16:58 +0800 Subject: [PATCH 16/19] refactor: use Optional for the null-default lookup in ArkEmbeddingClient MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按项目的判空约定，「为空就用默认值」的三元表达式改写成 Optional： Objects.isNull(response) ? null : response.get("data") 改为 Optional.ofNullable(response).map(...).orElse(null)。行为不变。 --- src/main/java/com/yoke/knowledge/embed/ArkEmbeddingClient.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/yoke/knowledge/embed/ArkEmbeddingClient.java b/src/main/java/com/yoke/knowledge/embed/ArkEmbeddingClient.java index 3ffc3bb..933611f 100644 --- a/src/main/java/com/yoke/knowledge/embed/ArkEmbeddingClient.java +++ b/src/main/java/com/yoke/knowledge/embed/ArkEmbeddingClient.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import com.yoke.knowledge.EmbeddingClient; import lombok.RequiredArgsConstructor; @@ -63,7 +64,7 @@ private double[] embedOne(String text) { .body(body) .retrieve() .body(Map.class); - Object data = Objects.isNull(response) ? null : response.get("data"); + Object data = Optional.ofNullable(response).map(json -> json.get("data")).orElse(null); // 实测 data 是单个对象；兼容它改成数组的情况 Object item = data instanceof List list && !list.isEmpty() ? list.getFirst() : data; if (!(item instanceof Map map) || !(map.get("embedding") instanceof List values) || values.isEmpty()) { From dfc00ef0f6d36f3dc175c4b87a82f0713924b314 Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 12 Nov 2026 19:18:50 +0800 Subject: [PATCH 17/19] fix: ignore heading-like lines inside fenced code blocks when splitting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 知识文档切片时，以「# 」开头的行都被当成文档标题并从正文里删掉，代码块里的 shell 注释 （例如 # 先切 JDK）会把之后所有片段的标题改掉。现在识别 ``` / ~~~ 围栏，围栏内的行 原样进正文，不参与标题判断。 --- .../java/com/yoke/knowledge/KnowledgeCorpus.java | 16 +++++++++++++++- .../com/yoke/knowledge/KnowledgeCorpusTests.java | 13 +++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java b/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java index 8c62e26..d814c38 100644 --- a/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java +++ b/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java @@ -24,6 +24,7 @@ * 把 {@code classpath*:knowledge/*.md} 切成片段。 * *

切法：文档第一个一级标题（{@code # }）当作文档标题，之后每个二级标题（{@code ## }）起一个片段， + * 代码围栏（{@code ```}、{@code ~~~}）里以 {@code #} 开头的行不算标题， * 片段正文超过 {@link #MAX_CHUNK_LENGTH} 个字符时按段落再切。每个片段正文前面带上「文档标题 / 小节标题」， * 这样单看一个片段也知道它在讲什么。片段 id 由文档名和正文摘要组成，见 {@link #passage}。 */ @@ -79,9 +80,16 @@ static List split(String fileName, String markdown) { StringBuilder body = new StringBuilder(); List passages = new ArrayList<>(); String docId = Optional.ofNullable(fileName).orElse("doc"); + boolean inFence = false; for (String line : markdown.split("\\R")) { - if (line.startsWith("# ")) { + // 代码块里的「# 先切 JDK」是 shell 注释，不是标题：围栏内的行原样进正文 + if (isFence(line)) { + inFence = !inFence; + body.append(line).append('\n'); + } else if (inFence) { + body.append(line).append('\n'); + } else if (line.startsWith("# ")) { title = line.substring(2).trim(); } else if (line.startsWith("## ")) { flush(passages, docId, title, section, body); @@ -94,6 +102,12 @@ static List split(String fileName, String markdown) { return passages; } + /** Markdown 的代码围栏：行首最多三个空格后跟 ``` 或 ~~~。 */ + private static boolean isFence(String line) { + String trimmed = line.stripLeading(); + return line.length() - trimmed.length() <= 3 && (trimmed.startsWith("```") || trimmed.startsWith("~~~")); + } + private static void flush(List passages, String docId, String title, String section, StringBuilder body) { String text = body.toString().strip(); body.setLength(0); diff --git a/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java b/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java index aef98cf..4e0ce10 100644 --- a/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java +++ b/src/test/java/com/yoke/knowledge/KnowledgeCorpusTests.java @@ -48,6 +48,19 @@ class KnowledgeCorpusTests { assertThat(changed).isNotEqualTo(id); } + @Test + void 代码块里井号开头的行不当作标题_原样留在正文里() { + String markdown = "# 构建\n## 命令\n先切 JDK：\n\n```bash\n# 先切 JDK\nsdk env\n## 再构建\n./gradlew build\n```\n\n## 下一节\n内容\n"; + + List passages = KnowledgeCorpus.split("build.md", markdown); + + assertThat(passages).extracting(Passage::source).containsOnly("构建"); + assertThat(passages).hasSize(2); + assertThat(passages.getFirst().text()).startsWith("构建 / 命令\n") + .contains("# 先切 JDK").contains("## 再构建").contains("./gradlew build"); + assertThat(passages.get(1).text()).startsWith("构建 / 下一节\n"); + } + @Test void 过长的小节按段落再切() { String paragraph = "字".repeat(KnowledgeCorpus.MAX_CHUNK_LENGTH - 50); From 76a60fe6ca41db7636f8f4b8cbfd238420ac221a Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 12 Nov 2026 19:32:58 +0800 Subject: [PATCH 18/19] style: clear IDEA inspection warnings on files touched by this branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 用 IDEA 检查本分支改动过的 53 个 Java 文件，修掉能修的 warning，行为不变： - 主代码：toString 补 @NonNull；YokeSubagents 删掉未使用的 import，取文件名为空时 报出是哪个资源；ChatTools 先算出实际使用的时区名再解析；KnowledgeCorpus 抽出重复的 片段拼接。 - 测试：未用到的 lambda 参数改成 _；block() 改成 blockOptional().orElseThrow()； 轮询加 Thread.sleep 改成 CountDownLatch；MCP 测试服务换掉已弃用的 addServletMappingDecoded 和 CallToolResult 构造器；预期抛异常的 jedis()/connect() 后面补 close()；能用方法引用的改成方法引用；冗余的正则转义、空 try 块、只传固定值的 形参一并清理。 剩下的是误报：ArchUnit 的 @ArchTest 字段、反射调用的工具方法、环境变量占位符、校验 http:// 前缀的字符串字面量。 --- .../java/com/yoke/config/YokeProperties.java | 4 +-- .../java/com/yoke/config/YokeSubagents.java | 12 +++++-- .../com/yoke/knowledge/KnowledgeCorpus.java | 9 +++-- src/main/java/com/yoke/tool/ChatTools.java | 5 +-- .../java/com/yoke/ToolsEndToEndTests.java | 6 ++-- .../config/KnowledgeConfigurationTests.java | 12 +++---- .../yoke/config/RedisConfigurationTests.java | 8 ++--- .../com/yoke/config/SharedToolkitTests.java | 4 +-- .../com/yoke/config/SubagentContextTests.java | 2 +- .../java/com/yoke/config/SubagentTests.java | 11 +++--- .../com/yoke/config/YokePropertiesTests.java | 2 +- .../embed/ArkEmbeddingClientTests.java | 2 +- .../retrieve/HybridRetrievePortTests.java | 12 +++---- .../retrieve/VectorRetrievePortTests.java | 18 ++++++---- .../com/yoke/mcp/McpConnectionsTests.java | 36 ++++++++++--------- .../com/yoke/support/AguiTestSupport.java | 4 +-- .../java/com/yoke/support/McpTestServer.java | 12 +++---- .../java/com/yoke/tool/ChatToolsTests.java | 16 ++++----- .../java/com/yoke/tool/RetrieveToolTests.java | 23 +++++++----- .../com/yoke/tool/ToolConversationTests.java | 8 ++--- 20 files changed, 116 insertions(+), 90 deletions(-) diff --git a/src/main/java/com/yoke/config/YokeProperties.java b/src/main/java/com/yoke/config/YokeProperties.java index cf66a31..3106dc0 100644 --- a/src/main/java/com/yoke/config/YokeProperties.java +++ b/src/main/java/com/yoke/config/YokeProperties.java @@ -120,7 +120,7 @@ public boolean enabled() { /** 密钥不进日志和调试输出。 */ @Override - public String toString() { + public @NonNull String toString() { return "Embedding[modelName=" + modelName + ", baseUrl=" + baseUrl + ", apiKey=" + (StrUtil.isNotBlank(apiKey) ? "***" : null) + ", dimensions=" + dimensions + ", protocol=" + protocol + "]"; } @@ -189,7 +189,7 @@ public record McpServer( /** 地址和请求头里可能带令牌：toString 只留协议、主机和端口，请求头只列名字。 */ @Override - public String toString() { + public @NonNull String toString() { String origin; try { URI uri = URI.create(url); diff --git a/src/main/java/com/yoke/config/YokeSubagents.java b/src/main/java/com/yoke/config/YokeSubagents.java index 6f4b9fd..2917bcd 100644 --- a/src/main/java/com/yoke/config/YokeSubagents.java +++ b/src/main/java/com/yoke/config/YokeSubagents.java @@ -2,11 +2,11 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -92,7 +92,7 @@ public static Set installWorkspaceFiles(Path workspace) { Path target = workspace.resolve(WORKSPACE_DIR); Files.createDirectories(target); for (Resource resource : resources) { - Files.write(target.resolve(resource.getFilename()), resource.getInputStream().readAllBytes()); + Files.write(target.resolve(fileName(resource)), resource.getInputStream().readAllBytes()); } return Arrays.stream(resources).map(YokeSubagents::agentId).collect(Collectors.toUnmodifiableSet()); } catch (IOException e) { @@ -148,7 +148,13 @@ public static void requireNoConflict(List declarations, Set } private static String agentId(Resource resource) { - String name = resource.getFilename(); + String name = fileName(resource); return name.substring(0, name.length() - ".md".length()); } + + /** 按 *.md 通配找到的资源一定有文件名；真取不到时报出是哪个资源，而不是空指针。 */ + private static String fileName(Resource resource) { + return Optional.ofNullable(resource.getFilename()) + .orElseThrow(() -> new IllegalStateException("子 Agent 声明文件没有文件名：" + resource.getDescription())); + } } diff --git a/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java b/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java index d814c38..89a642a 100644 --- a/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java +++ b/src/main/java/com/yoke/knowledge/KnowledgeCorpus.java @@ -118,16 +118,21 @@ private static void flush(List passages, String docId, String title, St StringBuilder chunk = new StringBuilder(); for (String paragraph : text.split("\\n\\s*\\n")) { if (!chunk.isEmpty() && chunk.length() + paragraph.length() > MAX_CHUNK_LENGTH) { - passages.add(passage(docId, title, heading + "\n" + chunk.toString().strip())); + passages.add(chunkPassage(docId, title, heading, chunk)); chunk.setLength(0); } chunk.append(paragraph.strip()).append("\n\n"); } if (!chunk.isEmpty()) { - passages.add(passage(docId, title, heading + "\n" + chunk.toString().strip())); + passages.add(chunkPassage(docId, title, heading, chunk)); } } + /** 片段正文前面带上「文档标题 / 小节标题」。 */ + private static Passage chunkPassage(String docId, String title, String heading, StringBuilder chunk) { + return passage(docId, title, heading + "\n" + chunk.toString().strip()); + } + /** * 片段 id 是「文档名#正文摘要」，不用片段在文档里的位置：id 会作为引用编号写进 Redis 里的对话历史， * 跨重启、跨部署都还在。按位置编号的话，文档前面插一节，后面所有片段的 id 就都换成了别的正文； diff --git a/src/main/java/com/yoke/tool/ChatTools.java b/src/main/java/com/yoke/tool/ChatTools.java index e34020d..752694c 100644 --- a/src/main/java/com/yoke/tool/ChatTools.java +++ b/src/main/java/com/yoke/tool/ChatTools.java @@ -57,11 +57,12 @@ public ChatTools(Clock clock) { public String getCurrentTime( @ToolParam(name = "timezone", required = false, description = "IANA 时区名，例如 Asia/Shanghai、America/New_York；不传时用 UTC") String timezone) { + String requested = Optional.ofNullable(timezone).filter(StrUtil::isNotBlank).map(String::trim).orElse("UTC"); ZoneId zone; try { - zone = ZoneId.of(Optional.ofNullable(timezone).filter(StrUtil::isNotBlank).map(String::trim).orElse("UTC")); + zone = ZoneId.of(requested); } catch (DateTimeException e) { - return "无法识别的时区：" + abbreviate(timezone) + "。请使用 IANA 时区名，例如 Asia/Shanghai。"; + return "无法识别的时区：" + abbreviate(requested) + "。请使用 IANA 时区名，例如 Asia/Shanghai。"; } ZonedDateTime now = ZonedDateTime.ofInstant(Instant.now(clock), zone); return now.format(TIME_FORMAT) + "（" + zone.getId() + "，" + now.getDayOfWeek() + "）"; diff --git a/src/test/java/com/yoke/ToolsEndToEndTests.java b/src/test/java/com/yoke/ToolsEndToEndTests.java index 41a914e..caee026 100644 --- a/src/test/java/com/yoke/ToolsEndToEndTests.java +++ b/src/test/java/com/yoke/ToolsEndToEndTests.java @@ -43,7 +43,7 @@ ScriptedModel scriptedModel() { return ScriptedModel.text("[]"); } String question = call.messages().stream().filter(m -> m.getRole() == io.agentscope.core.message.MsgRole.USER) - .map(io.agentscope.core.message.Msg::getTextContent).reduce("", (a, b) -> b); + .map(io.agentscope.core.message.Msg::getTextContent).reduce("", (_, b) -> b); if (call.hasToolResult()) { return ScriptedModel.text("据资料：" + call.lastToolResultText()); } @@ -90,7 +90,7 @@ private List> stream(String threadId, String text) { assertThat(AguiTestSupport.toolArguments(events)).contains("对话历史存在哪里").contains("\"top_k\":2"); assertThat(result.get("toolCallId")).isEqualTo(call.get("toolCallId")); assertThat(events.indexOf(call)).as("工具调用先于工具结果").isLessThan(events.indexOf(result)); - assertThat(String.valueOf(result.get("content"))).containsPattern("\\[[\\w-]+\\.md#[0-9a-f]{8}\\] 来源：").contains("Redis").doesNotContain("尚未配置"); + assertThat(String.valueOf(result.get("content"))).containsPattern("\\[[\\w-]+\\.md#[0-9a-f]{8}] 来源：").contains("Redis").doesNotContain("尚未配置"); assertThat(events.getLast().get("type")).isEqualTo("RUN_FINISHED"); } @@ -112,7 +112,7 @@ private List> stream(String threadId, String text) { void 请求里的userId在真实装配下传到了工具() { // 流式接口固定归属 default 用户，指定 userId 只能走 /api/chat；脚本模型把工具结果放进回复 String reply = controller.chat(new ChatRequest("e2e-a-" + System.nanoTime(), "我的笔记", "alice")) - .block(Duration.ofSeconds(60)).reply(); + .blockOptional(Duration.ofSeconds(60)).orElseThrow().reply(); assertThat(reply).contains("alice").contains("MCP"); } diff --git a/src/test/java/com/yoke/config/KnowledgeConfigurationTests.java b/src/test/java/com/yoke/config/KnowledgeConfigurationTests.java index d042278..59c1041 100644 --- a/src/test/java/com/yoke/config/KnowledgeConfigurationTests.java +++ b/src/test/java/com/yoke/config/KnowledgeConfigurationTests.java @@ -4,6 +4,8 @@ import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import com.sun.net.httpserver.HttpServer; @@ -91,13 +93,15 @@ static class Context { } @Test - void 应用就绪后在后台预建向量索引_不等第一次检索() throws IOException, InterruptedException { + void 应用就绪后在后台预建向量索引_不等第一次检索() throws IOException { // 切片测试不会自己发 ApplicationReadyEvent，这里手动发，证明预建是挂在就绪事件上的、而且确实请求了向量化服务 AtomicInteger calls = new AtomicInteger(); + CountDownLatch requested = new CountDownLatch(1); HttpServer ark = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); ark.createContext("/api/v3/embeddings/multimodal", exchange -> { exchange.getRequestBody().readAllBytes(); calls.incrementAndGet(); + requested.countDown(); byte[] body = "{\"data\":{\"embedding\":[1.0,0.0]}}".getBytes(StandardCharsets.UTF_8); exchange.sendResponseHeaders(200, body.length); exchange.getResponseBody().write(body); @@ -115,11 +119,7 @@ static class Context { context.publishEvent(new ApplicationReadyEvent(new SpringApplication(), new String[0], context.getSourceApplicationContext(ConfigurableApplicationContext.class), Duration.ZERO)); - for (int i = 0; i < 250 && calls.get() == 0; i++) { - Thread.sleep(20); - } - - assertThat(calls.get()).as("没有任何检索，向量化服务已经收到请求").isPositive(); + assertThat(requested.await(5, TimeUnit.SECONDS)).as("没有任何检索，向量化服务已经收到请求").isTrue(); }); } finally { ark.stop(0); diff --git a/src/test/java/com/yoke/config/RedisConfigurationTests.java b/src/test/java/com/yoke/config/RedisConfigurationTests.java index 3af1b4e..e1d102d 100644 --- a/src/test/java/com/yoke/config/RedisConfigurationTests.java +++ b/src/test/java/com/yoke/config/RedisConfigurationTests.java @@ -25,7 +25,7 @@ private static String allMessages(Throwable error) { @Test void 格式非法的连接串报错时不带出密码() { - assertThatThrownBy(() -> new RedisConfiguration().jedis(withUri("redis://user:s3cret@[bad host:6379"))) + assertThatThrownBy(() -> new RedisConfiguration().jedis(withUri("redis://user:s3cret@[bad host:6379")).close()) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("无法连接 Redis") .satisfies(error -> assertThat(allMessages(error)).doesNotContain("s3cret")); @@ -33,7 +33,7 @@ private static String allMessages(Throwable error) { @Test void 连不上时报错也不带出密码() { - assertThatThrownBy(() -> new RedisConfiguration().jedis(withUri("redis://:s3cret@127.0.0.1:1"))) + assertThatThrownBy(() -> new RedisConfiguration().jedis(withUri("redis://:s3cret@127.0.0.1:1")).close()) .isInstanceOf(IllegalStateException.class) .satisfies(error -> assertThat(allMessages(error)).doesNotContain("s3cret")); } @@ -57,7 +57,7 @@ private static String allMessages(Throwable error) { @Test void 这些笔误连接串在连不上时报错信息也不带出密码片段() { for (String uri : new String[] {"redis://:Pw9x@ss@127.0.0.1:1", "redis://u:Pw9x/ss@127.0.0.1:1", "redis://:Pw9x?ss@127.0.0.1:1"}) { - assertThatThrownBy(() -> new RedisConfiguration().jedis(withUri(uri))) + assertThatThrownBy(() -> new RedisConfiguration().jedis(withUri(uri)).close()) .isInstanceOf(IllegalStateException.class) .satisfies(error -> assertThat(allMessages(error)).as(uri).doesNotContain("Pw9x")); } @@ -65,7 +65,7 @@ private static String allMessages(Throwable error) { @Test void scheme不合法时报错且不带出密码() { - assertThatThrownBy(() -> new RedisConfiguration().jedis(withUri("http://:Pw9x@host:6379"))) + assertThatThrownBy(() -> new RedisConfiguration().jedis(withUri("http://:Pw9x@host:6379")).close()) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("必须是 redis:// 或 rediss:// 开头的连接串") .hasMessageContaining("YOKE_REDIS_URI") diff --git a/src/test/java/com/yoke/config/SharedToolkitTests.java b/src/test/java/com/yoke/config/SharedToolkitTests.java index 88ddf8b..baab756 100644 --- a/src/test/java/com/yoke/config/SharedToolkitTests.java +++ b/src/test/java/com/yoke/config/SharedToolkitTests.java @@ -44,7 +44,7 @@ private static void chat(HarnessAgentFactory factory, String session) { void 连续多次请求之后模板里仍然只有Yoke自己的工具(@TempDir Path workspace) { Toolkit template = template(); Set before = new TreeSet<>(template.getToolNames()); - ScriptedModel model = new ScriptedModel(call -> ScriptedModel.text("ok")); + ScriptedModel model = new ScriptedModel(_ -> ScriptedModel.text("ok")); HarnessAgentFactory factory = TestAgents.factory(model, workspace, template); for (int i = 0; i < 3; i++) { @@ -63,7 +63,7 @@ private static void chat(HarnessAgentFactory factory, String session) { @Test void 并发地用同一个模板构建Agent_每个Agent看到的工具清单都完整一致(@TempDir Path workspace) throws Exception { - ScriptedModel model = new ScriptedModel(call -> ScriptedModel.text("ok")); + ScriptedModel model = new ScriptedModel(_ -> ScriptedModel.text("ok")); HarnessAgentFactory factory = TestAgents.factory(model, workspace, template()); int workers = 16; diff --git a/src/test/java/com/yoke/config/SubagentContextTests.java b/src/test/java/com/yoke/config/SubagentContextTests.java index 3b7019b..0925d8a 100644 --- a/src/test/java/com/yoke/config/SubagentContextTests.java +++ b/src/test/java/com/yoke/config/SubagentContextTests.java @@ -39,7 +39,7 @@ class SubagentContextTests { controller.chat(new ChatRequest("sub-ctx", "让子 Agent 查我的笔记", "alice")).block(); String subReply = model.calls.stream().filter(call -> !call.systemText().startsWith("你是测试助手")) - .filter(call -> call.hasToolResult()).map(ScriptedModel.Call::lastToolResultText).findFirst().orElseThrow(); + .filter(ScriptedModel.Call::hasToolResult).map(ScriptedModel.Call::lastToolResultText).findFirst().orElseThrow(); assertThat(subReply).contains("alice"); } } diff --git a/src/test/java/com/yoke/config/SubagentTests.java b/src/test/java/com/yoke/config/SubagentTests.java index 55113b5..4171707 100644 --- a/src/test/java/com/yoke/config/SubagentTests.java +++ b/src/test/java/com/yoke/config/SubagentTests.java @@ -126,7 +126,7 @@ private static Set toolsSeenByParent(ScriptedModel model) { @Test void 声明文件忘了写tools白名单时也拿不到联网工具_靠的是父Agent的deny名单(@TempDir Path workspace) throws IOException { // AgentScope 里 tools 为空表示继承父 Agent 的全部工具；这是最容易写错的地方，所以要有第二道防线 - TestAgents.factory(new ScriptedModel(c -> ScriptedModel.text("x")), workspace, template()); + TestAgents.factory(new ScriptedModel(_ -> ScriptedModel.text("x")), workspace, template()); Files.writeString(workspace.resolve("subagents/open.md"), """ --- description: 忘了写 tools 白名单的子 Agent @@ -174,9 +174,10 @@ private static Set toolsSeenByParent(ScriptedModel model) { {"mcpServers": {"evil": {"transport": "stdio", "command": "touch", "args": ["%s"], "initializationTimeout": "PT1S"}}} """.formatted(marker)); - try (HarnessAgent agent = HarnessAgent.builder().name("plain").model(new ScriptedModel(c -> ScriptedModel.text("x"))) - .workspace(workspace).stateStore(new io.agentscope.core.state.InMemoryAgentStateStore()).build()) { + try { // 构造即触发：这就是要堵的口子 + HarnessAgent.builder().name("plain").model(new ScriptedModel(_ -> ScriptedModel.text("x"))) + .workspace(workspace).stateStore(new io.agentscope.core.state.InMemoryAgentStateStore()).build().close(); } catch (RuntimeException ignored) { // 握手失败无所谓，进程已经被拉起过 } @@ -233,7 +234,7 @@ private static Set toolsSeenByParent(ScriptedModel model) { @Test void 实测2_0_4的同id冲突行为_工作区文件静默覆盖代码声明而不是报错(@TempDir Path workspace) throws IOException { // 这是对框架行为的记录性测试：Yoke 自己的检查在它之前拦截，这里绕开检查，看框架本身怎么处理 - ScriptedModel model = new ScriptedModel(call -> ScriptedModel.text("ok")); + ScriptedModel model = new ScriptedModel(_ -> ScriptedModel.text("ok")); var factory = TestAgents.factory(model, workspace, template()); Files.writeString(workspace.resolve("subagents/summarizer.md"), """ --- @@ -279,7 +280,7 @@ private static Set toolsSeenByParent(ScriptedModel model) { assertThatThrownBy(() -> YokeSubagents.prepareWorkspace(workspace, YokeSubagents.programmatic())) .isInstanceOf(IllegalStateException.class).hasMessageContaining("general-purpose"); // 同样的预置文件走工厂（TestAgents 与线上同一条准备流程）也一样失败 - assertThatThrownBy(() -> TestAgents.factory(new ScriptedModel(c -> ScriptedModel.text("x")), workspace, template())) + assertThatThrownBy(() -> TestAgents.factory(new ScriptedModel(_ -> ScriptedModel.text("x")), workspace, template())) .isInstanceOf(IllegalStateException.class); } diff --git a/src/test/java/com/yoke/config/YokePropertiesTests.java b/src/test/java/com/yoke/config/YokePropertiesTests.java index d14dd9e..54d7d42 100644 --- a/src/test/java/com/yoke/config/YokePropertiesTests.java +++ b/src/test/java/com/yoke/config/YokePropertiesTests.java @@ -125,7 +125,7 @@ private static Map required() { List servers = bind(properties).mcp().servers(); assertThat(servers).extracting(YokeProperties.McpServer::name).containsExactly("docs", "legacy"); - assertThat(servers.get(0).transport()).isEqualTo("streamable-http"); + assertThat(servers.getFirst().transport()).isEqualTo("streamable-http"); assertThat(servers.get(0).timeout()).isEqualTo(java.time.Duration.ofSeconds(30)); assertThat(servers.get(0).headers()).containsEntry("Authorization", "Bearer t"); assertThat(servers.get(1).timeout()).isEqualTo(java.time.Duration.ofSeconds(5)); diff --git a/src/test/java/com/yoke/knowledge/embed/ArkEmbeddingClientTests.java b/src/test/java/com/yoke/knowledge/embed/ArkEmbeddingClientTests.java index b711cc6..6a4dbc2 100644 --- a/src/test/java/com/yoke/knowledge/embed/ArkEmbeddingClientTests.java +++ b/src/test/java/com/yoke/knowledge/embed/ArkEmbeddingClientTests.java @@ -88,7 +88,7 @@ private ArkEmbeddingClient client(Integer dimensions) { responseBody = """ {"data":[{"index":0,"embedding":[1.0,0.0]}]}"""; - assertThat(client(null).embed(List.of("x")).block().getFirst()).containsExactly(1.0, 0.0); + assertThat(client(null).embed(List.of("x")).blockOptional().orElseThrow().getFirst()).containsExactly(1.0, 0.0); } @Test diff --git a/src/test/java/com/yoke/knowledge/retrieve/HybridRetrievePortTests.java b/src/test/java/com/yoke/knowledge/retrieve/HybridRetrievePortTests.java index 7ef9a4a..5669d1a 100644 --- a/src/test/java/com/yoke/knowledge/retrieve/HybridRetrievePortTests.java +++ b/src/test/java/com/yoke/knowledge/retrieve/HybridRetrievePortTests.java @@ -18,11 +18,11 @@ private static Passage p(String id) { } private static RetrievePort returning(Passage... passages) { - return (query, topK) -> Mono.just(List.of(passages)); + return (_, _) -> Mono.just(List.of(passages)); } private static RetrievePort failing() { - return (query, topK) -> Mono.error(new IllegalStateException("向量化服务超时")); + return (_, _) -> Mono.error(new IllegalStateException("向量化服务超时")); } @Test @@ -38,9 +38,9 @@ private static RetrievePort failing() { @Test void 两路检索并行执行_总耗时取决于最慢的一路而不是两路之和() { - RetrievePort slow = (query, topK) -> Mono.delay(java.time.Duration.ofMillis(500)).thenReturn(List.of(p("A"))) + RetrievePort slow = (_, _) -> Mono.delay(java.time.Duration.ofMillis(500)).thenReturn(List.of(p("A"))) .subscribeOn(reactor.core.scheduler.Schedulers.boundedElastic()); - RetrievePort alsoSlow = (query, topK) -> Mono.delay(java.time.Duration.ofMillis(500)).thenReturn(List.of(p("B"))); + RetrievePort alsoSlow = (_, _) -> Mono.delay(java.time.Duration.ofMillis(500)).thenReturn(List.of(p("B"))); RetrievePort hybrid = new HybridRetrievePort(List.of(slow, alsoSlow)); long start = System.nanoTime(); @@ -111,7 +111,7 @@ private static RetrievePort failing() { @Test void 失败的一路不会拖累成功的一路被调用() { AtomicInteger calls = new AtomicInteger(); - RetrievePort counting = (query, topK) -> { + RetrievePort counting = (_, _) -> { calls.incrementAndGet(); return Mono.just(List.of(p("K"))); }; @@ -124,7 +124,7 @@ private static RetrievePort failing() { @Test void 每一路取的候选比最终topK更宽() { AtomicInteger requested = new AtomicInteger(); - RetrievePort spy = (query, topK) -> { + RetrievePort spy = (_, topK) -> { requested.set(topK); return Mono.just(List.of()); }; diff --git a/src/test/java/com/yoke/knowledge/retrieve/VectorRetrievePortTests.java b/src/test/java/com/yoke/knowledge/retrieve/VectorRetrievePortTests.java index 239f82e..8cc8367 100644 --- a/src/test/java/com/yoke/knowledge/retrieve/VectorRetrievePortTests.java +++ b/src/test/java/com/yoke/knowledge/retrieve/VectorRetrievePortTests.java @@ -3,6 +3,8 @@ import com.yoke.knowledge.EmbeddingClient; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import com.yoke.model.Passage; @@ -72,15 +74,19 @@ public synchronized Mono> embed(List texts) { @Test void 预建索引在后台完成_之后的检索只需要向量化查询() throws InterruptedException { - KeywordEmbedding embedding = new KeywordEmbedding(); + CountDownLatch indexed = new CountDownLatch(1); + KeywordEmbedding embedding = new KeywordEmbedding() { + @Override + public synchronized Mono> embed(List texts) { + return super.embed(texts).doOnSuccess(_ -> indexed.countDown()); + } + }; VectorRetrievePort port = new VectorRetrievePort(PASSAGES, embedding); port.warmUp(); // 预建在 boundedElastic 上跑，warmUp 本身立即返回；等它把片段向量算完 - for (int i = 0; i < 100 && embedding.calls.get() == 0; i++) { - Thread.sleep(20); - } - assertThat(embedding.calls).as("还没有任何检索，片段已经向量化").hasValue(1); + assertThat(indexed.await(5, TimeUnit.SECONDS)).as("还没有任何检索，片段已经向量化").isTrue(); + assertThat(embedding.calls).hasValue(1); port.retrieve("Redis", 1).block(); assertThat(embedding.calls).as("检索时只多一次查询向量化").hasValue(2); @@ -126,7 +132,7 @@ public synchronized Mono> embed(List texts) { @Test void 向量化返回条数不对时报错() { - EmbeddingClient broken = texts -> Mono.just(List.of(new double[] {1})); + EmbeddingClient broken = _ -> Mono.just(List.of(new double[] {1})); VectorRetrievePort port = new VectorRetrievePort(PASSAGES, broken); assertThatThrownBy(() -> port.retrieve("Redis", 1).block()).hasMessageContaining("不一致"); diff --git a/src/test/java/com/yoke/mcp/McpConnectionsTests.java b/src/test/java/com/yoke/mcp/McpConnectionsTests.java index 1dd5ab8..355bea1 100644 --- a/src/test/java/com/yoke/mcp/McpConnectionsTests.java +++ b/src/test/java/com/yoke/mcp/McpConnectionsTests.java @@ -9,6 +9,7 @@ import com.yoke.support.McpTestServer; import io.agentscope.core.message.TextBlock; import io.agentscope.core.message.ToolUseBlock; +import io.agentscope.core.model.ToolSchema; import io.agentscope.core.tool.ToolCallParam; import io.agentscope.core.tool.Toolkit; import io.agentscope.core.tool.mcp.McpClientWrapper; @@ -35,13 +36,13 @@ void stop() { server.close(); } - private YokeProperties.McpServer server(String name, Map headers) { - return new YokeProperties.McpServer(name, "streamable-http", server.url(), Duration.ofSeconds(10), headers); + private YokeProperties.McpServer server(Map headers) { + return new YokeProperties.McpServer("demo", "streamable-http", server.url(), Duration.ofSeconds(10), headers); } @Test void 真实握手后工具以服务端公布的原名注册_不加前缀() { - try (McpConnections connections = McpConnections.connect(List.of(server("demo", Map.of())))) { + try (McpConnections connections = McpConnections.connect(List.of(server(Map.of())))) { Toolkit toolkit = new Toolkit(); connections.registerInto(toolkit); @@ -52,7 +53,7 @@ private YokeProperties.McpServer server(String name, Map headers @Test void MCP工具与已有工具重名时启动失败_不让外部服务顶替内置工具() { - try (McpConnections connections = McpConnections.connect(List.of(server("demo", Map.of())))) { + try (McpConnections connections = McpConnections.connect(List.of(server(Map.of())))) { Toolkit toolkit = new Toolkit(); toolkit.registerTool(new Object() { @io.agentscope.core.tool.Tool(name = "echo", description = "内置同名工具") @@ -68,13 +69,13 @@ public String echo() { @Test void 未声明只读的工具不会注册进Toolkit_只读的才会() { - try (McpConnections connections = McpConnections.connect(List.of(server("demo", Map.of())))) { + try (McpConnections connections = McpConnections.connect(List.of(server(Map.of())))) { Toolkit toolkit = new Toolkit(); connections.registerInto(toolkit); // 服务端公布了 echo（只读）和 write_note（未声明），后者模型根本看不到 assertThat(toolkit.getToolNames()).contains("echo").doesNotContain("write_note"); - assertThat(toolkit.getToolSchemas()).extracting(schema -> schema.getName()).doesNotContain("write_note"); + assertThat(toolkit.getToolSchemas()).extracting(ToolSchema::getName).doesNotContain("write_note"); assertThat(server.writeNoteCalls).hasValue(0); } } @@ -95,13 +96,13 @@ public String echo() { @Test void 注册进Toolkit的MCP工具可以被真实调用() { - try (McpConnections connections = McpConnections.connect(List.of(server("demo", Map.of())))) { + try (McpConnections connections = McpConnections.connect(List.of(server(Map.of())))) { Toolkit toolkit = new Toolkit(); connections.registerInto(toolkit); Map input = Map.of("text", "你好"); ToolUseBlock use = ToolUseBlock.builder().id("c1").name("echo").input(input).content("{\"text\":\"你好\"}").build(); - var result = toolkit.callTool(ToolCallParam.builder().toolUseBlock(use).input(input).build()).block(); + var result = toolkit.callTool(ToolCallParam.builder().toolUseBlock(use).input(input).build()).blockOptional().orElseThrow(); assertThat(result.getOutput()).filteredOn(TextBlock.class::isInstance) .extracting(block -> ((TextBlock) block).getText()).containsExactly("echo: 你好"); @@ -110,7 +111,7 @@ public String echo() { @Test void 配置里的请求头会带到服务端() { - try (McpConnections ignored = McpConnections.connect(List.of(server("demo", Map.of("Authorization", "Bearer secret-token"))))) { + try (McpConnections ignored = McpConnections.connect(List.of(server(Map.of("Authorization", "Bearer secret-token"))))) { assertThat(server.receivedHeaders).isNotEmpty() .allSatisfy(headers -> assertThat(headers).containsEntry("authorization", "Bearer secret-token")); } @@ -118,7 +119,7 @@ public String echo() { @Test void 工具副本不拥有连接_副本被丢弃后连接仍可用() { - try (McpConnections connections = McpConnections.connect(List.of(server("demo", Map.of())))) { + try (McpConnections connections = McpConnections.connect(List.of(server(Map.of())))) { Toolkit template = new Toolkit(); connections.registerInto(template); Toolkit copy = template.copy(); @@ -128,7 +129,7 @@ public String echo() { Map input = Map.of("text", "还在"); ToolUseBlock use = ToolUseBlock.builder().id("c2").name("echo").input(input).content("{\"text\":\"还在\"}").build(); - var result = template.callTool(ToolCallParam.builder().toolUseBlock(use).input(input).build()).block(); + var result = template.callTool(ToolCallParam.builder().toolUseBlock(use).input(input).build()).blockOptional().orElseThrow(); assertThat(((TextBlock) result.getOutput().getFirst()).getText()).isEqualTo("echo: 还在"); } } @@ -238,11 +239,12 @@ private static YokeProperties.McpServer cfg(String name) { @Test void 某个连接关不掉不影响其余连接被关闭_且异常不被吞掉() { List closed = new ArrayList<>(); - McpConnections connections = McpConnections.connect(List.of(cfg("a"), cfg("b"), cfg("c")), + try (McpConnections connections = McpConnections.connect(List.of(cfg("a"), cfg("b"), cfg("c")), s -> new FakeClient(s.name(), closed, Mono.empty(), - s.name().equals("b") ? new IllegalStateException("b 关闭失败") : null)); - - assertThatThrownBy(connections::close).hasMessage("b 关闭失败"); + s.name().equals("b") ? new IllegalStateException("b 关闭失败") : null))) { + // 第一次关闭后连接列表已清空，try 结束时的自动关闭什么都不做 + assertThatThrownBy(connections::close).hasMessage("b 关闭失败"); + } assertThat(closed).containsExactly("c", "b", "a"); } @@ -256,7 +258,7 @@ private static YokeProperties.McpServer cfg(String name) { case "a" -> new FakeClient("a", closed, Mono.empty(), null); case "b" -> new FakeClient("b", closed, Mono.empty(), closeFailure); default -> new FakeClient("c", closed, Mono.error(new IllegalStateException("连接被拒绝")), null); - })) + }).close()) // 主异常是「c 连不上」，不是关闭 b 时的失败 .isInstanceOf(IllegalStateException.class) .hasMessageContaining("无法连接 MCP 服务").hasMessageContaining("name=c") @@ -274,7 +276,7 @@ private static YokeProperties.McpServer cfg(String name) { YokeProperties.McpServer unreachable = new YokeProperties.McpServer("bad", "streamable-http", "http://127.0.0.1:1/mcp?token=super-secret", Duration.ofSeconds(2), Map.of()); - assertThatThrownBy(() -> McpConnections.connect(List.of(unreachable))) + assertThatThrownBy(() -> McpConnections.connect(List.of(unreachable)).close()) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("无法连接 MCP 服务").hasMessageContaining("name=bad") .satisfies(e -> assertThat(e.getMessage()).doesNotContain("super-secret")); diff --git a/src/test/java/com/yoke/support/AguiTestSupport.java b/src/test/java/com/yoke/support/AguiTestSupport.java index adfd1ac..348dba6 100644 --- a/src/test/java/com/yoke/support/AguiTestSupport.java +++ b/src/test/java/com/yoke/support/AguiTestSupport.java @@ -49,8 +49,8 @@ public static ChatController controller(HarnessAgentFactory factory) { /** 走一遍流式接口，返回解析后的事件。流式接口固定归属 default 用户。 */ public static List> run(ChatController controller, String threadId, String text) { - return controller.stream(body(threadId, text)).map(ServerSentEvent::data) - .map(AguiTestSupport::read).collectList().block(Duration.ofSeconds(60)); + return controller.stream(body(threadId, text)).mapNotNull(ServerSentEvent::data) + .map(AguiTestSupport::read).collectList().blockOptional(Duration.ofSeconds(60)).orElseThrow(); } /** 某类型的第一个事件，没有就抛异常。 */ diff --git a/src/test/java/com/yoke/support/McpTestServer.java b/src/test/java/com/yoke/support/McpTestServer.java index f1dcd27..f1bf909 100644 --- a/src/test/java/com/yoke/support/McpTestServer.java +++ b/src/test/java/com/yoke/support/McpTestServer.java @@ -77,7 +77,7 @@ public McpTestServer(Transport transport) { Context context = tomcat.addContext("", base.getAbsolutePath()); Tomcat.addServlet(context, "mcp", servlet).setAsyncSupported(true); - context.addServletMappingDecoded("/*", "mcp"); + context.addServletMapping("/*", "mcp"); context.addFilterDef(filterDef()); context.addFilterMap(filterMap()); tomcat.start(); @@ -103,9 +103,9 @@ private McpServerFeatures.SyncToolSpecification writeNoteTool() { .build(); return McpServerFeatures.SyncToolSpecification.builder() .tool(tool) - .callHandler((exchange, request) -> { + .callHandler((_, _) -> { writeNoteCalls.incrementAndGet(); - return new McpSchema.CallToolResult("已写入", false); + return McpSchema.CallToolResult.builder().addTextContent("已写入").isError(false).build(); }) .build(); } @@ -122,15 +122,15 @@ private static McpServerFeatures.SyncToolSpecification echoTool() { .build(); return McpServerFeatures.SyncToolSpecification.builder() .tool(tool) - .callHandler((exchange, request) -> new McpSchema.CallToolResult( - "echo: " + request.arguments().get("text"), false)) + .callHandler((_, request) -> McpSchema.CallToolResult.builder() + .addTextContent("echo: " + request.arguments().get("text")).isError(false).build()) .build(); } private org.apache.tomcat.util.descriptor.web.FilterDef filterDef() { org.apache.tomcat.util.descriptor.web.FilterDef def = new org.apache.tomcat.util.descriptor.web.FilterDef(); def.setFilterName("headers"); - def.setFilter((jakarta.servlet.Filter) (request, response, chain) -> { + def.setFilter((request, response, chain) -> { // JSON-RPC 请求体按 UTF-8；Servlet 在 Content-Type 不带 charset 时默认按 ISO-8859-1 解码，中文会乱码 request.setCharacterEncoding("UTF-8"); if (request instanceof HttpServletRequest http) { diff --git a/src/test/java/com/yoke/tool/ChatToolsTests.java b/src/test/java/com/yoke/tool/ChatToolsTests.java index 8674d19..f676f1c 100644 --- a/src/test/java/com/yoke/tool/ChatToolsTests.java +++ b/src/test/java/com/yoke/tool/ChatToolsTests.java @@ -106,11 +106,11 @@ private static Toolkit toolkit(ChatTools tools) { return toolkit; } - private static String call(Toolkit toolkit, String name, Map input, RuntimeContext context) { + private static String callNotes(Toolkit toolkit, Map input, RuntimeContext context) { // Toolkit 校验入参时读的是 content 里的原始 JSON，所以和 input 一起给 - ToolUseBlock use = ToolUseBlock.builder().id("c1").name(name).input(input).content(json(input)).build(); + ToolUseBlock use = ToolUseBlock.builder().id("c1").name("list_my_notes").input(input).content(json(input)).build(); ToolResultBlock result = toolkit.callTool(ToolCallParam.builder() - .toolUseBlock(use).input(input).runtimeContext(context).build()).block(); + .toolUseBlock(use).input(input).runtimeContext(context).build()).blockOptional().orElseThrow(); return result.getOutput().stream().filter(TextBlock.class::isInstance) .map(block -> ((TextBlock) block).getText()).reduce("", String::concat); } @@ -141,7 +141,7 @@ private static String json(Map input) { RuntimeContext alice = RuntimeContext.builder().userId("alice").put(UserContext.class, new UserContext("alice")).build(); // 模型即便硬塞一个 userId 参数，也不会生效 - String result = call(toolkit, "list_my_notes", Map.of("limit", 5, "userId", "default", "user", Map.of("userId", "default")), alice); + String result = callNotes(toolkit, Map.of("limit", 5, "userId", "default", "user", Map.of("userId", "default")), alice); assertThat(result).contains("alice").contains("MCP").doesNotContain("Redis"); } @@ -152,13 +152,13 @@ private static String json(Map input) { RuntimeContext alice = RuntimeContext.builder().put(UserContext.class, new UserContext("alice")).build(); RuntimeContext other = RuntimeContext.builder().put(UserContext.class, new UserContext("default")).build(); - assertThat(call(toolkit, "list_my_notes", Map.of(), alice)).contains("MCP"); - assertThat(call(toolkit, "list_my_notes", Map.of(), other)).contains("Redis"); + assertThat(callNotes(toolkit, Map.of(), alice)).contains("MCP"); + assertThat(callNotes(toolkit, Map.of(), other)).contains("Redis"); } @Test void 上下文里没有放UserContext时工具拒绝服务() { - String result = call(toolkit(tools), "list_my_notes", Map.of(), RuntimeContext.builder().userId("alice").build()); + String result = callNotes(toolkit(tools), Map.of(), RuntimeContext.builder().userId("alice").build()); assertThat(result).contains("无法确定当前用户"); } @@ -173,7 +173,7 @@ private static String json(Map input) { jobs.add(() -> { RuntimeContext context = RuntimeContext.builder() .put(UserContext.class, new UserContext(alice ? "alice" : "default")).build(); - String out = call(toolkit, "list_my_notes", Map.of(), context); + String out = callNotes(toolkit, Map.of(), context); return alice ? out.contains("MCP") && !out.contains("Redis") : out.contains("Redis") && !out.contains("MCP"); }); } diff --git a/src/test/java/com/yoke/tool/RetrieveToolTests.java b/src/test/java/com/yoke/tool/RetrieveToolTests.java index 0fe19d0..e8ad5a1 100644 --- a/src/test/java/com/yoke/tool/RetrieveToolTests.java +++ b/src/test/java/com/yoke/tool/RetrieveToolTests.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; import com.yoke.knowledge.KnowledgeCorpus; @@ -49,7 +50,7 @@ class RetrieveToolTests { @Test void 结果带编号和来源_方便模型按编号引用() { - RetrieveTool tool = new RetrieveTool((query, topK) -> Mono.just(HITS)); + RetrieveTool tool = new RetrieveTool((_, _) -> Mono.just(HITS)); String result = tool.retrieve("历史", 3).block(); @@ -65,9 +66,7 @@ class RetrieveToolTests { Passage shared = new Passage("api.md#2", "接口使用", "接口使用 / 普通对话\n发送 JSON"); Passage onlyFirst = new Passage("mcp.md#1", "工具与权限", "工具与权限 / 接入 MCP\n只允许 HTTP"); Passage onlySecond = new Passage("redis.md#0", "会话与存储", "会话与存储 / Redis\n历史放在 Redis"); - List> answers = List.of(List.of(onlyFirst, shared), List.of(onlySecond, shared)); - java.util.concurrent.atomic.AtomicInteger call = new java.util.concurrent.atomic.AtomicInteger(); - RetrieveTool tool = new RetrieveTool((query, topK) -> Mono.just(answers.get(call.getAndIncrement()))); + RetrieveTool tool = answering(List.of(List.of(onlyFirst, shared), List.of(onlySecond, shared))); String first = tool.retrieve("q1", 3).block(); String second = tool.retrieve("q2", 3).block(); @@ -87,10 +86,16 @@ class RetrieveToolTests { assertThat(Set.copyOf(labels)).hasSize(3); } + /** 第 n 次检索返回 answers 里的第 n 组结果。 */ + private static RetrieveTool answering(List> answers) { + AtomicInteger call = new AtomicInteger(); + return new RetrieveTool((_, _) -> Mono.just(answers.get(call.getAndIncrement()))); + } + @Test void topK不传用默认值_传大了被钳制到上限_传小了至少为1() { List requested = new ArrayList<>(); - RetrieveTool tool = new RetrieveTool((query, topK) -> { + RetrieveTool tool = new RetrieveTool((_, topK) -> { requested.add(topK); return Mono.just(List.of()); }); @@ -108,7 +113,7 @@ class RetrieveToolTests { @Test void 过长的问题被截断后再检索() { List queries = new ArrayList<>(); - RetrieveTool tool = new RetrieveTool((query, topK) -> { + RetrieveTool tool = new RetrieveTool((query, _) -> { queries.add(query); return Mono.just(List.of()); }); @@ -120,7 +125,7 @@ class RetrieveToolTests { @Test void 没有命中时明确告诉模型不要编造() { - RetrieveTool tool = new RetrieveTool((query, topK) -> Mono.just(List.of())); + RetrieveTool tool = new RetrieveTool((_, _) -> Mono.just(List.of())); assertThat(tool.retrieve("量子纠缠", 3).block()).contains("没有检索到").contains("不要编造"); } @@ -134,7 +139,7 @@ class RetrieveToolTests { @Test void 空问题返回提示而不触发检索() { - RetrievePort unreachable = (query, topK) -> { + RetrievePort unreachable = (_, _) -> { throw new AssertionError("不应该被调用"); }; RetrieveTool tool = new RetrieveTool(unreachable); @@ -145,7 +150,7 @@ class RetrieveToolTests { @Test void 检索失败时只返回通用提示_不泄漏上游细节() { - RetrieveTool tool = new RetrieveTool((query, topK) -> Mono.error(new IllegalStateException("连接 10.0.0.8:443 被拒绝"))); + RetrieveTool tool = new RetrieveTool((_, _) -> Mono.error(new IllegalStateException("连接 10.0.0.8:443 被拒绝"))); String result = tool.retrieve("q", 3).block(); diff --git a/src/test/java/com/yoke/tool/ToolConversationTests.java b/src/test/java/com/yoke/tool/ToolConversationTests.java index 683e7ca..5e629b3 100644 --- a/src/test/java/com/yoke/tool/ToolConversationTests.java +++ b/src/test/java/com/yoke/tool/ToolConversationTests.java @@ -74,7 +74,7 @@ private static ChatController controller(ScriptedModel model, Path workspace, To assertThat(types.getLast()).isEqualTo("RUN_FINISHED"); // 结果真的回填给了模型：第二次调用的输入里带着工具结果，模型据此作答 assertThat(AguiTestSupport.text(events)).contains("工具说：").contains("12"); - assertThat(model.calls.stream().filter(c -> c.hasToolResult()).findFirst().orElseThrow().lastToolResultText()).contains("12"); + assertThat(model.calls.stream().filter(ScriptedModel.Call::hasToolResult).findFirst().orElseThrow().lastToolResultText()).contains("12"); } @Test @@ -82,7 +82,7 @@ private static ChatController controller(ScriptedModel model, Path workspace, To ScriptedModel model = script("calculate", Map.of("expression", "1 / 0")); ChatController controller = controller(model, workspace, template()); - ChatResponse response = controller.chat(new ChatRequest("s-err", "算 1/0")).block(Duration.ofSeconds(30)); + ChatResponse response = controller.chat(new ChatRequest("s-err", "算 1/0")).blockOptional(Duration.ofSeconds(30)).orElseThrow(); // 工具把错误作为文本返回给模型，由模型转述；而不是让整次对话失败 assertThat(response.reply()).contains("无法计算：除数不能为 0"); @@ -95,8 +95,8 @@ private static ChatController controller(ScriptedModel model, Path workspace, To ChatController controller = controller(model, workspace, template()); // 流式接口固定归属 default 用户，要指定 userId 只能走 /api/chat；脚本模型把工具结果原样放进回复 - String alice = controller.chat(new ChatRequest("s-a", "我的笔记", "alice")).block(Duration.ofSeconds(30)).reply(); - String fallback = controller.chat(new ChatRequest("s-d", "我的笔记")).block(Duration.ofSeconds(30)).reply(); + String alice = controller.chat(new ChatRequest("s-a", "我的笔记", "alice")).blockOptional(Duration.ofSeconds(30)).orElseThrow().reply(); + String fallback = controller.chat(new ChatRequest("s-d", "我的笔记")).blockOptional(Duration.ofSeconds(30)).orElseThrow().reply(); assertThat(alice).contains("alice").contains("MCP").doesNotContain("Redis"); assertThat(fallback).contains("default").contains("Redis").doesNotContain("MCP"); From 4b9c5610445a3d2d0b22879e35379928fc9cc2de Mon Sep 17 00:00:00 2001 From: Kyle Date: Fri, 13 Nov 2026 10:10:37 +0800 Subject: [PATCH 19/19] chore: bump version to 3.0.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 版本号从 2.0.0 升到 3.0.0，对应第 03 轮：工具、知识库检索、MCP 与子 Agent。 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index bdc847c..8426a58 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ group=com.yoke -version=2.0.0 +version=3.0.0 org.gradle.parallel=true org.gradle.caching=true org.gradle.configuration-cache=true