diff --git a/hermit/adapter/templates/ahe/hook_hosts.py b/hermit/adapter/templates/ahe/hook_hosts.py index 2c0cdae5..9f912f34 100644 --- a/hermit/adapter/templates/ahe/hook_hosts.py +++ b/hermit/adapter/templates/ahe/hook_hosts.py @@ -16,9 +16,9 @@ the wires, and the same file re-runs both CLIs on a scripted model when they are installed and `--integration` is given. -Pi reads no hook file: its deployment lists the hook commands in AGENTS.md for the agent to run by hand. Run so, with -no payload (the `pi` wire), session_start still hands over the memory; the other hooks have no tool call, prompt or -stop to act on. +Pi and DeepSeek Harness read no hook file: their deployments list the hook commands in AGENTS.md for the agent to +run by hand. Run so, with no payload (the `pi` and `dsh` wires), session_start still hands over the memory; the other +hooks have no tool call, prompt or stop to act on. Stdlib only, beside hook_runner.py in the adapter's copy: nothing of the framework is installed where the agent runs. """ @@ -480,8 +480,14 @@ class Pi(Wire): tool_kinds = ClaudeCode.tool_kinds -CODEX, CLAUDE_CODE, PI = Codex(), ClaudeCode(), Pi() -HOSTS = {CODEX.name: CODEX, CLAUDE_CODE.name: CLAUDE_CODE, PI.name: PI} +class Dsh(Pi): + """DeepSeek Harness runs no hook file of its own either: its deployment lists each hook command in AGENTS.md, as + Pi's does, and a command run so has no payload.""" + name, label = "dsh", "DeepSeek Harness" + + +CODEX, CLAUDE_CODE, PI, DSH = Codex(), ClaudeCode(), Pi(), Dsh() +HOSTS = {CODEX.name: CODEX, CLAUDE_CODE.name: CLAUDE_CODE, PI.name: PI, DSH.name: DSH} HOOK_HOSTS = (CODEX.name, CLAUDE_CODE.name) # the hosts that run the hooks themselves diff --git a/hermit/agent/acting/agent.py b/hermit/agent/acting/agent.py index 8cafd23b..fb66edf0 100644 --- a/hermit/agent/acting/agent.py +++ b/hermit/agent/acting/agent.py @@ -7,7 +7,7 @@ def run(self, task: dict, adapter: Path | None) -> RunOutput A config names the agent: agent: - name: react # a built-in: react | codex | claude-code | pi + name: react # a built-in: react | codex | claude-code | pi | dsh name: agent/agent.py:CLBenchAgent # or the task's own class (file.py:Class, module:Class) name: agent/pipeline.py:run # or a plain function run(task, adapter), wrapped as a FunctionAgent concurrency: 4 @@ -75,9 +75,9 @@ def builtin(name: str): if name == "react": from hermit.agent.acting.react import ReactAgent return ReactAgent - if name in ("cli", "codex", "claude-code", "claude", "pi"): + if name in ("cli", "codex", "claude-code", "claude", "pi", "dsh"): from hermit.agent.acting.workspace import CliAgent - backend = {"codex": "codex", "claude-code": "claude", "claude": "claude", "pi": "pi"}.get(name) + backend = {"codex": "codex", "claude-code": "claude", "claude": "claude", "pi": "pi", "dsh": "dsh"}.get(name) if backend is None: return CliAgent return lambda params, base: CliAgent({"backend": backend, **(params or {})}, base) @@ -87,7 +87,7 @@ def builtin(name: str): return None -BUILTIN_NAMES = ("react", "codex", "claude-code", "pi", "cli", "harbor") +BUILTIN_NAMES = ("react", "codex", "claude-code", "pi", "dsh", "cli", "harbor") def load_agent(name: str, params: dict | None, base: Path) -> Agent: diff --git a/hermit/agent/acting/workspace.py b/hermit/agent/acting/workspace.py index f280c344..ad27e1c8 100644 --- a/hermit/agent/acting/workspace.py +++ b/hermit/agent/acting/workspace.py @@ -1,5 +1,5 @@ -"""The `codex`, `claude-code` and `pi` agents: one task run by the agent CLI in a workspace the adapter is deployed -into. They are the framework's one way to act with a coding-agent CLI; a task adds only what is its own. +"""The `codex`, `claude-code`, `pi` and `dsh` agents: one task run by the agent CLI in a workspace the adapter is +deployed into. They are the framework's one way to act with a coding-agent CLI; a task adds only what is its own. agent: name: codex # or claude-code; `cli` with params.backend is the same @@ -69,7 +69,7 @@ from hermit.agent.base.login import LOGINS from hermit.task import RunOutput, load_callable -BACKENDS = ("codex", "claude", "pi") +BACKENDS = ("codex", "claude", "pi", "dsh") DEPLOYS = ("host",) ISOLATIONS = ("none", "container") INPUT_FIELDS = ("question", "prompt", "instruction", "input", "query") @@ -144,6 +144,9 @@ def deploy(adapter: Path, workspace: Path, backend: str, *, context: str | None if backend == "pi": from hermit.hosts.pi import prepare as prepare_pi return prepare_pi(adapter, workspace, sys.executable, context=context) + if backend == "dsh": + from hermit.hosts.dsh import prepare as prepare_dsh + return prepare_dsh(adapter, workspace, sys.executable, context=context) from hermit.hosts.claude_code import prepare return prepare(adapter, workspace, sys.executable, context=context, serve_tools=serve_tools) @@ -170,9 +173,12 @@ def _deploy(task: dict, adapter: Path | None, work: Path, backend: str, params: def _session_args(backend: str, deployed: dict, work: Path) -> list[str]: """What the session needs to use what the deployment connected: the workspace's own Codex configuration (its tools, roles and hooks) trusted, as a person trusts a deployed project, and its sub-agents enabled; Pi's project - configuration (.pi/mcp.json, .pi/skills) trusted for the session (--approve), without which Pi skips it.""" + configuration (.pi/mcp.json, .pi/skills) trusted for the session (--approve), without which Pi skips it; dsh's + patch layer of the adapter's tools (.dsh/hermit.patch.yml), which dsh loads only when it is named (--patch).""" if backend == "pi": return ["--approve"] + if backend == "dsh": + return ["--patch", str(deployed["patch"])] if deployed.get("patch") else [] if backend == "codex": return ["-c", f'projects={{"{work}" = {{ trust_level = "trusted" }} }}', *(["--enable", "multi_agent"] if deployed.get("roles") else [])] @@ -277,11 +283,17 @@ def _check(backend: str, params: dict) -> str: if not isinstance(params.get("web_search"), bool): raise ValueError("agent.params.web_search must be set, true or false: Codex's hosted search runs at the " "provider, past any sandbox or proxy, so it is never left to a default") - if backend == "pi": # Pi signs in through a model reference, and runs on this machine + if backend in ("pi", "dsh"): # they sign in through a model reference (dsh: or its own), on this machine if params.get("login") is not None: - raise ValueError("agent pi takes no login: its model is a / reference") + raise ValueError(f"agent {backend} takes no login: its model is a / reference" + + (", or a model of dsh's own provider" if backend == "dsh" else "")) if params.get("isolation") == "container": - raise ValueError("isolation: container runs Codex or Claude Code; agent pi takes isolation: none") + raise ValueError(f"isolation: container runs Codex or Claude Code; agent {backend} takes isolation: none") + if backend == "dsh" and params.get("permission_mode") is not None: + from hermit.agent.base.cli import DSH_PERMISSION_MODES + if params["permission_mode"] not in DSH_PERMISSION_MODES: + raise ValueError(f"agent.dsh.permission_mode is one of {', '.join(DSH_PERMISSION_MODES)} (dsh's own " + f"sandbox), not {params['permission_mode']!r}") return backend login, label = params.get("login"), {"codex": "Codex", "claude": "Claude Code"}[backend] if login is not None and login not in LOGINS: @@ -311,6 +323,8 @@ def base_agent(backend: str, params: dict) -> BaseAgent: if k in params and (params[k] is not None or k == "plan_max_wait_s")} if params.get("claude_permission_mode"): settings["permission_mode"] = params["claude_permission_mode"] + if backend == "dsh" and params.get("permission_mode"): # dsh's own sandbox (agent.dsh.permission_mode) + settings["permission_mode"] = params["permission_mode"] return make(backend, {**settings, "web_search": params["web_search"]}) diff --git a/hermit/agent/base/cli.py b/hermit/agent/base/cli.py index ef6e2152..1aba60eb 100644 --- a/hermit/agent/base/cli.py +++ b/hermit/agent/base/cli.py @@ -1,4 +1,5 @@ -"""`codex`, `claude` and `pi`: a coding-agent CLI as a Base Agent (docs/framework/agent-layers.md#layer-2-base-agent). +"""`codex`, `claude`, `pi` and `dsh`: a coding-agent CLI as a Base Agent +(docs/framework/agent-layers.md#layer-2-base-agent). One `run` is one session of the CLI in print mode: the prompt is given, the CLI works in `workspace` with its own tools and the call's `tools` (served over MCP by the framework's tool server), and its last message is the reply. @@ -69,6 +70,15 @@ image_input (true: the bound model takes images, such as a tool's result; default: text only, Pi's default for a model it does not know), append_system_prompt, effort (its --thinking level). Pi takes no roles, has no web search and runs under isolation none or sandbox. + DeepSeek Harness (dsh) only: provider (its pi-ai route for a bound model, from the binding: Binding.dsh_provider), + image_input (as Pi's), effort (the default model's reasoningEffort; YAML's bare `off`, a boolean, is `off`: dsh + then asks a bound model for no reasoning level, which leaves its server's default, and turns thinking off on its + own provider), permission_mode (its sandbox: read-only, workspace-write, the default, or danger-full-access; under + workspace-write the `writable` paths are granted to it with --add-dir), instruction_files (false: no AGENTS.md or + CLAUDE.md of the working directory or below it is given to the model; default true). dsh takes no roles here and + runs under isolation none or sandbox; `args` may hold its own options. The caller's DSH_* variables, + DEEPSEEK_BASE_URL and DEEPSEEK_SEARCH_BASE_URL never reach a session, whatever a sandbox lets through: `env` + gives them (Dsh.dropped_env). Limits are enforced by the isolation, or the call is refused (Unsupported): @@ -93,12 +103,13 @@ import threading import time from typing import Any, Sequence +import urllib.parse from hermit.agent.base import container, login as codex_login from hermit.agent.base.interface import BaseAgent, Limits, Mount, Result, Unsupported -from hermit.agent.base.records import parse_claude, parse_codex, parse_pi, price_usage +from hermit.agent.base.records import parse_claude, parse_codex, parse_dsh, parse_pi, price_usage from hermit.agent.base.tool import Tool -from hermit.agent.base.trajectory import claude_steps, codex_steps, events, pi_steps +from hermit.agent.base.trajectory import claude_steps, codex_steps, dsh_steps, events, pi_steps from hermit.agent.model.backends import CLAUDE_ENV ISOLATIONS = ("none", "sandbox", "container") @@ -108,6 +119,8 @@ r"error sending request|too many requests|service unavailable|bad gateway|overloaded|" r"model is at capacity|" r"^(?:429|5\d\d): |" # Pi's report of a provider's status + # dsh's, by its codes: those it retries itself, and a stream the provider closed + r"^dsh: (?:RATE_LIMIT|SERVER|TIMEOUT|TRANSPORT|EMPTY_RESPONSE|STREAM_CLOSED): |" r"error response from daemon|cannot connect to the docker daemon|" + re.escape(container.MISSING), re.I | re.M) CODEX_HOME_SETTINGS = ["features.shell_snapshot = false", # a snapshot would bring back the whole environment @@ -552,8 +565,8 @@ def adjust_usage(self, home: Path, handle, usage: list, resumed: bool) -> list: return usage # Without isolation, a mount inside the workspace is a link to its source (`_link`); a CLI whose searches do not - # follow links (Claude Code's Glob and Grep run ripgrep without --follow; `grep -r` and `find` in its Bash neither) - # gets it bound there instead, a real directory, as a container gives it (`_bound`). + # follow links (Claude Code's Glob and Grep and dsh's glob and grep run ripgrep without --follow; `grep -r` and + # `find` in a shell neither) gets it bound there instead, a real directory, as a container gives it (`_bound`). binds_mounts = False # ---------- what the call may do ---------- @@ -651,7 +664,8 @@ def prompt_text(self, prompt: str) -> str: def outcome(self, raw: str, stream: Path, last_file: Path) -> dict: """What the session's stream says: `reply`, `served`, `unavailable` (a provider that could not serve), `refused` - (the plan's limit), and for what stops short of that `failed`, `failure` (why) and the rest of Result.extra.""" + (the plan's limit), and for what stops short of that `failed`, `failure` (why) and the rest of Result.extra; + `started` for a CLI that writes an event before it has a session (absent: it started once it wrote one).""" raise NotImplementedError # ---------- the home ---------- @@ -959,6 +973,7 @@ def _session(self, prompt, *, workspace=None, tools: Sequence[Tool] = (), limits raw = readable.read_text(encoding="utf-8", errors="replace") if readable is stream else "" said = self.outcome(raw, readable, last_file) said["failed"] = said.get("failed", False) and not timed_out and not cancelled + began = said.pop("started", None) # the CLI's own word, else: it wrote an event handle, usage = self.parse(readable, (s.get("binding") or {}).get("ref") or s.get("model") or self.backend) usage = self.adjust_usage(home, handle, usage, resumed=bool(session)) price_usage(usage, s) @@ -968,7 +983,8 @@ def _session(self, prompt, *, workspace=None, tools: Sequence[Tool] = (), limits return Result(reply=said.pop("reply"), finish_reason=said.pop("finish_reason", None), trajectory=steps, usage=usage, session=handle, exit_code=code, timed_out=timed_out, log=stream, extra={"failed": False, "login_failed": False, "rejected": False, **said, - "started": any(line.lstrip().startswith("{") for line in raw.splitlines()), + "started": any(line.lstrip().startswith("{") for line in raw.splitlines()) + if began is None else bool(began), "cancelled": cancelled, "isolation": isolation, "wall_s": round(wall_s, 1), "mounts": placed, "output_tail": raw[-2000:]}) @@ -1462,8 +1478,311 @@ def _pi_session(home: Path, handle) -> Path | None: return sorted(found)[-1] if found else None +DSH_PROVIDER = "hermit" # the pi-ai route (`llm-pi-ai`) a bound model is given under +DSH_ROUTE = "deepseek-official" # dsh's own route, for a model of its own provider (no binding) +DSH_PATCH = "hermit.patch.yml" # the session's patch layer, in its home (`--patch`) +# Rows of dsh's tree off in every session: no telemetry export, no title request, no plugin list sent along with a +# request to DeepSeek's own API (nor the session log, a row of its own below; such a request still carries dsh's +# anonymous user id and the session's id), so the session's requests are the only ones; and no sub-agents or +# workflows, whose usage dsh's stream does not report (sub-agents are Codex's only when a role asks for them) +DSH_OFF = ("session-telemetry-otel", "session-title-llm", "plugin-package-inventory-deepseek", "tool-subagent", + "tool-subagent-fork", "tool-subagent-control", "tool-workflow") +DSH_PERMISSION_MODES = ("read-only", "workspace-write", "danger-full-access") +DSH_EFFORTS = ("off", "minimal", "low", "medium", "high", "xhigh", "max") # the reasoning levels of dsh's pi-ai routes +DSH_OWN_EFFORTS = ("off", "low", "high", "max") # those of its own route (DSH_ROUTE, DeepSeek's API): no binding +# The variable a route names for a server that needs no key: a route that names none gets no key from pi-ai under +# the name `hermit`, and dsh refuses the request (PI_AI_ERROR "No API key") +DSH_NO_KEY = "HERMIT_DSH_NO_KEY" +# dsh's cap on the project's instructions as the model is given them (agent-instructions' maxBytes), and what dsh puts +# around one AGENTS.md at the project's root within it (its block, the block's opening sentences and +# the file's heading, 0.2.2): an AGENTS.md longer than the difference is cut, which dsh tells the model only +DSH_INSTRUCTION_BYTES = 65536 +DSH_INSTRUCTION_FRAME = 304 +# dsh's report of an MCP row that did not start (` (): `): the session then runs without its tools +DSH_TOOLS_DOWN = re.compile(r"^(mcp-[A-Za-z0-9_-]{1,32}) \(@deepseek-ai/dsh-mcp-client\): (.*)$", re.M) +# The session's transcripts as plain JSON lines (dsh compresses them with zstd), so that hermit.redact scrubs a key a +# session wrote from its home as from every record; the row's root is dsh's own, an expression (`!!js`) of the home +# as dsh sees it, which a sandbox shows at another path than this process's +DSH_TRANSCRIPTS = ('{"id": "session-persistence-jsonl", "config": {"root": !!js "dshHomePath(\'sessions\')", ' + '"compression": "none"}}') +# Where dsh's web search goes, whatever the session's model: DeepSeek's own API, on this key +DSH_SEARCH_HOST, DSH_SEARCH_KEY = "api.deepseek.com", "DEEPSEEK_API_KEY" + + +def dsh_problems(settings: dict, who: str) -> list[str]: + """What every session of a dsh agent would refuse of its settings, found before the first one rather than at each + session, after its task's or its learner step's rollouts: an effort its model's route has not (dsh's own, for a + model of no binding, has fewer levels than a bound model's pi-ai route) or one with no model to take it; a sandbox + of dsh's own it has not. A setting the agent does not take is refused rather than dropped: the Evolver runs dsh's + sandbox off and gives its sessions no instruction file (AgentEvolver.settings), and the adapter's context reaches + the acting agent as its AGENTS.md. `settings` are the agent's params as bound (the run's default model in; the + Evolver's method's own params under the run's); `who` is its section of the run config, agent or evolver.""" + effort, mode = settings.get("effort"), settings.get("permission_mode") + effort = "off" if effort is False else effort # YAML reads a bare `off` as false + levels, route = (DSH_EFFORTS, "a bound model") if settings.get("provider") else \ + (DSH_OWN_EFFORTS, "a model of dsh's own provider (DeepSeek's API)") + out = [] + if effort is not None and str(effort) not in levels: + out.append(f"{who}.model.effort is one of {', '.join(levels)} for {route}, not {effort!r}") + if effort is not None and not settings.get("model"): + out.append(f"{who}.model.effort needs {who}.model for dsh: it is that model's reasoningEffort") + if who == "agent" and mode is not None and mode not in DSH_PERMISSION_MODES: + out.append(f"agent.dsh.permission_mode is one of {', '.join(DSH_PERMISSION_MODES)} (dsh's own sandbox), " + f"not {mode!r}") + if who == "evolver" and mode not in (None, "danger-full-access"): + out.append("evolver.dsh.permission_mode is not taken: the dsh Evolver runs dsh's own sandbox off " + "(danger-full-access), the evolver sandbox being the boundary, as for Codex") + if settings.get("instruction_files") not in (None, who == "agent"): # what each takes: its default + out.append(f"{who}.dsh.instruction_files is not taken: " + ( + "the adapter's context reaches the acting agent as its AGENTS.md" if who == "agent" else + "the dsh Evolver's sessions read no AGENTS.md (an adapter's is the acting agent's policy)")) + return out + + +class Dsh(CliAgent): + """DeepSeek Harness (`dsh --profile headless --json`). Its home is DSH_HOME and its HOME (dsh reads ~/.agents), + where dsh makes the headless profile on first use. The session's settings are a patch layer of dsh's own (rows + of its plugin tree, `--patch`), written in the home: a bound model as the pi-ai route `hermit`, its key read by + dsh from the backend's variable, made the default model; the call's tools as an MCP server, whose tools the model + is offered as its own (`mcp____`); web search and fetch off unless asked for (search is DeepSeek's + own service: with a model of another backend, fetch only); no telemetry export and no title request, no + sub-agents (DSH_OFF); the workspace as the project, as in the run's sandbox: its own skills (.dsh/skills) and + instructions only, never those of a folder above it (a repository it lies in). dsh runs commands in its own + sandbox (`permission_mode`, default workspace-write) and is never asked for an approval (`--approval-policy + never`: what would need one is refused). It exits 0 for a run SIGTERM stopped too: whether the run failed is read + from its end (`outcome`).""" + name = backend = program = "dsh" + label = "DeepSeek Harness" + home_var, user_folder = "DSH_HOME", ".dsh" + home_files = (".env",) # dsh reads the keys of its own provider from $DSH_HOME/.env + turn_done = "final" + binds_mounts = True # dsh's glob and grep run ripgrep without --follow + + def home_env(self, home): + """The variables that set where dsh reads its own files from, here, so that they pass the sandbox's filter: + its home, HOME, and the shared agents' folder dsh reads skills and AGENTS.md from (~/.agents unless + DSH_AGENTS_HOME says another), so none of the user's own is read; telemetry off; the placeholder key of a + server that needs none (DSH_NO_KEY). dsh also reads a `.env` in the working directory (one that sets a name + dsh keeps for its launch, such as PATH or PYTHONPATH, stops it before it writes an event) and in its home.""" + from hermit.agent.model.backends import PLACEHOLDER_KEY + return {"DSH_HOME": str(home), "HOME": str(home), "DSH_AGENTS_HOME": str(home / ".agents"), + "DSH_TELEMETRY_DISABLED": "1", DSH_NO_KEY: PLACEHOLDER_KEY} + + @classmethod + def user_home(cls) -> Path: + return Path(os.environ.get("DSH_HOME") or Path.home() / cls.user_folder) + + def dropped_env(self): + """The caller's own dsh variables (`DSH_*`: another permission mode, a folder of skills, a telemetry endpoint, + a web provider, ...; and the hosts of DeepSeek's API it reaches, DEEPSEEK_BASE_URL for its own provider's + requests and DEEPSEEK_SEARCH_BASE_URL for its web search), which would change the session without the + sandbox: its own are home_env's. They are dropped whatever a sandbox lets through (sandbox.evolver.env, + environment.env), which may be for the run's other processes: hermit's own DEEPSEEK_BASE_URL moves its + `deepseek` backend, a chat completions root, and dsh reads it as its own provider's Messages root. A variable + the session is given (`env`, where `$DEEPSEEK_BASE_URL` is this process's own) still passes.""" + return tuple(name for name in os.environ + if name.startswith("DSH_") or name in ("DEEPSEEK_BASE_URL", "DEEPSEEK_SEARCH_BASE_URL")) + + def prepare_home(self, home, roles, tools_url, tools_name): + """The patch layer (DSH_PATCH): dsh's own rows, by their ids. A row's `config` replaces the row's, so a row is + given whole (the instructions' maxBytes is dsh's own). No host path is written in it: a sandbox translates + only the command, the environment and the prompt; the skills' folder is the working directory's, the + transcripts' root dsh's own expression (DSH_TRANSCRIPTS).""" + if roles: + raise Unsupported("dsh: roles are not passed to DeepSeek Harness; give it none") + if tools_url and not re.fullmatch(r"[A-Za-z0-9_-]{1,32}", tools_name): + raise ValueError(f"dsh takes an MCP server name of 32 characters at most ([A-Za-z0-9_-]), not " + f"{tools_name!r}") + s = self.settings + rows: list[dict] = [*({"id": row, "disabled": True} for row in DSH_OFF), + {"id": "session-log-deepseek", "config": {"enabled": False}}, + # The workspace is the project, wherever it lies: the skills of its own .dsh/skills only + # (not those of the nearest folder above with a .git, a repository the run sits in, nor + # the user's), and the instructions of the working directory, the workspace (no root + # marker: dsh reads none above it, a repository's or a deployment's around it); none with + # instruction_files false (dsh also adds a folder's AGENTS.md below the working directory + # when its file tools touch the folder) + {"id": "skill-filesystem", "config": {"includeDefaultRoots": False, + "customSkillDirs": [".dsh/skills"]}}, + {"id": "agent-instructions", "config": { + "maxBytes": DSH_INSTRUCTION_BYTES if s.get("instruction_files", True) else 0, + "projectRootMarkers": []}}] + binding = s.get("binding") or {} + if not s.get("web_search"): + rows.append({"id": "tool-web", "disabled": True}) # web search and fetch, the model's tools + elif s.get("provider") and (urllib.parse.urlparse(str(binding.get("base_url") or "")).hostname, + binding.get("api_key_env")) != (DSH_SEARCH_HOST, DSH_SEARCH_KEY): + # dsh's search is DeepSeek's own service (its Messages API on DEEPSEEK_API_KEY), whatever the session's + # model: with a model reached elsewhere or on another key it would search on another provider and key + # than the run's, outside its usage. Fetch only, as Codex's hosted search is off for a backend other than + # OpenAI's + rows.append({"id": "tool-web", "config": {"search": False, "fetch": True}}) + effort = "off" if s.get("effort") is False else str(s["effort"]) if s.get("effort") else None + levels = DSH_EFFORTS if s.get("provider") else DSH_OWN_EFFORTS # a bound model's route, or dsh's own + if effort and effort not in levels: + raise ValueError(f"dsh: an effort is one of {', '.join(levels)}" + f"{'' if s.get('provider') else ' for a model of its own provider'}, not {effort!r}") + if effort and not s.get("model"): + raise ValueError("dsh: an effort needs its model (the `model` setting): it is that model's reasoningEffort") + default, chosen = {"provider": DSH_ROUTE}, {} + if s.get("provider"): + provider = json.loads(json.dumps(s["provider"])) + provider.setdefault("apiKeyEnv", DSH_NO_KEY) # a server that needs no key: the placeholder + for model in provider.get("models") or []: + if effort: # a declared model is asked only the levels it lists; off: none, its server's default + model.setdefault("reasoningEfforts", False if effort == "off" else {effort: effort}) + if s.get("image_input"): # else dsh sends the model no image (a text-only model refuses them) + model["input"] = ["text", "image"] + if model.get("id") == s.get("model"): + chosen = model + rows.append({"id": "llm-pi-ai", "config": {"providers": {DSH_PROVIDER: provider}}}) + default["provider"] = DSH_PROVIDER + if s.get("model"): + default["model"] = str(s["model"]) + if effort and chosen.get("reasoningEfforts") is not False: # a model declared with no levels takes none + default["reasoningEffort"] = effort + rows.append({"id": "agent-default-model", "config": default}) + elif s.get("provider"): + raise ValueError("dsh: a provider route needs its model (the `model` setting)") + if tools_url: + # failOnStartupError: a server dsh cannot reach fails its row at once (dsh says so on stderr and goes on, + # and `outcome` fails the session) rather than being retried while the session runs without the tools + rows.append({"insert": [{"id": f"mcp-{tools_name}", "name": "@deepseek-ai/dsh-mcp-client", + "config": {"serverName": tools_name, "transport": "streamable-http", + "url": tools_url, "toolCallTimeoutMs": 1_800_000, + "failOnStartupError": True}}]}) + _clear(home / DSH_PATCH) + # JSON is YAML, which dsh reads: a flow sequence, to which the transcripts' row adds its `!!js` root + text = json.dumps(rows, indent=1) + (home / DSH_PATCH).write_text(f"{text[:text.rindex(']')].rstrip()},\n {DSH_TRANSCRIPTS}\n]\n") + + @classmethod + def web_search_args(cls, enabled: bool) -> list[str]: + """dsh's web search and fetch are a row of its tree, set in the session's patch layer (prepare_home).""" + return [] + + def system_settings(self, system, prompt): + """dsh takes no system prompt for one run, and its persona text (the system-prompt row) takes only the + variables `{{provider}}` and `{{model}}`: any other `{{...}}`, which instructions may hold, fails the model's + step. The instructions go at the top of the prompt instead, as a marked block.""" + return {}, f"# SYSTEM INSTRUCTIONS\n\n{system}\n\n# END\n\n{prompt}" + + @staticmethod + def _reason(raw: str) -> dict: + """The last `final` event's reason ({} when there is none: the run never finished).""" + finals = [e for e in _objects(raw)[0] if e.get("type") == "final"] + reason = finals[-1].get("reason") if finals else None + return reason if isinstance(reason, dict) else {} + + @classmethod + def served(cls, raw): + """The run's last turn completed (its `final` reason).""" + return cls._reason(raw).get("kind") == "completed" + + def outcome(self, raw, stream, last_file): + """dsh exits 0 for a run whose `final` event is `completed`, and 1 for any other end (an error, an abort, the + token limit, ...), but also 0 when SIGTERM stops it: a run without a completed end has failed, whatever its + exit code. So has one whose tools did not start (an MCP row dsh reports inactive): it ran without them. The + session started once dsh wrote its `session` event (a usage error is an `error` event before any session).""" + notes, done = self.notices(raw), self.served(raw) + down = DSH_TOOLS_DOWN.findall(notes) + failure = f"{down[0][0]} did not start, so the session ran without its tools: {down[0][1]}" if down else \ + notes or "the run did not finish (no completed `final` event)" + if "CONTEXT_WINDOW_EXCEEDED" in notes and self.settings.get("binding") and \ + not self.settings["binding"].get("context_window"): # dsh's error line, then what to do + hint = ("; the backend declares no context_window, so dsh sized the model by its own catalog or, for a " + "model it does not know, as 1,000,000 tokens with 128,000 output tokens a request, which a smaller " + "server refuses: declare the backend's context_window") + said = next(line for line in notes.splitlines() if "CONTEXT_WINDOW_EXCEEDED" in line) + failure = said[:600 - len(hint)] + hint + return {"reply": self.last_message(stream, last_file), "served": done, + "unavailable": bool(UNAVAILABLE.search(notes)), "refused": False, "failed": not done or bool(down), + "failure": failure[-600:], "started": any(e.get("type") == "session" for e in _objects(raw)[0])} + + @classmethod + def errors(cls, event): + """A run that did not complete: an error, as dsh writes it on stderr (`dsh: : `: the code is + how a provider that could not serve is told, UNAVAILABLE), or any other end (aborted, max-tokens, blocked, + interrupted, ...); a failure outside a turn.""" + kind = event.get("type") + reason = event.get("reason") if isinstance(event.get("reason"), dict) else {} + if kind == "final" and reason.get("kind") not in (None, "completed"): + error = reason.get("error") if isinstance(reason.get("error"), dict) else {} + if error.get("code"): + return [f"dsh: {error['code']}: {error.get('message') or ''}"] + return [str(error.get("message") or f"run {reason['kind']}")] + if kind == "error": + return [str(event.get("message") or "")] + return [] + + @classmethod + def last_message(cls, log, last=None): + """The `final` event's answer (dsh's last assistant text, never cut).""" + text = "" + for event in events(log): + if event.get("type") == "final" and isinstance(event.get("text"), str): + text = event["text"] + return text + + @classmethod + def parse(cls, log, model): + return parse_dsh(log, model) + + @classmethod + def steps(cls, stream): + return dsh_steps(stream) + + def command(self, prompt, workspace, last_file, *, session=None, roles=(), tools_url=None): + """The launcher's options first (the profile, the `--patch` layers: those in `args`, a deployment's, then + the session's own, which is applied last and so wins), then the headless app's; dsh's launcher reads its + own options only before the first one it does not know. Under workspace-write the `writable` paths are + granted to dsh's own sandbox by their real paths, as it requires (an existing directory, no link).""" + s, home = self.settings, last_file.parent # the session's home + mode = s.get("permission_mode") or "workspace-write" + if mode not in DSH_PERMISSION_MODES: + raise ValueError(f"dsh: permission_mode is one of {DSH_PERMISSION_MODES}, not {mode!r}") + added = [] + for path in s.get("writable") or () if mode == "workspace-write" else (): + real = os.path.realpath(path) + if not os.path.isdir(real): + raise ValueError(f"dsh: a writable path is an existing directory, not {str(path)!r}") + added += ["--add-dir", real] + patches, rest = _dsh_args(s.get("args") or []) + resume = _dsh_session(home, session) if session else None + return [str(s.get("bin") or self.program), "--profile", "headless", *patches, + "--patch", str(home / DSH_PATCH), "--json", "--permission-mode", mode, + "--approval-policy", "never", *added, + *(["--session-id", resume] if resume else []), *rest, + *(["-"] if s.get("stdin") else ["--", prompt])] + + +def _dsh_args(args) -> tuple[list[str], list[str]]: + """(the `--patch` layers among `args`, the other arguments): dsh's launcher takes a layer only before the app's + options, so the layers go first, in their order.""" + args, patches, rest, i = [str(a) for a in args], [], [], 0 + while i < len(args): + if args[i] == "--patch" and i + 1 < len(args): + patches += args[i:i + 2] + i += 2 + continue + (patches if args[i].startswith("--patch=") else rest).append(args[i]) + i += 1 + return patches, rest + + +def _dsh_session(home: Path, handle) -> str | None: + """Session `handle` when `home` keeps it (sessions///): dsh refuses an id it does not know, so + a call whose session is gone starts a new one. dsh also resumes a session only in the directory it ran in, which + an AgentEvolver's calls share (a resume elsewhere would fail rather than start over).""" + if not re.fullmatch(r"session-[A-Za-z0-9-]+", str(handle)): + return None + root = Path(home) / "sessions" + found = [p for p in root.glob(f"*/{handle}") if p.is_dir() and not p.is_symlink() and not p.parent.is_symlink()] + return str(handle) if found else None + + # Every CLI Base Agent, by each name make() takes for it. -CLIS: dict[str, type[CliAgent]] = {n: c for c in (Codex, Claude, Pi) for n in (c.backend, *c.aliases)} +CLIS: dict[str, type[CliAgent]] = {n: c for c in (Codex, Claude, Pi, Dsh) for n in (c.backend, *c.aliases)} def _this_call(home: Path, handle: str, reported, resumed: bool): @@ -1538,8 +1857,8 @@ def _bound(command: list[str], mounts: list[Mount], workspace: Path, hidden: lis if inside and "linked" not in _WARNED: _WARNED.add("linked") print("hermit: bubblewrap cannot make a mount namespace here, so a task's data in the workspace is linked, " - "not bound: a search that walks the tree (`grep -r`, `find`, Claude Code's Glob and Grep) does not " - "follow a link", + "not bound: a search that walks the tree (`grep -r`, `find`, Claude Code's Glob and Grep, dsh's glob " + "and grep) does not follow a link", file=sys.stderr, flush=True) if hidden and "plan" not in _WARNED: _WARNED.add("plan") diff --git a/hermit/agent/base/login.py b/hermit/agent/base/login.py index e6141a77..8778f465 100644 --- a/hermit/agent/base/login.py +++ b/hermit/agent/base/login.py @@ -603,18 +603,23 @@ def logins_of(data: dict, evolver_params: dict | None = None) -> list[tuple[str, def evolver_problems(params: dict | None) -> list[str]: - """What an agent Evolver's own sign-in cannot run, found before the first step rather than at each step's session - (after its rollouts): a login that is not its CLI's (`backend`), a Claude login with no model name to run.""" + """What an agent Evolver's own sign-in or settings cannot run, found before the first step rather than at each + step's session (after its rollouts): a login that is not its CLI's (`backend`), a Claude login with no model name + to run, a dsh setting dsh refuses (cli.dsh_problems). `params` are the Evolver's as the Trainer gives them: its + method's own under the run's, the model bound.""" params = params or {} login = params.get("login") + out = [] + if params.get("backend") == "dsh": + from hermit.agent.base.cli import dsh_problems + out += dsh_problems(params, "evolver") if not login: - return [] + return out own = LOGINS.get(login) backend = {"claude-code": "claude"}.get(params.get("backend"), params.get("backend")) - out = [] - if backend == "pi": - out.append(f"login: {login} is a Codex or Claude Code sign-in; the pi Evolver signs in through its model " - "reference (evolver.model: /)") + if backend in ("pi", "dsh"): + out.append(f"login: {login} is a Codex or Claude Code sign-in; the {backend} Evolver signs in through its " + "model reference (evolver.model: /)") if own is not None and backend in ("codex", "claude") and own != backend: out.append(f"login: {login} is {'Codex' if own == 'codex' else 'Claude Code'}'s sign-in; the Evolver's " f"sessions run {'Codex' if backend == 'codex' else 'Claude Code'}, which takes login: " diff --git a/hermit/agent/base/records.py b/hermit/agent/base/records.py index 225360d0..d2657989 100644 --- a/hermit/agent/base/records.py +++ b/hermit/agent/base/records.py @@ -12,7 +12,9 @@ def price_usage(usage: list[Usage], params: dict) -> None: """Fill in each usage's cost from `price` (USD per million tokens: {input, cached_input, output}) where the agent reported none. Claude Code reports a cost at Anthropic's prices; for a model bound to another backend that is not the bill, so the backend's price replaces it (no price: the cost is unknown). On a subscription (`login`) that - figure is the same estimate at list prices, not what the session costs: none is kept.""" + figure is the same estimate at list prices, not what the session costs: none is kept. A usage that holds calls + whose tokens the CLI did not report (`unreported_calls`) keeps an unknown cost: its reported tokens' price would + be too little.""" price = params.get("price") binding = params.get("binding") or {} foreign = bool(params.get("login")) or \ @@ -20,7 +22,7 @@ def price_usage(usage: list[Usage], params: dict) -> None: for u in usage: if foreign: u.cost_usd = None - if price and u.cost_usd is None: + if price and u.cost_usd is None and not u.unreported_calls: u.cost_usd = ((u.input_tokens - u.cached_input_tokens) * price.get("input", 0) + u.cached_input_tokens * price.get("cached_input", price.get("input", 0)) + u.output_tokens * price.get("output", 0)) / 1e6 @@ -124,6 +126,64 @@ def parse_pi(log: Path, model: str) -> tuple[str | None, list[Usage]]: return _parse(log, model, _pi_event) +def _dsh_reader(): + """DeepSeek Harness: the session id (its opening `session` event), and each model step's usage (`status` events + of phase `step_end`): the sum of its attempts' (a request dsh retried is an attempt). dsh's `inputTokens` leaves + the cache out, so the input counted is input, cache reads and cache writes, as Codex and Claude Code count it. dsh + reports no cost: it is priced from the backend's price (price_usage), which prices a cache write as input. + + A step that produced nothing (its requests all failed) is no call; one that produced a reply, a thought or a tool + call is, even when its tokens are not reported: the provider reported no usage (dsh then reports zeros), or an + attempt reported none at all (a stream that ended before its last event, a failed request of dsh's own provider: + dsh then leaves the step's usage out). Such a step is an unreported call: it adds no tokens, and the session's + cost is unknown (price_usage), not the price of what was reported. + + A step one of whose attempts reported no cache count (a request the provider refused, as for a rate limit dsh + retried, reports zeros, and dsh keeps a cache count only when it is not 0) has its cache counts left out by dsh, + which its `totalTokens` still holds. What the total holds beyond the counts reported is counted as input, priced + as a cache write is: the step's price is what it would be with the counts reported, or more if those tokens were + read from the cache (a price whose cached input is no dearer than its input), never less. The total is the sum + of the counts on every route but the Responses API's, where it is the server's own `total_tokens`, trusted: what + it holds beyond the counts is priced as input, and a server that reports none leaves those tokens out. + + What dsh's stream leaves out is not counted: a compaction's summary request, web searches (each a request of + DeepSeek's own API), the title request (off in a session, hermit.agent.base.cli.DSH_OFF), sub-agents, what a + provider billed for an attempt it reported as zeros (one cut off mid-stream, before its usage), and the last + step of a session a stop (SIGTERM: a timeout, a cancelled call) cut short before that step's end: dsh writes a + step's usage at its end, after its tool calls ran, and a stopped session writes nothing more.""" + produced = False + + def read(event: dict, usage: Usage, session): + nonlocal produced + kind = event.get("type") + if kind == "session": + session = event.get("sessionId") or session + if kind in ("text", "thinking", "tool_call"): + produced = True + if kind != "status" or event.get("phase") != "step_end": + return session + made, produced = produced, False + u = event.get("usage") if isinstance(event.get("usage"), dict) else {} + cached, output = int(u.get("cacheReadTokens") or 0), int(u.get("outputTokens") or 0) + tokens = int(u.get("inputTokens") or 0) + cached + int(u.get("cacheWriteTokens") or 0) + dropped = max(int(u.get("totalTokens") or 0) - tokens - output, 0) # cache counts dsh left out + if not tokens and not output and not dropped: + usage.calls += 1 if made else 0 + usage.unreported_calls += 1 if made else 0 + return session + usage.input_tokens += tokens + dropped + usage.cached_input_tokens += cached + usage.output_tokens += output + usage.calls += 1 + return session + return read + + +def parse_dsh(log: Path, model: str) -> tuple[str | None, list[Usage]]: + """DeepSeek Harness's `--json` events: the session id, and every model step's usage (_dsh_reader).""" + return _parse(log, model, _dsh_reader()) + + def parse_codex(log: Path, model: str) -> tuple[str | None, list[Usage]]: """Codex's `exec --json` events: the thread id, and each completed turn's usage.""" return _parse(log, model, _codex_event) @@ -135,5 +195,5 @@ def parse_claude(log: Path, model: str) -> tuple[str | None, list[Usage]]: def parse_log(log: Path, backend: str, model: str) -> tuple[str | None, list[Usage]]: - """Session handle and usage from the agent's JSON event stream: Codex's, else Claude Code's.""" - return (parse_codex if backend == "codex" else parse_claude)(log, model) + """Session handle and usage from the agent's JSON event stream: Codex's, dsh's, else Claude Code's.""" + return {"codex": parse_codex, "dsh": parse_dsh}.get(backend, parse_claude)(log, model) diff --git a/hermit/agent/base/trajectory.py b/hermit/agent/base/trajectory.py index 8cb731df..91a4a321 100644 --- a/hermit/agent/base/trajectory.py +++ b/hermit/agent/base/trajectory.py @@ -9,8 +9,8 @@ text(steps, message=1200, output=600) # one line per step, each field clipped from its start The streams read: Codex's `codex exec --json` events (`agent-codex.jsonl`), Claude Code's stream-json -(`agent-claude.jsonl`) and Pi's `--mode json` events (`agent-pi.jsonl`), also under another name below a log -directory (Harbor's `agent/codex.txt`). A normalised +(`agent-claude.jsonl`), Pi's `--mode json` events (`agent-pi.jsonl`) and DeepSeek Harness's `--json` events +(`agent-dsh.jsonl`), also under another name below a log directory (Harbor's `agent/codex.txt`). A normalised message list (`RunOutput.messages`) is the pipeline's own rendering and is left to each Evolver. """ from __future__ import annotations @@ -18,11 +18,14 @@ from dataclasses import dataclass import json from pathlib import Path +import re CODEX_STREAM = "agent-codex.jsonl" CLAUDE_STREAM = "agent-claude.jsonl" PI_STREAM = "agent-pi.jsonl" -STREAM_HOSTS = {CODEX_STREAM: "codex", CLAUDE_STREAM: "claude", PI_STREAM: "pi"} # the file a host's log keeps +DSH_STREAM = "agent-dsh.jsonl" +STREAM_HOSTS = {CODEX_STREAM: "codex", CLAUDE_STREAM: "claude", PI_STREAM: "pi", # the file a host's log keeps + DSH_STREAM: "dsh"} STREAMS = tuple(STREAM_HOSTS) OTHER_STREAMS = ("agent/codex.txt", "codex.txt") # a Harbor trial's Codex events: from the trial or its agent/ @@ -213,12 +216,89 @@ def pi_steps(stream: list[dict]) -> list[Step]: return steps +DSH_FILE_TOOLS = ("write", "edit") +# How dsh's bash tool ends a command's result (its own parseExitStatus reads it so, anchored at the very end, as +# JavaScript's `$` is: \Z here): a nonzero exit or a signal is marked last, a clean exit is not; a command stopped by +# its timeout or from outside is marked before that (a command that traps the signal may still exit 0), and one still +# running was moved to the background, unfinished (its marker is the whole result when it printed nothing yet). A +# timeout is printed as dsh applied it, a JavaScript number: the model's (`1000.5`, `1e-7`), else 60000, at most 600000 +DSH_EXIT = re.compile(r"\n\[exit code: (\d+)\]\Z") +DSH_SIGNAL = re.compile(r"\n\[killed by signal: [^\]\n]+\]\Z") +DSH_STOPPED = re.compile( + r"\n\[(?:timed out after \d+(?:\.\d+)?(?:e[+-]\d+)?ms|stopped: [^\]\n]*)\](?:\n\[[^\]\n]*\])*\Z") +DSH_BACKGROUND = re.compile(r"\[still running after \d+(?:\.\d+)?(?:e[+-]\d+)?ms; moved to background job [^\]\n]+\]\Z") +DSH_NO_OUTPUT = "(no output)" # a finished command's empty output, alone or before its markers +# dsh's --json keeps 8 KiB of each string and marks the event `truncated`: the end, where the marker was, is gone +DSH_CUT = "\n[the rest was cut: dsh's --json keeps 8 KiB of a string]" + + +def dsh_steps(stream: list[dict]) -> list[Step]: + """DeepSeek Harness's `--json` events: its thinking and text as each model step committed them, and each + `tool_call` completed by its `tool_result`: bash as a command whose exit code and output are read from the + result as dsh reads them (`[exit code: N]` after the output; no marker: 0; a signal, a timeout, a stop, a command + moved to or started in the background, a result dsh's --json cut (its marker is gone) or a call that failed to + run: unknown; `(no output)`: empty), write and edit as file changes, an MCP tool (`mcp____`) as a + tool call with its server and arguments and its result as `output`, any other tool as a tool call; a run that did + not complete (the `final` event's reason: an error, an abort, the token limit, ...), or failed outside a turn (an + `error` event), as an error. A text cut by --json says so at its end (DSH_CUT). Status events and usage are left + out.""" + steps, pending, background = [], {}, set() + for event in stream: + kind = event.get("type") + cut = DSH_CUT if event.get("truncated") else "" + if kind == "thinking" and str(event.get("text") or "").strip(): + steps.append(Step("reasoning", str(event["text"]) + cut)) + elif kind == "text" and str(event.get("text") or "").strip(): + steps.append(Step("message", str(event["text"]) + cut)) + elif kind == "tool_call": + name = str(event.get("tool") or "") + args = event.get("input") if isinstance(event.get("input"), dict) else {} + if name == "bash": + step = Step("command", command=str(args.get("command") or "")) + if args.get("run_in_background"): # "started background job ": no exit status yet + background.add(event.get("callId")) + elif name in DSH_FILE_TOOLS: + step = Step("file_change", json.dumps({"tool": name, "path": args.get("file_path")}, + ensure_ascii=False, default=str)) + elif name.startswith("mcp__") and name.count("__") >= 2: + _, server, tool = name.split("__", 2) + step = Step("tool_call", json.dumps({"arguments": args, "server": server, "tool": tool}, + ensure_ascii=False, default=str)) + else: + step = Step("tool_call", json.dumps({"tool": name, "input": event.get("input")}, ensure_ascii=False, + default=str)) + steps.append(step) + pending[event.get("callId")] = step + elif kind == "tool_result": + step = pending.pop(event.get("callId"), None) + if step is None: + continue + result = str(event.get("result") or "") + if step.kind == "command": + found = None if cut else DSH_EXIT.search(result) + output = result[:found.start()] if found else result + step.output = ("" if output == DSH_NO_OUTPUT else output) + cut + unknown = cut or event.get("status") != "completed" or event.get("callId") in background or \ + DSH_SIGNAL.search(result) or DSH_STOPPED.search(result) or DSH_BACKGROUND.search(result) + step.exit_code = None if unknown else int(found.group(1)) if found else 0 + elif step.kind == "tool_call": + step.output = result + cut + elif kind == "final" and isinstance(event.get("reason"), dict) and \ + event["reason"].get("kind") not in (None, "completed"): + error = event["reason"].get("error") if isinstance(event["reason"].get("error"), dict) else {} + steps.append(Step("error", str(error.get("message") or f"run {event['reason']['kind']}"))) + elif kind == "error": + steps.append(Step("error", str(event.get("message") or ""))) + return steps + + # Each host's stream: the reader of its events, and the event types that mark it (tried in this order when a stream # is found by its events). A host's stream is added here and in STREAM_HOSTS. -READERS = {"codex": codex_steps, "claude": claude_steps, "pi": pi_steps} +READERS = {"codex": codex_steps, "claude": claude_steps, "pi": pi_steps, "dsh": dsh_steps} MARKS = (("codex", {"thread.started", "turn.started", "turn.completed", "turn.failed", "item.completed"}), ("claude", {"system", "assistant", "user", "result"}), - ("pi", {"agent_start", "agent_settled", "message_end", "tool_execution_end"})) + ("pi", {"agent_start", "agent_settled", "message_end", "tool_execution_end"}), + ("dsh", {"status", "tool_call", "tool_result", "final"})) def _host(stream: list[dict]) -> str | None: diff --git a/hermit/agent/evolver/agent.py b/hermit/agent/evolver/agent.py index 9c6595f0..79e20051 100644 --- a/hermit/agent/evolver/agent.py +++ b/hermit/agent/evolver/agent.py @@ -1,13 +1,14 @@ -"""AgentEvolver: a built-in agent (Codex, Claude Code or Pi) run on the workspace with a prompt; CommandEvolver: any -program. +"""AgentEvolver: a built-in agent (Codex, Claude Code, Pi or DeepSeek Harness) run on the workspace with a prompt; +CommandEvolver: any program. AgentEvolver params: - backend: codex | claude | pi + backend: codex | claude | pi | dsh model: the model passed to the agent (optional: the agent's default); with `login`, the CLI's model name on the user's subscription (hermit.agent.base.login: `chatgpt` for Codex, `claude` for Claude Code). A `/` reference (see hermit.agent.model.backends) binds it to that backend: the provider settings go into `args` (Codex), `env` - (Claude Code) or `provider` (Pi's models.json entry) and `price` is taken from the backend + (Claude Code) or `provider` (Pi's models.json entry, DeepSeek Harness's pi-ai route) and `price` is taken + from the backend login, plan_dir, plan_max_wait_s, plan_rerun, usage_credits: the CLI's subscription and its plan (hermit.agent.base.cli, hermit.agent.base.login; claude: the token in CLAUDE_CODE_OAUTH_TOKEN) env: environment variables set for the agent; a `$NAME` value is expanded from the environment at launch (Claude @@ -22,9 +23,10 @@ tools: the only Trainer tools the agent may use (for example [check_spec]); null: all of them. With a list, the task message names no endpoint URL either web_search: the CLI's web search, true or false -This is Layer 3 over the `codex`, `claude` or `pi` Base Agent (docs/framework/agent-layers.md): it fills the prompt, -picks the Trainer's tools and resumes the last session; the Base Agent runs the CLI in the run's evolver sandbox (the -workspace under /w, the paths in the prompt relative to it), with the tools as one MCP server (`evolver_tools`). +This is Layer 3 over the `codex`, `claude`, `pi` or `dsh` Base Agent (docs/framework/agent-layers.md): it fills the +prompt, picks the Trainer's tools and resumes the last session; the Base Agent runs the CLI in the run's evolver +sandbox (the workspace under /w, the paths in the prompt relative to it), with the tools as one MCP server +(`evolver_tools`). """ from __future__ import annotations @@ -78,7 +80,7 @@ class AgentEvolver(Evolver): def step(self, ctx: StepContext) -> StepResult: params = ctx.params backend = params.get("backend", "codex") - if backend not in ("codex", "claude", "pi"): + if backend not in ("codex", "claude", "pi", "dsh"): return StepResult("failed", f"unknown backend {backend!r}") if not isinstance(params.get("web_search"), bool): return StepResult("failed", "evolver.params.web_search must be set, true or false") @@ -130,6 +132,10 @@ def settings(self, ctx: Context, backend: str) -> dict: "web_search": params["web_search"], "env": env, "isolation": "sandbox" if ctx.sandbox is not None else "none", "sandbox": ctx.sandbox, "codex_sandbox": "danger-full-access", "shell_env": "all", + # dsh's own sandbox off, as Codex's; and no instruction file of the evolver workspace: dsh adds a + # folder's AGENTS.md when its file tools touch it (an adapter's is the acting agent's policy), which + # Codex's learner is never given + **({"permission_mode": "danger-full-access", "instruction_files": False} if backend == "dsh" else {}), "home": ctx.workspace.state.parent / "agent_home", "home_files": params.get("home"), "log": ctx.workspace.output / f"agent-{backend}.jsonl", "timeout_s": None, "cancel": ctx.cancel, "tools_name": "evolver_tools"} diff --git a/hermit/agent/evolver/methods/ahe_adapter/debugger.py b/hermit/agent/evolver/methods/ahe_adapter/debugger.py index 39ba9e9b..b457cec4 100644 --- a/hermit/agent/evolver/methods/ahe_adapter/debugger.py +++ b/hermit/agent/evolver/methods/ahe_adapter/debugger.py @@ -181,7 +181,12 @@ def trace_source(log: Path | None) -> tuple[Path, str] | None: ("the raw Codex event stream", "the raw Claude Code event stream"), ("the Codex event stream", "the Claude Code event stream"), ("`codex exec --json` event stream", - "`claude -p --output-format stream-json` event stream"))} + "`claude -p --output-format stream-json` event stream")), + "agent-dsh.jsonl": (("agent-codex.jsonl", "agent-dsh.jsonl"), + ("the raw Codex event stream", "the raw DeepSeek Harness event stream"), + ("the Codex event stream", "the DeepSeek Harness event stream"), + ("`codex exec --json` event stream", + "`dsh --profile headless --json` event stream"))} def stream_of(logs) -> str: diff --git a/hermit/agent/evolver/methods/dsh/RESUME.md b/hermit/agent/evolver/methods/dsh/RESUME.md new file mode 100644 index 00000000..08d6fc0b --- /dev/null +++ b/hermit/agent/evolver/methods/dsh/RESUME.md @@ -0,0 +1,4 @@ +A new step. {message} +Read {learning_material}/current/ (the rollouts of the adapter you handed back last time, and check_feedback.json if +the Trainer had to repair it), then improve {adapter} again as before: change only the adapter, call `check_spec`, +and write {output}/report.md (first line: what you changed and why). diff --git a/hermit/agent/evolver/methods/dsh/START.md b/hermit/agent/evolver/methods/dsh/START.md new file mode 100644 index 00000000..5560934e --- /dev/null +++ b/hermit/agent/evolver/methods/dsh/START.md @@ -0,0 +1,15 @@ +You are improving a harness adapter: the files an acting agent receives (context, tools, skills) when it works on a +task. Your job in this call is to change the adapter so the acting agent does better on tasks like the ones in the +learning material, within the adapter's specification. + +{message} +How to work: +1. Read the task description ({task_info}/README.md) and the adapter's guide ({adapter}/.hermit/EVOLVER.md): it says which + files you may change and in what format. +2. Read the learning material of this step ({learning_material}/current/): each task's input, the agent's output, the + judge's score and feedback, and the full trajectory. Earlier steps are next to it; index.jsonl shows the trend. +3. Decide what to change, and change only files in {adapter}. Keep what still helps; generalise from the material + instead of writing down facts about individual tasks. +4. Call the `check_spec` tool before you finish; fix what it reports. +5. Write {output}/report.md: first line one sentence on what you changed and why, then the details. +You may keep notes for later calls in {state}. Do not touch anything else. diff --git a/hermit/agent/evolver/methods/dsh/evolver.yaml b/hermit/agent/evolver/methods/dsh/evolver.yaml new file mode 100644 index 00000000..9496baa2 --- /dev/null +++ b/hermit/agent/evolver/methods/dsh/evolver.yaml @@ -0,0 +1,13 @@ +description: A DeepSeek Harness (dsh) agent improves the adapter from the step's learning material. +class: hermit.agent.evolver.agent:AgentEvolver +needs: [] +params: + backend: dsh + prompt: START.md + resume_prompt: RESUME.md + args: [] + model: null # a / reference (dsh's pi-ai route, the key read from its variable) + price: null # USD per million tokens {input, cached_input, output}; null: the backend's + home: [] # no file of ~/.dsh is copied: dsh signs in through the model reference + tools: null # the only Trainer tools the agent may use, e.g. [check_spec]; null: all + web_search: false # dsh's web search and fetch tools; the evolver sandbox reaches only the model diff --git a/hermit/agent/evolver/methods/mwm/evolver.py b/hermit/agent/evolver/methods/mwm/evolver.py index 4ae8ac0d..2d87393c 100644 --- a/hermit/agent/evolver/methods/mwm/evolver.py +++ b/hermit/agent/evolver/methods/mwm/evolver.py @@ -456,6 +456,9 @@ def _notes(admission: dict | None) -> str: # ------------------------------------------------------------------ steps def setup(self, ctx: Context) -> None: + if ctx.params.get("backend") == "dsh": # its session texts, homes and sign-ins are Codex's or Claude Code's + raise ValueError("the mwm Evolver's learner is a Codex or Claude Code session: evolver.params.backend is " + "codex or claude, not 'dsh'") if "network_isolation" in ctx.params: raise ValueError("evolver.params.network_isolation was removed: the learner runs in the run's evolver " "sandbox, whose network reaches only its models, the tool endpoint and " diff --git a/hermit/agent/evolver/methods/rrsi_adapter/evolver.py b/hermit/agent/evolver/methods/rrsi_adapter/evolver.py index 0ac28a4e..ed26de5f 100644 --- a/hermit/agent/evolver/methods/rrsi_adapter/evolver.py +++ b/hermit/agent/evolver/methods/rrsi_adapter/evolver.py @@ -417,6 +417,9 @@ def setup(self, ctx) -> None: """What every step needs, checked once: a broken hyperparameter file or domain, or an unset web_search, stops the run here rather than failing each step.""" _require_rrsi() + if ctx.params.get("backend") == "dsh": # its roles' settings are Codex's or Claude Code's + raise ValueError("the RRSI Adapter's proposer and roles are Codex or Claude Code sessions: " + "evolver.params.backend is codex or claude, not 'dsh'") if not isinstance(ctx.params.get("web_search"), bool): raise ValueError("evolver.params.web_search must be set, true or false") role = ctx.params.get("roles") or {} diff --git a/hermit/agent/model/backends.py b/hermit/agent/model/backends.py index 78b02ce8..1956e501 100644 --- a/hermit/agent/model/backends.py +++ b/hermit/agent/model/backends.py @@ -61,8 +61,10 @@ def user_file() -> Path: "context_window", "developer_role", "models", "description"} DEVELOPER_ROLES = ("keep", "system") # What each consumer speaks: the dialect a backend must serve for it. -NEEDS = {"chat": "chat", "react": "chat", "llm": "chat", "codex": "responses", "claude": "messages", "pi": "chat"} -# Pi speaks several dialects: the one it is given is the first the backend serves (Binding.pi_dialect). +NEEDS = {"chat": "chat", "react": "chat", "llm": "chat", "codex": "responses", "claude": "messages", "pi": "chat", + "dsh": "chat"} +# Pi speaks several dialects: the one it is given is the first the backend serves (Binding.pi_dialect). DeepSeek +# Harness (dsh) reaches a model through pi-ai too, so it speaks the same ones, chosen the same way. PI_APIS = {"chat": "openai-completions", "messages": "anthropic-messages", "responses": "openai-responses"} # The variable litellm reads the key from, per route prefix (the examples' CL-Bench host calls through litellm). LITELLM_KEY_ENV = {"fireworks_ai": "FIREWORKS_AI_API_KEY", "openai": "OPENAI_API_KEY", "hosted_vllm": "HOSTED_VLLM_API_KEY", @@ -325,6 +327,24 @@ def pi_provider(self) -> dict: provider["compat"] = {"supportsDeveloperRole": False} # the backend refuses the developer role return provider + def dsh_provider(self) -> dict: + """DeepSeek Harness's route for this model (a `providers` value of its pi-ai adapter, `llm-pi-ai`): the + backend's URL in the dialect dsh speaks to it (pi-ai's, as Pi's), the key's variable by name (`apiKeyEnv`, + which dsh reads at each request, so no file holds the key), and the model with its context window. dsh + reports no cost: the backend's price is applied to its usage (`price`).""" + api = self.pi_dialect() + model: dict = {"id": self.model} + if self.backend.context_window: + model["contextWindow"] = int(self.backend.context_window) + provider = {"api": PI_APIS[api], "models": [model], + "baseURL": (self.backend.messages_root if api == "messages" else self.base_url).rstrip("/")} + key_env = self.key_env_if_set() + if key_env: + provider["apiKeyEnv"] = key_env + if api == "chat" and self.backend.developer_role == "system": + provider["compat"] = {"supportsDeveloperRole": False} # the backend refuses the developer role + return provider + CLAUDE_ENV = ("ANTHROPIC_BASE_URL", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_API_KEY", "ANTHROPIC_MODEL", "ANTHROPIC_DEFAULT_OPUS_MODEL", "ANTHROPIC_DEFAULT_SONNET_MODEL", "ANTHROPIC_DEFAULT_HAIKU_MODEL", @@ -396,11 +416,11 @@ def _spec_of(value, binding: Binding) -> dict: def bind_agent_params(params: dict, kind: str, backends: dict[str, Backend], default=None) -> dict: - """The params of a Codex (`kind` codex), Claude Code (`kind` claude) or Pi (`kind` pi) agent or Evolver with its - model bound: `model` becomes the id the endpoint expects, the provider settings go into `args` (Codex), `env` - (Claude Code) ahead of the user's own, or `provider` (Pi's models.json entry), `price` is filled from the - backend, and `binding` records the whole. A `model` that is no reference (the CLI's own provider) is left - alone.""" + """The params of a Codex (`kind` codex), Claude Code (`kind` claude), Pi (`kind` pi) or DeepSeek Harness (`kind` + dsh) agent or Evolver with its model bound: `model` becomes the id the endpoint expects, the provider settings go + into `args` (Codex), `env` (Claude Code) ahead of the user's own, or `provider` (Pi's models.json entry, dsh's + pi-ai route), `price` is filled from the backend, and `binding` records the whole. A `model` that is no + reference (the CLI's own provider) is left alone.""" binding = as_binding(params.get("model") or default, backends) if binding is None: return params @@ -412,6 +432,8 @@ def bind_agent_params(params: dict, kind: str, backends: dict[str, Backend], def out["env"] = {**binding.claude_env(), **(params.get("env") or {})} elif kind == "pi": out["provider"] = params.get("provider") or binding.pi_provider() + elif kind == "dsh": + out["provider"] = params.get("provider") or binding.dsh_provider() else: raise ValueError(f"kind must be one of {CLI_KINDS}, not {kind!r}") if not out.get("price"): @@ -422,19 +444,19 @@ def bind_agent_params(params: dict, kind: str, backends: dict[str, Backend], def # ---- the run config ------------------------------------------------------------------------------------------- -CLI_AGENTS = {"codex": "codex", "claude-code": "claude", "claude": "claude", "pi": "pi", "cli": None} -CLI_KINDS = ("codex", "claude", "pi") # the CLI Base Agents a model is bound for (bind_agent_params) +CLI_AGENTS = {"codex": "codex", "claude-code": "claude", "claude": "claude", "pi": "pi", "dsh": "dsh", "cli": None} +CLI_KINDS = ("codex", "claude", "pi", "dsh") # the CLI Base Agents a model is bound for (bind_agent_params) def needs(kind: str, binding: Binding) -> str: """The dialect `kind` (a consumer of NEEDS) speaks to `binding`'s backend.""" - return binding.pi_dialect() if kind == "pi" else NEEDS.get(kind, "responses") + return binding.pi_dialect() if kind in ("pi", "dsh") else NEEDS.get(kind, "responses") def _cli_kind(agent: dict, params: dict) -> str: - """codex, claude or pi for an acting agent that is a CLI (by its name, `cli`'s backend, or a task's own agent's - agent.base), else "".""" - names = {"codex": "codex", "claude": "claude", "claude-code": "claude", "pi": "pi"} + """codex, claude, pi or dsh for an acting agent that is a CLI (by its name, `cli`'s backend, or a task's own + agent's agent.base), else "".""" + names = {"codex": "codex", "claude": "claude", "claude-code": "claude", "pi": "pi", "dsh": "dsh"} kind = CLI_AGENTS.get(str(agent.get("name") or ""), "") if kind is None: # `cli`: the backend is a param return names.get(params.get("backend", "codex"), "") diff --git a/hermit/agent/model/usage.py b/hermit/agent/model/usage.py index 096498a2..7845f44c 100644 --- a/hermit/agent/model/usage.py +++ b/hermit/agent/model/usage.py @@ -12,3 +12,4 @@ class Usage: output_tokens: int = 0 calls: int = 0 cost_usd: float | None = None # None when no price applies (for example a subscription) + unreported_calls: int = 0 # calls whose tokens the CLI did not report: no price makes a cost of them diff --git a/hermit/cli.py b/hermit/cli.py index d2e9390f..2e6b0975 100644 --- a/hermit/cli.py +++ b/hermit/cli.py @@ -30,11 +30,12 @@ def main(argv=None) -> int: p.add_argument("--adapter", required=True, help="final | root | none | a tag | a commit | a local path") if name == "check": p.add_argument("--no-probe", action="store_true", help="skip running one task through the pipeline") - dep = sub.add_parser("deploy", help="deploy an adapter's exports into a Codex, Claude Code or Pi workspace") + dep = sub.add_parser("deploy", help="deploy an adapter's exports into a Codex, Claude Code, Pi or DeepSeek Harness " + "workspace") dep.add_argument("--adapter", type=Path, required=True) dep.add_argument("--workspace", "--output", dest="workspace", type=Path, required=True, help="the project directory the agent will work in (--output is the older spelling)") - dep.add_argument("--host", choices=["codex", "claude-code", "pi"], default="codex") + dep.add_argument("--host", choices=["codex", "claude-code", "pi", "dsh"], default="codex") dep.add_argument("--python", default=sys.executable) dep.add_argument("--context-limit", type=int, default=None, help="codex: the project_doc_max_bytes your Codex config sets (default 32768, Codex's own); " @@ -97,6 +98,8 @@ def main(argv=None) -> int: from hermit.hosts.codex import prepare elif args.host == "pi": from hermit.hosts.pi import prepare + elif args.host == "dsh": + from hermit.hosts.dsh import prepare else: from hermit.hosts.claude_code import prepare try: diff --git a/hermit/config.py b/hermit/config.py index 70e65649..02c7428a 100644 --- a/hermit/config.py +++ b/hermit/config.py @@ -44,10 +44,10 @@ def _merge(base: dict, extra: dict, prefix: str = "") -> dict: FREE = ("params", "llm", "backends", "container", "resources", # sections whose keys the framework does not know - "model", *("codex", "claude", "pi", "react", "direct")) # a layer's settings (docs/framework/agent-layers.md) -BASES = {"codex": "codex", "claude": "claude-code", "claude-code": "claude-code", "pi": "pi", "react": "react", - "direct": None} -CLI_BASES = ("codex", "claude-code", "pi") # the bases that are a CLI (their Layer 1 settings: _model) + "model", *("codex", "claude", "pi", "dsh", "react", "direct")) # a layer's settings (agent-layers.md) +BASES = {"codex": "codex", "claude": "claude-code", "claude-code": "claude-code", "pi": "pi", "dsh": "dsh", + "react": "react", "direct": None} +CLI_BASES = ("codex", "claude-code", "pi", "dsh") # the bases that are a CLI (their Layer 1 settings: _model) MODEL_KEYS = ("login", "effort", "price") # Layer 1 settings a CLI takes beside its model # A login's plan settings: Layer 2's (agent., evolver.), refused beside the model's name. PLAN_KEYS = {"plan_dir", "plan_max_wait_s", "plan_rerun", "usage_credits", "login_root"} @@ -114,14 +114,14 @@ def layered(data: dict) -> None: agent = data["agent"] base = agent.pop("base", None) model = agent.pop("model", None) - sections = {b: agent.pop(b, None) for b in ("codex", "claude", "pi", "react", "direct")} + sections = {b: agent.pop(b, None) for b in ("codex", "claude", "pi", "dsh", "react", "direct")} if base is None: if model is not None or any(sections.values()): - raise ValueError("agent.model and agent. need agent.base (codex, claude, pi, react or direct)") + raise ValueError("agent.model and agent. need agent.base (codex, claude, pi, dsh, react or direct)") else: if base not in BASES: raise ValueError(f"agent.base must be one of {sorted(BASES)}, not {base!r}") - builtin = {"codex": "codex", "claude-code": "claude-code", "claude": "claude-code", "pi": "pi", + builtin = {"codex": "codex", "claude-code": "claude-code", "claude": "claude-code", "pi": "pi", "dsh": "dsh", "react": "react", "cli": None} if not agent.get("name"): if BASES[base] is None: @@ -142,13 +142,13 @@ def layered(data: dict) -> None: evolver = data["evolver"] base = evolver.pop("base", None) model = evolver.pop("model", None) - sections = {b: evolver.pop(b, None) for b in ("codex", "claude", "pi", "react", "direct")} + sections = {b: evolver.pop(b, None) for b in ("codex", "claude", "pi", "dsh", "react", "direct")} if base is None: if model is not None or any(sections.values()): - raise ValueError("evolver.model and evolver. need evolver.base (codex, claude, pi or direct)") + raise ValueError("evolver.model and evolver. need evolver.base (codex, claude, pi, dsh or direct)") return - if base not in ("codex", "claude", "claude-code", "pi", "direct"): - raise ValueError(f"evolver.base must be codex, claude, pi or direct, not {base!r}") + if base not in ("codex", "claude", "claude-code", "pi", "dsh", "direct"): + raise ValueError(f"evolver.base must be codex, claude, pi, dsh or direct, not {base!r}") params = dict(evolver.get("params") or {}) if base == "direct": if model is not None: diff --git a/hermit/defaults.yaml b/hermit/defaults.yaml index 04e4243c..5d6e8b22 100644 --- a/hermit/defaults.yaml +++ b/hermit/defaults.yaml @@ -9,15 +9,18 @@ data: shuffle_seed: 0 task_info: null agent: - name: null # react | codex | claude-code | pi, or file.py:Class / file.py:function of the task's own + name: null # react | codex | claude-code | pi | dsh, or file.py:Class / file.py:function + # of the task's own concurrency: 4 timeout_min: null # a sandboxed task call that runs longer is stopped and recorded as an error; null: none params: {} # Layer 3: the Acting Agent's own settings (docs/framework/agent-layers.md) - base: null # Layer 2: codex | claude | pi | react | direct; its settings go in a section of its name + base: null # Layer 2: codex | claude | pi | dsh | react | direct; its settings go in a section of its + # name model: null # Layer 1: a / reference, or {ref | name, login, effort, price, ...} codex: null claude: null pi: null + dsh: null react: null direct: null pipeline: # the older spelling of `agent`: run -> name @@ -53,11 +56,12 @@ evolver: name: null resume: false params: {} # Layer 3: the method's settings - base: null # Layer 2: codex | claude | pi | direct (the Base Agent the method runs on) + base: null # Layer 2: codex | claude | pi | dsh | direct (the Base Agent the method runs on) model: null # Layer 1: as agent.model; with direct it is the run's `llm` codex: null claude: null pi: null + dsh: null direct: null evolver_access: environment: [] diff --git a/hermit/hosts/dsh.py b/hermit/hosts/dsh.py new file mode 100644 index 00000000..1007f438 --- /dev/null +++ b/hermit/hosts/dsh.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Deploy an adapter into a DeepSeek Harness (dsh) workspace: the whole adapter under hermit//, and its exports +connected to dsh's own places: AGENTS.md context, .dsh/skills, and a patch layer, .dsh/hermit.patch.yml. + +dsh reads, without any trust prompt, the instruction files (AGENTS.md, CLAUDE.md and their .local overlays) of every +folder from the project's root down to the working directory, and the skills of the root's .dsh/skills and +.agents/skills; the root is the nearest folder above with a `.git`, else the working directory. It has no project +file for MCP servers or for more skill folders: they are rows of its plugin tree, so the patch layer holds the +adapter's tool server (an `insert` row: a server dsh cannot start is reported on its stderr, not retried in the +background, and the `dsh` acting agent then fails the session; dsh itself goes on without the tools) and names the +workspace's .dsh/skills (the `skill-filesystem` row's customSkillDirs, so that a workspace below a repository's root +keeps its skills; the row replaces any `skill-filesystem` settings of the user's own while the layer is loaded, and +dsh's web profile turns the row off). A session loads the layer from the workspace with `dsh --profile headless +--patch .dsh/hermit.patch.yml`, as the `dsh` acting agent does. dsh gives the model 65,536 bytes of a project's +instructions (its agent-instructions maxBytes), its own frame around them included, and cuts the rest, so a longer +AGENTS.md is refused: exactly for a session whose project root is the workspace and which holds no other instruction +file (the acting agent's); a session below a repository's root also counts that repository's AGENTS.md files. dsh +runs no hook file of its own (its experimental bridge to Claude Code's hooks is not used) and has no role file: an +adapter's roles and hooks stay files of its copy, which AGENTS.md names with what to do with them +(hermit.hosts.inputs.resources), as it names the other files. dsh's own settings ($DSH_HOME) are never changed. +""" +import argparse +import json +from pathlib import Path +import re +import subprocess +import sys + +from hermit.adapter.loader import agent_tools, hook_definitions, load_manifest, not_portable +from hermit.agent.base.cli import DSH_INSTRUCTION_BYTES, DSH_INSTRUCTION_FRAME +from hermit.hosts.context import PROJECT_CONTEXT, START, append_context +from hermit.hosts.inputs import (MARKER, PACKAGE_ROOT, _adapter_sandbox_env, adapter_files, adapter_folder, + check_workspace, copy_adapter, copy_skills, folder_note, render_exports, resources, + server_name, source_label) + +NATIVE = {"skills"} +PATCH = Path(".dsh") / "hermit.patch.yml" # the workspace's patch layer: the adapter's tool server and skills +SERVER_NAME = re.compile(r"[A-Za-z0-9_-]{1,32}") # what dsh takes as an MCP server's name + + +def prepare(adapter: Path, output: Path, python: str, *, learning_material: Path | None = None, + context: str | None = None) -> dict: + """`context`: the adapter's context for one task, which the harness computed, in place of the one rendered + here; an adapter whose context is per task deploys only so.""" + adapter, output = adapter.resolve(), output.resolve() + refused = not_portable(load_manifest(adapter), "A DeepSeek Harness workspace") # before anything is written + if refused and context is None: + raise ValueError(refused) + previous = check_workspace(adapter, output, "dsh") + manifest = load_manifest(adapter) + has_tools = bool(agent_tools(manifest)) # the context tool is the harness's, not served + if has_tools and not SERVER_NAME.fullmatch(server_name(manifest)): + raise ValueError(f"dsh takes an MCP server name of 32 characters at most ([A-Za-z0-9_-]), not " + f"{server_name(manifest)!r} (the adapter's id)") + files = adapter_files(adapter, output, manifest, previous) + exports = render_exports(adapter, python) + listed, placed = resources(adapter, output, manifest, exports, hook_definitions(adapter, manifest), NATIVE) + if has_tools: + subprocess.run([python, "-c", "import mcp.server"], check=True, capture_output=True) + output.mkdir(parents=True, exist_ok=True) + context_path = output / "AGENTS.md" + existing = context_path.read_text() if context_path.exists() else PROJECT_CONTEXT + if existing.startswith(START): + existing = PROJECT_CONTEXT + "\n\n" + existing + context_text = append_context(existing, "\n\n".join(t for t in ( + folder_note(output, manifest), + exports["context"] if context is None else context, listed) if t), + source=source_label(manifest), + version=manifest["version"], adapter_id=manifest["id"]) + # as dsh gives it to the model: in its frame, a `` of the text escaped by one more character + size = len(context_text.encode("utf-8")) + if size + DSH_INSTRUCTION_FRAME + context_text.count("") > DSH_INSTRUCTION_BYTES: + raise ValueError(f"AGENTS.md would be {size} bytes: with the {DSH_INSTRUCTION_FRAME} bytes dsh puts around " + f"it, past the {DSH_INSTRUCTION_BYTES} bytes dsh gives the model of a project's instructions " + "(its agent-instructions maxBytes): dsh would cut the rest, telling only the model. Shorten " + "the adapter's context") + audit = output / "tool-calls.jsonl" + server = None + if has_tools: + server = {"serverName": server_name(manifest), "transport": "stdio", "command": python, + "args": ["-m", "hermit.runtime.mcp_server", "--adapter", str(adapter), "--audit-log", str(audit)], + "env": {"PYTHONPATH": PACKAGE_ROOT, **_adapter_sandbox_env(python)}, + "toolCallTimeoutMs": 120_000, # Codex's tool_timeout_sec, not dsh's 60 s + # a server dsh cannot start fails its row, which dsh reports on stderr and the dsh acting agent + # reads as a failed session, rather than retrying it in the background while the session runs + # without it (dsh itself goes on either way) + "failOnStartupError": True} + if learning_material is not None: + server["args"].extend(["--learning-material", str(Path(learning_material).resolve())]) + skills = copy_skills(adapter, manifest, output / ".dsh" / "skills") + rows = [*([{"insert": [{"id": f"mcp-{server_name(manifest)}", "name": "@deepseek-ai/dsh-mcp-client", + "config": server}]}] if has_tools else []), + # the workspace's own skills wherever it lies, beside dsh's default roots (the row's settings replace + # those of the user's own home layer while the layer is loaded: a row's `config` is replaced whole) + *([{"id": "skill-filesystem", "config": {"customSkillDirs": [".dsh/skills"]}}] if skills else [])] + (output / PATCH.parent).mkdir(exist_ok=True) + # JSON is YAML: dsh reads the layer as YAML, and nothing in it is a `!!js` expression + (output / PATCH).write_text(json.dumps(rows, indent=2) + "\n") + context_path.write_text(context_text) + copy_adapter(adapter, output, manifest, files, previous) + result = {"host": "dsh", "adapter_id": manifest["id"], "version": manifest["version"], "adapter": str(adapter), + "workspace": str(output), "context_injected_via": "AGENTS.md", + "server_name": server_name(manifest) if has_tools else None, "server": server, + "patch": str(output / PATCH) if rows else None, "roles": {}, "role_name_map": {}, + "skills_dir": str(output / ".dsh" / "skills") if skills else None, "skills": skills, "hooks": None, + "audit_log": str(audit) if has_tools else None, + "adapter_dir": str(adapter_folder(output, manifest)), "files": files, "placed": placed} + (output / MARKER).write_text(json.dumps(result, indent=2) + "\n") + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--adapter", type=Path, required=True) + parser.add_argument("--workspace", "--output", dest="workspace", type=Path, required=True) + parser.add_argument("--python", default=sys.executable) + args = parser.parse_args() + print(json.dumps(prepare(args.adapter, args.workspace, str(Path(args.python).absolute())), indent=2)) diff --git a/hermit/isolate.py b/hermit/isolate.py index 293d5c98..b32e8fb0 100644 --- a/hermit/isolate.py +++ b/hermit/isolate.py @@ -165,8 +165,8 @@ def validate(config) -> None: login = params.get("login") if kind and login: # wherever the pipeline runs: else every session would be refused, one task at a time own = codex_login.LOGINS.get(login) - if kind == "pi": - raise ValueError("agent pi takes no login: its model is a / reference (agent.model)") + if kind in ("pi", "dsh"): + raise ValueError(f"agent {kind} takes no login: its model is a / reference (agent.model)") if own is None: raise ValueError(f"agent.params.login is {' or '.join(sorted(codex_login.LOGINS))} or absent, not " f"{login!r}") @@ -182,6 +182,11 @@ def validate(config) -> None: if params["effort"] not in Claude.efforts: raise ValueError(f"agent.model.effort is one of {', '.join(Claude.efforts)} for Claude Code, not " f"{params['effort']!r}") + if kind == "dsh": # dsh's own settings, here too: else every session would be refused, one task at a time + from hermit.agent.base.cli import dsh_problems + problems = dsh_problems(params, "agent") # the dsh Evolver's: login.evolver_problems, before its steps + if problems: + raise ValueError("; ".join(problems)) if env.get("image"): if kind not in ("codex", "claude"): raise ValueError("environment.image runs a CLI acting agent (codex or claude) in a container: agent.base " diff --git a/hermit/settings_check.py b/hermit/settings_check.py index cd31a021..f2787aa0 100644 --- a/hermit/settings_check.py +++ b/hermit/settings_check.py @@ -57,7 +57,10 @@ def report(item: str, problems: list[str], note: str = ""): cls = load_class(spec) _, evolver_params = instantiate(spec, d["evolver"]["params"], backends=registry(d.get("backends")), default_model=d.get("model")) - report("evolver", needs_errors(config, spec.needs, cls.supports_resume), f"{spec.name} ({spec.class_path})") + # an Evolver's own settings (with a login, its problems are the login's, below) + report("evolver", needs_errors(config, spec.needs, cls.supports_resume) + + ([] if evolver_params.get("login") else subscription.evolver_problems(evolver_params)), + f"{spec.name} ({spec.class_path})") except Exception as exc: report("evolver", [str(exc)]) # Models: every model the run calls, the backend each is bound to, and whether that backend speaks the caller's diff --git a/tests/README.md b/tests/README.md index 4cbeb802..0ce1d975 100644 --- a/tests/README.md +++ b/tests/README.md @@ -35,7 +35,7 @@ ChatGPT-login directory and Claude plan for every test, and no copy of the user' models), `toy_task.py` (the toy task the Trainer tests run on), `claude_stand.py` (the stand-in `claude` that replays `fixtures/claude_streams/`, and the probe for a working bubblewrap, for core's and the methods' Claude tests), `fixtures/` (`pi/`: Pi's recorded streams, read by -`core/test_pi_agent.py`). +`core/test_pi_agent.py`; `dsh/`: DeepSeek Harness's, read by `core/test_dsh_agent.py`). A new or changed test follows the [test checklist](../.agents/skills/prepare-pr/references/test-checklist.md): where it goes, how it is named and documented, what it asserts, and what it may touch. diff --git a/tests/core/test_context_tool.py b/tests/core/test_context_tool.py index 5cfcddbd..d08001d3 100644 --- a/tests/core/test_context_tool.py +++ b/tests/core/test_context_tool.py @@ -121,13 +121,13 @@ def test_the_codex_agent_gives_each_task_its_own_context(adapter, tmp_path): # ---------- plugin-portable: whether a template's adapters can be deployed or converted ---------- def test_a_template_that_is_not_plugin_portable_is_refused_by_deploy_and_convert(adapter, tmp_path): - from hermit.hosts import claude_code, codex, pi + from hermit.hosts import claude_code, codex, dsh, pi from hermit.plugins import TARGETS, Unsupported, build for target in TARGETS: with pytest.raises(Unsupported, match="plugin-portable: false"): build(target, adapter, tmp_path / f"plugin-{target}", sys.executable) assert not (tmp_path / f"plugin-{target}").exists() - for host in (codex, claude_code, pi): + for host in (codex, claude_code, pi, dsh): workspace = tmp_path / f"ws-{host.__name__.rsplit('.', 1)[-1]}" with pytest.raises(ValueError, match="plugin-portable: false"): host.prepare(adapter, workspace, sys.executable) diff --git a/tests/core/test_deploy.py b/tests/core/test_deploy.py index 69dc8b79..6b59d84d 100644 --- a/tests/core/test_deploy.py +++ b/tests/core/test_deploy.py @@ -1,4 +1,4 @@ -"""Deploying the Lean template into Codex, Claude Code and Pi workspaces.""" +"""Deploying the Lean template into Codex, Claude Code, Pi and DeepSeek Harness workspaces.""" import json from pathlib import Path import sys @@ -8,6 +8,7 @@ from hermit.hosts.claude_code import prepare as claude_prepare from hermit.hosts.codex import prepare as codex_prepare +from hermit.hosts.dsh import prepare as dsh_prepare from hermit.hosts.pi import prepare as pi_prepare LEAN = Path(__file__).resolve().parents[2] / "hermit" / "adapter" / "templates" / "lean_collab" @@ -51,6 +52,69 @@ def test_pi(tmp_path): codex_prepare(LEAN, tmp_path / "p", sys.executable) +def test_dsh(tmp_path): + """dsh has no project file for MCP servers or skill folders: the tools are an `insert` row of the workspace's + patch layer, which a session loads with --patch, and the layer names the workspace's .dsh/skills (dsh reads a + project's skills only at the nearest folder with a .git); roles stay files named in AGENTS.md; a context longer + than dsh gives the model of a project's instructions is refused.""" + result = dsh_prepare(LEAN, tmp_path / "d", sys.executable) + layer = json.loads((tmp_path / "d" / ".dsh" / "hermit.patch.yml").read_text()) + [server] = [r for r in layer if "insert" in r] + assert {"id": "skill-filesystem", "config": {"customSkillDirs": [".dsh/skills"]}} in layer + [row] = server["insert"] + assert row["name"] == "@deepseek-ai/dsh-mcp-client" and row["config"]["serverName"] == "lean_collab" + assert row["config"]["transport"] == "stdio" and row["config"]["args"][:3] == ["-m", "hermit.runtime.mcp_server", + "--adapter"] + assert row["config"]["toolCallTimeoutMs"] == 120_000 # Codex's tool_timeout_sec, not dsh's 60 s + assert row["config"]["failOnStartupError"] is True # a server that does not start fails the session + assert result["patch"] == str(tmp_path / "d" / ".dsh" / "hermit.patch.yml") + assert "hermit:lean-collab:start" in (tmp_path / "d" / "AGENTS.md").read_text() + assert (tmp_path / "d" / ".dsh" / "skills" / "good-map" / "SKILL.md").exists() + assert result["context_injected_via"] == "AGENTS.md" and result["roles"] == {} + dsh_prepare(LEAN, tmp_path / "d", sys.executable) + assert (tmp_path / "d" / "AGENTS.md").read_text().count("hermit:lean-collab:start") == 1 + with pytest.raises(ValueError, match="generated for another host"): + pi_prepare(LEAN, tmp_path / "d", sys.executable) + a = subagent_adapter(tmp_path) + result = dsh_prepare(a, tmp_path / "s", sys.executable) + assert result["placed"] == {"agents": "files"} and result["patch"] is None # no tools: no server row + assert json.loads((tmp_path / "s" / ".dsh" / "hermit.patch.yml").read_text()) == [] + assert "`reviewer`: Check a result against the task requirements." in (tmp_path / "s" / "AGENTS.md").read_text() + big = subagent_adapter(tmp_path / "big") + (big / "ADAPTER.md").write_text("Check the units.\n" * 4000) # 68,000 bytes + with pytest.raises(ValueError, match="past the 65536 bytes dsh gives the model of a project's instructions"): + dsh_prepare(big, tmp_path / "b", sys.executable) + assert not (tmp_path / "b" / "AGENTS.md").exists() + + +def test_dsh_refuses_an_agents_md_dsh_would_cut_and_a_server_name_it_would_refuse(tmp_path): + """dsh's 65,536 bytes are the instructions as the model is given them, in dsh's frame (304 bytes around one + AGENTS.md; a `` in the text one byte longer): an AGENTS.md up to the difference is deployed, one + byte more is refused (it was deployed, and dsh cut its end, telling only the model). An adapter id that is no MCP + server name dsh takes is refused before anything is written.""" + import shutil + from hermit.agent.base.cli import DSH_INSTRUCTION_BYTES, DSH_INSTRUCTION_FRAME + limit = DSH_INSTRUCTION_BYTES - DSH_INSTRUCTION_FRAME + edge = subagent_adapter(tmp_path / "edge") + dsh_prepare(edge, tmp_path / "e0", sys.executable) + rest = len((tmp_path / "e0" / "AGENTS.md").read_bytes()) - len("Check the units.\n") + (edge / "ADAPTER.md").write_text("x" * (limit - rest - 1) + "\n") + dsh_prepare(edge, tmp_path / "e1", sys.executable) + assert len((tmp_path / "e1" / "AGENTS.md").read_bytes()) == limit + for text in ("x" * (limit - rest) + "\n", "" + "x" * (limit - rest - 19) + "\n"): + (edge / "ADAPTER.md").write_text(text) + with pytest.raises(ValueError, match=f"with the {DSH_INSTRUCTION_FRAME} bytes dsh puts around it"): + dsh_prepare(edge, tmp_path / "e2", sys.executable) + assert not (tmp_path / "e2" / "AGENTS.md").exists() + long_id = tmp_path / "long" + shutil.copytree(LEAN, long_id) + manifest = long_id / ".hermit" / "adapter.yaml" + manifest.write_text(manifest.read_text().replace("id: lean-collab", "id: lean-collab-" + "x" * 30)) + with pytest.raises(ValueError, match="dsh takes an MCP server name of 32 characters at most"): + dsh_prepare(long_id, tmp_path / "l", sys.executable) + assert not (tmp_path / "l").exists() + + def subagent_adapter(tmp_path): a = tmp_path / "adapter" (a / "agents").mkdir(parents=True) diff --git a/tests/core/test_dsh_agent.py b/tests/core/test_dsh_agent.py new file mode 100644 index 00000000..ffe42d57 --- /dev/null +++ b/tests/core/test_dsh_agent.py @@ -0,0 +1,1229 @@ +"""The `dsh` Base Agent (DeepSeek Harness, hermit/agent/base/cli.py), its model binding +(hermit/agent/model/backends.py), its stream's reader (hermit/agent/base/trajectory.py) and the Layer 3 code that runs +it: the acting agent (hermit/agent/acting/workspace.py) with its deployment (hermit/hosts/dsh.py), the config's `dsh` +base, the `dsh` Evolver. + +Stand-ins: a fake `dsh` on PATH that replays a stream recorded from DeepSeek Harness 0.2.2-rc.1 (tests/fixtures/dsh, +whose README says how) and records what it was given; and, for the tests marked integration, the installed `dsh` +against a scripted chat-completions model on the loopback (`serve_chat`), never a provider.""" +import json +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import tempfile +import threading + +import pytest + +from hermit.agent.base import cli as base_cli +from hermit.agent.base import trajectory +from hermit.agent.base.interface import Unsupported, make +from hermit.agent.base.tool import Tool +from hermit.agent.model.backends import bind_agent_params, registry + +ROOT = Path(__file__).resolve().parents[2] +STREAMS = ROOT / "tests" / "fixtures" / "dsh" +FILE_ADAPTER = ROOT / "docs" / "_examples" / "file_adapter" +SESSION = "session-b2fbf9bf-a321-4065-8f8a-ee344b66b5f3" # the recorded success stream's session + +FAKE_DSH = f'''#!{sys.executable} +"""A stand-in for `dsh --profile headless ...`: records its arguments, dsh's variables, the patch layers it is given +and its stdin, runs FAKE_DSH_RUN in its working directory, then prints FAKE_DSH_STREAM and exits FAKE_DSH_EXIT.""" +import json, os, subprocess, sys +args = sys.argv[1:] +patches = {{args[i + 1]: open(args[i + 1]).read() for i, a in enumerate(args[:-1]) if a == "--patch"}} +names = ("DSH_HOME", "HOME", "DSH_AGENTS_HOME", "DSH_TELEMETRY_DISABLED", "{base_cli.DSH_NO_KEY}", "FAKE_KEY", + "DSH_BUNDLED_SKILL_DIR", "DEEPSEEK_BASE_URL", "DEEPSEEK_SEARCH_BASE_URL") +seen = {{"args": args, "env": {{k: os.environ.get(k) for k in names}}, "patches": patches, "stdin": sys.stdin.read()}} +with open(os.environ["FAKE_DSH_SEEN"], "a") as f: + f.write(json.dumps(seen) + "\\n") +if os.environ.get("FAKE_DSH_RUN"): + subprocess.run(os.environ["FAKE_DSH_RUN"], shell=True) +sys.stdout.write(open(os.environ["FAKE_DSH_STREAM"]).read()) +sys.exit(int(os.environ.get("FAKE_DSH_EXIT", "0"))) +''' + + +@pytest.fixture +def fake(tmp_path, monkeypatch): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + exe = bin_dir / "dsh" + exe.write_text(FAKE_DSH) + exe.chmod(exe.stat().st_mode | stat.S_IEXEC) + monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ['PATH']}") + monkeypatch.setenv("FAKE_DSH_SEEN", str(tmp_path / "seen.jsonl")) + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "success.jsonl")) + monkeypatch.setenv("FAKE_KEY", "sk-fake") + return tmp_path + + +def seen(tmp_path) -> list[dict]: + return [json.loads(line) for line in (tmp_path / "seen.jsonl").read_text().splitlines()] + + +def layer_rows(text: str) -> list[dict]: + """A patch layer's rows, read as dsh reads it (YAML), a `!!js` expression as {"!!js": its text}.""" + import yaml + + class Loader(yaml.SafeLoader): + pass + Loader.add_constructor("tag:yaml.org,2002:js", lambda loader, node: {"!!js": loader.construct_scalar(node)}) + return yaml.load(text, Loader=Loader) + + +def rows(call: dict, home: Path) -> list[dict]: + """The session's own patch layer, as the fake read it.""" + return layer_rows(call["patches"][str(home / base_cli.DSH_PATCH)]) + + +def row(layer: list[dict], row_id: str) -> dict: + [found] = [r for r in layer if r.get("id") == row_id] + return found + + +def bound(**backend) -> dict: + """A `dsh` agent's params with its model bound to a backend of its own (`fake/m-1`).""" + spec = {"base_url": "http://127.0.0.1:9/v1", "api_key_env": "FAKE_KEY", "apis": ["chat"], + "prices": {"m-1": {"input": 1.0, "cached_input": 0.1, "output": 2.0}}, **backend} + return bind_agent_params({"model": "fake/m-1"}, "dsh", registry({"fake": spec})) + + +def echo() -> Tool: + return Tool("echo", "Echo the text back.", lambda text="": {"echoed": text}, + input_schema={"type": "object", "properties": {"text": {"type": "string"}}, "required": ["text"]}) + + +def test_a_bound_model_is_a_dsh_route_with_its_key_variable_in_the_dialect_the_backend_serves(): + """The key reaches dsh only as the name of its variable (`apiKeyEnv`): the records keep the name, never the key.""" + params = bound() + assert params["model"] == "m-1" and params["binding"]["ref"] == "fake/m-1" + assert params["provider"] == {"api": "openai-completions", "baseURL": "http://127.0.0.1:9/v1", + "apiKeyEnv": "FAKE_KEY", "models": [{"id": "m-1"}]} + assert params["price"] == {"input": 1.0, "cached_input": 0.1, "output": 2.0} + messages = bound(apis=["messages"])["provider"] + assert messages["api"] == "anthropic-messages" and messages["baseURL"] == "http://127.0.0.1:9" + system = bound(developer_role="system", context_window=32000)["provider"] + assert system["compat"] == {"supportsDeveloperRole": False} and system["models"][0]["contextWindow"] == 32000 + free = bound(api_key_env="NO_SUCH_KEY_VAR", key_required=False)["provider"] + assert "apiKeyEnv" not in free # a server that needs no key: the session names the placeholder + + +def test_dsh_runs_headless_in_its_home_with_its_model_and_the_tools_as_its_own(fake, monkeypatch): + """Without the sandbox a session gets this process's environment: a dsh variable of the caller's (here a folder + of skills, and where DeepSeek's API is reached, for dsh's own provider and for its web search) does not reach it, + so the session's settings are its own home's; one the session is given (`env`, here the caller's by name) does. + The call's tools are an MCP row whose server, if dsh cannot start it, fails the session (failOnStartupError), and + whose calls may take as long as a Trainer tool's rollouts.""" + tmp_path = fake + monkeypatch.setenv("DSH_BUNDLED_SKILL_DIR", str(tmp_path / "callers-skills")) + monkeypatch.setenv("DEEPSEEK_BASE_URL", "http://127.0.0.1:9") + monkeypatch.setenv("DEEPSEEK_SEARCH_BASE_URL", "http://127.0.0.1:9") + home, work = tmp_path / "home", tmp_path / "work" + work.mkdir() + agent = make("dsh", {**bound(), "effort": "high", "web_search": False, "home": str(home), + "log": str(tmp_path / "agent-dsh.jsonl")}) + result = agent.run("-a prompt that starts with a dash", workspace=work, tools=[echo()]) + [call] = seen(tmp_path) + assert call["args"][:4] == ["--profile", "headless", "--patch", str(home / base_cli.DSH_PATCH)] + assert call["args"][4:9] == ["--json", "--permission-mode", "workspace-write", "--approval-policy", "never"] + assert call["args"][-2:] == ["--", "-a prompt that starts with a dash"] + assert call["env"] == {"DSH_HOME": str(home), "HOME": str(home), "DSH_AGENTS_HOME": str(home / ".agents"), + "DSH_TELEMETRY_DISABLED": "1", base_cli.DSH_NO_KEY: "local", + "FAKE_KEY": "sk-fake", "DSH_BUNDLED_SKILL_DIR": None, # none of the user's own skills + "DEEPSEEK_BASE_URL": None, "DEEPSEEK_SEARCH_BASE_URL": None} + layer = rows(call, home) + assert "sk-fake" not in json.dumps(layer) + assert {r["id"] for r in layer if r.get("disabled")} == {"session-telemetry-otel", "session-title-llm", + "plugin-package-inventory-deepseek", "tool-subagent", + "tool-subagent-fork", "tool-subagent-control", + "tool-workflow", "tool-web"} + assert row(layer, "session-log-deepseek")["config"] == {"enabled": False} + route = row(layer, "llm-pi-ai")["config"]["providers"][base_cli.DSH_PROVIDER] + assert route["apiKeyEnv"] == "FAKE_KEY" and route["models"] == [{"id": "m-1", "reasoningEfforts": {"high": "high"}}] + assert row(layer, "agent-default-model")["config"] == {"provider": base_cli.DSH_PROVIDER, "model": "m-1", + "reasoningEffort": "high"} + [server] = [r for r in layer if "insert" in r] + [entry] = server["insert"] + assert entry["name"] == "@deepseek-ai/dsh-mcp-client" and entry["config"]["serverName"] == "hermit_tools" + assert entry["config"]["transport"] == "streamable-http" and entry["config"]["url"].startswith("http://127.0.0.1:") + assert entry["config"]["failOnStartupError"] is True and entry["config"]["toolCallTimeoutMs"] == 1_800_000 + assert result.reply == "the answer" and result.exit_code == 0 and result.extra["served"] + assert not result.extra["failed"] and result.session == SESSION + [usage] = result.usage + assert (usage.input_tokens, usage.cached_input_tokens, usage.output_tokens, usage.calls) == (6000, 2400, 120, 6) + assert usage.cost_usd == pytest.approx(0.00408) # dsh reports none: the backend's price + kinds = [s.kind for s in result.trajectory if s.kind != "reasoning"] + assert kinds == ["command", "command", "tool_call", "file_change", "file_change", "message"] + failed, passed, tool = result.trajectory[1], result.trajectory[3], json.loads(result.trajectory[5].text) + assert (failed.command, failed.exit_code, failed.output) == ("echo hi && exit 3", 3, "hi") + assert (passed.command, passed.exit_code, passed.output) == ("echo ok", 0, "ok\n") # no marker: exit 0 + assert tool == {"arguments": {"text": "from dsh"}, "server": "hermit_tools", "tool": "echo"} + assert json.loads(result.trajectory[5].output) == {"echoed": "from dsh"} + assert result.trajectory[0] == trajectory.Step("reasoning", "Thinking it over.") + agent.configured(env={"DEEPSEEK_BASE_URL": "$DEEPSEEK_BASE_URL"}).run("go", workspace=work) + assert seen(tmp_path)[-1]["env"]["DEEPSEEK_BASE_URL"] == "http://127.0.0.1:9" + + +def test_an_effort_is_a_level_the_bound_model_lists_and_off_asks_it_for_none(fake): + """dsh refuses an effort its route does not list for the model (UNSUPPORTED_REASONING_EFFORT), and a list of + `off` alone (INVALID_CONFIG): effort off declares the model with no levels and gives no effort, so dsh asks for + no reasoning level, which leaves the server's default (only dsh's own provider has a way to turn thinking off).""" + work = fake / "work" + work.mkdir() + agent = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}) + for effort, efforts, default in (("high", {"high": "high"}, "high"), ("off", False, None), (None, None, None)): + agent.configured(effort=effort).run("go", workspace=work) + layer = rows(seen(fake)[-1], fake / "home") + [model] = row(layer, "llm-pi-ai")["config"]["providers"][base_cli.DSH_PROVIDER]["models"] + assert model.get("reasoningEfforts") == efforts + assert row(layer, "agent-default-model")["config"].get("reasoningEffort") == default + + +def test_a_failed_run_is_an_error_and_only_an_outage_is_unavailable(fake, monkeypatch): + """dsh exits 1 for a run that did not complete; its error code tells an outage (the codes it retries, and a + stream the provider closed) from a refusal, and only an outage is `unavailable`: the acting agent runs the task + again only when the provider could not serve it.""" + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "failure.jsonl")) + monkeypatch.setenv("FAKE_DSH_EXIT", "1") + work = fake / "work" + work.mkdir() + result = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}).run("go", workspace=work) + assert result.exit_code == 1 and result.extra["failed"] and not result.extra["served"] + assert result.extra["unavailable"] and result.extra["failure"].startswith("dsh: SERVER: 503: ") + assert result.usage == [] # retried requests that used no tokens are no calls + assert [s.kind for s in result.trajectory] == ["error"] and result.trajectory[0].text.startswith("503: ") + refused = (STREAMS / "failure.jsonl").read_text().replace('"SERVER"', '"AUTH"').replace("dsh: SERVER:", + "dsh: AUTH:") + assert not base_cli.UNAVAILABLE.search(base_cli.notices("dsh", refused)) # a bad key is no outage + for code in ("RATE_LIMIT", "SERVER", "TIMEOUT", "TRANSPORT", "EMPTY_RESPONSE", "STREAM_CLOSED"): + outage = (STREAMS / "failure.jsonl").read_text().replace("SERVER", code) + assert base_cli.UNAVAILABLE.search(base_cli.notices("dsh", outage)), code + + +def test_a_run_dsh_was_stopped_in_is_no_success_although_dsh_exits_0(fake, monkeypatch, tmp_path): + """dsh exits 0 when SIGTERM stops it (recorded: stopped.jsonl, a session stopped while its command ran): its own + rule for a completed run is exit 0 and a completed `final`, so a run without one has failed. The acting agent + records no answer, and an AgentEvolver's step is a failed one; so is a session whose tools did not start. The + stopped step's request is not counted (dsh writes a step's usage at its end, which a stopped session never + writes): the session has no usage.""" + from hermit.agent.acting import workspace as acting + from hermit.agent.evolver.agent import AgentEvolver + from hermit.agent.evolver.base import Access, StepContext, Workspace + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "stopped.jsonl")) + work = fake / "work" + work.mkdir() + result = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}).run("go", workspace=work) + assert result.exit_code == 0 and result.extra["failed"] and not result.extra["served"] and result.extra["started"] + assert not result.extra["unavailable"] and result.reply == "" and "closed handle" in result.extra["failure"] + assert [s.kind for s in result.trajectory] == ["command", "error"] and result.trajectory[0].exit_code is None + assert result.usage == [] + out = acting.run({"id": "q1"}, None, params={**bound(), "backend": "dsh", "web_search": False, "prompt": "go", + "workspaces": "ws"}, base=fake) + assert out.output is None and "closed handle" in out.extra["session_failed"] + ws = Workspace(*(tmp_path / "evolver" / n for n in ("adapter", "lm", "info", "hist", "state", "scratch", "out"))) + for folder in (ws.adapter, ws.state, ws.output): + folder.mkdir(parents=True) + (tmp_path / "START.md").write_text("go") + evolver = AgentEvolver() + evolver.directory = tmp_path + for stream, says in (("stopped.jsonl", "closed handle"), ("tools_down.jsonl", "did not start"), + ("success.jsonl", None)): + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / stream)) + step = evolver.step(StepContext(workspace=ws, access=Access(), tools={}, params={ + **bound(), "backend": "dsh", "web_search": False, "prompt": "START.md", "home": []})) + assert (step.status, says is None or says in step.summary) == ("failed" if says else "done", True) + + +def test_a_resumed_session_is_adopted_by_its_id_and_counts_only_its_new_usage(fake, monkeypatch): + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "resume.jsonl")) + home, work = fake / "home", fake / "work" + work.mkdir() + (home / "sessions" / "--run-work--" / SESSION).mkdir(parents=True) + agent = make("dsh", {**bound(), "web_search": False, "home": str(home)}) + result = agent.run("again", workspace=work, session=SESSION) + args = seen(fake)[0]["args"] + assert args[args.index("--session-id") + 1] == SESSION + [usage] = result.usage + assert (usage.input_tokens, usage.calls) == (1000, 1) + agent.run("again", workspace=work, session="session-gone") # no such session kept: a new one + agent.run("again", workspace=work, session="*") # nor a glob + assert all("--session-id" not in call["args"] for call in seen(fake)[1:]) + + +def test_dsh_refuses_roles_and_a_container_and_searches_only_on_deepseeks_own_provider(fake): + """Roles and a container are refused, never ignored. Web search on: dsh's search is DeepSeek's own service (its + Messages API on DEEPSEEK_API_KEY), so a session whose model is another backend's would search on a provider and + key the run did not bind, outside its usage: it gets fetch only, as Codex's hosted search is off for a backend + other than OpenAI's. A model of DeepSeek's API, or of dsh's own provider, keeps dsh's web tools as they are.""" + work = fake / "work" + work.mkdir() + agent = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}) + role = {"name": "reviewer", "description": "Review.", "instructions": "Check it."} + with pytest.raises(Unsupported, match="roles"): + agent.run("go", workspace=work, roles=[role]) + with pytest.raises(Unsupported, match="takes isolation none or sandbox"): + agent.configured(isolation="container").run("go", workspace=work) + with pytest.raises(ValueError, match="permission_mode"): + agent.configured(permission_mode="bypass").run("go", workspace=work) + agent.configured(web_search=True).run("go", workspace=work) + assert row(rows(seen(fake)[-1], fake / "home"), "tool-web") == {"id": "tool-web", "config": {"search": False, + "fetch": True}} + deepseek = bind_agent_params({"model": "deepseek/deepseek-v4-flash"}, "dsh", registry({})) + named = bind_agent_params({"model": "ds/deepseek-v4-flash"}, "dsh", registry({"ds": {"from": "deepseek"}})) + for params in (deepseek, named, {}): # DeepSeek's API on its key, under any name; dsh's own + make("dsh", {**params, "web_search": True, "home": str(fake / "home")}).run("go", workspace=work) + assert all(r.get("id") != "tool-web" for r in rows(seen(fake)[-1], fake / "home")) + relayed = bind_agent_params({"model": "deepseek/deepseek-v4-flash"}, "dsh", registry( + {}, environ={"DEEPSEEK_BASE_URL": "http://127.0.0.1:9/v1"})) # DEEPSEEK_BASE_URL moves the backend + make("dsh", {**relayed, "web_search": True, "home": str(fake / "home")}).run("go", workspace=work) + assert row(rows(seen(fake)[-1], fake / "home"), "tool-web")["config"] == {"search": False, "fetch": True} + + +def test_dsh_takes_no_login_and_no_container_wherever_the_run_is_checked(): + """dsh signs in through a model reference (or a model of its own provider): a ChatGPT or Claude login is refused + by the acting agent, the run config and the Evolver's check, never passed on or ignored; a container runs Codex + or Claude Code.""" + from hermit.agent.acting import workspace as acting + from hermit.agent.base.login import evolver_problems + from hermit.config import Config + with pytest.raises(ValueError, match="agent dsh takes no login"): + acting._check("dsh", {"web_search": False, "login": "chatgpt"}) + with pytest.raises(ValueError, match="agent dsh takes isolation: none"): + acting._check("dsh", {"web_search": False, "isolation": "container"}) + assert acting._check("dsh", {"web_search": False}) == "dsh" + assert any("the dsh Evolver signs in through its model reference" in problem + for problem in evolver_problems({"backend": "dsh", "login": "claude"})) + run = ROOT / "docs" / "_examples" / "first_training" / "run.yaml" + fake = "backends={fake: {base_url: 'http://127.0.0.1:9/v1', api_key_env: FAKE_KEY, apis: [chat]}}" + with pytest.raises(ValueError, match="agent dsh takes no login"): + Config.load(run, [fake, "agent.name=null", "agent.base=dsh", "agent.model={ref: fake/m-1, login: chatgpt}", + "agent.params={prompt: go, web_search: false}"]) + + +def test_an_effort_or_a_sandbox_dsh_has_not_is_refused_before_any_session(tmp_path, monkeypatch): + """dsh's effort levels and permission modes are its own: a value it has not would fail every session, one task + (or one learner step, after its rollouts were paid for) at a time. The acting agent's are refused as the run + config loads (as a Claude Code effort is); the dsh Evolver's by `hermit check` and before `train`'s first step, + from its params as the Trainer has them, whichever way the config names it (evolver.base: dsh, or the dsh method + with its params) and with the run's model as its own. dsh's own provider (a model of no binding) has fewer + levels than a bound model's route. YAML reads a bare `off` as false, which is `off`. A setting an agent does not + take is refused, not dropped: the Evolver runs dsh's own sandbox off and gives its sessions no AGENTS.md (its own + danger-full-access and instruction_files false are taken, another value refused), and the acting agent's + sessions read the adapter's context as their AGENTS.md (instruction_files false is refused).""" + from hermit.agent.base.login import evolver_problems + from hermit.agent.evolver.loader import find, instantiate + from hermit.config import Config + from hermit.settings_check import run_check + from hermit.trainer.trainer import Trainer + monkeypatch.setenv("HA_BACKENDS", str(tmp_path / "no-user-file.yaml")) + monkeypatch.setenv("FAKE_KEY", "sk-fake") + run = ROOT / "docs" / "_examples" / "first_training" / "run.yaml" + fake = "backends={fake: {base_url: 'http://127.0.0.1:9/v1', api_key_env: FAKE_KEY, apis: [chat]}}" + load = lambda *more: Config.load(run, [fake, f"trainer.output_dir={tmp_path / 'runs'}", "agent.name=null", + "agent.base=dsh", "agent.params={prompt: go, web_search: false}", *more]) + loaded = load("agent.model={ref: fake/m-1, effort: xhigh}", "agent.dsh={permission_mode: read-only}") + assert loaded.data["agent"]["params"]["effort"] == "xhigh" + assert load("agent.model={ref: fake/m-1, effort: off}").data["agent"]["params"]["effort"] is False + with pytest.raises(ValueError, match="agent.model.effort is one of off, minimal, low, medium, high, xhigh, max"): + load("agent.model={ref: fake/m-1, effort: ultra}") + with pytest.raises(ValueError, match="one of off, low, high, max for a model of dsh's own provider"): + load("agent.model={name: deepseek-v4-flash, effort: medium}", "sandbox.enabled=false") + assert load("agent.model={name: deepseek-v4-flash, effort: max}", "sandbox.enabled=false") + with pytest.raises(ValueError, match="agent.dsh.permission_mode is one of read-only, workspace-write"): + load("agent.model=fake/m-1", "agent.dsh={permission_mode: bypass}") + assert load("agent.model=fake/m-1", "agent.dsh={instruction_files: true}") + with pytest.raises(ValueError, match="agent.dsh.instruction_files is not taken: the adapter's context reaches"): + load("agent.model=fake/m-1", "agent.dsh={instruction_files: false}") + + def problems(*more) -> list[str]: # the Evolver's, from its params as the Trainer and the check have them + config = load("agent.model=fake/m-1", *more) + d = config.data + _, params = instantiate(find(d["evolver"]["name"], config.base), d["evolver"]["params"], + backends=registry(d.get("backends")), default_model=d.get("model")) + return evolver_problems(params) + + layered, flat = ("evolver.name=dsh", "evolver.base=dsh"), ("evolver.name=dsh",) + assert problems(*layered, "evolver.model={ref: fake/m-1, effort: off}") == [] + assert problems(*flat, "evolver.params={model: fake/m-1, effort: xhigh}") == [] + assert problems("model=fake/m-1", *layered, "evolver.dsh={effort: high}") == [] # the run's model, its own + assert problems(*layered, "evolver.model=fake/m-1", # what the Evolver uses + "evolver.dsh={permission_mode: danger-full-access, instruction_files: false}") == [] + for more, said in (((*layered, "evolver.model={ref: fake/m-1, effort: ultra}"), "is one of off, minimal, low"), + ((*flat, "evolver.params={model: fake/m-1, effort: ultra}"), "is one of off, minimal, low"), + ((*flat, "evolver.params={effort: high}"), "evolver.model.effort needs evolver.model"), + ((*flat, "evolver.params={model: deepseek-v4-flash, effort: medium}"), + "is one of off, low, high, max for a model of dsh's own provider"), + ((*layered, "evolver.model=fake/m-1", "evolver.dsh={permission_mode: workspace-write}"), + "evolver.dsh.permission_mode is not taken"), + ((*flat, "evolver.params={model: fake/m-1, instruction_files: true}"), + "evolver.dsh.instruction_files is not taken")): + assert any(said in problem for problem in problems(*more)), more + flat_ultra = load("agent.model=fake/m-1", *flat, "evolver.params={model: fake/m-1, effort: ultra}") + lines = [] + assert run_check(flat_ultra, probe=False, log=lines.append) is False + at = lines.index("[PROBLEM] evolver") + assert "evolver.model.effort is one of off, minimal, low, medium, high, xhigh, max" in lines[at + 1] + with pytest.raises(SystemExit, match="train refuses to start: evolver: evolver.model.effort is one of off, min"): + Trainer(flat_ultra, log=lambda *_: None)._prepare() + lines.clear() + run_check(load("agent.model=fake/m-1", "model=fake/m-1", *layered, "evolver.dsh={effort: high}"), probe=False, + log=lines.append) + assert any(line.startswith("[ok] evolver — dsh") for line in lines) + + +def test_a_model_that_takes_images_is_declared_so_to_dsh(fake): + """dsh's pi-ai route offers a model it does not know text only, as Pi does: `image_input` (agent.dsh or + evolver.dsh) declares the bound model's images, through the acting agent's and the Evolver's settings.""" + from hermit.agent.acting.workspace import base_agent + from hermit.agent.evolver.loader import find, instantiate + from hermit.config import layered + data = {"agent": {"base": "dsh", "model": "fake/m-1", "dsh": {"image_input": True}}, + "evolver": {"name": "dsh", "base": "dsh", "model": "fake/m-1", "dsh": {"image_input": True}}, "llm": None} + layered(data) + assert base_agent("dsh", {**data["agent"]["params"], **bound(), "web_search": False}).settings["image_input"] + _, params = instantiate(find("dsh", ROOT), data["evolver"]["params"], default_model="fake/m-1", + backends=registry({"fake": {"base_url": "http://127.0.0.1:9/v1", "apis": ["chat"]}})) + assert params["image_input"] is True + work = fake / "work" + work.mkdir() + for image_input, declared in ((True, ["text", "image"]), (False, None)): + make("dsh", {**bound(), "web_search": False, "home": str(fake / "home"), "image_input": image_input}).run( + "go", workspace=work) + route = row(rows(seen(fake)[-1], fake / "home"), "llm-pi-ai")["config"]["providers"][base_cli.DSH_PROVIDER] + assert route["models"][0].get("input") == declared + + +def test_system_instructions_go_at_the_top_of_the_prompt(fake): + """dsh's persona rows interpolate `{{name}}`, which instructions may hold: they are given in the prompt.""" + work = fake / "work" + work.mkdir() + agent = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}) + own, prompt = agent.system_settings("Fill {{name}} in.", "go") + assert own == {} and prompt.startswith("# SYSTEM INSTRUCTIONS\n\nFill {{name}} in.\n\n# END") and \ + prompt.endswith("\n\ngo") + agent.configured(stdin=True).run(prompt, workspace=work) + call = seen(fake)[0] + assert call["args"][-1] == "-" and call["stdin"] == prompt + + +def test_the_learners_sessions_take_no_instruction_file_and_every_transcript_is_plain_text(fake, tmp_path): + """dsh adds a folder's AGENTS.md (or CLAUDE.md) when its file tools touch a file there: the learner, which reads + and edits an adapter whose AGENTS.md is the acting agent's policy, would take that policy, or its own text, for + instructions (Codex's learner is given no such file). The dsh Evolver's sessions take none. dsh keeps its + transcripts zstd-compressed, which hermit.redact cannot scrub a key from: every session keeps them in plain JSON + lines, under dsh's own root for them (its home's sessions folder, as dsh sees it in a sandbox too).""" + from hermit.agent.evolver.agent import AgentEvolver + + class Ctx: + params = {**bound(), "web_search": False} + sandbox = cancel = None + workspace = type("W", (), {"state": tmp_path / "state", "output": tmp_path / "output"}) + settings = AgentEvolver().settings(Ctx(), "dsh") + assert settings["instruction_files"] is False and "instruction_files" not in AgentEvolver().settings(Ctx(), "pi") + work = fake / "work" + work.mkdir() + for given, budget in (({}, 65536), ({"instruction_files": False}, 0)): + make("dsh", {**bound(), "web_search": False, "home": str(fake / "home"), **given}).run("go", workspace=work) + layer = rows(seen(fake)[-1], fake / "home") + assert row(layer, "agent-instructions")["config"]["maxBytes"] == budget + assert row(layer, "session-persistence-jsonl")["config"] == {"root": {"!!js": "dshHomePath('sessions')"}, + "compression": "none"} + + +def test_the_workspace_is_the_sessions_project_wherever_it_lies(fake): + """dsh takes the nearest folder above with a .git for the project: a run's workspace inside a repository would + get the repository's skills (.dsh/skills, .agents/skills) and instructions, which outrank the adapter's, and lose + its own skills. The session's layer gives dsh the workspace's .dsh/skills alone, and no root marker, so its + instructions are the working directory's: none above it, a repository's or a deployment's (any host's deployment + leaves a .hermit-deploy.json, which a run without an adapter below it took for its root), as the run's sandbox + shows nothing above the workspace.""" + work = fake / "work" + work.mkdir() + make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}).run("go", workspace=work) + layer = rows(seen(fake)[0], fake / "home") + assert row(layer, "skill-filesystem")["config"] == {"includeDefaultRoots": False, "customSkillDirs": [".dsh/skills"]} + assert row(layer, "agent-instructions")["config"] == {"maxBytes": 65536, "projectRootMarkers": []} + + +def test_what_dsh_cannot_use_is_refused_rather_than_dropped(fake, monkeypatch, tmp_path): + """An effort without a model reaches no model (it is the default model's reasoningEffort), and dsh's own provider + (a model of no binding, DeepSeek's API) takes fewer levels than a bound model's route; a writable path dsh's + sandbox cannot take (a link, a missing folder) failed every command and write of the session: dsh requires an + existing directory by its real path, which the session is given. The user's own dsh home is where its .env is + copied from.""" + work = fake / "work" + work.mkdir() + with pytest.raises(ValueError, match="an effort needs its model"): + make("dsh", {"effort": "high", "web_search": False, "home": str(fake / "home")}).run("go", workspace=work) + with pytest.raises(ValueError, match="an effort is one of off, minimal, low, medium, high, xhigh, max"): + make("dsh", {**bound(), "effort": "ultra", "web_search": False, "home": str(fake / "home")}).run( + "go", workspace=work) + with pytest.raises(ValueError, match="an effort is one of off, low, high, max for a model of its own provider"): + make("dsh", {"model": "deepseek-v4-flash", "effort": "medium", "web_search": False, + "home": str(fake / "home")}).run("go", workspace=work) + make("dsh", {"model": "deepseek-v4-flash", "effort": "max", "web_search": False, "home": str(fake / "home")}).run( + "go", workspace=work) + assert row(rows(seen(fake)[-1], fake / "home"), "agent-default-model")["config"] == { + "provider": base_cli.DSH_ROUTE, "model": "deepseek-v4-flash", "reasoningEffort": "max"} + make("dsh", {**bound(), "effort": False, "web_search": False, "home": str(fake / "home")}).run( + "go", workspace=work) # YAML's bare `off` + [model] = row(rows(seen(fake)[-1], fake / "home"), "llm-pi-ai")["config"]["providers"][base_cli.DSH_PROVIDER][ + "models"] + assert model["reasoningEfforts"] is False + real = tmp_path / "data" + real.mkdir() + (tmp_path / "link").symlink_to(real) + agent = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}) + agent.configured(writable=[str(tmp_path / "link")]).run("go", workspace=work) + args = seen(fake)[-1]["args"] + assert args[args.index("--add-dir") + 1] == os.path.realpath(real) + with pytest.raises(ValueError, match="a writable path is an existing directory"): + agent.configured(writable=[str(tmp_path / "missing")]).run("go", workspace=work) + monkeypatch.setenv("DSH_HOME", str(tmp_path / "users-dsh")) + assert base_cli.Dsh.user_home() == tmp_path / "users-dsh" + + +def test_a_tasks_data_is_bound_in_the_workspace_where_bubblewrap_can_bind_it(fake, monkeypatch, tmp_path): + """dsh's glob and grep run ripgrep without --follow: without isolation a task's data inside the workspace is + bound there, a real read-only directory that a search walks into (as for Claude Code), not linked.""" + from hermit.agent.base.interface import Limits, Mount + if base_cli._bwrap_binds() is None: + pytest.skip("needs a working bubblewrap") + data = tmp_path / "data" + (data / "deep").mkdir(parents=True) + (data / "deep" / "a.txt").write_text("1934 557\n") + work = fake / "work" + work.mkdir() + monkeypatch.setenv("FAKE_DSH_RUN", 'find docs -name "*.txt" > found.txt; [ -L docs ] && echo link >> found.txt') + result = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}).run( + "go", workspace=work, limits=Limits(readable=[Mount(data, Path("docs"))])) + assert result.extra["mounts"] == {str(work / "docs"): "bound"} + assert (work / "found.txt").read_text().split() == ["docs/deep/a.txt"] + + +def test_a_run_whose_tools_did_not_start_has_failed_although_dsh_completed_it(fake, monkeypatch): + """dsh reports an MCP row it could not start on stderr and runs on without its tools (recorded: tools_down.jsonl, + a server dsh could not reach, exit 0): the session ran without what it was given, so it has failed.""" + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "tools_down.jsonl")) + work = fake / "work" + work.mkdir() + result = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}).run("go", workspace=work, + tools=[echo()]) + assert result.exit_code == 0 and result.extra["served"] and result.extra["failed"] + assert result.extra["failure"].startswith("mcp-hermit_tools did not start, so the session ran without its tools") + + +def test_a_run_dsh_refused_to_start_is_no_session(fake, monkeypatch): + """A usage error is an `error` event before any session (recorded: bad_option.jsonl), and a .env in the working + directory that sets a name dsh keeps for its launch stops it before any event (dotenv.jsonl): neither started + a session (the usage error's event read as one), so the acting agent reports a CLI that exited before starting + one (each task's error record says so), rather than a session that failed.""" + from hermit.agent.acting import workspace as acting + monkeypatch.setenv("FAKE_DSH_EXIT", "1") + work = fake / "work" + work.mkdir() + agent = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}) + for stream, says in (("bad_option.jsonl", "unknown option"), ("dotenv.jsonl", 'sets "PYTHONPATH"')): + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / stream)) + result = agent.run("go", workspace=work) + assert result.exit_code == 1 and not result.extra["started"] and says in result.extra["output_tail"] + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "bad_option.jsonl")) + with pytest.raises(RuntimeError, match="before starting a session"): + acting.run({"id": "q1"}, None, params={**bound(), "backend": "dsh", "web_search": False, "prompt": "go", + "workspaces": "ws"}, base=fake) + + +def test_dsh_permission_mode_reaches_the_acting_agents_session_or_is_refused(fake): + """agent.dsh.permission_mode picks dsh's own sandbox for the acting agent (it was dropped, so the session always + ran workspace-write); a mode dsh has not is refused when the call is checked.""" + from hermit.agent.acting import workspace as acting + from hermit.config import layered + data = {"agent": {"base": "dsh", "model": "fake/m-1", "dsh": {"permission_mode": "danger-full-access"}, + "params": {"prompt": "go"}}, + "evolver": {"name": "dsh", "base": "dsh", "model": "fake/m-1"}, "llm": None} + layered(data) + params = {**data["agent"]["params"], **bound(), "backend": "dsh", "web_search": False, "workspaces": "ws"} + acting.run({"id": "q1"}, None, params=params, base=fake) + args = seen(fake)[-1]["args"] + assert args[args.index("--permission-mode") + 1] == "danger-full-access" + with pytest.raises(ValueError, match="permission_mode is one of read-only, workspace-write"): + acting._check("dsh", {**params, "permission_mode": "bypassPermissions"}) + + +def test_a_dsh_stream_is_read_back_by_its_name_or_by_its_events(tmp_path): + """The learning material keeps the acting agent's log: an Evolver reads dsh's steps from it like Codex's.""" + named = tmp_path / "log" + named.mkdir() + shutil.copy(STREAMS / "success.jsonl", named / trajectory.DSH_STREAM) + other = tmp_path / "events.jsonl" + shutil.copy(STREAMS / "success.jsonl", other) + for log in (named, other): + assert [s.kind for s in trajectory.log_steps(log) if s.kind != "reasoning"] == \ + ["command", "command", "tool_call", "file_change", "file_change", "message"] + raw = (STREAMS / "failure.jsonl").read_text() + assert not base_cli.served("dsh", raw) and base_cli.notices("dsh", raw).startswith("dsh: SERVER: 503: ") + assert base_cli.served("dsh", (STREAMS / "success.jsonl").read_text()) + + +def test_a_bash_result_is_read_as_dsh_marks_its_end(): + """dsh marks a nonzero exit (`[exit code: N]`) or a signal after a command's output and nothing after a clean + exit; `(no output)` stands for an empty output, also before a marker. A command stopped by its timeout has no + exit code to trust, even one that trapped the signal and exited 0 (it read as a clean exit before), as a + signal's has none: unknown, as Codex reports it. dsh prints the timeout it applied (the model's, else 60 s, at + most 600 s), which need not be a whole number of milliseconds (`1000.5ms`, read as a clean exit before).""" + stream = [json.loads(line) for line in (STREAMS / "bash.jsonl").read_text().splitlines()] + commands = [(s.command, s.exit_code, s.output) for s in trajectory.dsh_steps(stream) if s.kind == "command"] + assert commands == [("exit 3", 3, ""), ("printf ''", 0, ""), ("true", 0, ""), ("echo out; exit 4", 4, "out"), + ("kill -9 $$", None, "(no output)\n[killed by signal: SIGKILL]"), + ("printf 'no newline'; exit 5", 5, "no newline"), + ("trap 'exit 0' TERM; sleep 5 & wait", None, "(no output)\n[timed out after 1000ms]"), + ("sleep 5", None, "(no output)\n[timed out after 1000ms]\n[killed by signal: SIGTERM]"), + ("trap 'exit 0' TERM; sleep 5 & wait", None, "(no output)\n[timed out after 1000.5ms]")] + + +def test_a_command_whose_end_dsh_did_not_report_has_no_exit_code(): + """dsh's --json keeps 8 KiB of a string: a long result loses its end, where the marker is (recorded: + big_bash.jsonl, 1,500 lines then exit 2), and read as exit 0 before; a command started in the background + (background.jsonl) has no status yet, nor has one still running at its timeout (60 s by default), which dsh moves + to a background job in a session unless its job limit is reached (promoted.jsonl): only DSH_BACKGROUND keeps it + from reading as exit 0, also when its marker is the whole result (the command printed nothing yet) and when the + timeout is no whole number of milliseconds (`1000.5ms`, read as exit 0 before). A command stopped from outside + the call (the job stopped in dsh's UI) is marked `[stopped: ]` before its signal or exit marker, and may + have trapped the signal and exited 0: no headless session writes one, so its results are written here as dsh's + bash tool renders them (tool-bash's render.ts, dsh 0.2.2-rc.1). A command that printed a marker of its own and + exited 0 (fake_marker.jsonl) is a clean exit, as dsh reads it: the marker counts only at the very end. A call + that failed to run (a result dsh marks `error`) has no exit status either, as dsh's own view of a bash result has + none for it.""" + def commands(name): + stream = [json.loads(line) for line in (STREAMS / name).read_text().splitlines() if line.startswith("{")] + return [s for s in trajectory.dsh_steps(stream) if s.kind == "command"] + + def command(result, status="completed"): + [step] = trajectory.dsh_steps([ + {"type": "tool_call", "callId": "call_0", "tool": "bash", "input": {"command": "make"}}, + {"type": "tool_result", "callId": "call_0", "status": status, "result": result}]) + return step + [cut] = commands("big_bash.jsonl") + assert cut.exit_code is None and cut.output.startswith("line 1 of a long build log\n") + assert cut.output.endswith(trajectory.DSH_CUT) and "[exit code" not in cut.output + [background] = commands("background.jsonl") + assert background.exit_code is None and background.output.startswith("started background job ") + [building, silent, fraction] = commands("promoted.jsonl") + for step, start in ((building, "building\n[still running after 1000ms"), (silent, "[still running after 1000ms"), + (fraction, "building\n[still running after 1000.5ms")): + assert step.exit_code is None and step.output.startswith(f"{start}; moved to background job bash-"), start + for stopped in ("(no output)\n[stopped: cancelled by the user]", + "cleaning up\n[stopped: cancelled by the user]\n[exit code: 143]"): + assert command(stopped).exit_code is None, stopped + [printed] = commands("fake_marker.jsonl") + assert (printed.exit_code, printed.output) == (0, "built\n[exit code: 7]\n") + unrun = command("bash could not run", status="error") + assert (unrun.kind, unrun.exit_code, unrun.output) == ("command", None, "bash could not run") + + +def test_a_session_is_resumed_only_from_a_directory_of_the_home_itself(tmp_path): + """dsh keeps a session at sessions/// in its home, which the session's commands may write: a + link there (in place of the session's folder, or of the folder it lies in) is no session to resume, so a call + starts a new one; so is a handle that is no session id.""" + home, elsewhere = tmp_path / "home", tmp_path / "elsewhere" + (home / "sessions" / "project" / SESSION).mkdir(parents=True) + (elsewhere / SESSION).mkdir(parents=True) + assert base_cli._dsh_session(home, SESSION) == SESSION + assert base_cli._dsh_session(home, f"../{SESSION}") is None + linked = tmp_path / "linked" + (linked / "sessions" / "project").mkdir(parents=True) + (linked / "sessions" / "project" / SESSION).symlink_to(elsewhere / SESSION) + assert base_cli._dsh_session(linked, SESSION) is None + parent = tmp_path / "parent" + (parent / "sessions").mkdir(parents=True) + (parent / "sessions" / "project").symlink_to(elsewhere) + assert base_cli._dsh_session(parent, SESSION) is None + + +def without_usage(stream: str, tmp_path: Path) -> Path: + """A recorded stream whose steps report no usage, as dsh writes a step one of whose attempts reported none (its + json-stream.ts omits the partial sum): a stream that ended before its last event, a failed request of dsh's own + provider.""" + out = tmp_path / f"no-usage-{stream}" + out.write_text("".join(json.dumps({k: v for k, v in json.loads(line).items() if k != "usage"}) + "\n" + if '"step_end"' in line else line + "\n" + for line in (STREAMS / stream).read_text().splitlines())) + return out + + +def test_a_step_counts_as_a_call_whenever_it_made_one(tmp_path): + """A step whose requests all failed produced nothing and is no call (failure.jsonl), also when dsh reports no + usage for it; one that answered is a call even when the provider reported no usage, which dsh reports as zeros + (no_usage.jsonl), and so is one dsh reports no usage for at all. A step dsh retried after the provider refused a + request (retried.jsonl: a 503, then 1000 prompt tokens, 900 of them cached) reports its input and output but not + its cache counts, which its total holds: those tokens count as input, and the call as one whose tokens are known. + parse_log reads a dsh stream as dsh's (it read every stream but Codex's as Claude Code's).""" + from hermit.agent.base.records import parse_dsh, parse_log + assert parse_dsh(STREAMS / "failure.jsonl", "m")[1] == [] + assert parse_dsh(without_usage("failure.jsonl", tmp_path), "m")[1] == [] + _, [answered] = parse_dsh(STREAMS / "no_usage.jsonl", "m") + assert (answered.calls, answered.unreported_calls, answered.input_tokens, answered.output_tokens) == (1, 1, 0, 0) + session, [resumed] = parse_log(STREAMS / "resume.jsonl", "dsh", "m") + assert session == SESSION and (resumed.calls, resumed.unreported_calls, resumed.input_tokens) == (1, 0, 1000) + [unknown] = parse_dsh(without_usage("resume.jsonl", tmp_path), "m")[1] + assert (unknown.calls, unknown.unreported_calls, unknown.input_tokens) == (1, 1, 0) + [retried] = parse_dsh(STREAMS / "retried.jsonl", "m")[1] + assert (retried.calls, retried.unreported_calls, retried.input_tokens, retried.cached_input_tokens, + retried.output_tokens) == (1, 0, 1000, 0, 10) + + + + +def test_a_step_is_a_call_when_it_called_a_tool_or_thought_and_none_when_its_requests_then_failed(tmp_path): + """Which steps whose tokens dsh did not report are calls, from the recorded streams: a step whose only output is a + tool call (bash.jsonl's nine, recorded without reasoning) is one, and so is a step whose only output is a + thought (resume.jsonl's step without its reply); a step whose requests all failed (failure.jsonl's) after a step + that produced something is none: what a step produced does not carry over to the next.""" + from hermit.agent.base.records import parse_dsh + + def stream(name: str, events: list[dict]) -> Path: + path = tmp_path / name + path.write_text("".join(json.dumps(e) + "\n" for e in events)) + return path + + def read(name: str) -> list[dict]: + return [json.loads(line) for line in (STREAMS / name).read_text().splitlines() if line.startswith("{")] + [tools] = parse_dsh(without_usage("bash.jsonl", tmp_path), "m")[1] + assert (tools.calls, tools.unreported_calls) == (10, 10) + resume = read("resume.jsonl") + thought = [{k: v for k, v in e.items() if k != "usage"} for e in resume if e.get("type") != "text"] + [thinking] = parse_dsh(stream("thought.jsonl", thought), "m")[1] + assert (thinking.calls, thinking.unreported_calls) == (1, 1) + end = next(i for i, e in enumerate(resume) if e.get("phase") == "step_end") + failed = resume[:end + 1] + [e for e in read("failure.jsonl") if e.get("phase") in ("step_start", "step_end")] + [after] = parse_dsh(stream("failed_after.jsonl", failed), "m")[1] + assert (after.calls, after.unreported_calls, after.input_tokens) == (1, 0, 1000) + + +def test_calls_whose_tokens_were_not_reported_have_an_unknown_cost_not_a_free_one(fake, monkeypatch, tmp_path): + """dsh reports no cost, so the backend's price makes one from the tokens. A session with a call whose tokens it + did not report (a provider that reports no usage, an attempt that reported none) has an unknown cost (None), not + $0.00 nor the price of the rest, which may be far less (the successful attempt of a step whose usage dsh left out + goes unreported too): the run's budget counts its calls as of unknown cost, and none of its dollars. A session + whose tokens dsh reported is priced from them; a step whose cache counts dsh left out after a retry is priced + with those tokens as input, as a cache write is, never less than with the counts (the cached input being no + dearer than the input), so that a USD budget counts the session. Only dsh's reader marks a call unreported: + another CLI's call of no tokens (Claude Code's result of a refused request) keeps its price, $0.00.""" + from hermit.trainer.budget import Budget + from hermit.agent.base.records import price_usage + from hermit.agent.model.usage import Usage + work = fake / "work" + work.mkdir() + agent = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}) + partial = tmp_path / "partial.jsonl" + lines = (STREAMS / "success.jsonl").read_text().splitlines() + first = next(i for i, line in enumerate(lines) if '"step_end"' in line) + lines[first] = json.dumps({k: v for k, v in json.loads(lines[first]).items() if k != "usage"}) + partial.write_text("\n".join(lines) + "\n") # one of six steps without usage + for stream, tokens in ((STREAMS / "no_usage.jsonl", 0), (without_usage("success.jsonl", tmp_path), 0), + (partial, 5000)): + monkeypatch.setenv("FAKE_DSH_STREAM", str(stream)) + [usage] = agent.run("go", workspace=work).usage + assert usage.unreported_calls and usage.input_tokens == tokens and usage.cost_usd is None + refused = [Usage(model="claude-x", calls=1)] + price_usage(refused, {"price": {"input": 1.0, "output": 2.0}}) + assert refused[0].cost_usd == 0.0 + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "retried.jsonl")) + [retried] = agent.run("go", workspace=work).usage + assert not retried.unreported_calls and retried.cost_usd == pytest.approx((1000 * 1.0 + 10 * 2.0) / 1e6) + budget = Budget(usd=0.001) + budget.add("rollout", retried) + assert budget.exceeded() and budget.unknown_cost_calls == 0 + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "resume.jsonl")) + [usage] = agent.run("go", workspace=work).usage + assert usage.input_tokens and usage.cost_usd == pytest.approx( + ((usage.input_tokens - usage.cached_input_tokens) * 1.0 + usage.cached_input_tokens * 0.1 + + usage.output_tokens * 2.0) / 1e6) + + +def test_a_model_whose_backend_declares_no_context_window_fails_with_what_to_declare(fake, monkeypatch): + """A model dsh's catalog does not know is taken for 1,000,000 tokens, asked for up to 128,000 output tokens at a + time: a server sized below that (vLLM refuses a request past its model's length) fails every request (recorded: + context_window.jsonl). The failure says to declare the backend's context_window, which dsh then sizes requests + by; a backend that declared one gets dsh's error alone.""" + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "context_window.jsonl")) + monkeypatch.setenv("FAKE_DSH_EXIT", "1") + work = fake / "work" + work.mkdir() + said = base_cli.notices("dsh", (STREAMS / "context_window.jsonl").read_text()) # its error, twice + for backend, hinted in (({}, True), ({"context_window": 32768}, False)): + result = make("dsh", {**bound(**backend), "web_search": False, "home": str(fake / "home")}).run( + "go", workspace=work) + assert result.extra["failed"] and "CONTEXT_WINDOW_EXCEEDED" in result.extra["failure"] + assert said.splitlines()[0] in result.extra["failure"] # dsh's error line (this one fits whole) + assert ("declare the backend's context_window" in result.extra["failure"]) is hinted + assert result.usage == [] # the refused requests used nothing + + +def test_a_run_that_ends_at_the_token_limit_is_an_error_the_learner_can_read(): + """dsh exits 1 for a run that ends `max-tokens` and writes nothing on stderr: the run is no success and no + outage, and its end is a step of the trajectory and a notice (only errors and aborts were, before).""" + raw = (STREAMS / "max_tokens.jsonl").read_text() + assert not base_cli.served("dsh", raw) and base_cli.notices("dsh", raw) == "run max-tokens" + assert not base_cli.UNAVAILABLE.search(base_cli.notices("dsh", raw)) + steps = trajectory.dsh_steps([json.loads(line) for line in raw.splitlines()]) + assert [(s.kind, s.text) for s in steps] == [("message", "partial answ"), ("error", "run max-tokens")] + + +def test_a_server_that_needs_no_key_gets_the_placeholder_key(fake): + """A route that names no key variable gets no key from pi-ai under the name `hermit`, and dsh refuses every + request (PI_AI_ERROR "No API key"): the session names a variable that holds the placeholder key instead.""" + work = fake / "work" + work.mkdir() + params = bound(api_key_env="NO_SUCH_KEY_VAR", key_required=False) + make("dsh", {**params, "web_search": False, "home": str(fake / "home")}).run("go", workspace=work) + [call] = seen(fake) + route = row(rows(call, fake / "home"), "llm-pi-ai")["config"]["providers"][base_cli.DSH_PROVIDER] + assert route["apiKeyEnv"] == base_cli.DSH_NO_KEY and call["env"][base_cli.DSH_NO_KEY] == "local" + + +def test_the_dsh_acting_agent_deploys_the_adapter_and_loads_its_patch_layer(fake): + """dsh has no project file for MCP servers: the deployment's patch layer (.dsh/hermit.patch.yml) declares the + adapter's tool server, and the session loads it with --patch, before its own layer, which wins.""" + from hermit.agent.acting import workspace as acting + params = {**bound(), "backend": "dsh", "web_search": False, "prompt": "go", "workspaces": "ws"} + out = acting.run({"id": "q1"}, FILE_ADAPTER, params=params, base=fake) + work = Path(out.extra["workspace"]) + [call] = seen(fake) + patches = [call["args"][i + 1] for i, a in enumerate(call["args"]) if a == "--patch"] + assert patches == [str(work / ".dsh" / "hermit.patch.yml"), str(work.parent / "home" / base_cli.DSH_PATCH)] + assert call["args"].index("--json") > call["args"].index(patches[-1]) # the launcher's options come first + layer = json.loads((work / ".dsh" / "hermit.patch.yml").read_text()) + [server] = [r["insert"][0] for r in layer if "insert" in r] + assert server["config"]["transport"] == "stdio" and server["config"]["args"][:2] == ["-m", + "hermit.runtime.mcp_server"] + assert (work / ".dsh" / "skills" / "verify-output" / "SKILL.md").is_file() + assert (work / ".hermit-deploy.json").is_file() # the deployment's record + assert "do it yourself" in (work / "AGENTS.md").read_text() # roles stay files: none is passed to dsh + assert out.output == "the answer" and out.extra["backend"] == "dsh" + assert (out.log / "agent-dsh.jsonl").is_file() + + +def test_dsh_is_a_base_of_the_config_and_the_dsh_evolver_binds_its_model_with_its_sandbox_off(): + """In the evolver sandbox the run's isolation is the boundary, so dsh's own sandbox is off, as Codex's is.""" + from hermit.agent.evolver.agent import AgentEvolver + from hermit.agent.evolver.loader import find, instantiate + from hermit.config import layered + data = {"agent": {"base": "dsh", "model": {"ref": "fake/m-1", "effort": "high"}, "dsh": {"timeout_s": 60}, + "params": {"prompt": "go"}}, + "evolver": {"name": "dsh", "base": "dsh", "model": "fake/m-1", "params": {}}, "llm": None} + layered(data) + assert data["agent"]["name"] == "dsh" and data["agent"]["params"]["effort"] == "high" + assert data["agent"]["params"]["timeout_s"] == 60 and data["evolver"]["params"]["backend"] == "dsh" + spec = find("dsh", ROOT) + backends = registry({"fake": {"base_url": "http://127.0.0.1:9/v1", "api_key_env": "FAKE_KEY", "apis": ["chat"]}}) + _, params = instantiate(spec, data["evolver"]["params"], backends=backends, default_model="fake/m-1") + assert params["backend"] == "dsh" and params["home"] == [] and params["provider"]["apiKeyEnv"] == "FAKE_KEY" + + class Ctx: # the fields AgentEvolver.settings reads + def __init__(self, root): + self.params, self.sandbox, self.cancel = params, None, None + self.workspace = type("W", (), {"state": root / "state", "output": root / "output"}) + settings = AgentEvolver().settings(Ctx(Path("/w")), "dsh") + assert settings["permission_mode"] == "danger-full-access" and settings["provider"] == params["provider"] + assert "permission_mode" not in AgentEvolver().settings(Ctx(Path("/w")), "claude") + + +# ---------- the installed dsh (integration) ---------- + +def serve_chat(script, keys=None): + """A chat-completions model on the loopback: `script(request body) -> (tool call (name, arguments) | None)`; + None answers `the answer`. Every response is streamed, with usage. `keys` collects each request's + Authorization header.""" + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + def chunk(body): + return f"data: {json.dumps(body)}\n\n".encode() + + class Model(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers.get("content-length") or 0)) or b"{}") + if keys is not None: + keys.append(self.headers.get("authorization")) + call = script(body) + delta = {"role": "assistant", "tool_calls": [{"index": 0, "id": "call_1", "type": "function", + "function": {"name": call[0], + "arguments": json.dumps(call[1])}}]} \ + if call else {"role": "assistant", "content": "the answer"} + data = b"".join([chunk({"choices": [{"index": 0, "delta": delta, "finish_reason": None}]}), + chunk({"choices": [{"index": 0, "delta": {}, "finish_reason": + "tool_calls" if call else "stop"}]}), + chunk({"choices": [], "usage": {"prompt_tokens": 100, "completion_tokens": 5}}), + b"data: [DONE]\n\n"]) + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Model) + threading.Thread(target=server.serve_forever, daemon=True).start() + return server, f"http://127.0.0.1:{server.server_address[1]}/v1" + + +def _bwrap_works() -> bool: + return bool(shutil.which("bwrap")) and subprocess.run( + ["bwrap", "--ro-bind", "/", "/", "--unshare-net", "true"], capture_output=True).returncode == 0 + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +@pytest.mark.parametrize("isolation", ["none", "sandbox"]) +def test_with_the_dsh_cli_a_session_reaches_its_model_and_its_tools(tmp_path, monkeypatch, isolation): + """The installed dsh, bound to a scripted model: it calls the call's MCP tool, writes a file, answers; in the + run's sandbox (isolated network) only its model and the tool server are reached.""" + from hermit.sandbox import Sandbox + if isolation == "sandbox" and not _bwrap_works(): + pytest.skip("needs a working bubblewrap") + requests = [] + + def script(body): + requests.append(body) + done = sum(1 for m in body.get("messages") or [] if m.get("role") == "tool") + names = [t["function"]["name"] for t in body.get("tools") or []] + plan = [("mcp__hermit_tools__echo", {"text": "hi"}), ("write", {"file_path": "note.txt", "content": "ok\n"})] + assert plan[0][0] in names and "web_search" not in names # the tool offered as dsh's own; no web tools + return plan[done] if done < len(plan) else None + + server, url = serve_chat(script) + monkeypatch.setenv("MOCK_KEY", "sk-mock") + params = bind_agent_params({"model": "mock/m-1"}, "dsh", + registry({"mock": {"base_url": url, "api_key_env": "MOCK_KEY", "apis": ["chat"]}})) + work = tmp_path / "work" + work.mkdir() + settings = {**params, "web_search": False, "home": str(tmp_path / "home"), "isolation": isolation, + "timeout_s": 300} + if isolation == "sandbox": + settings["sandbox"] = Sandbox(network="isolated", env_allow=[], writable=[work], role="agent", + record_dir=tmp_path / "records") + try: + result = make("dsh", settings).run("Do the task.", workspace=work, tools=[echo()]) + finally: + server.shutdown() + assert result.exit_code == 0 and result.extra["served"] and result.reply == "the answer" + assert (work / "note.txt").read_text() == "ok\n" + assert [s.kind for s in result.trajectory] == ["tool_call", "file_change", "message"] + assert json.loads(result.trajectory[0].output) == {"echoed": "hi"} + assert result.usage[0].calls == 3 == len(requests) # no request besides the session's own + if isolation == "sandbox": + [record] = (tmp_path / "records").glob("agent-*.json") + data = json.loads(record.read_text()) + assert data["network"] == "isolated" and data["probe"]["passed"] + assert {"DSH_HOME", "HOME", "MOCK_KEY"} <= set(data["environment"]) + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +def test_with_the_dsh_cli_a_command_runs_in_dshs_own_sandbox_inside_the_runs(tmp_path, monkeypatch): + """The acting agent keeps dsh's own sandbox (workspace-write) inside the run's: dsh starts it nested in the run's + bubblewrap sandbox (bubblewrap, else Landlock), and a command of the session writes its workspace there, not the + session's home, which the run's sandbox lets the session write (the home lies outside /tmp, which workspace-write + lets a command write). Where neither can start, dsh refuses every command (SANDBOX_UNAVAILABLE), which this would + show.""" + from hermit.sandbox import Sandbox + if not _bwrap_works(): + pytest.skip("needs a working bubblewrap") + base = Path.home() / ".cache" / "hermit-tests" # outside /tmp, an empty tmpfs under bubblewrap + base.mkdir(parents=True, exist_ok=True) + home = Path(tempfile.mkdtemp(prefix="dsh-home-", dir=base)) + outside = home / "outside.txt" + commands = [{"command": "echo made > made.txt && cat made.txt", "description": "Write a file"}, + {"command": f"echo out > {outside}", "description": "Write the home"}] + + def script(body): + done = sum(1 for m in body.get("messages") or [] if m.get("role") == "tool") + return ("bash", commands[done]) if done < len(commands) else None + + server, url = serve_chat(script) + monkeypatch.setenv("MOCK_KEY", "sk-mock") + params = bind_agent_params({"model": "mock/m-1"}, "dsh", + registry({"mock": {"base_url": url, "api_key_env": "MOCK_KEY", "apis": ["chat"]}})) + work = tmp_path / "work" + work.mkdir() + box = Sandbox(network="isolated", env_allow=[], writable=[work], role="agent", record_dir=tmp_path / "records") + try: + result = make("dsh", {**params, "web_search": False, "home": str(home), "isolation": "sandbox", + "sandbox": box, "timeout_s": 300}).run("Write the files.", workspace=work) + escaped = outside.exists() + finally: + server.shutdown() + shutil.rmtree(home, ignore_errors=True) + assert result.exit_code == 0 and result.extra["served"] and not result.extra["failed"] + [inside, written] = [s for s in result.trajectory if s.kind == "command"] + assert (inside.exit_code, inside.output) == (0, "made\n") and (work / "made.txt").read_text() == "made\n" + assert written.exit_code != 0 and not escaped + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +@pytest.mark.skipif(not shutil.which("git"), reason="needs git") +def test_with_the_dsh_cli_a_deployed_adapter_brings_its_skills_and_tools_into_a_git_repository(tmp_path, monkeypatch): + """The acting agent with the installed dsh, its workspaces inside a git repository (a config kept in a project, + run without the sandbox). dsh takes the repository for the project: before, the deployed skill was not offered, + and the repository's own skills and AGENTS.md were, ahead of the adapter's. Now the session's layer makes the + workspace the project. The adapter's own tool server (the deployment's stdio server, loaded from its patch layer) + answers a call.""" + from hermit.agent.acting import workspace as acting + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", str(repo)], check=True) + (repo / "AGENTS.md").write_text("REPO-INSTRUCTIONS-MARKER\n") + for folder in (".dsh", ".agents"): + (repo / folder / "skills" / "repo-skill").mkdir(parents=True) + (repo / folder / "skills" / "repo-skill" / "SKILL.md").write_text( + f"---\nname: repo-skill{folder}\ndescription: REPO-SKILL-MARKER\n---\nbody\n") + requests = [] + + def script(body): + requests.append(body) + done = sum(1 for m in body.get("messages") or [] if m.get("role") == "tool") + return ("mcp__file_example__count_words", {"text": "one two three"}) if done == 0 else None + + server, url = serve_chat(script) + monkeypatch.setenv("MOCK_KEY", "sk-mock") + params = {**bind_agent_params({"model": "mock/m-1"}, "dsh", + registry({"mock": {"base_url": url, "api_key_env": "MOCK_KEY", "apis": ["chat"]}})), + "backend": "dsh", "web_search": False, "prompt": "go", "workspaces": "ws", "timeout_s": 300, + "retries": 1} + try: + out = acting.run({"id": "q1"}, FILE_ADAPTER, params=params, base=repo) + finally: + server.shutdown() + assert out.output == "the answer" and len(requests) == 2 + assert "Check a completed result against the task requirements." in json.dumps(requests[0]) # the skill + assert "REPO-SKILL-MARKER" not in json.dumps(requests[0]) and "REPO-INSTRUCTIONS-MARKER" not in json.dumps( + requests[0]) # nothing of the repository the run lies in + assert "mcp__file_example__count_words" in [t["function"]["name"] for t in requests[0].get("tools") or []] + [answer] = [m for m in requests[1]["messages"] if m.get("role") == "tool"] + assert json.loads(answer["content"] if isinstance(answer["content"], str) else + "".join(p.get("text", "") for p in answer["content"])) == {"count": 3} + + +def mock_model(monkeypatch, url: str) -> dict: + """The params of a `dsh` agent bound to the scripted model at `url`.""" + monkeypatch.setenv("MOCK_KEY", "sk-mock") + return bind_agent_params({"model": "mock/m-1"}, "dsh", + registry({"mock": {"base_url": url, "api_key_env": "MOCK_KEY", "apis": ["chat"]}})) + + +def tool_texts(body: dict) -> list[str]: + """The tool results a request carries, as text.""" + return [m["content"] if isinstance(m["content"], str) else "".join(p.get("text", "") for p in m["content"]) + for m in body.get("messages") or [] if m.get("role") == "tool"] + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +def test_with_the_dsh_cli_a_tasks_data_is_found_by_dshs_own_search(tmp_path, monkeypatch): + """dsh's glob and grep run ripgrep without --follow, so a task's data linked into the workspace was never listed + or matched. Without isolation it is bound there, a real directory: dsh's own search finds it, and dsh's own + sandbox, nested in that mount namespace, runs a command that reads it.""" + from hermit.agent.base.interface import Limits, Mount + if base_cli._bwrap_binds() is None: + pytest.skip("needs a working bubblewrap") + data, work = tmp_path / "data", tmp_path / "work" + (data / "deep").mkdir(parents=True) + work.mkdir() + (data / "deep" / "facts.txt").write_text("1934 557\n") + last = [] + + def script(body): + last[:] = tool_texts(body) + plan = [("glob", {"pattern": "**/*.txt"}), ("grep", {"pattern": "1934"}), + ("bash", {"command": "cat docs/deep/facts.txt", "description": "Read the facts"})] + return plan[len(last)] if len(last) < len(plan) else None + + server, url = serve_chat(script) + try: + result = make("dsh", {**mock_model(monkeypatch, url), "web_search": False, "home": str(tmp_path / "home"), + "timeout_s": 300}).run("Find the facts.", workspace=work, + limits=Limits(readable=[Mount(data, Path("docs"))])) + finally: + server.shutdown() + assert result.extra["served"] and result.extra["mounts"] == {str(work / "docs"): "bound"} + found, matched, read = last + assert "docs/deep/facts.txt" in found and "1934 557" in matched and "1934 557" in read + [command] = [s for s in result.trajectory if s.kind == "command"] + assert (command.exit_code, command.output) == (0, "1934 557\n") + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +def test_with_the_dsh_cli_a_deployed_tool_server_that_does_not_start_fails_the_session(tmp_path, monkeypatch): + """The deployment's tool server is a stdio row of its patch layer: one dsh could not start was retried in the + background while the session ran on without the adapter's tools, and its answer was taken like any other. The + row now fails at the session's start, which dsh reports, so the session has failed.""" + from hermit.hosts.dsh import prepare + work = tmp_path / "ws" + prepare(FILE_ADAPTER, work, sys.executable) + patch = work / ".dsh" / "hermit.patch.yml" + layer = json.loads(patch.read_text()) + [server] = [r["insert"][0] for r in layer if "insert" in r] + server["config"]["command"] = str(tmp_path / "no-such-python") # a server that cannot start + patch.write_text(json.dumps(layer)) + server_http, url = serve_chat(lambda body: None) + try: + result = make("dsh", {**mock_model(monkeypatch, url), "web_search": False, "home": str(tmp_path / "home"), + "args": ["--patch", str(patch)], "timeout_s": 300}).run("go", workspace=work) + finally: + server_http.shutdown() + assert result.extra["failed"] and result.extra["failure"].startswith("mcp-file_example did not start") + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +def test_with_the_dsh_cli_no_instructions_above_the_workspace_are_read(tmp_path, monkeypatch): + """A session whose workspace holds no deployment (a run without an adapter, an Evolver's) below a folder that + does (a project deployed into with `hermit deploy`, of any host): dsh took that folder, the nearest with a + .hermit-deploy.json, for the project's root and read its AGENTS.md too. Without a root marker the working + directory is the root, as the run's sandbox shows nothing above it.""" + above = tmp_path / "deployed" + work = above / "runs" / "work" + work.mkdir(parents=True) + (above / ".hermit-deploy.json").write_text("{}\n") + (above / "AGENTS.md").write_text("ABOVE-INSTRUCTIONS-MARKER\n") + (work / "AGENTS.md").write_text("OWN-INSTRUCTIONS-MARKER\n") + requests = [] + server, url = serve_chat(lambda body: requests.append(body)) + try: + make("dsh", {**mock_model(monkeypatch, url), "web_search": False, "home": str(tmp_path / "home"), + "timeout_s": 300}).run("go", workspace=work) + finally: + server.shutdown() + sent = json.dumps(requests[0]) + assert "OWN-INSTRUCTIONS-MARKER" in sent and "ABOVE-INSTRUCTIONS-MARKER" not in sent + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +def test_with_the_dsh_cli_a_learners_session_takes_no_instruction_file_and_its_transcript_is_scrubbed(tmp_path, + monkeypatch): + """The installed dsh adds a folder's AGENTS.md when its read tool touches a file there; a session with + instruction_files false (the dsh Evolver's) is given none. Every session's transcript is plain JSON lines in its + home, so a key a session wrote there is scrubbed (hermit.redact), and the session is resumed from it.""" + from hermit.redact import redact_path + work = tmp_path / "work" + (work / "adapter").mkdir(parents=True) + (work / "adapter" / "AGENTS.md").write_text("NESTED-INSTRUCTIONS-MARKER\n") + (work / "adapter" / "notes.txt").write_text("the notes\n") + secret = "sk-mock-0123456789abcdef" + requests = [] + + def script(body): + requests.append(body) + return ("read", {"file_path": "adapter/notes.txt"}) if not tool_texts(body) else None + + server, url = serve_chat(script) + params = mock_model(monkeypatch, url) + monkeypatch.setenv("MOCK_KEY", secret) + try: + for given in ({}, {"instruction_files": False}): + home = tmp_path / f"home-{len(given)}" + agent = make("dsh", {**params, "web_search": False, "home": str(home), "timeout_s": 300, **given}) + first = len(requests) + result = agent.run(f"Read the notes; the key is {secret}.", workspace=work) + sent = json.dumps(requests[first:]) + assert result.extra["served"] and ("NESTED-INSTRUCTIONS-MARKER" in sent) is not bool(given) + [transcript] = (home / "sessions").rglob("session.v4.jsonl") + assert secret in transcript.read_text() and not list((home / "sessions").rglob("*.zstd")) + redact_path(home) + assert secret not in transcript.read_text() and "" in transcript.read_text() + again = agent.run("Again.", workspace=work, session=result.session) + finally: + server.shutdown() + assert again.extra["served"] and again.session == result.session + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +def test_with_the_dsh_cli_an_agents_md_at_the_deploy_limit_reaches_the_model_whole(tmp_path, monkeypatch): + """The deployment refuses an AGENTS.md past what dsh gives the model of a project's instructions, its frame + included (DSH_INSTRUCTION_FRAME): one at that size reaches the model whole, and one byte more is cut, which dsh + tells the model in a note of its budget.""" + from hermit.hosts.dsh import prepare + adapter = tmp_path / "adapter" + (adapter / ".hermit").mkdir(parents=True) + (adapter / ".hermit" / "adapter.yaml").write_text("template: {id: edge, version: '1'}\n" + "structure: {learnable: [exports.yaml, ADAPTER.md]}\n") + (adapter / "exports.yaml").write_text("{context: ADAPTER.md}\n") + (adapter / "ADAPTER.md").write_text("LAST-LINE-MARKER\n") + prepare(adapter, tmp_path / "small", sys.executable) + rest = len((tmp_path / "small" / "AGENTS.md").read_bytes()) - len("LAST-LINE-MARKER\n") + limit = base_cli.DSH_INSTRUCTION_BYTES - base_cli.DSH_INSTRUCTION_FRAME + (adapter / "ADAPTER.md").write_text("x" * (limit - rest - len("\nLAST-LINE-MARKER\n")) + "\nLAST-LINE-MARKER\n") + prepare(adapter, tmp_path / "edge", sys.executable) + whole = tmp_path / "edge" / "AGENTS.md" + assert len(whole.read_bytes()) == limit + over = tmp_path / "over" + over.mkdir() + (over / "AGENTS.md").write_text(whole.read_text().replace("x", "xx", 1)) # one byte more, by hand + requests = [] + server, url = serve_chat(lambda body: requests.append(body)) + try: + for work in (tmp_path / "edge", over): + make("dsh", {**mock_model(monkeypatch, url), "web_search": False, "home": str(tmp_path / "home"), + "timeout_s": 300}).run("go", workspace=work) + finally: + server.shutdown() + edge, cut = json.dumps(requests[0]), json.dumps(requests[1]) + assert "LAST-LINE-MARKER" in edge and "Workspace instruction budget" not in edge + assert "Workspace instruction budget 65536 bytes: truncated AGENTS.md" in cut + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +def test_with_the_dsh_cli_a_server_that_needs_no_key_is_reached_with_the_placeholder(tmp_path, monkeypatch): + """A backend that needs no key (a local server): the installed dsh refused every request of a route that named + no key variable (PI_AI_ERROR "No API key"); the session's placeholder reaches the server, and a key of another + provider in the environment (OPENAI_API_KEY) is never sent.""" + keys = [] + server, url = serve_chat(lambda body: None, keys) + monkeypatch.setenv("OPENAI_API_KEY", "sk-another-provider") + params = bind_agent_params({"model": "local/m-1"}, "dsh", registry( + {"local": {"base_url": url, "api_key_env": "NO_SUCH_KEY_VAR", "key_required": False, "apis": ["chat"]}})) + work = tmp_path / "work" + work.mkdir() + try: + result = make("dsh", {**params, "web_search": False, "home": str(tmp_path / "home"), + "timeout_s": 300}).run("Answer.", workspace=work) + finally: + server.shutdown() + assert result.exit_code == 0 and result.reply == "the answer" and keys == ["Bearer local"] + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +def test_the_installed_dsh_has_every_row_the_session_patches(tmp_path): + """A patch row whose id dsh does not have is ignored, and dsh says so only when asked (--dump-config, which + starts nothing and reaches no network): every row the session's layer names (telemetry, title, sub-agents, the + web tools, the route, the default model) is one of the installed dsh's, so none is silently left on.""" + home = tmp_path / "home" + home.mkdir() + agent = make("dsh", {**bound(), "effort": "high", "web_search": True}) # web fetch only: a row's settings + agent.prepare_home(home, (), "http://127.0.0.1:9/mcp", "hermit_tools") + layer = layer_rows((home / base_cli.DSH_PATCH).read_text()) + assert {r["id"] for r in layer if "id" in r} >= {*base_cli.DSH_OFF, "session-log-deepseek", "tool-web", + "llm-pi-ai", "agent-default-model", "skill-filesystem", + "agent-instructions", "session-persistence-jsonl"} + work = tmp_path / "work" + work.mkdir() + env = {**os.environ, **agent.home_env(home)} + done = subprocess.run(["dsh", "--profile", "headless", "--patch", str(home / base_cli.DSH_PATCH), + "--dump-config"], cwd=work, env=env, capture_output=True, text=True, timeout=120) + assert done.returncode == 0, done.stderr + assert "not found" not in done.stderr and "id: llm-pi-ai" in done.stdout + assert " config:\n search: false\n fetch: true\n" in done.stdout # web fetch, no search + assert " projectRootMarkers: []\n" in done.stdout + assert " root: !!js dshHomePath('sessions')\n compression: none\n" in done.stdout # dsh's own root, plain diff --git a/tests/core/test_isolate.py b/tests/core/test_isolate.py index c321cef1..f8a9f9d4 100644 --- a/tests/core/test_isolate.py +++ b/tests/core/test_isolate.py @@ -686,6 +686,10 @@ def test_a_cli_deploy_renders_the_adapter_in_a_sandbox(tmp_path, monkeypatch): assert not planted.exists() # the module could not read or write outside toml = (tmp_path / "ws" / ".codex" / "config.toml") assert "HA_ADAPTER_SANDBOX" in toml.read_text() # the tool server's Session is sandboxed too + from hermit.hosts.dsh import prepare as prepare_dsh # dsh's server takes it in its patch layer + prepare_dsh(adapter, tmp_path / "ws-dsh", sys.executable) + assert not planted.exists() + assert "HA_ADAPTER_SANDBOX" in (tmp_path / "ws-dsh" / ".dsh" / "hermit.patch.yml").read_text() def test_a_rollout_that_replaced_its_home_by_a_link_deletes_no_login_outside(tmp_path, monkeypatch, shared_login): diff --git a/tests/core/test_sandbox.py b/tests/core/test_sandbox.py index 3298a4c4..8fb696ac 100644 --- a/tests/core/test_sandbox.py +++ b/tests/core/test_sandbox.py @@ -362,7 +362,7 @@ def test_a_uv_alias_inside_a_granted_checkout_is_bound_once(tmp_path, monkeypatc @pytest.mark.integration # the CLI installed on the machine @pytest.mark.skipif(not _bwrap_works(), reason="needs a working bubblewrap (unprivileged user namespaces)") -@pytest.mark.parametrize("cli", ["codex", "claude", "pi"]) +@pytest.mark.parametrize("cli", ["codex", "claude", "pi", "dsh"]) def test_an_installed_cli_starts_with_only_its_own_paths(cli): """The built-in CLI agents: the CLI as installed (an nvm package, a ~/bin wrapper) starts inside, with no home.""" from hermit.sandbox import program_paths diff --git a/tests/core/test_selection.py b/tests/core/test_selection.py index 4a6d33aa..a3c6c505 100644 --- a/tests/core/test_selection.py +++ b/tests/core/test_selection.py @@ -8,7 +8,7 @@ from selection import METHODS, READ_BY, NoBase, changed_paths, groups_for, select ROOT = Path(__file__).resolve().parents[2] -BUILT_IN = {"codex", "claude", "pi"} # the generic AgentEvolvers: tests/core +BUILT_IN = {"codex", "claude", "pi", "dsh"} # the generic AgentEvolvers: tests/core NO_METHOD = {"prompt_only", "lean_collab"} # templates without an Evolver of their own: tests/core diff --git a/tests/fixtures/dsh/README.md b/tests/fixtures/dsh/README.md new file mode 100644 index 00000000..695d77cb --- /dev/null +++ b/tests/fixtures/dsh/README.md @@ -0,0 +1,54 @@ +# DeepSeek Harness streams + +Recorded from DeepSeek Harness 0.2.2-rc.1 (`npm install -g @deepseek-ai/dsh@0.2.2-rc.1`, on Node 24.22.2, Linux) run +by hermit's `dsh` Base Agent (`make("dsh").run`, isolation none), against a scripted chat-completions model on the +loopback, bound as a `/` reference with `effort: high`; the model streams a `reasoning_content` +delta before each answer when it is asked for a reasoning effort. + +- `success.jsonl`: a bash call that exits 3, a bash call that exits 0, a call of the call's MCP tool `echo` + (`hermit_tools`), a write, an edit, then the answer `the answer`; six requests of 1000 prompt tokens (400 cached) + and 20 output tokens each. +- `resume.jsonl`: the same session resumed (`--session-id`): one request. +- `failure.jsonl`: every request answered 503; dsh retries five times, ends the run with the error `SERVER`, writes + `dsh: SERVER: ...` on stderr (the last line, which the Base Agent keeps in the stream) and exits 1. +- `promoted.jsonl`: bash `echo building; sleep 5` and `sleep 5` with `timeoutMs: 1000`, and the first again with + `timeoutMs: 1000.5`, then the answer. The Base Agent's layer leaves the `tool-bash` row's `promoteOnTimeout` at + dsh's default, true, so at its timeout dsh moves each command to a background job: the result is the output so + far, then `[still running after ms; moved to background job ]` (the marker alone for `sleep 5`, + which printed nothing), with no exit marker. + +Recorded by running dsh directly (`dsh --profile headless --json --permission-mode danger-full-access`, with a patch +layer like the Base Agent's) against a scripted chat-completions model: + +- `bash.jsonl`: how dsh's bash tool marks a command's end. The model calls bash nine times: `exit 3`, `printf ''`, + `true`, `echo out; exit 4`, `kill -9 $$`, `printf 'no newline'; exit 5`, then with `timeoutMs: 1000` + `trap 'exit 0' TERM; sleep 5 & wait` (stopped by its timeout, it exits 0) and `sleep 5` (killed), and the first of + these again with `timeoutMs: 1000.5` (dsh prints the timeout it applied, here the model's), then answers `done`. + The layer also sets the `tool-bash` row's `promoteOnTimeout: false`, so a timeout stops the command rather than + moving it to a background job. +- `max_tokens.jsonl`: the model's answer ends at its token limit (`finish_reason: length`): the run ends `max-tokens` + and dsh exits 1, writing nothing on stderr. +- `big_bash.jsonl`: bash prints 1,500 lines (about 42 KB) and exits 2; dsh's `--json` keeps 8 KiB of the result and + marks the event `truncated`, so the exit marker is cut off. +- `background.jsonl`: bash with `run_in_background: true` returns `started background job `, no exit marker. +- `fake_marker.jsonl`: bash prints `built` and a line `[exit code: 7]` of its own, then exits 0. +- `no_usage.jsonl`: the model sends no usage; dsh reports the step's usage as zeros. +- `tools_down.jsonl`: the session's MCP row (`failOnStartupError: true`) points at a port nothing listens on: dsh + writes `warning: 1 entry did not activate` and the row's error on stderr, then completes the run and exits 0. +- `bad_option.jsonl`: an unknown option after `--json`: one `error` event, no session, exit 1. +- `dotenv.jsonl`: the working directory's `.env` sets `PYTHONPATH`: dsh stops before it writes an event (its + error on stderr) and exits 1. +- `stopped.jsonl`: the model calls bash `sleep 60`; two seconds after dsh writes the call, it gets SIGTERM (as a + session's timeout or the Trainer's stop sends it): no `final` event, an `error` event and its stderr line + (`flush on a closed handle`), exit 0. (With a task's data bound in the workspace, the binding's bubblewrap is what + the signal ends, and dsh with it: no event at all.) +- `context_window.jsonl`: the route declares no `contextWindow`, and the server refuses a request whose prompt and + output tokens pass its model's 32,768, with vLLM's error: dsh asks for 128,000 output tokens, then 65,536, and ends + the run `CONTEXT_WINDOW_EXCEEDED` (exit 1). With `contextWindow: 32768` declared it asked for 22,503 and completed. +- `retried.jsonl`: the server answers the first request 503, then the retry with 1000 prompt tokens (900 of them + cached) and 10 output tokens: dsh sums the two attempts' usage into the step's, and leaves out the cache count the + refused attempt did not report (its zeros), so the step reports `inputTokens` 100 and `totalTokens` 1010. + +The streams are dsh's stdout unchanged, followed by its stderr's lines where it wrote any, as the Base Agent keeps +them in one stream; the run's directory was `/run` (`/run/` for the streams recorded directly, `/tmp/rec3` for +`stopped.jsonl`, `/tmp/r4` for `context_window.jsonl`, `/tmp/r5` for `retried.jsonl`). diff --git a/tests/fixtures/dsh/background.jsonl b/tests/fixtures/dsh/background.jsonl new file mode 100644 index 00000000..65afd5d3 --- /dev/null +++ b/tests/fixtures/dsh/background.jsonl @@ -0,0 +1,11 @@ +{"type":"session","sessionId":"session-ad7761cf-4b99-4881-a5b7-d309a5d0ceb0","cwd":"/run/background/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"tool_call","callId":"call_0","tool":"bash","input":{"command":"sleep 2; echo done","run_in_background":true,"description":"edge"}} +{"type":"tool_result","callId":"call_0","status":"completed","result":"started background job bash-b055b99ea4af4d91"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":2} +{"type":"text","text":"done"} +{"type":"status","phase":"step_end","turn":1,"step":2,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-ad7761cf-4b99-4881-a5b7-d309a5d0ceb0","reason":{"kind":"completed"},"text":"done"} diff --git a/tests/fixtures/dsh/bad_option.jsonl b/tests/fixtures/dsh/bad_option.jsonl new file mode 100644 index 00000000..63172c20 --- /dev/null +++ b/tests/fixtures/dsh/bad_option.jsonl @@ -0,0 +1 @@ +{"type":"error","message":"unknown option '--no-such-option'"} diff --git a/tests/fixtures/dsh/bash.jsonl b/tests/fixtures/dsh/bash.jsonl new file mode 100644 index 00000000..a05f2d20 --- /dev/null +++ b/tests/fixtures/dsh/bash.jsonl @@ -0,0 +1,43 @@ +{"type":"session","sessionId":"session-9fc3acf0-ab99-4d1f-8770-d448a699fe26","cwd":"/run/bash/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"tool_call","callId":"call_0","tool":"bash","input":{"command":"exit 3","description":"edge"}} +{"type":"tool_result","callId":"call_0","status":"completed","result":"(no output)\n[exit code: 3]"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":2} +{"type":"tool_call","callId":"call_1","tool":"bash","input":{"command":"printf ''","description":"edge"}} +{"type":"tool_result","callId":"call_1","status":"completed","result":"(no output)"} +{"type":"status","phase":"step_end","turn":1,"step":2,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":3} +{"type":"tool_call","callId":"call_2","tool":"bash","input":{"command":"true","description":"edge"}} +{"type":"tool_result","callId":"call_2","status":"completed","result":"(no output)"} +{"type":"status","phase":"step_end","turn":1,"step":3,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":4} +{"type":"tool_call","callId":"call_3","tool":"bash","input":{"command":"echo out; exit 4","description":"edge"}} +{"type":"tool_result","callId":"call_3","status":"completed","result":"out\n[exit code: 4]"} +{"type":"status","phase":"step_end","turn":1,"step":4,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":5} +{"type":"tool_call","callId":"call_4","tool":"bash","input":{"command":"kill -9 $$","description":"edge"}} +{"type":"tool_result","callId":"call_4","status":"completed","result":"(no output)\n[killed by signal: SIGKILL]"} +{"type":"status","phase":"step_end","turn":1,"step":5,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":6} +{"type":"tool_call","callId":"call_5","tool":"bash","input":{"command":"printf 'no newline'; exit 5","description":"edge"}} +{"type":"tool_result","callId":"call_5","status":"completed","result":"no newline\n[exit code: 5]"} +{"type":"status","phase":"step_end","turn":1,"step":6,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":7} +{"type":"tool_call","callId":"call_6","tool":"bash","input":{"command":"trap 'exit 0' TERM; sleep 5 & wait","timeoutMs":1000,"description":"edge"}} +{"type":"tool_result","callId":"call_6","status":"completed","result":"(no output)\n[timed out after 1000ms]"} +{"type":"status","phase":"step_end","turn":1,"step":7,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":8} +{"type":"tool_call","callId":"call_7","tool":"bash","input":{"command":"sleep 5","timeoutMs":1000,"description":"edge"}} +{"type":"tool_result","callId":"call_7","status":"completed","result":"(no output)\n[timed out after 1000ms]\n[killed by signal: SIGTERM]"} +{"type":"status","phase":"step_end","turn":1,"step":8,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":9} +{"type":"tool_call","callId":"call_8","tool":"bash","input":{"command":"trap 'exit 0' TERM; sleep 5 & wait","timeoutMs":1000.5,"description":"edge"}} +{"type":"tool_result","callId":"call_8","status":"completed","result":"(no output)\n[timed out after 1000.5ms]"} +{"type":"status","phase":"step_end","turn":1,"step":9,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":10} +{"type":"text","text":"done"} +{"type":"status","phase":"step_end","turn":1,"step":10,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-9fc3acf0-ab99-4d1f-8770-d448a699fe26","reason":{"kind":"completed"},"text":"done"} diff --git a/tests/fixtures/dsh/big_bash.jsonl b/tests/fixtures/dsh/big_bash.jsonl new file mode 100644 index 00000000..b44c9894 --- /dev/null +++ b/tests/fixtures/dsh/big_bash.jsonl @@ -0,0 +1,11 @@ +{"type":"session","sessionId":"session-2055df01-dada-4a0a-a7ee-13ae0996ff9a","cwd":"/run/bigbash/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"tool_call","callId":"call_0","tool":"bash","input":{"command":"for i in $(seq 1 1500); do echo \"line $i of a long build log\"; done; exit 2","description":"edge"}} +{"type":"tool_result","callId":"call_0","status":"completed","result":"line 1 of a long build log\nline 2 of a long build log\nline 3 of a long build log\nline 4 of a long build log\nline 5 of a long build log\nline 6 of a long build log\nline 7 of a long build log\nline 8 of a long build log\nline 9 of a long build log\nline 10 of a long build log\nline 11 of a long build log\nline 12 of a long build log\nline 13 of a long build log\nline 14 of a long build log\nline 15 of a long build log\nline 16 of a long build log\nline 17 of a long build log\nline 18 of a long build log\nline 19 of a long build log\nline 20 of a long build log\nline 21 of a long build log\nline 22 of a long build log\nline 23 of a long build log\nline 24 of a long build log\nline 25 of a long build log\nline 26 of a long build log\nline 27 of a long build log\nline 28 of a long build log\nline 29 of a long build log\nline 30 of a long build log\nline 31 of a long build log\nline 32 of a long build log\nline 33 of a long build log\nline 34 of a long build log\nline 35 of a long build log\nline 36 of a long build log\nline 37 of a long build log\nline 38 of a long build log\nline 39 of a long build log\nline 40 of a long build log\nline 41 of a long build log\nline 42 of a long build log\nline 43 of a long build log\nline 44 of a long build log\nline 45 of a long build log\nline 46 of a long build log\nline 47 of a long build log\nline 48 of a long build log\nline 49 of a long build log\nline 50 of a long build log\nline 51 of a long build log\nline 52 of a long build log\nline 53 of a long build log\nline 54 of a long build log\nline 55 of a long build log\nline 56 of a long build log\nline 57 of a long build log\nline 58 of a long build log\nline 59 of a long build log\nline 60 of a long build log\nline 61 of a long build log\nline 62 of a long build log\nline 63 of a long build log\nline 64 of a long build log\nline 65 of a long build log\nline 66 of a long build log\nline 67 of a long build log\nline 68 of a long build log\nline 69 of a long build log\nline 70 of a long build log\nline 71 of a long build log\nline 72 of a long build log\nline 73 of a long build log\nline 74 of a long build log\nline 75 of a long build log\nline 76 of a long build log\nline 77 of a long build log\nline 78 of a long build log\nline 79 of a long build log\nline 80 of a long build log\nline 81 of a long build log\nline 82 of a long build log\nline 83 of a long build log\nline 84 of a long build log\nline 85 of a long build log\nline 86 of a long build log\nline 87 of a long build log\nline 88 of a long build log\nline 89 of a long build log\nline 90 of a long build log\nline 91 of a long build log\nline 92 of a long build log\nline 93 of a long build log\nline 94 of a long build log\nline 95 of a long build log\nline 96 of a long build log\nline 97 of a long build log\nline 98 of a long build log\nline 99 of a long build log\nline 100 of a long build log\nline 101 of a long build log\nline 102 of a long build log\nline 103 of a long build log\nline 104 of a long build log\nline 105 of a long build log\nline 106 of a long build log\nline 107 of a long build log\nline 108 of a long build log\nline 109 of a long build log\nline 110 of a long build log\nline 111 of a long build log\nline 112 of a long build log\nline 113 of a long build log\nline 114 of a long build log\nline 115 of a long build log\nline 116 of a long build log\nline 117 of a long build log\nline 118 of a long build log\nline 119 of a long build log\nline 120 of a long build log\nline 121 of a long build log\nline 122 of a long build log\nline 123 of a long build log\nline 124 of a long build log\nline 125 of a long build log\nline 126 of a long build log\nline 127 of a long build log\nline 128 of a long build log\nline 129 of a long build log\nline 130 of a long build log\nline 131 of a long build log\nline 132 of a long build log\nline 133 of a long build log\nline 134 of a long build log\nline 135 of a long build log\nline 136 of a long build log\nline 137 of a long build log\nline 138 of a long build log\nline 139 of a long build log\nline 140 of a long build log\nline 141 of a long build log\nline 142 of a long build log\nline 143 of a long build log\nline 144 of a long build log\nline 145 of a long build log\nline 146 of a long build log\nline 147 of a long build log\nline 148 of a long build log\nline 149 of a long build log\nline 150 of a long build log\nline 151 of a long build log\nline 152 of a long build log\nline 153 of a long build log\nline 154 of a long build log\nline 155 of a long build log\nline 156 of a long build log\nline 157 of a long build log\nline 158 of a long build log\nline 159 of a long build log\nline 160 of a long build log\nline 161 of a long build log\nline 162 of a long build log\nline 163 of a long build log\nline 164 of a long build log\nline 165 of a long build log\nline 166 of a long build log\nline 167 of a long build log\nline 168 of a long build log\nline 169 of a long build log\nline 170 of a long build log\nline 171 of a long build log\nline 172 of a long build log\nline 173 of a long build log\nline 174 of a long build log\nline 175 of a long build log\nline 176 of a long build log\nline 177 of a long build log\nline 178 of a long build log\nline 179 of a long build log\nline 180 of a long build log\nline 181 of a long build log\nline 182 of a long build log\nline 183 of a long build log\nline 184 of a long build log\nline 185 of a long build log\nline 186 of a long build log\nline 187 of a long build log\nline 188 of a long build log\nline 189 of a long build log\nline 190 of a long build log\nline 191 of a long build log\nline 192 of a long build log\nline 193 of a long build log\nline 194 of a long build log\nline 195 of a long build log\nline 196 of a long build log\nline 197 of a long build log\nline 198 of a long build log\nline 199 of a long build log\nline 200 of a long build log\nline 201 of a long build log\nline 202 of a long build log\nline 203 of a long build log\nline 204 of a long build log\nline 205 of a long build log\nline 206 of a long build log\nline 207 of a long build log\nline 208 of a long build log\nline 209 of a long build log\nline 210 of a long build log\nline 211 of a long build log\nline 212 of a long build log\nline 213 of a long build log\nline 214 of a long build log\nline 215 of a long build log\nline 216 of a long build log\nline 217 of a long build log\nline 218 of a long build log\nline 219 of a long build log\nline 220 of a long build log\nline 221 of a long build log\nline 222 of a long build log\nline 223 of a long build log\nline 224 of a long build log\nline 225 of a long build log\nline 226 of a long build log\nline 227 of a long build log\nline 228 of a long build log\nline 229 of a long build log\nline 230 of a long build log\nline 231 of a long build log\nline 232 of a long build log\nline 233 of a long build log\nline 234 of a long build log\nline 235 of a long build log\nline 236 of a long build log\nline 237 of a long build log\nline 238 of a long build log\nline 239 of a long build log\nline 240 of a long build log\nline 241 of a long build log\nline 242 of a long build log\nline 243 of a long build log\nline 244 of a long build log\nline 245 of a long build log\nline 246 of a long build log\nline 247 of a long build log\nline 248 of a long build log\nline 249 of a long build log\nline 250 of a long build log\nline 251 of a long build log\nline 252 of a long build log\nline 253 of a long build log\nline 254 of a long build log\nline 255 of a long build log\nline 256 of a long build log\nline 257 of a long build log\nline 258 of a long build log\nline 259 of a long build log\nline 260 of a long build log\nline 261 of a long build log\nline 262 of a long build log\nline 263 of a long build log\nline 264 of a long build log\nline 265 of a long build log\nline 266 of a long build log\nline 267 of a long build log\nline 268 of a long build log\nline 269 of a long build log\nline 270 of a long build log\nline 271 of a long build log\nline 272 of a long build log\nline 273 of a long build log\nline 274 of a long build log\nline 275 of a long build log\nline 276 of a long build log\nline 277 of a long build log\nline 278 of a long build log\nline 279 of a long build log\nline 280 of a long build log\nline 281 of a long build log\nline 282 of a long build log\nline 283 of a long build log\nline 284 of a long build log\nline 285 of a long build log\nline 286 of a long build log\nline 2","truncated":true} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":2} +{"type":"text","text":"done"} +{"type":"status","phase":"step_end","turn":1,"step":2,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-2055df01-dada-4a0a-a7ee-13ae0996ff9a","reason":{"kind":"completed"},"text":"done"} diff --git a/tests/fixtures/dsh/context_window.jsonl b/tests/fixtures/dsh/context_window.jsonl new file mode 100644 index 00000000..f86c7987 --- /dev/null +++ b/tests/fixtures/dsh/context_window.jsonl @@ -0,0 +1,7 @@ +{"type":"session","sessionId":"session-483b5af4-c1a2-46d9-b2d7-68a9b00a6a7a","cwd":"/tmp/r4/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":0,"outputTokens":0,"totalTokens":0}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"error","error":{"message":"400: {\"object\":\"error\",\"message\":\"This model's maximum context length is 32768 tokens. However, you requested 128743 tokens (743 in the messages, 128000 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}","code":"CONTEXT_WINDOW_EXCEEDED"}}} +{"type":"final","sessionId":"session-483b5af4-c1a2-46d9-b2d7-68a9b00a6a7a","reason":{"kind":"error","error":{"message":"400: {\"object\":\"error\",\"message\":\"This model's maximum context length is 32768 tokens. However, you requested 128743 tokens (743 in the messages, 128000 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}","code":"CONTEXT_WINDOW_EXCEEDED"}},"text":""} +dsh: CONTEXT_WINDOW_EXCEEDED: 400: {"object":"error","message":"This model's maximum context length is 32768 tokens. However, you requested 128743 tokens (743 in the messages, 128000 in the completion). Please reduce the length of the messages or completion.","type":"BadRequestError","param":null,"code":400} diff --git a/tests/fixtures/dsh/dotenv.jsonl b/tests/fixtures/dsh/dotenv.jsonl new file mode 100644 index 00000000..e810e492 --- /dev/null +++ b/tests/fixtures/dsh/dotenv.jsonl @@ -0,0 +1,12 @@ +file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/dsh-app-boot/lib/index.js:4468 + throw new Error(`${binName}: ${path} sets "${name}", which only the launching environment may set (it decides how this process starts, where its code and instructions load from, or how it reaches the network); ${remedy}`); + ^ + +Error: dsh: /run/dotenv/work/.env sets "PYTHONPATH", which only the launching environment may set (it decides how this process starts, where its code and instructions load from, or how it reaches the network); export PYTHONPATH instead of putting it in a .env file + at readEnvLayer (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/dsh-app-boot/lib/index.js:4468:9) + at loadLayeredEnv (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/dsh-app-boot/lib/index.js:4489:18) + at runCli (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/lib/bin.js:216:19) + at process.processTicksAndRejections (node:internal/process/task_queues:104:5) + at async file:///opt/node/lib/node_modules/@deepseek-ai/dsh/lib/bin.js:252:23 + +Node.js v24.22.2 diff --git a/tests/fixtures/dsh/failure.jsonl b/tests/fixtures/dsh/failure.jsonl new file mode 100644 index 00000000..61c0dc84 --- /dev/null +++ b/tests/fixtures/dsh/failure.jsonl @@ -0,0 +1,7 @@ +{"type":"session","sessionId":"session-23637470-e079-42ef-bbc7-7a40042ccad7","cwd":"/run/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":0,"outputTokens":0,"totalTokens":0}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"error","error":{"message":"503: {\"message\":\"the service is unavailable\"}","code":"SERVER"}}} +{"type":"final","sessionId":"session-23637470-e079-42ef-bbc7-7a40042ccad7","reason":{"kind":"error","error":{"message":"503: {\"message\":\"the service is unavailable\"}","code":"SERVER"}},"text":""} +dsh: SERVER: 503: {"message":"the service is unavailable"} diff --git a/tests/fixtures/dsh/fake_marker.jsonl b/tests/fixtures/dsh/fake_marker.jsonl new file mode 100644 index 00000000..6ec9470a --- /dev/null +++ b/tests/fixtures/dsh/fake_marker.jsonl @@ -0,0 +1,11 @@ +{"type":"session","sessionId":"session-20cfb756-ebba-4348-a685-deb4e93f7d22","cwd":"/run/fakemarker/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"tool_call","callId":"call_0","tool":"bash","input":{"command":"printf 'built\\n[exit code: 7]\\n'","description":"edge"}} +{"type":"tool_result","callId":"call_0","status":"completed","result":"built\n[exit code: 7]\n"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":2} +{"type":"text","text":"done"} +{"type":"status","phase":"step_end","turn":1,"step":2,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-20cfb756-ebba-4348-a685-deb4e93f7d22","reason":{"kind":"completed"},"text":"done"} diff --git a/tests/fixtures/dsh/max_tokens.jsonl b/tests/fixtures/dsh/max_tokens.jsonl new file mode 100644 index 00000000..d09e0102 --- /dev/null +++ b/tests/fixtures/dsh/max_tokens.jsonl @@ -0,0 +1,7 @@ +{"type":"session","sessionId":"session-5411e0f3-4792-4c94-84b0-bbd018ccbf6b","cwd":"/run/max_tokens/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"text","text":"partial answ"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"max-tokens"}} +{"type":"final","sessionId":"session-5411e0f3-4792-4c94-84b0-bbd018ccbf6b","reason":{"kind":"max-tokens"},"text":"partial answ"} diff --git a/tests/fixtures/dsh/no_usage.jsonl b/tests/fixtures/dsh/no_usage.jsonl new file mode 100644 index 00000000..3f27e935 --- /dev/null +++ b/tests/fixtures/dsh/no_usage.jsonl @@ -0,0 +1,7 @@ +{"type":"session","sessionId":"session-4340185a-604a-41cc-b6f0-460879480637","cwd":"/run/nousage/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"text","text":"done"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":0,"outputTokens":0,"totalTokens":0}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-4340185a-604a-41cc-b6f0-460879480637","reason":{"kind":"completed"},"text":"done"} diff --git a/tests/fixtures/dsh/promoted.jsonl b/tests/fixtures/dsh/promoted.jsonl new file mode 100644 index 00000000..c5a0cb3e --- /dev/null +++ b/tests/fixtures/dsh/promoted.jsonl @@ -0,0 +1,23 @@ +{"type":"session","sessionId":"session-89dbecd8-5170-42c8-85d6-b721d96c3364","cwd":"/run/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"thinking","text":"Thinking it over."} +{"type":"tool_call","callId":"call_1","tool":"bash","input":{"command":"echo building; sleep 5","description":"Build","timeoutMs":1000}} +{"type":"tool_result","callId":"call_1","status":"completed","result":"building\n[still running after 1000ms; moved to background job bash-215d1b678a6b4016]"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"step_start","turn":1,"step":2} +{"type":"thinking","text":"Thinking it over."} +{"type":"tool_call","callId":"call_2","tool":"bash","input":{"command":"sleep 5","description":"Wait","timeoutMs":1000}} +{"type":"tool_result","callId":"call_2","status":"completed","result":"[still running after 1000ms; moved to background job bash-05097e6c1e3f423d]"} +{"type":"status","phase":"step_end","turn":1,"step":2,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"step_start","turn":1,"step":3} +{"type":"thinking","text":"Thinking it over."} +{"type":"tool_call","callId":"call_3","tool":"bash","input":{"command":"echo building; sleep 5","description":"Build","timeoutMs":1000.5}} +{"type":"tool_result","callId":"call_3","status":"completed","result":"building\n[still running after 1000.5ms; moved to background job bash-edd1522dc3ff48cf]"} +{"type":"status","phase":"step_end","turn":1,"step":3,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"step_start","turn":1,"step":4} +{"type":"thinking","text":"Thinking it over."} +{"type":"text","text":"the answer"} +{"type":"status","phase":"step_end","turn":1,"step":4,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-89dbecd8-5170-42c8-85d6-b721d96c3364","reason":{"kind":"completed"},"text":"the answer"} diff --git a/tests/fixtures/dsh/resume.jsonl b/tests/fixtures/dsh/resume.jsonl new file mode 100644 index 00000000..ec52c0ad --- /dev/null +++ b/tests/fixtures/dsh/resume.jsonl @@ -0,0 +1,8 @@ +{"type":"session","sessionId":"session-b2fbf9bf-a321-4065-8f8a-ee344b66b5f3","cwd":"/run/work"} +{"type":"status","phase":"turn_start","turn":2} +{"type":"status","phase":"step_start","turn":2,"step":1} +{"type":"thinking","text":"Thinking it over."} +{"type":"text","text":"the answer"} +{"type":"status","phase":"step_end","turn":2,"step":1,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"turn_end","turn":2,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-b2fbf9bf-a321-4065-8f8a-ee344b66b5f3","reason":{"kind":"completed"},"text":"the answer"} diff --git a/tests/fixtures/dsh/retried.jsonl b/tests/fixtures/dsh/retried.jsonl new file mode 100644 index 00000000..0a15113f --- /dev/null +++ b/tests/fixtures/dsh/retried.jsonl @@ -0,0 +1,7 @@ +{"type":"session","sessionId":"session-4a46d7a7-7e83-4f43-b5c9-ab2c48c086b8","cwd":"/tmp/r5/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"text","text":"the answer"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":100,"outputTokens":10,"totalTokens":1010}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-4a46d7a7-7e83-4f43-b5c9-ab2c48c086b8","reason":{"kind":"completed"},"text":"the answer"} diff --git a/tests/fixtures/dsh/stopped.jsonl b/tests/fixtures/dsh/stopped.jsonl new file mode 100644 index 00000000..72d5dd12 --- /dev/null +++ b/tests/fixtures/dsh/stopped.jsonl @@ -0,0 +1,6 @@ +{"type":"session","sessionId":"session-acf190ca-aea6-48ce-b85f-181115db4d61","cwd":"/tmp/rec3/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"tool_call","callId":"call_0","tool":"bash","input":{"command":"sleep 60","description":"Wait"}} +{"type":"error","message":"session \"session-acf190ca-aea6-48ce-b85f-181115db4d61\": flush on a closed handle"} +dsh: session "session-acf190ca-aea6-48ce-b85f-181115db4d61": flush on a closed handle diff --git a/tests/fixtures/dsh/success.jsonl b/tests/fixtures/dsh/success.jsonl new file mode 100644 index 00000000..617182b3 --- /dev/null +++ b/tests/fixtures/dsh/success.jsonl @@ -0,0 +1,33 @@ +{"type":"session","sessionId":"session-b2fbf9bf-a321-4065-8f8a-ee344b66b5f3","cwd":"/run/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"thinking","text":"Thinking it over."} +{"type":"tool_call","callId":"call_1","tool":"bash","input":{"command":"echo hi && exit 3","description":"Print hi and fail"}} +{"type":"tool_result","callId":"call_1","status":"completed","result":"hi\n[exit code: 3]"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"step_start","turn":1,"step":2} +{"type":"thinking","text":"Thinking it over."} +{"type":"tool_call","callId":"call_2","tool":"bash","input":{"command":"echo ok","description":"Print ok"}} +{"type":"tool_result","callId":"call_2","status":"completed","result":"ok\n"} +{"type":"status","phase":"step_end","turn":1,"step":2,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"step_start","turn":1,"step":3} +{"type":"thinking","text":"Thinking it over."} +{"type":"tool_call","callId":"call_3","tool":"mcp__hermit_tools__echo","input":{"text":"from dsh"}} +{"type":"tool_result","callId":"call_3","status":"completed","result":"{\"echoed\": \"from dsh\"}"} +{"type":"status","phase":"step_end","turn":1,"step":3,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"step_start","turn":1,"step":4} +{"type":"thinking","text":"Thinking it over."} +{"type":"tool_call","callId":"call_4","tool":"write","input":{"file_path":"note.txt","content":"ok\n"}} +{"type":"tool_result","callId":"call_4","status":"completed","result":"/run/work/note.txt\nfile\n\nCreated file\n"} +{"type":"status","phase":"step_end","turn":1,"step":4,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"step_start","turn":1,"step":5} +{"type":"thinking","text":"Thinking it over."} +{"type":"tool_call","callId":"call_5","tool":"edit","input":{"file_path":"note.txt","old_string":"ok","new_string":"fine"}} +{"type":"tool_result","callId":"call_5","status":"completed","result":"The file /run/work/note.txt has been updated successfully."} +{"type":"status","phase":"step_end","turn":1,"step":5,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"step_start","turn":1,"step":6} +{"type":"thinking","text":"Thinking it over."} +{"type":"text","text":"the answer"} +{"type":"status","phase":"step_end","turn":1,"step":6,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-b2fbf9bf-a321-4065-8f8a-ee344b66b5f3","reason":{"kind":"completed"},"text":"the answer"} diff --git a/tests/fixtures/dsh/tools_down.jsonl b/tests/fixtures/dsh/tools_down.jsonl new file mode 100644 index 00000000..99239792 --- /dev/null +++ b/tests/fixtures/dsh/tools_down.jsonl @@ -0,0 +1,30 @@ +{"type":"session","sessionId":"session-4b51d3c1-98de-46aa-aeed-8f59862527de","cwd":"/run/toolsdown/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"text","text":"done"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-4b51d3c1-98de-46aa-aeed-8f59862527de","reason":{"kind":"completed"},"text":"done"} +dsh: warning: 1 entry did not activate +mcp-hermit_tools (@deepseek-ai/dsh-mcp-client): Error: mcp-client(hermit_tools): initial connection or tool synchronization failed + at Object.apply [as callback] (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/dsh-mcp-client/lib/index.js:833:67) + at async Fiber._reload (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/cordis/lib/index.js:677:41) +SdkError: Version negotiation probe failed: fetch failed + at classifyNetworkError (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@modelcontextprotocol/client/dist/index.mjs:2508:10) + at classifyProbeOutcome (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@modelcontextprotocol/client/dist/index.mjs:2417:32) + at probe (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@modelcontextprotocol/client/dist/index.mjs:2815:20) + at async negotiateEra (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@modelcontextprotocol/client/dist/index.mjs:2846:12) + at async Client._connectNegotiated (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@modelcontextprotocol/client/dist/index.mjs:3386:38) + at async connectGeneration (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/dsh-mcp-client/lib/index.js:644:4) + at async Object.apply [as callback] (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/dsh-mcp-client/lib/index.js:832:18) + at async Fiber._reload (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/cordis/lib/index.js:677:41) + at async Fiber.await (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/cordis/lib/index.js:728:24) +TypeError: fetch failed +Error: bad port + at makeNetworkError (node:internal/deps/undici/undici:11898:35) + at mainFetch (node:internal/deps/undici/undici:13280:22) + at fetching (node:internal/deps/undici/undici:13267:7) + at fetch (node:internal/deps/undici/undici:13131:20) + at fetch (node:internal/deps/undici/undici:17998:10) + at fetch (node:internal/bootstrap/web/exposed-window-or-worker:83:12) + at StreamableHTTPClientTransport._send (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@modelcontextprotocol/client/dist/index.mjs:5684:49) diff --git a/tests/methods/ahe/test_ahe_hosts.py b/tests/methods/ahe/test_ahe_hosts.py index 441b1805..3c4ab97a 100644 --- a/tests/methods/ahe/test_ahe_hosts.py +++ b/tests/methods/ahe/test_ahe_hosts.py @@ -4,9 +4,9 @@ ones' shape, the runner takes the host and the workspace from the deployment (so a Claude Code `cd` moves neither the logs nor the task), runs before_model where Claude Code has no event for it and hands its context over when Codex's would be, keeps a sub-agent's queue apart, the capability table EVOLVER.md shows is the module's own, both deployments -install the hooks that the template check runs, a Pi agent running a hook by hand gets the memory, and the Evolver's -texts name the acting agent's event stream. With the CLIs installed and --integration, both hosts run on a scripted -local model (no model, no Docker, no network).""" +install the hooks that the template check runs, a Pi or dsh agent running a hook by hand gets the memory, and the +Evolver's texts name the acting agent's event stream. With the CLIs installed and --integration, both hosts run on a +scripted local model (no model, no Docker, no network).""" import importlib.util import json import os @@ -57,7 +57,7 @@ def fixtures(host): def test_the_hooks_file_registers_every_event_of_every_host_and_nothing_else(): hooks = json.loads((TEMPLATE / "hooks.json").read_text())["hooks"] assert set(hooks) == set(CODEX.events) | set(CLAUDE.events) == set(wires.PI.events) - assert set(wires.HOSTS) == {"codex", "claude-code", "pi"} and wires.HOOK_HOSTS == ("codex", "claude-code") + assert set(wires.HOSTS) == {"codex", "claude-code", "pi", "dsh"} and wires.HOOK_HOSTS == ("codex", "claude-code") for wire in wires.HOSTS.values(): assert set(wire.flush) <= set(wire.events) and set(wire.specific) <= set(wire.events) assert {f for f in wire.events.values() if f} <= set(runner.hooks.MIDDLEWARE_EVENTS) @@ -484,13 +484,16 @@ def test_text_that_cannot_be_encoded_is_escaped_not_a_crash(tmp_path, capsys): assert _audit(work)[-1]["error"] is None -def test_on_pi_a_hook_run_by_hand_hands_the_memory_over(tmp_path): - """Pi runs no hooks: the deployment tells the agent to run each hook command itself, without a payload. The - SessionStart one prints the memory; the others have nothing to act on and print nothing.""" - from hermit.hosts.pi import prepare +@pytest.mark.parametrize("host", ["pi", "dsh"]) +def test_on_a_host_without_hooks_a_hook_run_by_hand_hands_the_memory_over(tmp_path, host): + """Pi and DeepSeek Harness run no hooks: the deployment tells the agent to run each hook command itself, without a + payload. The SessionStart one prints the memory; the others have nothing to act on and print nothing. (A host + the template has no wire for printed nothing at all, so the memory never reached the session.)""" + import importlib + prepare = importlib.import_module(f"hermit.hosts.{host}").prepare adapter = seed(tmp_path) (adapter / "MEMORY.md").write_text(T.MEMORY_WITH_ENTRIES) - work = tmp_path / "work-pi" + work = tmp_path / f"work-{host}" result = prepare(adapter, work, sys.executable) agents_md = (work / "AGENTS.md").read_text() line = next(l for l in agents_md.splitlines() if "On `SessionStart`" in l) @@ -502,7 +505,7 @@ def test_on_pi_a_hook_run_by_hand_hands_the_memory_over(tmp_path): stop = subprocess.run(["sh", "-c", command.replace("SessionStart", "Stop")], cwd=work, input="", capture_output=True, text=True) assert stop.returncode == 0 and stop.stdout == "" - assert json.loads((work / ".hermit-deploy.json").read_text())["host"] == "pi" and result["host"] == "pi" + assert json.loads((work / ".hermit-deploy.json").read_text())["host"] == host and result["host"] == host assert [l["event"] for l in _audit(work)] == ["SessionStart", "Stop"] and not any(l["error"] for l in _audit(work)) @@ -701,6 +704,21 @@ def test_the_evolvers_texts_name_the_acting_agents_stream(tmp_path): assert "`claude -p --output-format stream-json` event stream" in question and "codex exec --json" not in question +def test_the_evolvers_texts_name_a_dsh_acting_agents_stream(): + """A DeepSeek Harness acting agent's sessions keep their own stream (agent-dsh.jsonl), which rollout_stream finds: + the learners' texts name it, not Codex's file, which is not there.""" + from hermit.agent.base.trajectory import DSH_STREAM + from hermit.agent.evolver.methods.ahe_adapter import debugger as D + methods = T.ROOT / "hermit" / "agent" / "evolver" / "methods" / "ahe_adapter" + texts = [(methods / name).read_text() for name in ("START.md", "DEBUGGER.md")] + for text in texts: + words = D.for_stream(text, DSH_STREAM) + assert "agent-codex.jsonl" not in words and "Codex event stream" not in words + assert "codex exec --json" not in words + assert DSH_STREAM in D.for_stream(texts[0], DSH_STREAM) + assert "`dsh --profile headless --json` event stream" in D.for_stream(texts[1], DSH_STREAM) + + # --- the real CLIs, each on a scripted local model ---------------------------------------------------------------- # A session of each host in a workspace the framework deployed, its model a script served on 127.0.0.1 (every proxy # pointed at a closed port): which text reaches a request is what each channel delivers. Skipped without the CLI. diff --git a/tests/methods/mwm/test_mwm_evolver.py b/tests/methods/mwm/test_mwm_evolver.py index 687f028b..670c39de 100644 --- a/tests/methods/mwm/test_mwm_evolver.py +++ b/tests/methods/mwm/test_mwm_evolver.py @@ -148,6 +148,16 @@ def make(tmp: Path, monkeypatch, steps: int = 2, deployment_param: bool = False) return path +def test_a_dsh_learner_is_refused_at_setup(): + """mwm's session is Codex's or Claude Code's (its texts, homes and sign-ins): DeepSeek Harness, which the config + takes as evolver.base, is refused before any step rather than run on dsh's own provider instead of its bound + model (mwm's session settings carry no `provider`).""" + from types import SimpleNamespace + from hermit.agent.evolver.methods.mwm.evolver import MwmEvolver + with pytest.raises(ValueError, match="backend is codex or claude, not 'dsh'"): + MwmEvolver().setup(SimpleNamespace(params={"backend": "dsh"})) + + def test_bootstrap_review_mounts_and_records(tmp_path, monkeypatch): cfg = Config.load(make(tmp_path, monkeypatch), []) stale = tmp_path / "taskdir" / "evidence" / "block000" # a record already in the store: hidden from the bootstrap diff --git a/tests/methods/rrsi_adapter/test_rrsi_adapter.py b/tests/methods/rrsi_adapter/test_rrsi_adapter.py index 787b6782..eed28427 100644 --- a/tests/methods/rrsi_adapter/test_rrsi_adapter.py +++ b/tests/methods/rrsi_adapter/test_rrsi_adapter.py @@ -818,6 +818,8 @@ def test_setup_checks_what_every_step_needs(tmp_path): ev.setup(ctx) with pytest.raises(ValueError, match="web_search"): ev.setup(context(ws, 1, dom, tmp_path, web_search=None)) + with pytest.raises(ValueError, match="backend is codex or claude, not 'dsh'"): # its roles' settings are not dsh's + ev.setup(context(ws, 1, dom, tmp_path, backend="dsh")) with pytest.raises(Exception): ev.setup(context(ws, 1, str(tmp_path / "dom" / "missing.py") + ":ToyDomain", tmp_path)) with pytest.raises(ValueError, match="roles.model"): diff --git a/tests/selection.py b/tests/selection.py index 26a578b8..66f96f36 100644 --- a/tests/selection.py +++ b/tests/selection.py @@ -67,7 +67,7 @@ def fnmatch(path: str, pattern: str) -> bool: "tests/claude_stand.py", "tests/fixtures/claude_streams/*", # the Claude stand-in: core's and the methods' ".github/workflows/tests.yml"] # how CI runs them: a change to it runs them all # the rest of the framework, the templates no method group owns, the documentation's examples: the core group -CORE = ["hermit/*", "tests/core/*", "tests/fixtures/pi/*", "docs/_examples/*"] +CORE = ["hermit/*", "tests/core/*", "tests/fixtures/pi/*", "tests/fixtures/dsh/*", "docs/_examples/*"] # nothing to run: documentation, and files no test reads NONE = ["docs/*", "*.md", "tests/README.md", "LICENSE", "NOTICE", ".github/*", ".agents/*", "assets/*", ".gitignore"]