From 03fafff831d155d6abc246660e2f3e01eb72375f Mon Sep 17 00:00:00 2001 From: Albert Zhao Date: Sun, 8 Nov 2026 20:23:18 -0500 Subject: [PATCH 01/12] Add DeepSeek Harness (dsh) as a CLI Base Agent (acting agent and dsh Evolver) DeepSeek Harness (`dsh`, npm @deepseek-ai/dsh 0.2.2-rc.1) beside codex, claude and pi, in the same three layers as Pi (#89), on the runner's outcome() of #77. - Layer 1: a / reference becomes a route of dsh's pi-ai adapter (Binding.dsh_provider): the backend's URL in the first dialect it serves, the key by its variable's name (apiKeyEnv), the context window. dsh reports no cost: the backend's price applies. - Layer 2: Dsh(CliAgent) runs `dsh --profile headless --json` in a private DSH_HOME, HOME and DSH_AGENTS_HOME, telemetry off, the caller's own DSH_* variables dropped. The session's settings are a patch layer of dsh's own rows (--patch): the bound model as the default model (effort as its reasoningEffort, image_input as Pi's), the call's tools as an MCP server offered directly (1800 s per call), web search and fetch off unless asked for, no title request, session log, plugin list, sub-agents or workflows. A server that needs no key gets the placeholder key through HERMIT_DSH_NO_KEY (a route that names none is refused by dsh). Approvals are never asked (--approval-policy never); permission_mode picks dsh's own sandbox. Resume by --session-id when the home keeps the session; the workspace's .dsh/skills are also the session's own (dsh reads project skills only at the nearest folder with a .git). outcome(): a run without a completed `final` has failed, whatever dsh's exit code (SIGTERM exits 0). Usage from step_end events; the trajectory from the stream, a command's exit code from dsh's end markers (a timeout or a stop: unknown; "(no output)": empty); every unfinished end is an error step. dsh's outage codes (RATE_LIMIT, SERVER, TIMEOUT, TRANSPORT, EMPTY_RESPONSE, STREAM_CLOSED) join UNAVAILABLE. Roles, a login and container isolation are refused. - Layer 3: hermit/hosts/dsh.py (`hermit deploy --host dsh`): AGENTS.md, .dsh/skills and the adapter's tool server (120 s per call) as an insert row of .dsh/hermit.patch.yml, which the acting agent passes with --patch. The AgentEvolver, the loader and the config accept dsh; methods/dsh/ is the new Evolver (dsh's own sandbox off in the evolver sandbox, as Codex's is). - Tests: tests/core/test_dsh_agent.py over streams recorded from dsh 0.2.2-rc.1 (tests/fixtures/dsh) and, marked integration, the installed dsh: a session unsandboxed and in bubblewrap, the acting agent inside a git repository (the deployed skill offered, the adapter's own tool server answering), a keyless server, and --dump-config holding every row the session patches; the deploy host; selection entries. Checked against dsh's source at the tag dsh-v0.2.2-rc.1 and with the installed dsh. First built on #89's earlier head; rebuilt on main after #89 merged. Co-Authored-By: Claude Opus 5.5 --- hermit/agent/acting/agent.py | 8 +- hermit/agent/acting/workspace.py | 21 +- hermit/agent/base/cli.py | 229 ++++++- hermit/agent/base/login.py | 6 +- hermit/agent/base/records.py | 25 + hermit/agent/base/trajectory.py | 81 ++- hermit/agent/evolver/agent.py | 19 +- hermit/agent/evolver/methods/dsh/RESUME.md | 4 + hermit/agent/evolver/methods/dsh/START.md | 15 + hermit/agent/evolver/methods/dsh/evolver.yaml | 13 + hermit/agent/model/backends.py | 48 +- hermit/cli.py | 7 +- hermit/config.py | 22 +- hermit/defaults.yaml | 10 +- hermit/hosts/dsh.py | 95 +++ hermit/isolate.py | 4 +- tests/README.md | 2 +- tests/core/test_deploy.py | 28 +- tests/core/test_dsh_agent.py | 568 ++++++++++++++++++ tests/core/test_sandbox.py | 2 +- tests/core/test_selection.py | 2 +- tests/fixtures/dsh/README.md | 27 + tests/fixtures/dsh/bash.jsonl | 39 ++ tests/fixtures/dsh/failure.jsonl | 7 + tests/fixtures/dsh/max_tokens.jsonl | 7 + tests/fixtures/dsh/resume.jsonl | 8 + tests/fixtures/dsh/success.jsonl | 33 + tests/selection.py | 2 +- 28 files changed, 1265 insertions(+), 67 deletions(-) create mode 100644 hermit/agent/evolver/methods/dsh/RESUME.md create mode 100644 hermit/agent/evolver/methods/dsh/START.md create mode 100644 hermit/agent/evolver/methods/dsh/evolver.yaml create mode 100644 hermit/hosts/dsh.py create mode 100644 tests/core/test_dsh_agent.py create mode 100644 tests/fixtures/dsh/README.md create mode 100644 tests/fixtures/dsh/bash.jsonl create mode 100644 tests/fixtures/dsh/failure.jsonl create mode 100644 tests/fixtures/dsh/max_tokens.jsonl create mode 100644 tests/fixtures/dsh/resume.jsonl create mode 100644 tests/fixtures/dsh/success.jsonl diff --git a/hermit/agent/acting/agent.py b/hermit/agent/acting/agent.py index 8cafd23b..fb66edf0 100644 --- a/hermit/agent/acting/agent.py +++ b/hermit/agent/acting/agent.py @@ -7,7 +7,7 @@ def run(self, task: dict, adapter: Path | None) -> RunOutput A config names the agent: agent: - name: react # a built-in: react | codex | claude-code | pi + name: react # a built-in: react | codex | claude-code | pi | dsh name: agent/agent.py:CLBenchAgent # or the task's own class (file.py:Class, module:Class) name: agent/pipeline.py:run # or a plain function run(task, adapter), wrapped as a FunctionAgent concurrency: 4 @@ -75,9 +75,9 @@ def builtin(name: str): if name == "react": from hermit.agent.acting.react import ReactAgent return ReactAgent - if name in ("cli", "codex", "claude-code", "claude", "pi"): + if name in ("cli", "codex", "claude-code", "claude", "pi", "dsh"): from hermit.agent.acting.workspace import CliAgent - backend = {"codex": "codex", "claude-code": "claude", "claude": "claude", "pi": "pi"}.get(name) + backend = {"codex": "codex", "claude-code": "claude", "claude": "claude", "pi": "pi", "dsh": "dsh"}.get(name) if backend is None: return CliAgent return lambda params, base: CliAgent({"backend": backend, **(params or {})}, base) @@ -87,7 +87,7 @@ def builtin(name: str): return None -BUILTIN_NAMES = ("react", "codex", "claude-code", "pi", "cli", "harbor") +BUILTIN_NAMES = ("react", "codex", "claude-code", "pi", "dsh", "cli", "harbor") def load_agent(name: str, params: dict | None, base: Path) -> Agent: diff --git a/hermit/agent/acting/workspace.py b/hermit/agent/acting/workspace.py index f280c344..9efbf48e 100644 --- a/hermit/agent/acting/workspace.py +++ b/hermit/agent/acting/workspace.py @@ -1,5 +1,5 @@ -"""The `codex`, `claude-code` and `pi` agents: one task run by the agent CLI in a workspace the adapter is deployed -into. They are the framework's one way to act with a coding-agent CLI; a task adds only what is its own. +"""The `codex`, `claude-code`, `pi` and `dsh` agents: one task run by the agent CLI in a workspace the adapter is +deployed into. They are the framework's one way to act with a coding-agent CLI; a task adds only what is its own. agent: name: codex # or claude-code; `cli` with params.backend is the same @@ -69,7 +69,7 @@ from hermit.agent.base.login import LOGINS from hermit.task import RunOutput, load_callable -BACKENDS = ("codex", "claude", "pi") +BACKENDS = ("codex", "claude", "pi", "dsh") DEPLOYS = ("host",) ISOLATIONS = ("none", "container") INPUT_FIELDS = ("question", "prompt", "instruction", "input", "query") @@ -144,6 +144,9 @@ def deploy(adapter: Path, workspace: Path, backend: str, *, context: str | None if backend == "pi": from hermit.hosts.pi import prepare as prepare_pi return prepare_pi(adapter, workspace, sys.executable, context=context) + if backend == "dsh": + from hermit.hosts.dsh import prepare as prepare_dsh + return prepare_dsh(adapter, workspace, sys.executable, context=context) from hermit.hosts.claude_code import prepare return prepare(adapter, workspace, sys.executable, context=context, serve_tools=serve_tools) @@ -170,9 +173,12 @@ def _deploy(task: dict, adapter: Path | None, work: Path, backend: str, params: def _session_args(backend: str, deployed: dict, work: Path) -> list[str]: """What the session needs to use what the deployment connected: the workspace's own Codex configuration (its tools, roles and hooks) trusted, as a person trusts a deployed project, and its sub-agents enabled; Pi's project - configuration (.pi/mcp.json, .pi/skills) trusted for the session (--approve), without which Pi skips it.""" + configuration (.pi/mcp.json, .pi/skills) trusted for the session (--approve), without which Pi skips it; dsh's + patch layer of the adapter's tools (.dsh/hermit.patch.yml), which dsh loads only when it is named (--patch).""" if backend == "pi": return ["--approve"] + if backend == "dsh": + return ["--patch", str(deployed["patch"])] if deployed.get("patch") else [] if backend == "codex": return ["-c", f'projects={{"{work}" = {{ trust_level = "trusted" }} }}', *(["--enable", "multi_agent"] if deployed.get("roles") else [])] @@ -277,11 +283,12 @@ def _check(backend: str, params: dict) -> str: if not isinstance(params.get("web_search"), bool): raise ValueError("agent.params.web_search must be set, true or false: Codex's hosted search runs at the " "provider, past any sandbox or proxy, so it is never left to a default") - if backend == "pi": # Pi signs in through a model reference, and runs on this machine + if backend in ("pi", "dsh"): # they sign in through a model reference (dsh: or its own), on this machine if params.get("login") is not None: - raise ValueError("agent pi takes no login: its model is a / reference") + raise ValueError(f"agent {backend} takes no login: its model is a / reference" + + (", or a model of dsh's own provider" if backend == "dsh" else "")) if params.get("isolation") == "container": - raise ValueError("isolation: container runs Codex or Claude Code; agent pi takes isolation: none") + raise ValueError(f"isolation: container runs Codex or Claude Code; agent {backend} takes isolation: none") return backend login, label = params.get("login"), {"codex": "Codex", "claude": "Claude Code"}[backend] if login is not None and login not in LOGINS: diff --git a/hermit/agent/base/cli.py b/hermit/agent/base/cli.py index ef6e2152..aedd8ae2 100644 --- a/hermit/agent/base/cli.py +++ b/hermit/agent/base/cli.py @@ -1,4 +1,5 @@ -"""`codex`, `claude` and `pi`: a coding-agent CLI as a Base Agent (docs/framework/agent-layers.md#layer-2-base-agent). +"""`codex`, `claude`, `pi` and `dsh`: a coding-agent CLI as a Base Agent +(docs/framework/agent-layers.md#layer-2-base-agent). One `run` is one session of the CLI in print mode: the prompt is given, the CLI works in `workspace` with its own tools and the call's `tools` (served over MCP by the framework's tool server), and its last message is the reply. @@ -69,6 +70,10 @@ image_input (true: the bound model takes images, such as a tool's result; default: text only, Pi's default for a model it does not know), append_system_prompt, effort (its --thinking level). Pi takes no roles, has no web search and runs under isolation none or sandbox. + DeepSeek Harness (dsh) only: provider (its pi-ai route for a bound model, from the binding: Binding.dsh_provider), + image_input (as Pi's), effort (the default model's reasoningEffort), permission_mode (its sandbox: read-only, + workspace-write, the default, or danger-full-access; under workspace-write the `writable` paths are granted to it + with --add-dir). dsh takes no roles here and runs under isolation none or sandbox; `args` may hold its own options. Limits are enforced by the isolation, or the call is refused (Unsupported): @@ -96,9 +101,9 @@ from hermit.agent.base import container, login as codex_login from hermit.agent.base.interface import BaseAgent, Limits, Mount, Result, Unsupported -from hermit.agent.base.records import parse_claude, parse_codex, parse_pi, price_usage +from hermit.agent.base.records import parse_claude, parse_codex, parse_dsh, parse_pi, price_usage from hermit.agent.base.tool import Tool -from hermit.agent.base.trajectory import claude_steps, codex_steps, events, pi_steps +from hermit.agent.base.trajectory import claude_steps, codex_steps, dsh_steps, events, pi_steps from hermit.agent.model.backends import CLAUDE_ENV ISOLATIONS = ("none", "sandbox", "container") @@ -108,6 +113,8 @@ r"error sending request|too many requests|service unavailable|bad gateway|overloaded|" r"model is at capacity|" r"^(?:429|5\d\d): |" # Pi's report of a provider's status + # dsh's, by its codes: those it retries itself, and a stream the provider closed + r"^dsh: (?:RATE_LIMIT|SERVER|TIMEOUT|TRANSPORT|EMPTY_RESPONSE|STREAM_CLOSED): |" r"error response from daemon|cannot connect to the docker daemon|" + re.escape(container.MISSING), re.I | re.M) CODEX_HOME_SETTINGS = ["features.shell_snapshot = false", # a snapshot would bring back the whole environment @@ -1462,8 +1469,222 @@ def _pi_session(home: Path, handle) -> Path | None: return sorted(found)[-1] if found else None +DSH_PROVIDER = "hermit" # the pi-ai route (`llm-pi-ai`) a bound model is given under +DSH_ROUTE = "deepseek-official" # dsh's own route, for a model of its own provider (no binding) +DSH_PATCH = "hermit.patch.yml" # the session's patch layer, in its home (`--patch`) +# Rows of dsh's tree off in every session: no telemetry export, no title request and nothing sent along with a +# request to DeepSeek's own API (its plugin list; the session log, a row of its own below), so the session's requests +# are the only ones and carry only the session; and no sub-agents or workflows, whose usage dsh's stream does not +# report (sub-agents are Codex's only when a role asks for them) +DSH_OFF = ("session-telemetry-otel", "session-title-llm", "plugin-package-inventory-deepseek", "tool-subagent", + "tool-subagent-fork", "tool-subagent-control", "tool-workflow") +DSH_PERMISSION_MODES = ("read-only", "workspace-write", "danger-full-access") +# The variable a route names for a server that needs no key: a route that names none gets no key from pi-ai under +# the name `hermit`, and dsh refuses the request (PI_AI_ERROR "No API key") +DSH_NO_KEY = "HERMIT_DSH_NO_KEY" + + +class Dsh(CliAgent): + """DeepSeek Harness (`dsh --profile headless --json`). Its home is DSH_HOME and its HOME (dsh reads ~/.agents), + where dsh makes the headless profile on first use. The session's settings are a patch layer of dsh's own (rows + of its plugin tree, `--patch`), written in the home: a bound model as the pi-ai route `hermit`, its key read by + dsh from the backend's variable, made the default model; the call's tools as an MCP server, whose tools the model + is offered as its own (`mcp____`); web search and fetch off unless asked for; no telemetry export + and no title request, no sub-agents (DSH_OFF). dsh runs commands in its own sandbox (`permission_mode`, default + workspace-write) and is never asked for an approval (`--approval-policy never`: what would need one is + refused). It exits 0 for a run SIGTERM stopped too: whether the run failed is read from its end (`outcome`). + The workspace's project skills (.dsh/skills) are also the session's own (`_dsh_skills`), so that a workspace + inside a git repository keeps them.""" + name = backend = program = "dsh" + label = "DeepSeek Harness" + home_var, user_folder = "DSH_HOME", ".dsh" + home_files = (".env",) # dsh reads the keys of its own provider from $DSH_HOME/.env + turn_done = "final" + + def home_env(self, home): + """Every variable dsh reads goes here, so that it passes the sandbox's filter: its home, HOME, and the shared + agents' folder dsh reads skills and AGENTS.md from (~/.agents unless DSH_AGENTS_HOME says another), so none of + the user's own is read; telemetry off (no anonymous id is made either); the placeholder key of a server that + needs none (DSH_NO_KEY).""" + from hermit.agent.model.backends import PLACEHOLDER_KEY + return {"DSH_HOME": str(home), "HOME": str(home), "DSH_AGENTS_HOME": str(home / ".agents"), + "DSH_TELEMETRY_DISABLED": "1", DSH_NO_KEY: PLACEHOLDER_KEY} + + def dropped_env(self): + """The caller's own dsh variables (`DSH_*`: another permission mode, a folder of skills, a telemetry endpoint, + a web provider, ...), which would change the session without the sandbox: its own are home_env's.""" + return tuple(name for name in os.environ if name.startswith("DSH_")) + + def prepare_home(self, home, roles, tools_url, tools_name): + """The patch layer (DSH_PATCH): dsh's own rows, by their ids. A row's `config` replaces the row's, so the + default model is given whole. No host path is written in it: a sandbox translates only the command, the + environment and the prompt.""" + if roles: + raise Unsupported("dsh: roles are not passed to DeepSeek Harness; give it none") + if tools_url and not re.fullmatch(r"[A-Za-z0-9_-]{1,32}", tools_name): + raise ValueError(f"dsh takes an MCP server name of 32 characters at most ([A-Za-z0-9_-]), not " + f"{tools_name!r}") + s = self.settings + rows: list[dict] = [*({"id": row, "disabled": True} for row in DSH_OFF), + {"id": "session-log-deepseek", "config": {"enabled": False}}] + if not s.get("web_search"): + rows.append({"id": "tool-web", "disabled": True}) # web search and fetch, the model's tools + effort = str(s["effort"]) if s.get("effort") else None + default, chosen = {"provider": DSH_ROUTE}, {} + if s.get("provider"): + provider = json.loads(json.dumps(s["provider"])) + provider.setdefault("apiKeyEnv", DSH_NO_KEY) # a server that needs no key: the placeholder + for model in provider.get("models") or []: + if effort: # a declared model reasons only at the levels it lists; off: it does not reason + model.setdefault("reasoningEfforts", False if effort == "off" else {effort: effort}) + if s.get("image_input"): # else dsh sends the model no image (a text-only model refuses them) + model["input"] = ["text", "image"] + if model.get("id") == s.get("model"): + chosen = model + rows.append({"id": "llm-pi-ai", "config": {"providers": {DSH_PROVIDER: provider}}}) + default["provider"] = DSH_PROVIDER + if s.get("model"): + default["model"] = str(s["model"]) + if effort and chosen.get("reasoningEfforts") is not False: # a model that does not reason takes none + default["reasoningEffort"] = effort + rows.append({"id": "agent-default-model", "config": default}) + elif s.get("provider"): + raise ValueError("dsh: a provider route needs its model (the `model` setting)") + if tools_url: + rows.append({"insert": [{"id": f"mcp-{tools_name}", "name": "@deepseek-ai/dsh-mcp-client", + "config": {"serverName": tools_name, "transport": "streamable-http", + "url": tools_url, "toolCallTimeoutMs": 1_800_000, + "failOnStartupError": True}}]}) + _clear(home / DSH_PATCH) + (home / DSH_PATCH).write_text(json.dumps(rows, indent=1) + "\n") # JSON is YAML: dsh reads it as such + + @classmethod + def web_search_args(cls, enabled: bool) -> list[str]: + """dsh's web search and fetch are a row of its tree, set in the session's patch layer (prepare_home).""" + return [] + + def system_settings(self, system, prompt): + """dsh takes no system prompt for one run, and its persona text (the system-prompt row) takes only the + variables `{{provider}}` and `{{model}}`: any other `{{...}}`, which instructions may hold, fails the model's + step. The instructions go at the top of the prompt instead, as a marked block.""" + return {}, f"# SYSTEM INSTRUCTIONS\n\n{system}\n\n# END\n\n{prompt}" + + @staticmethod + def _reason(raw: str) -> dict: + """The last `final` event's reason ({} when there is none: the run never finished).""" + finals = [e for e in _objects(raw)[0] if e.get("type") == "final"] + reason = finals[-1].get("reason") if finals else None + return reason if isinstance(reason, dict) else {} + + @classmethod + def served(cls, raw): + """The run's last turn completed (its `final` reason).""" + return cls._reason(raw).get("kind") == "completed" + + def outcome(self, raw, stream, last_file): + """dsh exits 0 for a run whose `final` event is `completed`, and 1 for any other end (an error, an abort, the + token limit, ...), but also 0 when SIGTERM stops it: a run without a completed end has failed, whatever its + exit code.""" + notes, done = self.notices(raw), self.served(raw) + return {"reply": self.last_message(stream, last_file), "served": done, + "unavailable": bool(UNAVAILABLE.search(notes)), "refused": False, "failed": not done, + "failure": (notes or "the run did not finish (no completed `final` event)")[-600:]} + + @classmethod + def errors(cls, event): + """A run that did not complete: an error, as dsh writes it on stderr (`dsh: : `: the code is + how a provider that could not serve is told, UNAVAILABLE), or any other end (aborted, max-tokens, blocked, + interrupted, ...); a failure outside a turn.""" + kind = event.get("type") + reason = event.get("reason") if isinstance(event.get("reason"), dict) else {} + if kind == "final" and reason.get("kind") not in (None, "completed"): + error = reason.get("error") if isinstance(reason.get("error"), dict) else {} + if error.get("code"): + return [f"dsh: {error['code']}: {error.get('message') or ''}"] + return [str(error.get("message") or f"run {reason['kind']}")] + if kind == "error": + return [str(event.get("message") or "")] + return [] + + @classmethod + def last_message(cls, log, last=None): + """The `final` event's answer (dsh's last assistant text, never cut).""" + text = "" + for event in events(log): + if event.get("type") == "final" and isinstance(event.get("text"), str): + text = event["text"] + return text + + @classmethod + def parse(cls, log, model): + return parse_dsh(log, model) + + @classmethod + def steps(cls, stream): + return dsh_steps(stream) + + def command(self, prompt, workspace, last_file, *, session=None, roles=(), tools_url=None): + """The launcher's options first (the profile, the `--patch` layers: those in `args`, a deployment's, then + the session's own, which is applied last and so wins), then the headless app's; dsh's launcher reads its + own options only before the first one it does not know. The workspace's skills go into the home first.""" + s, home = self.settings, last_file.parent # the session's home + mode = s.get("permission_mode") or "workspace-write" + if mode not in DSH_PERMISSION_MODES: + raise ValueError(f"dsh: permission_mode is one of {DSH_PERMISSION_MODES}, not {mode!r}") + patches, rest = _dsh_args(s.get("args") or []) + resume = _dsh_session(home, session) if session else None + _dsh_skills(Path(workspace), home) + return [str(s.get("bin") or self.program), "--profile", "headless", *patches, + "--patch", str(home / DSH_PATCH), "--json", "--permission-mode", mode, + "--approval-policy", "never", + *[a for w in (s.get("writable") or () if mode == "workspace-write" else ()) + for a in ("--add-dir", str(Path(w).absolute()))], # what dsh's own sandbox lets it write + *(["--session-id", resume] if resume else []), *rest, + *(["-"] if s.get("stdin") else ["--", prompt])] + + +def _dsh_args(args) -> tuple[list[str], list[str]]: + """(the `--patch` layers among `args`, the other arguments): dsh's launcher takes a layer only before the app's + options, so the layers go first, in their order.""" + args, patches, rest, i = [str(a) for a in args], [], [], 0 + while i < len(args): + if args[i] == "--patch" and i + 1 < len(args): + patches += args[i:i + 2] + i += 2 + continue + (patches if args[i].startswith("--patch=") else rest).append(args[i]) + i += 1 + return patches, rest + + +def _dsh_skills(workspace: Path, home: Path) -> None: + """The workspace's project skills (.dsh/skills) as the session's own ($DSH_AGENTS_HOME/skills, in its home). dsh + reads a project's skills only at the project's root, the nearest folder above with a `.git`, so a workspace + inside a git repository (a run's workspaces under a config kept in a project) would lose them; it reads the + session's own wherever it runs, and offers a skill found in both places once. The last call's copy is replaced.""" + agents = home / ".agents" + if agents.is_symlink(): # a link the last session left would take the copy elsewhere + agents.unlink() + target = agents / "skills" + _clear(target) + target.unlink(missing_ok=True) + source = workspace / ".dsh" / "skills" + if source.is_dir() and not source.is_symlink(): + shutil.copytree(source, target, symlinks=True) + + +def _dsh_session(home: Path, handle) -> str | None: + """Session `handle` when `home` keeps it (sessions///): dsh refuses an id it does not know, so + a call whose session is gone starts a new one.""" + if not re.fullmatch(r"session-[A-Za-z0-9-]+", str(handle)): + return None + root = Path(home) / "sessions" + found = [p for p in root.glob(f"*/{handle}") if p.is_dir() and not p.is_symlink() and not p.parent.is_symlink()] + return str(handle) if found else None + + # Every CLI Base Agent, by each name make() takes for it. -CLIS: dict[str, type[CliAgent]] = {n: c for c in (Codex, Claude, Pi) for n in (c.backend, *c.aliases)} +CLIS: dict[str, type[CliAgent]] = {n: c for c in (Codex, Claude, Pi, Dsh) for n in (c.backend, *c.aliases)} def _this_call(home: Path, handle: str, reported, resumed: bool): diff --git a/hermit/agent/base/login.py b/hermit/agent/base/login.py index e6141a77..1d11e5ed 100644 --- a/hermit/agent/base/login.py +++ b/hermit/agent/base/login.py @@ -612,9 +612,9 @@ def evolver_problems(params: dict | None) -> list[str]: own = LOGINS.get(login) backend = {"claude-code": "claude"}.get(params.get("backend"), params.get("backend")) out = [] - if backend == "pi": - out.append(f"login: {login} is a Codex or Claude Code sign-in; the pi Evolver signs in through its model " - "reference (evolver.model: /)") + if backend in ("pi", "dsh"): + out.append(f"login: {login} is a Codex or Claude Code sign-in; the {backend} Evolver signs in through its " + "model reference (evolver.model: /)") if own is not None and backend in ("codex", "claude") and own != backend: out.append(f"login: {login} is {'Codex' if own == 'codex' else 'Claude Code'}'s sign-in; the Evolver's " f"sessions run {'Codex' if backend == 'codex' else 'Claude Code'}, which takes login: " diff --git a/hermit/agent/base/records.py b/hermit/agent/base/records.py index 225360d0..fd059fed 100644 --- a/hermit/agent/base/records.py +++ b/hermit/agent/base/records.py @@ -124,6 +124,31 @@ def parse_pi(log: Path, model: str) -> tuple[str | None, list[Usage]]: return _parse(log, model, _pi_event) +def _dsh_event(event: dict, usage: Usage, session): + """DeepSeek Harness: the session id (its opening `session` event), and each model step's usage (`status` events + of phase `step_end`). dsh's `inputTokens` leaves the cache out, so the input counted is input, cache reads and + cache writes, as Codex and Claude Code count it. A step that used no tokens (every request of it failed) is no + call. dsh reports no cost: it is priced from the backend's price (price_usage).""" + if event.get("type") == "session": + session = event.get("sessionId") or session + if event.get("type") == "status" and event.get("phase") == "step_end" and isinstance(event.get("usage"), dict): + u = event["usage"] + cached = int(u.get("cacheReadTokens") or 0) + tokens = int(u.get("inputTokens") or 0) + cached + int(u.get("cacheWriteTokens") or 0) + if not tokens and not u.get("outputTokens"): + return session + usage.input_tokens += tokens + usage.cached_input_tokens += cached + usage.output_tokens += int(u.get("outputTokens") or 0) + usage.calls += 1 + return session + + +def parse_dsh(log: Path, model: str) -> tuple[str | None, list[Usage]]: + """DeepSeek Harness's `--json` events: the session id, and every model step's usage.""" + return _parse(log, model, _dsh_event) + + def parse_codex(log: Path, model: str) -> tuple[str | None, list[Usage]]: """Codex's `exec --json` events: the thread id, and each completed turn's usage.""" return _parse(log, model, _codex_event) diff --git a/hermit/agent/base/trajectory.py b/hermit/agent/base/trajectory.py index 8cb731df..a29008ee 100644 --- a/hermit/agent/base/trajectory.py +++ b/hermit/agent/base/trajectory.py @@ -9,8 +9,8 @@ text(steps, message=1200, output=600) # one line per step, each field clipped from its start The streams read: Codex's `codex exec --json` events (`agent-codex.jsonl`), Claude Code's stream-json -(`agent-claude.jsonl`) and Pi's `--mode json` events (`agent-pi.jsonl`), also under another name below a log -directory (Harbor's `agent/codex.txt`). A normalised +(`agent-claude.jsonl`), Pi's `--mode json` events (`agent-pi.jsonl`) and DeepSeek Harness's `--json` events +(`agent-dsh.jsonl`), also under another name below a log directory (Harbor's `agent/codex.txt`). A normalised message list (`RunOutput.messages`) is the pipeline's own rendering and is left to each Evolver. """ from __future__ import annotations @@ -18,11 +18,14 @@ from dataclasses import dataclass import json from pathlib import Path +import re CODEX_STREAM = "agent-codex.jsonl" CLAUDE_STREAM = "agent-claude.jsonl" PI_STREAM = "agent-pi.jsonl" -STREAM_HOSTS = {CODEX_STREAM: "codex", CLAUDE_STREAM: "claude", PI_STREAM: "pi"} # the file a host's log keeps +DSH_STREAM = "agent-dsh.jsonl" +STREAM_HOSTS = {CODEX_STREAM: "codex", CLAUDE_STREAM: "claude", PI_STREAM: "pi", # the file a host's log keeps + DSH_STREAM: "dsh"} STREAMS = tuple(STREAM_HOSTS) OTHER_STREAMS = ("agent/codex.txt", "codex.txt") # a Harbor trial's Codex events: from the trial or its agent/ @@ -213,12 +216,80 @@ def pi_steps(stream: list[dict]) -> list[Step]: return steps +DSH_FILE_TOOLS = ("write", "edit") +# How dsh's bash tool ends a command's result (its own parseExitStatus reads it so): a nonzero exit or a signal is +# marked last, a clean exit is not; a command stopped by its timeout or from outside is marked before that (a command +# that traps the signal may still exit 0), and one still running was moved to the background, unfinished +DSH_EXIT = re.compile(r"\n\[exit code: (\d+)\]$") +DSH_SIGNAL = re.compile(r"\n\[killed by signal: [^\]\n]+\]$") +DSH_STOPPED = re.compile(r"\n\[(?:timed out after \d+ms|stopped: [^\]\n]*)\](?:\n\[[^\]\n]*\])*$") +DSH_BACKGROUND = re.compile(r"\[still running after \d+ms; moved to background job [^\]\n]+\]$") +DSH_NO_OUTPUT = "(no output)" # what the result holds before its marker when the command printed nothing + + +def dsh_steps(stream: list[dict]) -> list[Step]: + """DeepSeek Harness's `--json` events: its thinking and text as each model step committed them, and each + `tool_call` completed by its `tool_result`: bash as a command whose exit code and output are read from the + result as dsh reads them (`[exit code: N]` after the output; no marker: 0; a signal, a timeout, a stop, a command + moved to the background or a call that failed to run: unknown; `(no output)`: empty), write and edit as file + changes, an MCP tool (`mcp____`) as a tool call with its server and arguments and its result as + `output`, any other tool as a tool call; a run that did not complete (the `final` event's reason: an error, an + abort, the token limit, ...), or failed outside a turn (an `error` event), as an error. Status events and usage + are left out.""" + steps, pending = [], {} + for event in stream: + kind = event.get("type") + if kind == "thinking" and str(event.get("text") or "").strip(): + steps.append(Step("reasoning", str(event["text"]))) + elif kind == "text" and str(event.get("text") or "").strip(): + steps.append(Step("message", str(event["text"]))) + elif kind == "tool_call": + name = str(event.get("tool") or "") + args = event.get("input") if isinstance(event.get("input"), dict) else {} + if name == "bash": + step = Step("command", command=str(args.get("command") or "")) + elif name in DSH_FILE_TOOLS: + step = Step("file_change", json.dumps({"tool": name, "path": args.get("file_path")}, + ensure_ascii=False, default=str)) + elif name.startswith("mcp__") and name.count("__") >= 2: + _, server, tool = name.split("__", 2) + step = Step("tool_call", json.dumps({"arguments": args, "server": server, "tool": tool}, + ensure_ascii=False, default=str)) + else: + step = Step("tool_call", json.dumps({"tool": name, "input": event.get("input")}, ensure_ascii=False, + default=str)) + steps.append(step) + pending[event.get("callId")] = step + elif kind == "tool_result": + step = pending.pop(event.get("callId"), None) + if step is None: + continue + result = str(event.get("result") or "") + if step.kind == "command": + found = DSH_EXIT.search(result) + output = result[:found.start()] if found else result + step.output = "" if output == DSH_NO_OUTPUT else output + unknown = event.get("status") != "completed" or DSH_SIGNAL.search(result) or \ + DSH_STOPPED.search(result) or DSH_BACKGROUND.search(result) + step.exit_code = None if unknown else int(found.group(1)) if found else 0 + elif step.kind == "tool_call": + step.output = result + elif kind == "final" and isinstance(event.get("reason"), dict) and \ + event["reason"].get("kind") not in (None, "completed"): + error = event["reason"].get("error") if isinstance(event["reason"].get("error"), dict) else {} + steps.append(Step("error", str(error.get("message") or f"run {event['reason']['kind']}"))) + elif kind == "error": + steps.append(Step("error", str(event.get("message") or ""))) + return steps + + # Each host's stream: the reader of its events, and the event types that mark it (tried in this order when a stream # is found by its events). A host's stream is added here and in STREAM_HOSTS. -READERS = {"codex": codex_steps, "claude": claude_steps, "pi": pi_steps} +READERS = {"codex": codex_steps, "claude": claude_steps, "pi": pi_steps, "dsh": dsh_steps} MARKS = (("codex", {"thread.started", "turn.started", "turn.completed", "turn.failed", "item.completed"}), ("claude", {"system", "assistant", "user", "result"}), - ("pi", {"agent_start", "agent_settled", "message_end", "tool_execution_end"})) + ("pi", {"agent_start", "agent_settled", "message_end", "tool_execution_end"}), + ("dsh", {"status", "tool_call", "tool_result", "final"})) def _host(stream: list[dict]) -> str | None: diff --git a/hermit/agent/evolver/agent.py b/hermit/agent/evolver/agent.py index 9c6595f0..33080088 100644 --- a/hermit/agent/evolver/agent.py +++ b/hermit/agent/evolver/agent.py @@ -1,13 +1,14 @@ -"""AgentEvolver: a built-in agent (Codex, Claude Code or Pi) run on the workspace with a prompt; CommandEvolver: any -program. +"""AgentEvolver: a built-in agent (Codex, Claude Code, Pi or DeepSeek Harness) run on the workspace with a prompt; +CommandEvolver: any program. AgentEvolver params: - backend: codex | claude | pi + backend: codex | claude | pi | dsh model: the model passed to the agent (optional: the agent's default); with `login`, the CLI's model name on the user's subscription (hermit.agent.base.login: `chatgpt` for Codex, `claude` for Claude Code). A `/` reference (see hermit.agent.model.backends) binds it to that backend: the provider settings go into `args` (Codex), `env` - (Claude Code) or `provider` (Pi's models.json entry) and `price` is taken from the backend + (Claude Code) or `provider` (Pi's models.json entry, DeepSeek Harness's pi-ai route) and `price` is taken + from the backend login, plan_dir, plan_max_wait_s, plan_rerun, usage_credits: the CLI's subscription and its plan (hermit.agent.base.cli, hermit.agent.base.login; claude: the token in CLAUDE_CODE_OAUTH_TOKEN) env: environment variables set for the agent; a `$NAME` value is expanded from the environment at launch (Claude @@ -22,9 +23,10 @@ tools: the only Trainer tools the agent may use (for example [check_spec]); null: all of them. With a list, the task message names no endpoint URL either web_search: the CLI's web search, true or false -This is Layer 3 over the `codex`, `claude` or `pi` Base Agent (docs/framework/agent-layers.md): it fills the prompt, -picks the Trainer's tools and resumes the last session; the Base Agent runs the CLI in the run's evolver sandbox (the -workspace under /w, the paths in the prompt relative to it), with the tools as one MCP server (`evolver_tools`). +This is Layer 3 over the `codex`, `claude`, `pi` or `dsh` Base Agent (docs/framework/agent-layers.md): it fills the +prompt, picks the Trainer's tools and resumes the last session; the Base Agent runs the CLI in the run's evolver +sandbox (the workspace under /w, the paths in the prompt relative to it), with the tools as one MCP server +(`evolver_tools`). """ from __future__ import annotations @@ -78,7 +80,7 @@ class AgentEvolver(Evolver): def step(self, ctx: StepContext) -> StepResult: params = ctx.params backend = params.get("backend", "codex") - if backend not in ("codex", "claude", "pi"): + if backend not in ("codex", "claude", "pi", "dsh"): return StepResult("failed", f"unknown backend {backend!r}") if not isinstance(params.get("web_search"), bool): return StepResult("failed", "evolver.params.web_search must be set, true or false") @@ -130,6 +132,7 @@ def settings(self, ctx: Context, backend: str) -> dict: "web_search": params["web_search"], "env": env, "isolation": "sandbox" if ctx.sandbox is not None else "none", "sandbox": ctx.sandbox, "codex_sandbox": "danger-full-access", "shell_env": "all", + **({"permission_mode": "danger-full-access"} if backend == "dsh" else {}), # dsh's own sandbox "home": ctx.workspace.state.parent / "agent_home", "home_files": params.get("home"), "log": ctx.workspace.output / f"agent-{backend}.jsonl", "timeout_s": None, "cancel": ctx.cancel, "tools_name": "evolver_tools"} diff --git a/hermit/agent/evolver/methods/dsh/RESUME.md b/hermit/agent/evolver/methods/dsh/RESUME.md new file mode 100644 index 00000000..08d6fc0b --- /dev/null +++ b/hermit/agent/evolver/methods/dsh/RESUME.md @@ -0,0 +1,4 @@ +A new step. {message} +Read {learning_material}/current/ (the rollouts of the adapter you handed back last time, and check_feedback.json if +the Trainer had to repair it), then improve {adapter} again as before: change only the adapter, call `check_spec`, +and write {output}/report.md (first line: what you changed and why). diff --git a/hermit/agent/evolver/methods/dsh/START.md b/hermit/agent/evolver/methods/dsh/START.md new file mode 100644 index 00000000..5560934e --- /dev/null +++ b/hermit/agent/evolver/methods/dsh/START.md @@ -0,0 +1,15 @@ +You are improving a harness adapter: the files an acting agent receives (context, tools, skills) when it works on a +task. Your job in this call is to change the adapter so the acting agent does better on tasks like the ones in the +learning material, within the adapter's specification. + +{message} +How to work: +1. Read the task description ({task_info}/README.md) and the adapter's guide ({adapter}/.hermit/EVOLVER.md): it says which + files you may change and in what format. +2. Read the learning material of this step ({learning_material}/current/): each task's input, the agent's output, the + judge's score and feedback, and the full trajectory. Earlier steps are next to it; index.jsonl shows the trend. +3. Decide what to change, and change only files in {adapter}. Keep what still helps; generalise from the material + instead of writing down facts about individual tasks. +4. Call the `check_spec` tool before you finish; fix what it reports. +5. Write {output}/report.md: first line one sentence on what you changed and why, then the details. +You may keep notes for later calls in {state}. Do not touch anything else. diff --git a/hermit/agent/evolver/methods/dsh/evolver.yaml b/hermit/agent/evolver/methods/dsh/evolver.yaml new file mode 100644 index 00000000..9496baa2 --- /dev/null +++ b/hermit/agent/evolver/methods/dsh/evolver.yaml @@ -0,0 +1,13 @@ +description: A DeepSeek Harness (dsh) agent improves the adapter from the step's learning material. +class: hermit.agent.evolver.agent:AgentEvolver +needs: [] +params: + backend: dsh + prompt: START.md + resume_prompt: RESUME.md + args: [] + model: null # a / reference (dsh's pi-ai route, the key read from its variable) + price: null # USD per million tokens {input, cached_input, output}; null: the backend's + home: [] # no file of ~/.dsh is copied: dsh signs in through the model reference + tools: null # the only Trainer tools the agent may use, e.g. [check_spec]; null: all + web_search: false # dsh's web search and fetch tools; the evolver sandbox reaches only the model diff --git a/hermit/agent/model/backends.py b/hermit/agent/model/backends.py index 78b02ce8..1956e501 100644 --- a/hermit/agent/model/backends.py +++ b/hermit/agent/model/backends.py @@ -61,8 +61,10 @@ def user_file() -> Path: "context_window", "developer_role", "models", "description"} DEVELOPER_ROLES = ("keep", "system") # What each consumer speaks: the dialect a backend must serve for it. -NEEDS = {"chat": "chat", "react": "chat", "llm": "chat", "codex": "responses", "claude": "messages", "pi": "chat"} -# Pi speaks several dialects: the one it is given is the first the backend serves (Binding.pi_dialect). +NEEDS = {"chat": "chat", "react": "chat", "llm": "chat", "codex": "responses", "claude": "messages", "pi": "chat", + "dsh": "chat"} +# Pi speaks several dialects: the one it is given is the first the backend serves (Binding.pi_dialect). DeepSeek +# Harness (dsh) reaches a model through pi-ai too, so it speaks the same ones, chosen the same way. PI_APIS = {"chat": "openai-completions", "messages": "anthropic-messages", "responses": "openai-responses"} # The variable litellm reads the key from, per route prefix (the examples' CL-Bench host calls through litellm). LITELLM_KEY_ENV = {"fireworks_ai": "FIREWORKS_AI_API_KEY", "openai": "OPENAI_API_KEY", "hosted_vllm": "HOSTED_VLLM_API_KEY", @@ -325,6 +327,24 @@ def pi_provider(self) -> dict: provider["compat"] = {"supportsDeveloperRole": False} # the backend refuses the developer role return provider + def dsh_provider(self) -> dict: + """DeepSeek Harness's route for this model (a `providers` value of its pi-ai adapter, `llm-pi-ai`): the + backend's URL in the dialect dsh speaks to it (pi-ai's, as Pi's), the key's variable by name (`apiKeyEnv`, + which dsh reads at each request, so no file holds the key), and the model with its context window. dsh + reports no cost: the backend's price is applied to its usage (`price`).""" + api = self.pi_dialect() + model: dict = {"id": self.model} + if self.backend.context_window: + model["contextWindow"] = int(self.backend.context_window) + provider = {"api": PI_APIS[api], "models": [model], + "baseURL": (self.backend.messages_root if api == "messages" else self.base_url).rstrip("/")} + key_env = self.key_env_if_set() + if key_env: + provider["apiKeyEnv"] = key_env + if api == "chat" and self.backend.developer_role == "system": + provider["compat"] = {"supportsDeveloperRole": False} # the backend refuses the developer role + return provider + CLAUDE_ENV = ("ANTHROPIC_BASE_URL", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_API_KEY", "ANTHROPIC_MODEL", "ANTHROPIC_DEFAULT_OPUS_MODEL", "ANTHROPIC_DEFAULT_SONNET_MODEL", "ANTHROPIC_DEFAULT_HAIKU_MODEL", @@ -396,11 +416,11 @@ def _spec_of(value, binding: Binding) -> dict: def bind_agent_params(params: dict, kind: str, backends: dict[str, Backend], default=None) -> dict: - """The params of a Codex (`kind` codex), Claude Code (`kind` claude) or Pi (`kind` pi) agent or Evolver with its - model bound: `model` becomes the id the endpoint expects, the provider settings go into `args` (Codex), `env` - (Claude Code) ahead of the user's own, or `provider` (Pi's models.json entry), `price` is filled from the - backend, and `binding` records the whole. A `model` that is no reference (the CLI's own provider) is left - alone.""" + """The params of a Codex (`kind` codex), Claude Code (`kind` claude), Pi (`kind` pi) or DeepSeek Harness (`kind` + dsh) agent or Evolver with its model bound: `model` becomes the id the endpoint expects, the provider settings go + into `args` (Codex), `env` (Claude Code) ahead of the user's own, or `provider` (Pi's models.json entry, dsh's + pi-ai route), `price` is filled from the backend, and `binding` records the whole. A `model` that is no + reference (the CLI's own provider) is left alone.""" binding = as_binding(params.get("model") or default, backends) if binding is None: return params @@ -412,6 +432,8 @@ def bind_agent_params(params: dict, kind: str, backends: dict[str, Backend], def out["env"] = {**binding.claude_env(), **(params.get("env") or {})} elif kind == "pi": out["provider"] = params.get("provider") or binding.pi_provider() + elif kind == "dsh": + out["provider"] = params.get("provider") or binding.dsh_provider() else: raise ValueError(f"kind must be one of {CLI_KINDS}, not {kind!r}") if not out.get("price"): @@ -422,19 +444,19 @@ def bind_agent_params(params: dict, kind: str, backends: dict[str, Backend], def # ---- the run config ------------------------------------------------------------------------------------------- -CLI_AGENTS = {"codex": "codex", "claude-code": "claude", "claude": "claude", "pi": "pi", "cli": None} -CLI_KINDS = ("codex", "claude", "pi") # the CLI Base Agents a model is bound for (bind_agent_params) +CLI_AGENTS = {"codex": "codex", "claude-code": "claude", "claude": "claude", "pi": "pi", "dsh": "dsh", "cli": None} +CLI_KINDS = ("codex", "claude", "pi", "dsh") # the CLI Base Agents a model is bound for (bind_agent_params) def needs(kind: str, binding: Binding) -> str: """The dialect `kind` (a consumer of NEEDS) speaks to `binding`'s backend.""" - return binding.pi_dialect() if kind == "pi" else NEEDS.get(kind, "responses") + return binding.pi_dialect() if kind in ("pi", "dsh") else NEEDS.get(kind, "responses") def _cli_kind(agent: dict, params: dict) -> str: - """codex, claude or pi for an acting agent that is a CLI (by its name, `cli`'s backend, or a task's own agent's - agent.base), else "".""" - names = {"codex": "codex", "claude": "claude", "claude-code": "claude", "pi": "pi"} + """codex, claude, pi or dsh for an acting agent that is a CLI (by its name, `cli`'s backend, or a task's own + agent's agent.base), else "".""" + names = {"codex": "codex", "claude": "claude", "claude-code": "claude", "pi": "pi", "dsh": "dsh"} kind = CLI_AGENTS.get(str(agent.get("name") or ""), "") if kind is None: # `cli`: the backend is a param return names.get(params.get("backend", "codex"), "") diff --git a/hermit/cli.py b/hermit/cli.py index d2e9390f..2e6b0975 100644 --- a/hermit/cli.py +++ b/hermit/cli.py @@ -30,11 +30,12 @@ def main(argv=None) -> int: p.add_argument("--adapter", required=True, help="final | root | none | a tag | a commit | a local path") if name == "check": p.add_argument("--no-probe", action="store_true", help="skip running one task through the pipeline") - dep = sub.add_parser("deploy", help="deploy an adapter's exports into a Codex, Claude Code or Pi workspace") + dep = sub.add_parser("deploy", help="deploy an adapter's exports into a Codex, Claude Code, Pi or DeepSeek Harness " + "workspace") dep.add_argument("--adapter", type=Path, required=True) dep.add_argument("--workspace", "--output", dest="workspace", type=Path, required=True, help="the project directory the agent will work in (--output is the older spelling)") - dep.add_argument("--host", choices=["codex", "claude-code", "pi"], default="codex") + dep.add_argument("--host", choices=["codex", "claude-code", "pi", "dsh"], default="codex") dep.add_argument("--python", default=sys.executable) dep.add_argument("--context-limit", type=int, default=None, help="codex: the project_doc_max_bytes your Codex config sets (default 32768, Codex's own); " @@ -97,6 +98,8 @@ def main(argv=None) -> int: from hermit.hosts.codex import prepare elif args.host == "pi": from hermit.hosts.pi import prepare + elif args.host == "dsh": + from hermit.hosts.dsh import prepare else: from hermit.hosts.claude_code import prepare try: diff --git a/hermit/config.py b/hermit/config.py index 70e65649..02c7428a 100644 --- a/hermit/config.py +++ b/hermit/config.py @@ -44,10 +44,10 @@ def _merge(base: dict, extra: dict, prefix: str = "") -> dict: FREE = ("params", "llm", "backends", "container", "resources", # sections whose keys the framework does not know - "model", *("codex", "claude", "pi", "react", "direct")) # a layer's settings (docs/framework/agent-layers.md) -BASES = {"codex": "codex", "claude": "claude-code", "claude-code": "claude-code", "pi": "pi", "react": "react", - "direct": None} -CLI_BASES = ("codex", "claude-code", "pi") # the bases that are a CLI (their Layer 1 settings: _model) + "model", *("codex", "claude", "pi", "dsh", "react", "direct")) # a layer's settings (agent-layers.md) +BASES = {"codex": "codex", "claude": "claude-code", "claude-code": "claude-code", "pi": "pi", "dsh": "dsh", + "react": "react", "direct": None} +CLI_BASES = ("codex", "claude-code", "pi", "dsh") # the bases that are a CLI (their Layer 1 settings: _model) MODEL_KEYS = ("login", "effort", "price") # Layer 1 settings a CLI takes beside its model # A login's plan settings: Layer 2's (agent., evolver.), refused beside the model's name. PLAN_KEYS = {"plan_dir", "plan_max_wait_s", "plan_rerun", "usage_credits", "login_root"} @@ -114,14 +114,14 @@ def layered(data: dict) -> None: agent = data["agent"] base = agent.pop("base", None) model = agent.pop("model", None) - sections = {b: agent.pop(b, None) for b in ("codex", "claude", "pi", "react", "direct")} + sections = {b: agent.pop(b, None) for b in ("codex", "claude", "pi", "dsh", "react", "direct")} if base is None: if model is not None or any(sections.values()): - raise ValueError("agent.model and agent. need agent.base (codex, claude, pi, react or direct)") + raise ValueError("agent.model and agent. need agent.base (codex, claude, pi, dsh, react or direct)") else: if base not in BASES: raise ValueError(f"agent.base must be one of {sorted(BASES)}, not {base!r}") - builtin = {"codex": "codex", "claude-code": "claude-code", "claude": "claude-code", "pi": "pi", + builtin = {"codex": "codex", "claude-code": "claude-code", "claude": "claude-code", "pi": "pi", "dsh": "dsh", "react": "react", "cli": None} if not agent.get("name"): if BASES[base] is None: @@ -142,13 +142,13 @@ def layered(data: dict) -> None: evolver = data["evolver"] base = evolver.pop("base", None) model = evolver.pop("model", None) - sections = {b: evolver.pop(b, None) for b in ("codex", "claude", "pi", "react", "direct")} + sections = {b: evolver.pop(b, None) for b in ("codex", "claude", "pi", "dsh", "react", "direct")} if base is None: if model is not None or any(sections.values()): - raise ValueError("evolver.model and evolver. need evolver.base (codex, claude, pi or direct)") + raise ValueError("evolver.model and evolver. need evolver.base (codex, claude, pi, dsh or direct)") return - if base not in ("codex", "claude", "claude-code", "pi", "direct"): - raise ValueError(f"evolver.base must be codex, claude, pi or direct, not {base!r}") + if base not in ("codex", "claude", "claude-code", "pi", "dsh", "direct"): + raise ValueError(f"evolver.base must be codex, claude, pi, dsh or direct, not {base!r}") params = dict(evolver.get("params") or {}) if base == "direct": if model is not None: diff --git a/hermit/defaults.yaml b/hermit/defaults.yaml index 04e4243c..5d6e8b22 100644 --- a/hermit/defaults.yaml +++ b/hermit/defaults.yaml @@ -9,15 +9,18 @@ data: shuffle_seed: 0 task_info: null agent: - name: null # react | codex | claude-code | pi, or file.py:Class / file.py:function of the task's own + name: null # react | codex | claude-code | pi | dsh, or file.py:Class / file.py:function + # of the task's own concurrency: 4 timeout_min: null # a sandboxed task call that runs longer is stopped and recorded as an error; null: none params: {} # Layer 3: the Acting Agent's own settings (docs/framework/agent-layers.md) - base: null # Layer 2: codex | claude | pi | react | direct; its settings go in a section of its name + base: null # Layer 2: codex | claude | pi | dsh | react | direct; its settings go in a section of its + # name model: null # Layer 1: a / reference, or {ref | name, login, effort, price, ...} codex: null claude: null pi: null + dsh: null react: null direct: null pipeline: # the older spelling of `agent`: run -> name @@ -53,11 +56,12 @@ evolver: name: null resume: false params: {} # Layer 3: the method's settings - base: null # Layer 2: codex | claude | pi | direct (the Base Agent the method runs on) + base: null # Layer 2: codex | claude | pi | dsh | direct (the Base Agent the method runs on) model: null # Layer 1: as agent.model; with direct it is the run's `llm` codex: null claude: null pi: null + dsh: null direct: null evolver_access: environment: [] diff --git a/hermit/hosts/dsh.py b/hermit/hosts/dsh.py new file mode 100644 index 00000000..6f0b7d9e --- /dev/null +++ b/hermit/hosts/dsh.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Deploy an adapter into a DeepSeek Harness (dsh) workspace: the whole adapter under hermit//, and its exports +connected to dsh's own places: AGENTS.md context, .dsh/skills, and the tools as a patch layer, .dsh/hermit.patch.yml. + +dsh reads AGENTS.md at the project's root (the nearest folder above with a `.git`, else the working directory) and in +the working directory, and the skills of .dsh/skills at the project's root only, without any trust prompt: a +workspace that is not its project's root gets its skills only as the session's own, which the `dsh` acting agent +gives it (hermit.agent.base.cli._dsh_skills). It has no project file for MCP servers: they are rows of its +plugin tree, so the adapter's tool server is an `insert` row in a patch layer of the workspace, which a session +loads with `dsh --patch .dsh/hermit.patch.yml` (what the `dsh` acting agent passes). dsh reads no hook file and has +no role file: an adapter's roles and hooks stay files of its copy, which AGENTS.md names with what to do with them +(hermit.hosts.inputs.resources), as it names the other files. dsh's own settings ($DSH_HOME) are never changed. +""" +import argparse +import json +from pathlib import Path +import re +import subprocess +import sys + +from hermit.adapter.loader import agent_tools, hook_definitions, load_manifest, not_portable +from hermit.hosts.context import PROJECT_CONTEXT, START, append_context +from hermit.hosts.inputs import (MARKER, PACKAGE_ROOT, _adapter_sandbox_env, adapter_files, adapter_folder, + check_workspace, copy_adapter, copy_skills, folder_note, render_exports, resources, + server_name, source_label) + +NATIVE = {"skills"} +PATCH = Path(".dsh") / "hermit.patch.yml" # the workspace's patch layer: the adapter's tool server +SERVER_NAME = re.compile(r"[A-Za-z0-9_-]{1,32}") # what dsh takes as an MCP server's name + + +def prepare(adapter: Path, output: Path, python: str, *, learning_material: Path | None = None, + context: str | None = None) -> dict: + """`context`: the adapter's context for one task, which the harness computed, in place of the one rendered + here; an adapter whose context is per task deploys only so.""" + adapter, output = adapter.resolve(), output.resolve() + refused = not_portable(load_manifest(adapter), "A DeepSeek Harness workspace") # before anything is written + if refused and context is None: + raise ValueError(refused) + previous = check_workspace(adapter, output, "dsh") + manifest = load_manifest(adapter) + has_tools = bool(agent_tools(manifest)) # the context tool is the harness's, not served + if has_tools and not SERVER_NAME.fullmatch(server_name(manifest)): + raise ValueError(f"dsh takes an MCP server name of 32 characters at most ([A-Za-z0-9_-]), not " + f"{server_name(manifest)!r} (the adapter's id)") + files = adapter_files(adapter, output, manifest, previous) + exports = render_exports(adapter, python) + listed, placed = resources(adapter, output, manifest, exports, hook_definitions(adapter, manifest), NATIVE) + if has_tools: + subprocess.run([python, "-c", "import mcp.server"], check=True, capture_output=True) + output.mkdir(parents=True, exist_ok=True) + context_path = output / "AGENTS.md" + existing = context_path.read_text() if context_path.exists() else PROJECT_CONTEXT + if existing.startswith(START): + existing = PROJECT_CONTEXT + "\n\n" + existing + context_text = append_context(existing, "\n\n".join(t for t in ( + folder_note(output, manifest), + exports["context"] if context is None else context, listed) if t), + source=source_label(manifest), + version=manifest["version"], adapter_id=manifest["id"]) + audit = output / "tool-calls.jsonl" + server = None + if has_tools: + server = {"serverName": server_name(manifest), "transport": "stdio", "command": python, + "args": ["-m", "hermit.runtime.mcp_server", "--adapter", str(adapter), "--audit-log", str(audit)], + "env": {"PYTHONPATH": PACKAGE_ROOT, **_adapter_sandbox_env(python)}, + "toolCallTimeoutMs": 120_000} # Codex's tool_timeout_sec, not dsh's 60 s + if learning_material is not None: + server["args"].extend(["--learning-material", str(Path(learning_material).resolve())]) + rows = [{"insert": [{"id": f"mcp-{server_name(manifest)}", "name": "@deepseek-ai/dsh-mcp-client", + "config": server}]}] if has_tools else [] + (output / PATCH.parent).mkdir(exist_ok=True) + # JSON is YAML: dsh reads the layer as YAML, and nothing in it is a `!!js` expression + (output / PATCH).write_text(json.dumps(rows, indent=2) + "\n") + context_path.write_text(context_text) + skills = copy_skills(adapter, manifest, output / ".dsh" / "skills") + copy_adapter(adapter, output, manifest, files, previous) + result = {"host": "dsh", "adapter_id": manifest["id"], "version": manifest["version"], "adapter": str(adapter), + "workspace": str(output), "context_injected_via": "AGENTS.md", + "server_name": server_name(manifest) if has_tools else None, "server": server, + "patch": str(output / PATCH) if has_tools else None, "roles": {}, "role_name_map": {}, + "skills_dir": str(output / ".dsh" / "skills") if skills else None, "skills": skills, "hooks": None, + "audit_log": str(audit) if has_tools else None, + "adapter_dir": str(adapter_folder(output, manifest)), "files": files, "placed": placed} + (output / MARKER).write_text(json.dumps(result, indent=2) + "\n") + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--adapter", type=Path, required=True) + parser.add_argument("--workspace", "--output", dest="workspace", type=Path, required=True) + parser.add_argument("--python", default=sys.executable) + args = parser.parse_args() + print(json.dumps(prepare(args.adapter, args.workspace, str(Path(args.python).absolute())), indent=2)) diff --git a/hermit/isolate.py b/hermit/isolate.py index 293d5c98..a37b4ebe 100644 --- a/hermit/isolate.py +++ b/hermit/isolate.py @@ -165,8 +165,8 @@ def validate(config) -> None: login = params.get("login") if kind and login: # wherever the pipeline runs: else every session would be refused, one task at a time own = codex_login.LOGINS.get(login) - if kind == "pi": - raise ValueError("agent pi takes no login: its model is a / reference (agent.model)") + if kind in ("pi", "dsh"): + raise ValueError(f"agent {kind} takes no login: its model is a / reference (agent.model)") if own is None: raise ValueError(f"agent.params.login is {' or '.join(sorted(codex_login.LOGINS))} or absent, not " f"{login!r}") diff --git a/tests/README.md b/tests/README.md index 4cbeb802..0ce1d975 100644 --- a/tests/README.md +++ b/tests/README.md @@ -35,7 +35,7 @@ ChatGPT-login directory and Claude plan for every test, and no copy of the user' models), `toy_task.py` (the toy task the Trainer tests run on), `claude_stand.py` (the stand-in `claude` that replays `fixtures/claude_streams/`, and the probe for a working bubblewrap, for core's and the methods' Claude tests), `fixtures/` (`pi/`: Pi's recorded streams, read by -`core/test_pi_agent.py`). +`core/test_pi_agent.py`; `dsh/`: DeepSeek Harness's, read by `core/test_dsh_agent.py`). A new or changed test follows the [test checklist](../.agents/skills/prepare-pr/references/test-checklist.md): where it goes, how it is named and documented, what it asserts, and what it may touch. diff --git a/tests/core/test_deploy.py b/tests/core/test_deploy.py index 69dc8b79..e0b3f189 100644 --- a/tests/core/test_deploy.py +++ b/tests/core/test_deploy.py @@ -1,4 +1,4 @@ -"""Deploying the Lean template into Codex, Claude Code and Pi workspaces.""" +"""Deploying the Lean template into Codex, Claude Code, Pi and DeepSeek Harness workspaces.""" import json from pathlib import Path import sys @@ -8,6 +8,7 @@ from hermit.hosts.claude_code import prepare as claude_prepare from hermit.hosts.codex import prepare as codex_prepare +from hermit.hosts.dsh import prepare as dsh_prepare from hermit.hosts.pi import prepare as pi_prepare LEAN = Path(__file__).resolve().parents[2] / "hermit" / "adapter" / "templates" / "lean_collab" @@ -51,6 +52,31 @@ def test_pi(tmp_path): codex_prepare(LEAN, tmp_path / "p", sys.executable) +def test_dsh(tmp_path): + """dsh has no project file for MCP servers: the tools are an `insert` row of the workspace's patch layer, which a + session loads with --patch; the skills are dsh's project skills; roles stay files named in AGENTS.md.""" + result = dsh_prepare(LEAN, tmp_path / "d", sys.executable) + [layer] = json.loads((tmp_path / "d" / ".dsh" / "hermit.patch.yml").read_text()) + [row] = layer["insert"] + assert row["name"] == "@deepseek-ai/dsh-mcp-client" and row["config"]["serverName"] == "lean_collab" + assert row["config"]["transport"] == "stdio" and row["config"]["args"][:3] == ["-m", "hermit.runtime.mcp_server", + "--adapter"] + assert row["config"]["toolCallTimeoutMs"] == 120_000 # Codex's tool_timeout_sec, not dsh's 60 s + assert result["patch"] == str(tmp_path / "d" / ".dsh" / "hermit.patch.yml") + assert "hermit:lean-collab:start" in (tmp_path / "d" / "AGENTS.md").read_text() + assert (tmp_path / "d" / ".dsh" / "skills" / "good-map" / "SKILL.md").exists() + assert result["context_injected_via"] == "AGENTS.md" and result["roles"] == {} + dsh_prepare(LEAN, tmp_path / "d", sys.executable) + assert (tmp_path / "d" / "AGENTS.md").read_text().count("hermit:lean-collab:start") == 1 + with pytest.raises(ValueError, match="generated for another host"): + pi_prepare(LEAN, tmp_path / "d", sys.executable) + a = subagent_adapter(tmp_path) + result = dsh_prepare(a, tmp_path / "s", sys.executable) + assert result["placed"] == {"agents": "files"} and result["patch"] is None # no tools: no server row + assert json.loads((tmp_path / "s" / ".dsh" / "hermit.patch.yml").read_text()) == [] + assert "`reviewer`: Check a result against the task requirements." in (tmp_path / "s" / "AGENTS.md").read_text() + + def subagent_adapter(tmp_path): a = tmp_path / "adapter" (a / "agents").mkdir(parents=True) diff --git a/tests/core/test_dsh_agent.py b/tests/core/test_dsh_agent.py new file mode 100644 index 00000000..5acdd89d --- /dev/null +++ b/tests/core/test_dsh_agent.py @@ -0,0 +1,568 @@ +"""The `dsh` Base Agent (DeepSeek Harness, hermit/agent/base/cli.py), its model binding +(hermit/agent/model/backends.py), its stream's reader (hermit/agent/base/trajectory.py) and the Layer 3 code that runs +it: the acting agent (hermit/agent/acting/workspace.py) with its deployment (hermit/hosts/dsh.py), the config's `dsh` +base, the `dsh` Evolver. + +Stand-ins: a fake `dsh` on PATH that replays a stream recorded from DeepSeek Harness 0.2.2-rc.1 (tests/fixtures/dsh, +whose README says how) and records what it was given; and, for the tests marked integration, the installed `dsh` +against a scripted chat-completions model on the loopback (`serve_chat`), never a provider.""" +import json +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import threading + +import pytest + +from hermit.agent.base import cli as base_cli +from hermit.agent.base import trajectory +from hermit.agent.base.interface import Unsupported, make +from hermit.agent.base.tool import Tool +from hermit.agent.model.backends import bind_agent_params, registry + +ROOT = Path(__file__).resolve().parents[2] +STREAMS = ROOT / "tests" / "fixtures" / "dsh" +FILE_ADAPTER = ROOT / "docs" / "_examples" / "file_adapter" +SESSION = "session-b2fbf9bf-a321-4065-8f8a-ee344b66b5f3" # the recorded success stream's session + +FAKE_DSH = f'''#!{sys.executable} +"""A stand-in for `dsh --profile headless ...`: records its arguments, dsh's variables, the patch layers it is given +and its stdin, then prints FAKE_DSH_STREAM and exits FAKE_DSH_EXIT.""" +import json, os, sys +args = sys.argv[1:] +patches = {{args[i + 1]: open(args[i + 1]).read() for i, a in enumerate(args[:-1]) if a == "--patch"}} +names = ("DSH_HOME", "HOME", "DSH_AGENTS_HOME", "DSH_TELEMETRY_DISABLED", "{base_cli.DSH_NO_KEY}", "FAKE_KEY", + "DSH_BUNDLED_SKILL_DIR") +seen = {{"args": args, "env": {{k: os.environ.get(k) for k in names}}, "patches": patches, "stdin": sys.stdin.read()}} +with open(os.environ["FAKE_DSH_SEEN"], "a") as f: + f.write(json.dumps(seen) + "\\n") +sys.stdout.write(open(os.environ["FAKE_DSH_STREAM"]).read()) +sys.exit(int(os.environ.get("FAKE_DSH_EXIT", "0"))) +''' + + +@pytest.fixture +def fake(tmp_path, monkeypatch): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + exe = bin_dir / "dsh" + exe.write_text(FAKE_DSH) + exe.chmod(exe.stat().st_mode | stat.S_IEXEC) + monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ['PATH']}") + monkeypatch.setenv("FAKE_DSH_SEEN", str(tmp_path / "seen.jsonl")) + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "success.jsonl")) + monkeypatch.setenv("FAKE_KEY", "sk-fake") + return tmp_path + + +def seen(tmp_path) -> list[dict]: + return [json.loads(line) for line in (tmp_path / "seen.jsonl").read_text().splitlines()] + + +def rows(call: dict, home: Path) -> list[dict]: + """The session's own patch layer, as the fake read it.""" + return json.loads(call["patches"][str(home / base_cli.DSH_PATCH)]) + + +def row(layer: list[dict], row_id: str) -> dict: + [found] = [r for r in layer if r.get("id") == row_id] + return found + + +def bound(**backend) -> dict: + """A `dsh` agent's params with its model bound to a backend of its own (`fake/m-1`).""" + spec = {"base_url": "http://127.0.0.1:9/v1", "api_key_env": "FAKE_KEY", "apis": ["chat"], + "prices": {"m-1": {"input": 1.0, "cached_input": 0.1, "output": 2.0}}, **backend} + return bind_agent_params({"model": "fake/m-1"}, "dsh", registry({"fake": spec})) + + +def echo() -> Tool: + return Tool("echo", "Echo the text back.", lambda text="": {"echoed": text}, + input_schema={"type": "object", "properties": {"text": {"type": "string"}}, "required": ["text"]}) + + +def test_a_bound_model_is_a_dsh_route_with_its_key_variable_in_the_dialect_the_backend_serves(): + """The key reaches dsh only as the name of its variable (`apiKeyEnv`): the records keep the name, never the key.""" + params = bound() + assert params["model"] == "m-1" and params["binding"]["ref"] == "fake/m-1" + assert params["provider"] == {"api": "openai-completions", "baseURL": "http://127.0.0.1:9/v1", + "apiKeyEnv": "FAKE_KEY", "models": [{"id": "m-1"}]} + assert params["price"] == {"input": 1.0, "cached_input": 0.1, "output": 2.0} + messages = bound(apis=["messages"])["provider"] + assert messages["api"] == "anthropic-messages" and messages["baseURL"] == "http://127.0.0.1:9" + system = bound(developer_role="system", context_window=32000)["provider"] + assert system["compat"] == {"supportsDeveloperRole": False} and system["models"][0]["contextWindow"] == 32000 + free = bound(api_key_env="NO_SUCH_KEY_VAR", key_required=False)["provider"] + assert "apiKeyEnv" not in free # a server that needs no key: the session names the placeholder + + +def test_dsh_runs_headless_in_its_home_with_its_model_and_the_tools_as_its_own(fake, monkeypatch): + """Without the sandbox a session gets this process's environment: a dsh variable of the caller's (here a folder + of skills) does not reach it, so the session's settings are its own home's.""" + tmp_path = fake + monkeypatch.setenv("DSH_BUNDLED_SKILL_DIR", str(tmp_path / "callers-skills")) + home, work = tmp_path / "home", tmp_path / "work" + work.mkdir() + agent = make("dsh", {**bound(), "effort": "high", "web_search": False, "home": str(home), + "log": str(tmp_path / "agent-dsh.jsonl")}) + result = agent.run("-a prompt that starts with a dash", workspace=work, tools=[echo()]) + [call] = seen(tmp_path) + assert call["args"][:4] == ["--profile", "headless", "--patch", str(home / base_cli.DSH_PATCH)] + assert call["args"][4:9] == ["--json", "--permission-mode", "workspace-write", "--approval-policy", "never"] + assert call["args"][-2:] == ["--", "-a prompt that starts with a dash"] + assert call["env"] == {"DSH_HOME": str(home), "HOME": str(home), "DSH_AGENTS_HOME": str(home / ".agents"), + "DSH_TELEMETRY_DISABLED": "1", base_cli.DSH_NO_KEY: "local", + "FAKE_KEY": "sk-fake", "DSH_BUNDLED_SKILL_DIR": None} # none of the user's own skills + layer = rows(call, home) + assert "sk-fake" not in json.dumps(layer) + assert {r["id"] for r in layer if r.get("disabled")} == {*base_cli.DSH_OFF, "tool-web"} + assert {"tool-subagent", "session-telemetry-otel", "session-title-llm"} <= set(base_cli.DSH_OFF) + assert row(layer, "session-log-deepseek")["config"] == {"enabled": False} + route = row(layer, "llm-pi-ai")["config"]["providers"][base_cli.DSH_PROVIDER] + assert route["apiKeyEnv"] == "FAKE_KEY" and route["models"] == [{"id": "m-1", "reasoningEfforts": {"high": "high"}}] + assert row(layer, "agent-default-model")["config"] == {"provider": base_cli.DSH_PROVIDER, "model": "m-1", + "reasoningEffort": "high"} + [server] = [r for r in layer if "insert" in r] + [entry] = server["insert"] + assert entry["name"] == "@deepseek-ai/dsh-mcp-client" and entry["config"]["serverName"] == "hermit_tools" + assert entry["config"]["transport"] == "streamable-http" and entry["config"]["url"].startswith("http://127.0.0.1:") + assert result.reply == "the answer" and result.exit_code == 0 and result.extra["served"] + assert not result.extra["failed"] and result.session == SESSION + [usage] = result.usage + assert (usage.input_tokens, usage.cached_input_tokens, usage.output_tokens, usage.calls) == (6000, 2400, 120, 6) + assert usage.cost_usd == pytest.approx(0.00408) # dsh reports none: the backend's price + kinds = [s.kind for s in result.trajectory if s.kind != "reasoning"] + assert kinds == ["command", "command", "tool_call", "file_change", "file_change", "message"] + failed, passed, tool = result.trajectory[1], result.trajectory[3], json.loads(result.trajectory[5].text) + assert (failed.command, failed.exit_code, failed.output) == ("echo hi && exit 3", 3, "hi") + assert (passed.command, passed.exit_code, passed.output) == ("echo ok", 0, "ok\n") # no marker: exit 0 + assert tool == {"arguments": {"text": "from dsh"}, "server": "hermit_tools", "tool": "echo"} + assert json.loads(result.trajectory[5].output) == {"echoed": "from dsh"} + assert result.trajectory[0] == trajectory.Step("reasoning", "Thinking it over.") + + +def test_an_effort_is_a_level_the_bound_model_lists_and_off_is_a_model_that_does_not_reason(fake): + """dsh refuses an effort its route does not list for the model (UNSUPPORTED_REASONING_EFFORT), and a list of + `off` alone (INVALID_CONFIG): effort off declares a model that does not reason, and gives no effort.""" + work = fake / "work" + work.mkdir() + agent = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}) + for effort, efforts, default in (("high", {"high": "high"}, "high"), ("off", False, None), (None, None, None)): + agent.configured(effort=effort).run("go", workspace=work) + layer = rows(seen(fake)[-1], fake / "home") + [model] = row(layer, "llm-pi-ai")["config"]["providers"][base_cli.DSH_PROVIDER]["models"] + assert model.get("reasoningEfforts") == efforts + assert row(layer, "agent-default-model")["config"].get("reasoningEffort") == default + + +def test_a_failed_run_is_an_error_and_an_outage_is_retried(fake, monkeypatch): + """dsh exits 1 for a run that did not complete; its error code tells an outage (SERVER, RATE_LIMIT, ...) from + a refusal, so the acting agent runs the task again only when the provider could not serve it.""" + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "failure.jsonl")) + monkeypatch.setenv("FAKE_DSH_EXIT", "1") + work = fake / "work" + work.mkdir() + result = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}).run("go", workspace=work) + assert result.exit_code == 1 and result.extra["failed"] and not result.extra["served"] + assert result.extra["unavailable"] and result.extra["failure"].startswith("dsh: SERVER: 503: ") + assert result.usage == [] # retried requests that used no tokens are no calls + assert [s.kind for s in result.trajectory] == ["error"] and result.trajectory[0].text.startswith("503: ") + refused = (STREAMS / "failure.jsonl").read_text().replace('"SERVER"', '"AUTH"').replace("dsh: SERVER:", + "dsh: AUTH:") + assert not base_cli.UNAVAILABLE.search(base_cli.notices("dsh", refused)) # a bad key is no outage + closed = (STREAMS / "failure.jsonl").read_text().replace("SERVER", "STREAM_CLOSED") # dsh does not retry it + assert base_cli.UNAVAILABLE.search(base_cli.notices("dsh", closed)) + + +def test_a_run_dsh_was_stopped_in_is_no_success_although_dsh_exits_0(fake, monkeypatch, tmp_path): + """dsh exits 0 when SIGTERM stops it: its own rule for a completed run is exit 0 and a completed `final`, so a + run without one has failed (the acting agent records no answer, an AgentEvolver's step is not a done one).""" + cut = tmp_path / "cut.jsonl" + cut.write_text("".join((STREAMS / "success.jsonl").read_text().splitlines(keepends=True)[:12])) + monkeypatch.setenv("FAKE_DSH_STREAM", str(cut)) + work = fake / "work" + work.mkdir() + result = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}).run("go", workspace=work) + assert result.exit_code == 0 and result.extra["failed"] and not result.extra["served"] + assert not result.extra["unavailable"] and result.reply == "" and "did not finish" in result.extra["failure"] + + +def test_a_resumed_session_is_adopted_by_its_id_and_counts_only_its_new_usage(fake, monkeypatch): + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "resume.jsonl")) + home, work = fake / "home", fake / "work" + work.mkdir() + (home / "sessions" / "--run-work--" / SESSION).mkdir(parents=True) + agent = make("dsh", {**bound(), "web_search": False, "home": str(home)}) + result = agent.run("again", workspace=work, session=SESSION) + args = seen(fake)[0]["args"] + assert args[args.index("--session-id") + 1] == SESSION + [usage] = result.usage + assert (usage.input_tokens, usage.calls) == (1000, 1) + agent.run("again", workspace=work, session="session-gone") # no such session kept: a new one + agent.run("again", workspace=work, session="*") # nor a glob + assert all("--session-id" not in call["args"] for call in seen(fake)[1:]) + + +def test_dsh_refuses_roles_and_a_container_and_keeps_web_search_when_asked(fake): + """Roles and a container are refused, never ignored; web search on leaves dsh's web tools as they are.""" + work = fake / "work" + work.mkdir() + agent = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}) + role = {"name": "reviewer", "description": "Review.", "instructions": "Check it."} + with pytest.raises(Unsupported, match="roles"): + agent.run("go", workspace=work, roles=[role]) + with pytest.raises(Unsupported, match="takes isolation none or sandbox"): + agent.configured(isolation="container").run("go", workspace=work) + with pytest.raises(ValueError, match="permission_mode"): + agent.configured(permission_mode="bypass").run("go", workspace=work) + agent.configured(web_search=True).run("go", workspace=work) + assert all(r.get("id") != "tool-web" for r in rows(seen(fake)[-1], fake / "home")) + + +def test_dsh_takes_no_login_and_no_container_wherever_the_run_is_checked(): + """dsh signs in through a model reference (or a model of its own provider): a ChatGPT or Claude login is refused + by the acting agent, the run config and the Evolver's check, never passed on or ignored; a container runs Codex + or Claude Code.""" + from hermit.agent.acting import workspace as acting + from hermit.agent.base.login import evolver_problems + from hermit.config import Config + with pytest.raises(ValueError, match="agent dsh takes no login"): + acting._check("dsh", {"web_search": False, "login": "chatgpt"}) + with pytest.raises(ValueError, match="agent dsh takes isolation: none"): + acting._check("dsh", {"web_search": False, "isolation": "container"}) + assert acting._check("dsh", {"web_search": False}) == "dsh" + assert any("the dsh Evolver signs in through its model reference" in problem + for problem in evolver_problems({"backend": "dsh", "login": "claude"})) + run = ROOT / "docs" / "_examples" / "first_training" / "run.yaml" + fake = "backends={fake: {base_url: 'http://127.0.0.1:9/v1', api_key_env: FAKE_KEY, apis: [chat]}}" + with pytest.raises(ValueError, match="agent dsh takes no login"): + Config.load(run, [fake, "agent.name=null", "agent.base=dsh", "agent.model={ref: fake/m-1, login: chatgpt}", + "agent.params={prompt: go, web_search: false}"]) + + +def test_a_model_that_takes_images_is_declared_so_to_dsh(fake): + """dsh's pi-ai route offers a model it does not know text only, as Pi does: `image_input` (agent.dsh or + evolver.dsh) declares the bound model's images, through the acting agent's and the Evolver's settings.""" + from hermit.agent.acting.workspace import base_agent + from hermit.agent.evolver.loader import find, instantiate + from hermit.config import layered + data = {"agent": {"base": "dsh", "model": "fake/m-1", "dsh": {"image_input": True}}, + "evolver": {"name": "dsh", "base": "dsh", "model": "fake/m-1", "dsh": {"image_input": True}}, "llm": None} + layered(data) + assert base_agent("dsh", {**data["agent"]["params"], **bound(), "web_search": False}).settings["image_input"] + _, params = instantiate(find("dsh", ROOT), data["evolver"]["params"], default_model="fake/m-1", + backends=registry({"fake": {"base_url": "http://127.0.0.1:9/v1", "apis": ["chat"]}})) + assert params["image_input"] is True + work = fake / "work" + work.mkdir() + for image_input, declared in ((True, ["text", "image"]), (False, None)): + make("dsh", {**bound(), "web_search": False, "home": str(fake / "home"), "image_input": image_input}).run( + "go", workspace=work) + route = row(rows(seen(fake)[-1], fake / "home"), "llm-pi-ai")["config"]["providers"][base_cli.DSH_PROVIDER] + assert route["models"][0].get("input") == declared + + +def test_system_instructions_go_at_the_top_of_the_prompt(fake): + """dsh's persona rows interpolate `{{name}}`, which instructions may hold: they are given in the prompt.""" + work = fake / "work" + work.mkdir() + agent = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}) + own, prompt = agent.system_settings("Fill {{name}} in.", "go") + assert own == {} and prompt.startswith("# SYSTEM INSTRUCTIONS\n\nFill {{name}} in.\n\n# END") and \ + prompt.endswith("\n\ngo") + agent.configured(stdin=True).run(prompt, workspace=work) + call = seen(fake)[0] + assert call["args"][-1] == "-" and call["stdin"] == prompt + + +def test_the_workspaces_skills_are_the_sessions_own_and_the_last_calls_copy_is_replaced(fake): + """dsh reads a project's skills only at the project's root (the nearest folder above with a .git), so the + workspace's are copied to the session's own ($DSH_AGENTS_HOME/skills), which it reads wherever it runs; a skill + the last call copied is gone, and a link left in place of the folder is replaced, never written through.""" + home, work = fake / "home", fake / "work" + (work / ".dsh" / "skills" / "check").mkdir(parents=True) + (work / ".dsh" / "skills" / "check" / "SKILL.md").write_text("---\nname: check\ndescription: Check.\n---\n") + (home / ".agents" / "skills" / "old").mkdir(parents=True) + agent = make("dsh", {**bound(), "web_search": False, "home": str(home)}) + agent.run("go", workspace=work) + assert [p.name for p in (home / ".agents" / "skills").iterdir()] == ["check"] + outside = fake / "outside" + outside.mkdir() + shutil.rmtree(home / ".agents") + (home / ".agents").symlink_to(outside) + agent.run("go", workspace=work) + assert not (home / ".agents").is_symlink() and list(outside.iterdir()) == [] + assert (home / ".agents" / "skills" / "check" / "SKILL.md").is_file() + + +def test_a_dsh_stream_is_read_back_by_its_name_or_by_its_events(tmp_path): + """The learning material keeps the acting agent's log: an Evolver reads dsh's steps from it like Codex's.""" + named = tmp_path / "log" + named.mkdir() + shutil.copy(STREAMS / "success.jsonl", named / trajectory.DSH_STREAM) + other = tmp_path / "events.jsonl" + shutil.copy(STREAMS / "success.jsonl", other) + for log in (named, other): + assert [s.kind for s in trajectory.log_steps(log) if s.kind != "reasoning"] == \ + ["command", "command", "tool_call", "file_change", "file_change", "message"] + raw = (STREAMS / "failure.jsonl").read_text() + assert not base_cli.served("dsh", raw) and base_cli.notices("dsh", raw).startswith("dsh: SERVER: 503: ") + assert base_cli.served("dsh", (STREAMS / "success.jsonl").read_text()) + + +def test_a_bash_result_is_read_as_dsh_marks_its_end(): + """dsh marks a nonzero exit (`[exit code: N]`) or a signal after a command's output and nothing after a clean + exit; `(no output)` stands for an empty output, also before a marker. A command stopped by its timeout has no + exit code to trust, even one that trapped the signal and exited 0 (it read as a clean exit before), as a + signal's has none: unknown, as Codex reports it.""" + stream = [json.loads(line) for line in (STREAMS / "bash.jsonl").read_text().splitlines()] + commands = [(s.command, s.exit_code, s.output) for s in trajectory.dsh_steps(stream) if s.kind == "command"] + assert commands == [("exit 3", 3, ""), ("printf ''", 0, ""), ("true", 0, ""), ("echo out; exit 4", 4, "out"), + ("kill -9 $$", None, "(no output)\n[killed by signal: SIGKILL]"), + ("printf 'no newline'; exit 5", 5, "no newline"), + ("trap 'exit 0' TERM; sleep 5 & wait", None, "(no output)\n[timed out after 1000ms]"), + ("sleep 5", None, "(no output)\n[timed out after 1000ms]\n[killed by signal: SIGTERM]")] + + +def test_a_run_that_ends_at_the_token_limit_is_an_error_the_learner_can_read(): + """dsh exits 1 for a run that ends `max-tokens` and writes nothing on stderr: the run is no success and no + outage, and its end is a step of the trajectory and a notice (only errors and aborts were, before).""" + raw = (STREAMS / "max_tokens.jsonl").read_text() + assert not base_cli.served("dsh", raw) and base_cli.notices("dsh", raw) == "run max-tokens" + assert not base_cli.UNAVAILABLE.search(base_cli.notices("dsh", raw)) + steps = trajectory.dsh_steps([json.loads(line) for line in raw.splitlines()]) + assert [(s.kind, s.text) for s in steps] == [("message", "partial answ"), ("error", "run max-tokens")] + + +def test_a_server_that_needs_no_key_gets_the_placeholder_key(fake): + """A route that names no key variable gets no key from pi-ai under the name `hermit`, and dsh refuses every + request (PI_AI_ERROR "No API key"): the session names a variable that holds the placeholder key instead.""" + work = fake / "work" + work.mkdir() + params = bound(api_key_env="NO_SUCH_KEY_VAR", key_required=False) + make("dsh", {**params, "web_search": False, "home": str(fake / "home")}).run("go", workspace=work) + [call] = seen(fake) + route = row(rows(call, fake / "home"), "llm-pi-ai")["config"]["providers"][base_cli.DSH_PROVIDER] + assert route["apiKeyEnv"] == base_cli.DSH_NO_KEY and call["env"][base_cli.DSH_NO_KEY] == "local" + + +def test_the_dsh_acting_agent_deploys_the_adapter_and_loads_its_patch_layer(fake): + """dsh has no project file for MCP servers: the deployment's patch layer (.dsh/hermit.patch.yml) declares the + adapter's tool server, and the session loads it with --patch, before its own layer, which wins.""" + from hermit.agent.acting import workspace as acting + params = {**bound(), "backend": "dsh", "web_search": False, "prompt": "go", "workspaces": "ws"} + out = acting.run({"id": "q1"}, FILE_ADAPTER, params=params, base=fake) + work = Path(out.extra["workspace"]) + [call] = seen(fake) + patches = [call["args"][i + 1] for i, a in enumerate(call["args"]) if a == "--patch"] + assert patches == [str(work / ".dsh" / "hermit.patch.yml"), str(work.parent / "home" / base_cli.DSH_PATCH)] + assert call["args"].index("--json") > call["args"].index(patches[-1]) # the launcher's options come first + [layer] = json.loads((work / ".dsh" / "hermit.patch.yml").read_text()) + [server] = layer["insert"] + assert server["config"]["transport"] == "stdio" and server["config"]["args"][:2] == ["-m", + "hermit.runtime.mcp_server"] + skill = work / ".dsh" / "skills" / "verify-output" / "SKILL.md" + assert (work.parent / "home" / ".agents" / "skills" / "verify-output" / "SKILL.md").read_text() == \ + skill.read_text() # also the session's own: _dsh_skills + assert "do it yourself" in (work / "AGENTS.md").read_text() # roles stay files: none is passed to dsh + assert out.output == "the answer" and out.extra["backend"] == "dsh" + assert (out.log / "agent-dsh.jsonl").is_file() + + +def test_dsh_is_a_base_of_the_config_and_the_dsh_evolver_binds_its_model_with_its_sandbox_off(): + """In the evolver sandbox the run's isolation is the boundary, so dsh's own sandbox is off, as Codex's is.""" + from hermit.agent.evolver.agent import AgentEvolver + from hermit.agent.evolver.loader import find, instantiate + from hermit.config import layered + data = {"agent": {"base": "dsh", "model": {"ref": "fake/m-1", "effort": "high"}, "dsh": {"timeout_s": 60}, + "params": {"prompt": "go"}}, + "evolver": {"name": "dsh", "base": "dsh", "model": "fake/m-1", "params": {}}, "llm": None} + layered(data) + assert data["agent"]["name"] == "dsh" and data["agent"]["params"]["effort"] == "high" + assert data["agent"]["params"]["timeout_s"] == 60 and data["evolver"]["params"]["backend"] == "dsh" + spec = find("dsh", ROOT) + backends = registry({"fake": {"base_url": "http://127.0.0.1:9/v1", "api_key_env": "FAKE_KEY", "apis": ["chat"]}}) + _, params = instantiate(spec, data["evolver"]["params"], backends=backends, default_model="fake/m-1") + assert params["backend"] == "dsh" and params["home"] == [] and params["provider"]["apiKeyEnv"] == "FAKE_KEY" + + class Ctx: # the fields AgentEvolver.settings reads + def __init__(self, root): + self.params, self.sandbox, self.cancel = params, None, None + self.workspace = type("W", (), {"state": root / "state", "output": root / "output"}) + settings = AgentEvolver().settings(Ctx(Path("/w")), "dsh") + assert settings["permission_mode"] == "danger-full-access" and settings["provider"] == params["provider"] + assert "permission_mode" not in AgentEvolver().settings(Ctx(Path("/w")), "claude") + + +# ---------- the installed dsh (integration) ---------- + +def serve_chat(script, keys=None): + """A chat-completions model on the loopback: `script(request body) -> (tool call (name, arguments) | None)`; + None answers `the answer`. Every response is streamed, with usage. `keys` collects each request's + Authorization header.""" + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + def chunk(body): + return f"data: {json.dumps(body)}\n\n".encode() + + class Model(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers.get("content-length") or 0)) or b"{}") + if keys is not None: + keys.append(self.headers.get("authorization")) + call = script(body) + delta = {"role": "assistant", "tool_calls": [{"index": 0, "id": "call_1", "type": "function", + "function": {"name": call[0], + "arguments": json.dumps(call[1])}}]} \ + if call else {"role": "assistant", "content": "the answer"} + data = b"".join([chunk({"choices": [{"index": 0, "delta": delta, "finish_reason": None}]}), + chunk({"choices": [{"index": 0, "delta": {}, "finish_reason": + "tool_calls" if call else "stop"}]}), + chunk({"choices": [], "usage": {"prompt_tokens": 100, "completion_tokens": 5}}), + b"data: [DONE]\n\n"]) + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Model) + threading.Thread(target=server.serve_forever, daemon=True).start() + return server, f"http://127.0.0.1:{server.server_address[1]}/v1" + + +def _bwrap_works() -> bool: + return bool(shutil.which("bwrap")) and subprocess.run( + ["bwrap", "--ro-bind", "/", "/", "--unshare-net", "true"], capture_output=True).returncode == 0 + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +@pytest.mark.parametrize("isolation", ["none", "sandbox"]) +def test_with_the_dsh_cli_a_session_reaches_its_model_and_its_tools(tmp_path, monkeypatch, isolation): + """The installed dsh, bound to a scripted model: it calls the call's MCP tool, writes a file, answers; in the + run's sandbox (isolated network) only its model and the tool server are reached.""" + from hermit.sandbox import Sandbox + if isolation == "sandbox" and not _bwrap_works(): + pytest.skip("needs a working bubblewrap") + requests = [] + + def script(body): + requests.append(body) + done = sum(1 for m in body.get("messages") or [] if m.get("role") == "tool") + names = [t["function"]["name"] for t in body.get("tools") or []] + plan = [("mcp__hermit_tools__echo", {"text": "hi"}), ("write", {"file_path": "note.txt", "content": "ok\n"})] + assert plan[0][0] in names and "web_search" not in names # the tool offered as dsh's own; no web tools + return plan[done] if done < len(plan) else None + + server, url = serve_chat(script) + monkeypatch.setenv("MOCK_KEY", "sk-mock") + params = bind_agent_params({"model": "mock/m-1"}, "dsh", + registry({"mock": {"base_url": url, "api_key_env": "MOCK_KEY", "apis": ["chat"]}})) + work = tmp_path / "work" + work.mkdir() + settings = {**params, "web_search": False, "home": str(tmp_path / "home"), "isolation": isolation, + "timeout_s": 300} + if isolation == "sandbox": + settings["sandbox"] = Sandbox(network="isolated", env_allow=[], writable=[work], role="agent", + record_dir=tmp_path / "records") + try: + result = make("dsh", settings).run("Do the task.", workspace=work, tools=[echo()]) + finally: + server.shutdown() + assert result.exit_code == 0 and result.extra["served"] and result.reply == "the answer" + assert (work / "note.txt").read_text() == "ok\n" + assert [s.kind for s in result.trajectory] == ["tool_call", "file_change", "message"] + assert json.loads(result.trajectory[0].output) == {"echoed": "hi"} + assert result.usage[0].calls == 3 == len(requests) # no request besides the session's own + if isolation == "sandbox": + [record] = (tmp_path / "records").glob("agent-*.json") + data = json.loads(record.read_text()) + assert data["network"] == "isolated" and data["probe"]["passed"] + assert {"DSH_HOME", "HOME", "MOCK_KEY"} <= set(data["environment"]) + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +@pytest.mark.skipif(not shutil.which("git"), reason="needs git") +def test_with_the_dsh_cli_a_deployed_adapter_brings_its_skills_and_tools_into_a_git_repository(tmp_path, monkeypatch): + """The acting agent with the installed dsh, its workspaces inside a git repository (a config kept in a project). + dsh takes the repository for the project and reads project skills only from its .dsh/skills, so the deployed + skills reach it as the session's own (_dsh_skills; before, they were not offered there). The adapter's own tool + server (the deployment's stdio server, loaded from its patch layer) answers a call.""" + from hermit.agent.acting import workspace as acting + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", str(repo)], check=True) + requests = [] + + def script(body): + requests.append(body) + done = sum(1 for m in body.get("messages") or [] if m.get("role") == "tool") + return ("mcp__file_example__count_words", {"text": "one two three"}) if done == 0 else None + + server, url = serve_chat(script) + monkeypatch.setenv("MOCK_KEY", "sk-mock") + params = {**bind_agent_params({"model": "mock/m-1"}, "dsh", + registry({"mock": {"base_url": url, "api_key_env": "MOCK_KEY", "apis": ["chat"]}})), + "backend": "dsh", "web_search": False, "prompt": "go", "workspaces": "ws", "timeout_s": 300, + "retries": 1} + try: + out = acting.run({"id": "q1"}, FILE_ADAPTER, params=params, base=repo) + finally: + server.shutdown() + assert out.output == "the answer" and len(requests) == 2 + assert "Check a completed result against the task requirements." in json.dumps(requests[0]) # the skill + assert "mcp__file_example__count_words" in [t["function"]["name"] for t in requests[0].get("tools") or []] + [answer] = [m for m in requests[1]["messages"] if m.get("role") == "tool"] + assert json.loads(answer["content"] if isinstance(answer["content"], str) else + "".join(p.get("text", "") for p in answer["content"])) == {"count": 3} + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +def test_with_the_dsh_cli_a_server_that_needs_no_key_is_reached_with_the_placeholder(tmp_path, monkeypatch): + """A backend that needs no key (a local server): the installed dsh refused every request of a route that named + no key variable (PI_AI_ERROR "No API key"); the session's placeholder reaches the server, and a key of another + provider in the environment (OPENAI_API_KEY) is never sent.""" + keys = [] + server, url = serve_chat(lambda body: None, keys) + monkeypatch.setenv("OPENAI_API_KEY", "sk-another-provider") + params = bind_agent_params({"model": "local/m-1"}, "dsh", registry( + {"local": {"base_url": url, "api_key_env": "NO_SUCH_KEY_VAR", "key_required": False, "apis": ["chat"]}})) + work = tmp_path / "work" + work.mkdir() + try: + result = make("dsh", {**params, "web_search": False, "home": str(tmp_path / "home"), + "timeout_s": 300}).run("Answer.", workspace=work) + finally: + server.shutdown() + assert result.exit_code == 0 and result.reply == "the answer" and keys == ["Bearer local"] + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +def test_the_installed_dsh_has_every_row_the_session_patches(tmp_path): + """A patch row whose id dsh does not have is ignored, and dsh says so only when asked (--dump-config, which + starts nothing and reaches no network): every row the session's layer names (telemetry, title, sub-agents, the + web tools, the route, the default model) is one of the installed dsh's, so none is silently left on.""" + home = tmp_path / "home" + home.mkdir() + agent = make("dsh", {**bound(), "effort": "high", "web_search": False}) + agent.prepare_home(home, (), "http://127.0.0.1:9/mcp", "hermit_tools") + layer = json.loads((home / base_cli.DSH_PATCH).read_text()) + assert {r["id"] for r in layer if "id" in r} >= {*base_cli.DSH_OFF, "session-log-deepseek", "tool-web", + "llm-pi-ai", "agent-default-model"} + work = tmp_path / "work" + work.mkdir() + env = {**os.environ, **agent.home_env(home)} + done = subprocess.run(["dsh", "--profile", "headless", "--patch", str(home / base_cli.DSH_PATCH), + "--dump-config"], cwd=work, env=env, capture_output=True, text=True, timeout=120) + assert done.returncode == 0, done.stderr + assert "not found" not in done.stderr and "id: llm-pi-ai" in done.stdout diff --git a/tests/core/test_sandbox.py b/tests/core/test_sandbox.py index 3298a4c4..8fb696ac 100644 --- a/tests/core/test_sandbox.py +++ b/tests/core/test_sandbox.py @@ -362,7 +362,7 @@ def test_a_uv_alias_inside_a_granted_checkout_is_bound_once(tmp_path, monkeypatc @pytest.mark.integration # the CLI installed on the machine @pytest.mark.skipif(not _bwrap_works(), reason="needs a working bubblewrap (unprivileged user namespaces)") -@pytest.mark.parametrize("cli", ["codex", "claude", "pi"]) +@pytest.mark.parametrize("cli", ["codex", "claude", "pi", "dsh"]) def test_an_installed_cli_starts_with_only_its_own_paths(cli): """The built-in CLI agents: the CLI as installed (an nvm package, a ~/bin wrapper) starts inside, with no home.""" from hermit.sandbox import program_paths diff --git a/tests/core/test_selection.py b/tests/core/test_selection.py index 4a6d33aa..a3c6c505 100644 --- a/tests/core/test_selection.py +++ b/tests/core/test_selection.py @@ -8,7 +8,7 @@ from selection import METHODS, READ_BY, NoBase, changed_paths, groups_for, select ROOT = Path(__file__).resolve().parents[2] -BUILT_IN = {"codex", "claude", "pi"} # the generic AgentEvolvers: tests/core +BUILT_IN = {"codex", "claude", "pi", "dsh"} # the generic AgentEvolvers: tests/core NO_METHOD = {"prompt_only", "lean_collab"} # templates without an Evolver of their own: tests/core diff --git a/tests/fixtures/dsh/README.md b/tests/fixtures/dsh/README.md new file mode 100644 index 00000000..6929400d --- /dev/null +++ b/tests/fixtures/dsh/README.md @@ -0,0 +1,27 @@ +# DeepSeek Harness streams + +Recorded from DeepSeek Harness 0.2.2-rc.1 (`npm install -g @deepseek-ai/dsh@0.2.2-rc.1`, on Node 24.22.2, Linux) run +by hermit's `dsh` Base Agent (`make("dsh").run`, isolation none), against a scripted chat-completions model on the +loopback, bound as a `/` reference with `effort: high`; the model streams a `reasoning_content` +delta before each answer when it is asked for a reasoning effort. + +- `success.jsonl`: a bash call that exits 3, a bash call that exits 0, a call of the call's MCP tool `echo` + (`hermit_tools`), a write, an edit, then the answer `the answer`; six requests of 1000 prompt tokens (400 cached) + and 20 output tokens each. +- `resume.jsonl`: the same session resumed (`--session-id`): one request. +- `failure.jsonl`: every request answered 503; dsh retries five times, ends the run with the error `SERVER`, writes + `dsh: SERVER: ...` on stderr (the last line, which the Base Agent keeps in the stream) and exits 1. + +Recorded by running dsh directly (`dsh --profile headless --json --permission-mode danger-full-access`, with a patch +layer like the Base Agent's) against a scripted chat-completions model: + +- `bash.jsonl`: how dsh's bash tool marks a command's end. The model calls bash eight times: `exit 3`, `printf ''`, + `true`, `echo out; exit 4`, `kill -9 $$`, `printf 'no newline'; exit 5`, then with `timeoutMs: 1000` + `trap 'exit 0' TERM; sleep 5 & wait` (stopped by its timeout, it exits 0) and `sleep 5` (killed), then answers + `done`. The layer also sets the `tool-bash` row's `promoteOnTimeout: false`, so a timeout stops the command rather + than moving it to a background job. +- `max_tokens.jsonl`: the model's answer ends at its token limit (`finish_reason: length`): the run ends `max-tokens` + and dsh exits 1, writing nothing on stderr. + +The streams are dsh's output unchanged; the run's directory was `/run` (`/run/bash` and `/run/max_tokens` for the +last two). diff --git a/tests/fixtures/dsh/bash.jsonl b/tests/fixtures/dsh/bash.jsonl new file mode 100644 index 00000000..ca3b051d --- /dev/null +++ b/tests/fixtures/dsh/bash.jsonl @@ -0,0 +1,39 @@ +{"type":"session","sessionId":"session-b0463c76-d042-46aa-89d0-92ad134f0e29","cwd":"/run/bash/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"tool_call","callId":"call_0","tool":"bash","input":{"command":"exit 3","description":"edge"}} +{"type":"tool_result","callId":"call_0","status":"completed","result":"(no output)\n[exit code: 3]"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":2} +{"type":"tool_call","callId":"call_1","tool":"bash","input":{"command":"printf ''","description":"edge"}} +{"type":"tool_result","callId":"call_1","status":"completed","result":"(no output)"} +{"type":"status","phase":"step_end","turn":1,"step":2,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":3} +{"type":"tool_call","callId":"call_2","tool":"bash","input":{"command":"true","description":"edge"}} +{"type":"tool_result","callId":"call_2","status":"completed","result":"(no output)"} +{"type":"status","phase":"step_end","turn":1,"step":3,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":4} +{"type":"tool_call","callId":"call_3","tool":"bash","input":{"command":"echo out; exit 4","description":"edge"}} +{"type":"tool_result","callId":"call_3","status":"completed","result":"out\n[exit code: 4]"} +{"type":"status","phase":"step_end","turn":1,"step":4,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":5} +{"type":"tool_call","callId":"call_4","tool":"bash","input":{"command":"kill -9 $$","description":"edge"}} +{"type":"tool_result","callId":"call_4","status":"completed","result":"(no output)\n[killed by signal: SIGKILL]"} +{"type":"status","phase":"step_end","turn":1,"step":5,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":6} +{"type":"tool_call","callId":"call_5","tool":"bash","input":{"command":"printf 'no newline'; exit 5","description":"edge"}} +{"type":"tool_result","callId":"call_5","status":"completed","result":"no newline\n[exit code: 5]"} +{"type":"status","phase":"step_end","turn":1,"step":6,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":7} +{"type":"tool_call","callId":"call_6","tool":"bash","input":{"command":"trap 'exit 0' TERM; sleep 5 & wait","timeoutMs":1000,"description":"edge"}} +{"type":"tool_result","callId":"call_6","status":"completed","result":"(no output)\n[timed out after 1000ms]"} +{"type":"status","phase":"step_end","turn":1,"step":7,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":8} +{"type":"tool_call","callId":"call_7","tool":"bash","input":{"command":"sleep 5","timeoutMs":1000,"description":"edge"}} +{"type":"tool_result","callId":"call_7","status":"completed","result":"(no output)\n[timed out after 1000ms]\n[killed by signal: SIGTERM]"} +{"type":"status","phase":"step_end","turn":1,"step":8,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":9} +{"type":"text","text":"done"} +{"type":"status","phase":"step_end","turn":1,"step":9,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-b0463c76-d042-46aa-89d0-92ad134f0e29","reason":{"kind":"completed"},"text":"done"} diff --git a/tests/fixtures/dsh/failure.jsonl b/tests/fixtures/dsh/failure.jsonl new file mode 100644 index 00000000..61c0dc84 --- /dev/null +++ b/tests/fixtures/dsh/failure.jsonl @@ -0,0 +1,7 @@ +{"type":"session","sessionId":"session-23637470-e079-42ef-bbc7-7a40042ccad7","cwd":"/run/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":0,"outputTokens":0,"totalTokens":0}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"error","error":{"message":"503: {\"message\":\"the service is unavailable\"}","code":"SERVER"}}} +{"type":"final","sessionId":"session-23637470-e079-42ef-bbc7-7a40042ccad7","reason":{"kind":"error","error":{"message":"503: {\"message\":\"the service is unavailable\"}","code":"SERVER"}},"text":""} +dsh: SERVER: 503: {"message":"the service is unavailable"} diff --git a/tests/fixtures/dsh/max_tokens.jsonl b/tests/fixtures/dsh/max_tokens.jsonl new file mode 100644 index 00000000..d09e0102 --- /dev/null +++ b/tests/fixtures/dsh/max_tokens.jsonl @@ -0,0 +1,7 @@ +{"type":"session","sessionId":"session-5411e0f3-4792-4c94-84b0-bbd018ccbf6b","cwd":"/run/max_tokens/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"text","text":"partial answ"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"max-tokens"}} +{"type":"final","sessionId":"session-5411e0f3-4792-4c94-84b0-bbd018ccbf6b","reason":{"kind":"max-tokens"},"text":"partial answ"} diff --git a/tests/fixtures/dsh/resume.jsonl b/tests/fixtures/dsh/resume.jsonl new file mode 100644 index 00000000..ec52c0ad --- /dev/null +++ b/tests/fixtures/dsh/resume.jsonl @@ -0,0 +1,8 @@ +{"type":"session","sessionId":"session-b2fbf9bf-a321-4065-8f8a-ee344b66b5f3","cwd":"/run/work"} +{"type":"status","phase":"turn_start","turn":2} +{"type":"status","phase":"step_start","turn":2,"step":1} +{"type":"thinking","text":"Thinking it over."} +{"type":"text","text":"the answer"} +{"type":"status","phase":"step_end","turn":2,"step":1,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"turn_end","turn":2,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-b2fbf9bf-a321-4065-8f8a-ee344b66b5f3","reason":{"kind":"completed"},"text":"the answer"} diff --git a/tests/fixtures/dsh/success.jsonl b/tests/fixtures/dsh/success.jsonl new file mode 100644 index 00000000..617182b3 --- /dev/null +++ b/tests/fixtures/dsh/success.jsonl @@ -0,0 +1,33 @@ +{"type":"session","sessionId":"session-b2fbf9bf-a321-4065-8f8a-ee344b66b5f3","cwd":"/run/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"thinking","text":"Thinking it over."} +{"type":"tool_call","callId":"call_1","tool":"bash","input":{"command":"echo hi && exit 3","description":"Print hi and fail"}} +{"type":"tool_result","callId":"call_1","status":"completed","result":"hi\n[exit code: 3]"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"step_start","turn":1,"step":2} +{"type":"thinking","text":"Thinking it over."} +{"type":"tool_call","callId":"call_2","tool":"bash","input":{"command":"echo ok","description":"Print ok"}} +{"type":"tool_result","callId":"call_2","status":"completed","result":"ok\n"} +{"type":"status","phase":"step_end","turn":1,"step":2,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"step_start","turn":1,"step":3} +{"type":"thinking","text":"Thinking it over."} +{"type":"tool_call","callId":"call_3","tool":"mcp__hermit_tools__echo","input":{"text":"from dsh"}} +{"type":"tool_result","callId":"call_3","status":"completed","result":"{\"echoed\": \"from dsh\"}"} +{"type":"status","phase":"step_end","turn":1,"step":3,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"step_start","turn":1,"step":4} +{"type":"thinking","text":"Thinking it over."} +{"type":"tool_call","callId":"call_4","tool":"write","input":{"file_path":"note.txt","content":"ok\n"}} +{"type":"tool_result","callId":"call_4","status":"completed","result":"/run/work/note.txt\nfile\n\nCreated file\n"} +{"type":"status","phase":"step_end","turn":1,"step":4,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"step_start","turn":1,"step":5} +{"type":"thinking","text":"Thinking it over."} +{"type":"tool_call","callId":"call_5","tool":"edit","input":{"file_path":"note.txt","old_string":"ok","new_string":"fine"}} +{"type":"tool_result","callId":"call_5","status":"completed","result":"The file /run/work/note.txt has been updated successfully."} +{"type":"status","phase":"step_end","turn":1,"step":5,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"step_start","turn":1,"step":6} +{"type":"thinking","text":"Thinking it over."} +{"type":"text","text":"the answer"} +{"type":"status","phase":"step_end","turn":1,"step":6,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-b2fbf9bf-a321-4065-8f8a-ee344b66b5f3","reason":{"kind":"completed"},"text":"the answer"} diff --git a/tests/selection.py b/tests/selection.py index 26a578b8..66f96f36 100644 --- a/tests/selection.py +++ b/tests/selection.py @@ -67,7 +67,7 @@ def fnmatch(path: str, pattern: str) -> bool: "tests/claude_stand.py", "tests/fixtures/claude_streams/*", # the Claude stand-in: core's and the methods' ".github/workflows/tests.yml"] # how CI runs them: a change to it runs them all # the rest of the framework, the templates no method group owns, the documentation's examples: the core group -CORE = ["hermit/*", "tests/core/*", "tests/fixtures/pi/*", "docs/_examples/*"] +CORE = ["hermit/*", "tests/core/*", "tests/fixtures/pi/*", "tests/fixtures/dsh/*", "docs/_examples/*"] # nothing to run: documentation, and files no test reads NONE = ["docs/*", "*.md", "tests/README.md", "LICENSE", "NOTICE", ".github/*", ".agents/*", "assets/*", ".gitignore"] From 7a076fb88609aaddaa1151258ea4a30326c2df76 Mon Sep 17 00:00:00 2001 From: Albert Zhao Date: Sun, 8 Nov 2026 21:16:18 -0500 Subject: [PATCH 02/12] dsh: fixes from review round 1 (two independent reviewers on 03fafff) Each fix was checked against dsh's source at dsh-v0.2.2-rc.1 and, where it shows, against the installed dsh (new recorded streams in tests/fixtures/dsh); each has a test that fails without it. - Stream reading: a bash result dsh's --json cut at 8 KiB (`truncated`) lost its exit marker and read as exit 0; now unknown, and the text says it was cut. A command started in the background has no status yet (unknown). The end markers are anchored at the very end (\Z), as dsh's JavaScript reads them, so a command that printed a marker of its own and exited 0 is a clean exit. - Usage: a step that answered counts as a call even when the provider reported no usage (dsh reports zeros), and so does a step dsh reports no usage for (an attempt without any); what dsh never reports (compaction, a step's partial usage) is documented. parse_log reads Pi's and dsh's streams with their own readers. - outcome(): a session whose tools did not start (dsh writes the MCP row's error on stderr and runs on without them) has failed. A run that wrote only an `error` event (a usage error) or nothing (a working directory's .env that sets PATH, PYTHONPATH, ...) started no session, so the acting agent reports it instead of recording each task as a wrong answer: outcome() may now say `started` (shared runner; absent, as before). - The workspace is the project wherever it lies: the session's layer sets the skill-filesystem row to the workspace's .dsh/skills alone and the agent-instructions row to the deployment's marker (.hermit-deploy.json) as the project root, so a workspace inside a git repository gets its own skills and none of the repository's skills or AGENTS.md (they outranked the adapter's). This replaces copying the skills to the home. - `--add-dir` takes the real path of an existing directory, as dsh requires (a link failed every command and write); an effort without a model is refused; user_home() follows DSH_HOME. - The acting agent passes agent.dsh.permission_mode on (it was dropped) and refuses a mode dsh has not. - hermit deploy --host dsh: the patch layer also names the workspace's .dsh/skills (a workspace below a repository's root kept none), and a context past the 65,536 bytes dsh reads of a project's instructions is refused. - The AHE template has a `dsh` wire (as Pi's: hooks run by hand hand the memory over); mwm and RRSI refuse a backend other than Codex or Claude Code at setup (pi and dsh were accepted and ran without their model). - Docstrings say what dsh reads (.env files, the instruction chain) and sends (its own provider's ids). Co-Authored-By: Claude Opus 5.5 --- hermit/adapter/templates/ahe/hook_hosts.py | 16 +- hermit/agent/acting/workspace.py | 7 + hermit/agent/base/cli.py | 104 ++++++---- hermit/agent/base/records.py | 45 ++-- hermit/agent/base/trajectory.py | 47 +++-- hermit/agent/evolver/methods/mwm/evolver.py | 3 + .../evolver/methods/rrsi_adapter/evolver.py | 3 + hermit/hosts/dsh.py | 39 ++-- tests/core/test_context_tool.py | 4 +- tests/core/test_deploy.py | 17 +- tests/core/test_dsh_agent.py | 193 +++++++++++++++--- tests/core/test_isolate.py | 4 + tests/fixtures/dsh/README.md | 14 +- tests/fixtures/dsh/background.jsonl | 11 + tests/fixtures/dsh/bad_option.jsonl | 1 + tests/fixtures/dsh/big_bash.jsonl | 11 + tests/fixtures/dsh/dotenv.jsonl | 12 ++ tests/fixtures/dsh/fake_marker.jsonl | 11 + tests/fixtures/dsh/no_usage.jsonl | 7 + tests/fixtures/dsh/tools_down.jsonl | 30 +++ tests/methods/ahe/test_ahe_hosts.py | 23 ++- tests/methods/mwm/test_mwm_evolver.py | 10 + .../methods/rrsi_adapter/test_rrsi_adapter.py | 3 + 23 files changed, 474 insertions(+), 141 deletions(-) create mode 100644 tests/fixtures/dsh/background.jsonl create mode 100644 tests/fixtures/dsh/bad_option.jsonl create mode 100644 tests/fixtures/dsh/big_bash.jsonl create mode 100644 tests/fixtures/dsh/dotenv.jsonl create mode 100644 tests/fixtures/dsh/fake_marker.jsonl create mode 100644 tests/fixtures/dsh/no_usage.jsonl create mode 100644 tests/fixtures/dsh/tools_down.jsonl diff --git a/hermit/adapter/templates/ahe/hook_hosts.py b/hermit/adapter/templates/ahe/hook_hosts.py index 2c0cdae5..9f912f34 100644 --- a/hermit/adapter/templates/ahe/hook_hosts.py +++ b/hermit/adapter/templates/ahe/hook_hosts.py @@ -16,9 +16,9 @@ the wires, and the same file re-runs both CLIs on a scripted model when they are installed and `--integration` is given. -Pi reads no hook file: its deployment lists the hook commands in AGENTS.md for the agent to run by hand. Run so, with -no payload (the `pi` wire), session_start still hands over the memory; the other hooks have no tool call, prompt or -stop to act on. +Pi and DeepSeek Harness read no hook file: their deployments list the hook commands in AGENTS.md for the agent to +run by hand. Run so, with no payload (the `pi` and `dsh` wires), session_start still hands over the memory; the other +hooks have no tool call, prompt or stop to act on. Stdlib only, beside hook_runner.py in the adapter's copy: nothing of the framework is installed where the agent runs. """ @@ -480,8 +480,14 @@ class Pi(Wire): tool_kinds = ClaudeCode.tool_kinds -CODEX, CLAUDE_CODE, PI = Codex(), ClaudeCode(), Pi() -HOSTS = {CODEX.name: CODEX, CLAUDE_CODE.name: CLAUDE_CODE, PI.name: PI} +class Dsh(Pi): + """DeepSeek Harness runs no hook file of its own either: its deployment lists each hook command in AGENTS.md, as + Pi's does, and a command run so has no payload.""" + name, label = "dsh", "DeepSeek Harness" + + +CODEX, CLAUDE_CODE, PI, DSH = Codex(), ClaudeCode(), Pi(), Dsh() +HOSTS = {CODEX.name: CODEX, CLAUDE_CODE.name: CLAUDE_CODE, PI.name: PI, DSH.name: DSH} HOOK_HOSTS = (CODEX.name, CLAUDE_CODE.name) # the hosts that run the hooks themselves diff --git a/hermit/agent/acting/workspace.py b/hermit/agent/acting/workspace.py index 9efbf48e..ad27e1c8 100644 --- a/hermit/agent/acting/workspace.py +++ b/hermit/agent/acting/workspace.py @@ -289,6 +289,11 @@ def _check(backend: str, params: dict) -> str: + (", or a model of dsh's own provider" if backend == "dsh" else "")) if params.get("isolation") == "container": raise ValueError(f"isolation: container runs Codex or Claude Code; agent {backend} takes isolation: none") + if backend == "dsh" and params.get("permission_mode") is not None: + from hermit.agent.base.cli import DSH_PERMISSION_MODES + if params["permission_mode"] not in DSH_PERMISSION_MODES: + raise ValueError(f"agent.dsh.permission_mode is one of {', '.join(DSH_PERMISSION_MODES)} (dsh's own " + f"sandbox), not {params['permission_mode']!r}") return backend login, label = params.get("login"), {"codex": "Codex", "claude": "Claude Code"}[backend] if login is not None and login not in LOGINS: @@ -318,6 +323,8 @@ def base_agent(backend: str, params: dict) -> BaseAgent: if k in params and (params[k] is not None or k == "plan_max_wait_s")} if params.get("claude_permission_mode"): settings["permission_mode"] = params["claude_permission_mode"] + if backend == "dsh" and params.get("permission_mode"): # dsh's own sandbox (agent.dsh.permission_mode) + settings["permission_mode"] = params["permission_mode"] return make(backend, {**settings, "web_search": params["web_search"]}) diff --git a/hermit/agent/base/cli.py b/hermit/agent/base/cli.py index aedd8ae2..ac4980cf 100644 --- a/hermit/agent/base/cli.py +++ b/hermit/agent/base/cli.py @@ -658,7 +658,8 @@ def prompt_text(self, prompt: str) -> str: def outcome(self, raw: str, stream: Path, last_file: Path) -> dict: """What the session's stream says: `reply`, `served`, `unavailable` (a provider that could not serve), `refused` - (the plan's limit), and for what stops short of that `failed`, `failure` (why) and the rest of Result.extra.""" + (the plan's limit), and for what stops short of that `failed`, `failure` (why) and the rest of Result.extra; + `started` for a CLI that writes an event before it has a session (absent: it started once it wrote one).""" raise NotImplementedError # ---------- the home ---------- @@ -966,6 +967,7 @@ def _session(self, prompt, *, workspace=None, tools: Sequence[Tool] = (), limits raw = readable.read_text(encoding="utf-8", errors="replace") if readable is stream else "" said = self.outcome(raw, readable, last_file) said["failed"] = said.get("failed", False) and not timed_out and not cancelled + started = said.pop("started", None) # the CLI's own word, else: it wrote an event handle, usage = self.parse(readable, (s.get("binding") or {}).get("ref") or s.get("model") or self.backend) usage = self.adjust_usage(home, handle, usage, resumed=bool(session)) price_usage(usage, s) @@ -975,7 +977,8 @@ def _session(self, prompt, *, workspace=None, tools: Sequence[Tool] = (), limits return Result(reply=said.pop("reply"), finish_reason=said.pop("finish_reason", None), trajectory=steps, usage=usage, session=handle, exit_code=code, timed_out=timed_out, log=stream, extra={"failed": False, "login_failed": False, "rejected": False, **said, - "started": any(line.lstrip().startswith("{") for line in raw.splitlines()), + "started": any(line.lstrip().startswith("{") for line in raw.splitlines()) + if started is None else bool(started), "cancelled": cancelled, "isolation": isolation, "wall_s": round(wall_s, 1), "mounts": placed, "output_tail": raw[-2000:]}) @@ -1472,16 +1475,20 @@ def _pi_session(home: Path, handle) -> Path | None: DSH_PROVIDER = "hermit" # the pi-ai route (`llm-pi-ai`) a bound model is given under DSH_ROUTE = "deepseek-official" # dsh's own route, for a model of its own provider (no binding) DSH_PATCH = "hermit.patch.yml" # the session's patch layer, in its home (`--patch`) -# Rows of dsh's tree off in every session: no telemetry export, no title request and nothing sent along with a -# request to DeepSeek's own API (its plugin list; the session log, a row of its own below), so the session's requests -# are the only ones and carry only the session; and no sub-agents or workflows, whose usage dsh's stream does not -# report (sub-agents are Codex's only when a role asks for them) +# Rows of dsh's tree off in every session: no telemetry export, no title request, no plugin list sent along with a +# request to DeepSeek's own API (nor the session log, a row of its own below; such a request still carries dsh's +# anonymous user id and the session's id), so the session's requests are the only ones; and no sub-agents or +# workflows, whose usage dsh's stream does not report (sub-agents are Codex's only when a role asks for them) DSH_OFF = ("session-telemetry-otel", "session-title-llm", "plugin-package-inventory-deepseek", "tool-subagent", "tool-subagent-fork", "tool-subagent-control", "tool-workflow") DSH_PERMISSION_MODES = ("read-only", "workspace-write", "danger-full-access") # The variable a route names for a server that needs no key: a route that names none gets no key from pi-ai under # the name `hermit`, and dsh refuses the request (PI_AI_ERROR "No API key") DSH_NO_KEY = "HERMIT_DSH_NO_KEY" +DSH_INSTRUCTION_BYTES = 65536 # dsh's cap on a chain of instruction files (agent-instructions' maxBytes) +DSH_PROJECT_MARKER = ".hermit-deploy.json" # what a deployment writes at its workspace's root (hermit.hosts.inputs) +# dsh's report of an MCP row that did not start (` (): `): the session then runs without its tools +DSH_TOOLS_DOWN = re.compile(r"^(mcp-[A-Za-z0-9_-]{1,32}) \(@deepseek-ai/dsh-mcp-client\): (.*)$", re.M) class Dsh(CliAgent): @@ -1490,11 +1497,11 @@ class Dsh(CliAgent): of its plugin tree, `--patch`), written in the home: a bound model as the pi-ai route `hermit`, its key read by dsh from the backend's variable, made the default model; the call's tools as an MCP server, whose tools the model is offered as its own (`mcp____`); web search and fetch off unless asked for; no telemetry export - and no title request, no sub-agents (DSH_OFF). dsh runs commands in its own sandbox (`permission_mode`, default - workspace-write) and is never asked for an approval (`--approval-policy never`: what would need one is - refused). It exits 0 for a run SIGTERM stopped too: whether the run failed is read from its end (`outcome`). - The workspace's project skills (.dsh/skills) are also the session's own (`_dsh_skills`), so that a workspace - inside a git repository keeps them.""" + and no title request, no sub-agents (DSH_OFF); the workspace as the project, as in the run's sandbox: its own + skills (.dsh/skills) and instructions only, never those of a repository it lies in. dsh runs commands in its own + sandbox (`permission_mode`, default workspace-write) and is never asked for an approval (`--approval-policy + never`: what would need one is refused). It exits 0 for a run SIGTERM stopped too: whether the run failed is read + from its end (`outcome`).""" name = backend = program = "dsh" label = "DeepSeek Harness" home_var, user_folder = "DSH_HOME", ".dsh" @@ -1502,23 +1509,28 @@ class Dsh(CliAgent): turn_done = "final" def home_env(self, home): - """Every variable dsh reads goes here, so that it passes the sandbox's filter: its home, HOME, and the shared - agents' folder dsh reads skills and AGENTS.md from (~/.agents unless DSH_AGENTS_HOME says another), so none of - the user's own is read; telemetry off (no anonymous id is made either); the placeholder key of a server that - needs none (DSH_NO_KEY).""" + """The variables that set where dsh reads its own files from, here, so that they pass the sandbox's filter: + its home, HOME, and the shared agents' folder dsh reads skills and AGENTS.md from (~/.agents unless + DSH_AGENTS_HOME says another), so none of the user's own is read; telemetry off; the placeholder key of a + server that needs none (DSH_NO_KEY). dsh also reads a `.env` in the working directory (one that sets a name + dsh keeps for its launch, such as PATH or PYTHONPATH, stops it before it writes an event) and in its home.""" from hermit.agent.model.backends import PLACEHOLDER_KEY return {"DSH_HOME": str(home), "HOME": str(home), "DSH_AGENTS_HOME": str(home / ".agents"), "DSH_TELEMETRY_DISABLED": "1", DSH_NO_KEY: PLACEHOLDER_KEY} + @classmethod + def user_home(cls) -> Path: + return Path(os.environ.get("DSH_HOME") or Path.home() / cls.user_folder) + def dropped_env(self): """The caller's own dsh variables (`DSH_*`: another permission mode, a folder of skills, a telemetry endpoint, a web provider, ...), which would change the session without the sandbox: its own are home_env's.""" return tuple(name for name in os.environ if name.startswith("DSH_")) def prepare_home(self, home, roles, tools_url, tools_name): - """The patch layer (DSH_PATCH): dsh's own rows, by their ids. A row's `config` replaces the row's, so the - default model is given whole. No host path is written in it: a sandbox translates only the command, the - environment and the prompt.""" + """The patch layer (DSH_PATCH): dsh's own rows, by their ids. A row's `config` replaces the row's, so a row is + given whole (the instructions' maxBytes is dsh's own). No host path is written in it: a sandbox translates + only the command, the environment and the prompt; the skills' folder is the working directory's.""" if roles: raise Unsupported("dsh: roles are not passed to DeepSeek Harness; give it none") if tools_url and not re.fullmatch(r"[A-Za-z0-9_-]{1,32}", tools_name): @@ -1526,10 +1538,19 @@ def prepare_home(self, home, roles, tools_url, tools_name): f"{tools_name!r}") s = self.settings rows: list[dict] = [*({"id": row, "disabled": True} for row in DSH_OFF), - {"id": "session-log-deepseek", "config": {"enabled": False}}] + {"id": "session-log-deepseek", "config": {"enabled": False}}, + # The workspace is the project, wherever it lies: the skills of its own .dsh/skills only + # (not those of the nearest folder above with a .git, a repository the run sits in, nor + # the user's), and the instructions from its root down (a deployment marks the root) + {"id": "skill-filesystem", "config": {"includeDefaultRoots": False, + "customSkillDirs": [".dsh/skills"]}}, + {"id": "agent-instructions", "config": {"maxBytes": DSH_INSTRUCTION_BYTES, + "projectRootMarkers": [DSH_PROJECT_MARKER]}}] if not s.get("web_search"): rows.append({"id": "tool-web", "disabled": True}) # web search and fetch, the model's tools effort = str(s["effort"]) if s.get("effort") else None + if effort and not s.get("model"): + raise ValueError("dsh: an effort needs its model (the `model` setting): it is that model's reasoningEffort") default, chosen = {"provider": DSH_ROUTE}, {} if s.get("provider"): provider = json.loads(json.dumps(s["provider"])) @@ -1551,6 +1572,8 @@ def prepare_home(self, home, roles, tools_url, tools_name): elif s.get("provider"): raise ValueError("dsh: a provider route needs its model (the `model` setting)") if tools_url: + # failOnStartupError: a server dsh cannot reach fails its row at once (dsh says so, `outcome` reads it) + # rather than being retried while the session runs without the tools rows.append({"insert": [{"id": f"mcp-{tools_name}", "name": "@deepseek-ai/dsh-mcp-client", "config": {"serverName": tools_name, "transport": "streamable-http", "url": tools_url, "toolCallTimeoutMs": 1_800_000, @@ -1584,11 +1607,15 @@ def served(cls, raw): def outcome(self, raw, stream, last_file): """dsh exits 0 for a run whose `final` event is `completed`, and 1 for any other end (an error, an abort, the token limit, ...), but also 0 when SIGTERM stops it: a run without a completed end has failed, whatever its - exit code.""" + exit code. So has one whose tools did not start (an MCP row dsh reports inactive): it ran without them. The + session started once dsh wrote its `session` event (a usage error is an `error` event before any session).""" notes, done = self.notices(raw), self.served(raw) + down = DSH_TOOLS_DOWN.findall(notes) + failure = f"{down[0][0]} did not start, so the session ran without its tools: {down[0][1]}" if down else \ + notes or "the run did not finish (no completed `final` event)" return {"reply": self.last_message(stream, last_file), "served": done, - "unavailable": bool(UNAVAILABLE.search(notes)), "refused": False, "failed": not done, - "failure": (notes or "the run did not finish (no completed `final` event)")[-600:]} + "unavailable": bool(UNAVAILABLE.search(notes)), "refused": False, "failed": not done or bool(down), + "failure": failure[-600:], "started": any(e.get("type") == "session" for e in _objects(raw)[0])} @classmethod def errors(cls, event): @@ -1626,19 +1653,23 @@ def steps(cls, stream): def command(self, prompt, workspace, last_file, *, session=None, roles=(), tools_url=None): """The launcher's options first (the profile, the `--patch` layers: those in `args`, a deployment's, then the session's own, which is applied last and so wins), then the headless app's; dsh's launcher reads its - own options only before the first one it does not know. The workspace's skills go into the home first.""" + own options only before the first one it does not know. Under workspace-write the `writable` paths are + granted to dsh's own sandbox by their real paths, as it requires (an existing directory, no link).""" s, home = self.settings, last_file.parent # the session's home mode = s.get("permission_mode") or "workspace-write" if mode not in DSH_PERMISSION_MODES: raise ValueError(f"dsh: permission_mode is one of {DSH_PERMISSION_MODES}, not {mode!r}") + added = [] + for path in s.get("writable") or () if mode == "workspace-write" else (): + real = os.path.realpath(path) + if not os.path.isdir(real): + raise ValueError(f"dsh: a writable path is an existing directory, not {str(path)!r}") + added += ["--add-dir", real] patches, rest = _dsh_args(s.get("args") or []) resume = _dsh_session(home, session) if session else None - _dsh_skills(Path(workspace), home) return [str(s.get("bin") or self.program), "--profile", "headless", *patches, "--patch", str(home / DSH_PATCH), "--json", "--permission-mode", mode, - "--approval-policy", "never", - *[a for w in (s.get("writable") or () if mode == "workspace-write" else ()) - for a in ("--add-dir", str(Path(w).absolute()))], # what dsh's own sandbox lets it write + "--approval-policy", "never", *added, *(["--session-id", resume] if resume else []), *rest, *(["-"] if s.get("stdin") else ["--", prompt])] @@ -1657,25 +1688,10 @@ def _dsh_args(args) -> tuple[list[str], list[str]]: return patches, rest -def _dsh_skills(workspace: Path, home: Path) -> None: - """The workspace's project skills (.dsh/skills) as the session's own ($DSH_AGENTS_HOME/skills, in its home). dsh - reads a project's skills only at the project's root, the nearest folder above with a `.git`, so a workspace - inside a git repository (a run's workspaces under a config kept in a project) would lose them; it reads the - session's own wherever it runs, and offers a skill found in both places once. The last call's copy is replaced.""" - agents = home / ".agents" - if agents.is_symlink(): # a link the last session left would take the copy elsewhere - agents.unlink() - target = agents / "skills" - _clear(target) - target.unlink(missing_ok=True) - source = workspace / ".dsh" / "skills" - if source.is_dir() and not source.is_symlink(): - shutil.copytree(source, target, symlinks=True) - - def _dsh_session(home: Path, handle) -> str | None: """Session `handle` when `home` keeps it (sessions///): dsh refuses an id it does not know, so - a call whose session is gone starts a new one.""" + a call whose session is gone starts a new one. dsh also resumes a session only in the directory it ran in, which + an AgentEvolver's calls share (a resume elsewhere would fail rather than start over).""" if not re.fullmatch(r"session-[A-Za-z0-9-]+", str(handle)): return None root = Path(home) / "sessions" diff --git a/hermit/agent/base/records.py b/hermit/agent/base/records.py index fd059fed..5b477fc5 100644 --- a/hermit/agent/base/records.py +++ b/hermit/agent/base/records.py @@ -124,29 +124,47 @@ def parse_pi(log: Path, model: str) -> tuple[str | None, list[Usage]]: return _parse(log, model, _pi_event) -def _dsh_event(event: dict, usage: Usage, session): +def _dsh_reader(): """DeepSeek Harness: the session id (its opening `session` event), and each model step's usage (`status` events of phase `step_end`). dsh's `inputTokens` leaves the cache out, so the input counted is input, cache reads and - cache writes, as Codex and Claude Code count it. A step that used no tokens (every request of it failed) is no - call. dsh reports no cost: it is priced from the backend's price (price_usage).""" - if event.get("type") == "session": - session = event.get("sessionId") or session - if event.get("type") == "status" and event.get("phase") == "step_end" and isinstance(event.get("usage"), dict): - u = event["usage"] + cache writes, as Codex and Claude Code count it. dsh reports no cost: it is priced from the backend's price + (price_usage). A step whose requests all failed reports no tokens and is no call; one that produced a reply, a + thought or a tool call is, even when the provider reported no usage for it (dsh then reports zeros). + + What dsh's stream leaves out is not counted: a step one of whose attempts reported no usage carries none (dsh + omits a partial sum), so it counts as a call of unknown tokens, and a compaction's summary request, the title + request (off in a session, hermit.agent.base.cli.DSH_OFF) and sub-agents report nothing there.""" + produced = False + + def read(event: dict, usage: Usage, session): + nonlocal produced + kind = event.get("type") + if kind == "session": + session = event.get("sessionId") or session + if kind in ("text", "thinking", "tool_call"): + produced = True + if kind != "status" or event.get("phase") != "step_end": + return session + made, produced = produced, False + u = event.get("usage") + if not isinstance(u, dict): + usage.calls += 1 + return session cached = int(u.get("cacheReadTokens") or 0) tokens = int(u.get("inputTokens") or 0) + cached + int(u.get("cacheWriteTokens") or 0) - if not tokens and not u.get("outputTokens"): + if not tokens and not u.get("outputTokens") and not made: return session usage.input_tokens += tokens usage.cached_input_tokens += cached usage.output_tokens += int(u.get("outputTokens") or 0) usage.calls += 1 - return session + return session + return read def parse_dsh(log: Path, model: str) -> tuple[str | None, list[Usage]]: - """DeepSeek Harness's `--json` events: the session id, and every model step's usage.""" - return _parse(log, model, _dsh_event) + """DeepSeek Harness's `--json` events: the session id, and every model step's usage (_dsh_reader).""" + return _parse(log, model, _dsh_reader()) def parse_codex(log: Path, model: str) -> tuple[str | None, list[Usage]]: @@ -160,5 +178,6 @@ def parse_claude(log: Path, model: str) -> tuple[str | None, list[Usage]]: def parse_log(log: Path, backend: str, model: str) -> tuple[str | None, list[Usage]]: - """Session handle and usage from the agent's JSON event stream: Codex's, else Claude Code's.""" - return (parse_codex if backend == "codex" else parse_claude)(log, model) + """Session handle and usage from the agent's JSON event stream, read as its CLI writes it (Claude Code's for a + backend it does not know).""" + return {"codex": parse_codex, "pi": parse_pi, "dsh": parse_dsh}.get(backend, parse_claude)(log, model) diff --git a/hermit/agent/base/trajectory.py b/hermit/agent/base/trajectory.py index a29008ee..841b176b 100644 --- a/hermit/agent/base/trajectory.py +++ b/hermit/agent/base/trajectory.py @@ -217,37 +217,44 @@ def pi_steps(stream: list[dict]) -> list[Step]: DSH_FILE_TOOLS = ("write", "edit") -# How dsh's bash tool ends a command's result (its own parseExitStatus reads it so): a nonzero exit or a signal is -# marked last, a clean exit is not; a command stopped by its timeout or from outside is marked before that (a command -# that traps the signal may still exit 0), and one still running was moved to the background, unfinished -DSH_EXIT = re.compile(r"\n\[exit code: (\d+)\]$") -DSH_SIGNAL = re.compile(r"\n\[killed by signal: [^\]\n]+\]$") -DSH_STOPPED = re.compile(r"\n\[(?:timed out after \d+ms|stopped: [^\]\n]*)\](?:\n\[[^\]\n]*\])*$") -DSH_BACKGROUND = re.compile(r"\[still running after \d+ms; moved to background job [^\]\n]+\]$") +# How dsh's bash tool ends a command's result (its own parseExitStatus reads it so, anchored at the very end, as +# JavaScript's `$` is: \Z here): a nonzero exit or a signal is marked last, a clean exit is not; a command stopped by +# its timeout or from outside is marked before that (a command that traps the signal may still exit 0), and one still +# running was moved to the background, unfinished +DSH_EXIT = re.compile(r"\n\[exit code: (\d+)\]\Z") +DSH_SIGNAL = re.compile(r"\n\[killed by signal: [^\]\n]+\]\Z") +DSH_STOPPED = re.compile(r"\n\[(?:timed out after \d+ms|stopped: [^\]\n]*)\](?:\n\[[^\]\n]*\])*\Z") +DSH_BACKGROUND = re.compile(r"\[still running after \d+ms; moved to background job [^\]\n]+\]\Z") DSH_NO_OUTPUT = "(no output)" # what the result holds before its marker when the command printed nothing +# dsh's --json keeps 8 KiB of each string and marks the event `truncated`: the end, where the marker was, is gone +DSH_CUT = "\n[the rest was cut: dsh's --json keeps 8 KiB of a string]" def dsh_steps(stream: list[dict]) -> list[Step]: """DeepSeek Harness's `--json` events: its thinking and text as each model step committed them, and each `tool_call` completed by its `tool_result`: bash as a command whose exit code and output are read from the result as dsh reads them (`[exit code: N]` after the output; no marker: 0; a signal, a timeout, a stop, a command - moved to the background or a call that failed to run: unknown; `(no output)`: empty), write and edit as file - changes, an MCP tool (`mcp____`) as a tool call with its server and arguments and its result as - `output`, any other tool as a tool call; a run that did not complete (the `final` event's reason: an error, an - abort, the token limit, ...), or failed outside a turn (an `error` event), as an error. Status events and usage - are left out.""" - steps, pending = [], {} + moved to or started in the background, a result dsh's --json cut (its marker is gone) or a call that failed to + run: unknown; `(no output)`: empty), write and edit as file changes, an MCP tool (`mcp____`) as a + tool call with its server and arguments and its result as `output`, any other tool as a tool call; a run that did + not complete (the `final` event's reason: an error, an abort, the token limit, ...), or failed outside a turn (an + `error` event), as an error. A text cut by --json says so at its end (DSH_CUT). Status events and usage are left + out.""" + steps, pending, background = [], {}, set() for event in stream: kind = event.get("type") + cut = DSH_CUT if event.get("truncated") else "" if kind == "thinking" and str(event.get("text") or "").strip(): - steps.append(Step("reasoning", str(event["text"]))) + steps.append(Step("reasoning", str(event["text"]) + cut)) elif kind == "text" and str(event.get("text") or "").strip(): - steps.append(Step("message", str(event["text"]))) + steps.append(Step("message", str(event["text"]) + cut)) elif kind == "tool_call": name = str(event.get("tool") or "") args = event.get("input") if isinstance(event.get("input"), dict) else {} if name == "bash": step = Step("command", command=str(args.get("command") or "")) + if args.get("run_in_background"): # "started background job ": no exit status yet + background.add(event.get("callId")) elif name in DSH_FILE_TOOLS: step = Step("file_change", json.dumps({"tool": name, "path": args.get("file_path")}, ensure_ascii=False, default=str)) @@ -266,14 +273,14 @@ def dsh_steps(stream: list[dict]) -> list[Step]: continue result = str(event.get("result") or "") if step.kind == "command": - found = DSH_EXIT.search(result) + found = None if cut else DSH_EXIT.search(result) output = result[:found.start()] if found else result - step.output = "" if output == DSH_NO_OUTPUT else output - unknown = event.get("status") != "completed" or DSH_SIGNAL.search(result) or \ - DSH_STOPPED.search(result) or DSH_BACKGROUND.search(result) + step.output = ("" if output == DSH_NO_OUTPUT else output) + cut + unknown = cut or event.get("status") != "completed" or event.get("callId") in background or \ + DSH_SIGNAL.search(result) or DSH_STOPPED.search(result) or DSH_BACKGROUND.search(result) step.exit_code = None if unknown else int(found.group(1)) if found else 0 elif step.kind == "tool_call": - step.output = result + step.output = result + cut elif kind == "final" and isinstance(event.get("reason"), dict) and \ event["reason"].get("kind") not in (None, "completed"): error = event["reason"].get("error") if isinstance(event["reason"].get("error"), dict) else {} diff --git a/hermit/agent/evolver/methods/mwm/evolver.py b/hermit/agent/evolver/methods/mwm/evolver.py index 4ae8ac0d..9b3f473b 100644 --- a/hermit/agent/evolver/methods/mwm/evolver.py +++ b/hermit/agent/evolver/methods/mwm/evolver.py @@ -456,6 +456,9 @@ def _notes(admission: dict | None) -> str: # ------------------------------------------------------------------ steps def setup(self, ctx: Context) -> None: + if ctx.params.get("backend", "codex") not in ("codex", "claude"): # its session texts, homes and sign-ins + raise ValueError("the mwm Evolver's learner is a Codex or Claude Code session: evolver.params.backend is " + f"codex or claude, not {ctx.params.get('backend')!r}") if "network_isolation" in ctx.params: raise ValueError("evolver.params.network_isolation was removed: the learner runs in the run's evolver " "sandbox, whose network reaches only its models, the tool endpoint and " diff --git a/hermit/agent/evolver/methods/rrsi_adapter/evolver.py b/hermit/agent/evolver/methods/rrsi_adapter/evolver.py index 0ac28a4e..31add202 100644 --- a/hermit/agent/evolver/methods/rrsi_adapter/evolver.py +++ b/hermit/agent/evolver/methods/rrsi_adapter/evolver.py @@ -417,6 +417,9 @@ def setup(self, ctx) -> None: """What every step needs, checked once: a broken hyperparameter file or domain, or an unset web_search, stops the run here rather than failing each step.""" _require_rrsi() + if ctx.params.get("backend", "codex") not in ("codex", "claude"): # its roles' settings are those CLIs' + raise ValueError("the RRSI Adapter's proposer and roles are Codex or Claude Code sessions: " + f"evolver.params.backend is codex or claude, not {ctx.params.get('backend')!r}") if not isinstance(ctx.params.get("web_search"), bool): raise ValueError("evolver.params.web_search must be set, true or false") role = ctx.params.get("roles") or {} diff --git a/hermit/hosts/dsh.py b/hermit/hosts/dsh.py index 6f0b7d9e..bf24226e 100644 --- a/hermit/hosts/dsh.py +++ b/hermit/hosts/dsh.py @@ -1,15 +1,18 @@ #!/usr/bin/env python3 """Deploy an adapter into a DeepSeek Harness (dsh) workspace: the whole adapter under hermit//, and its exports -connected to dsh's own places: AGENTS.md context, .dsh/skills, and the tools as a patch layer, .dsh/hermit.patch.yml. +connected to dsh's own places: AGENTS.md context, .dsh/skills, and a patch layer, .dsh/hermit.patch.yml. -dsh reads AGENTS.md at the project's root (the nearest folder above with a `.git`, else the working directory) and in -the working directory, and the skills of .dsh/skills at the project's root only, without any trust prompt: a -workspace that is not its project's root gets its skills only as the session's own, which the `dsh` acting agent -gives it (hermit.agent.base.cli._dsh_skills). It has no project file for MCP servers: they are rows of its -plugin tree, so the adapter's tool server is an `insert` row in a patch layer of the workspace, which a session -loads with `dsh --patch .dsh/hermit.patch.yml` (what the `dsh` acting agent passes). dsh reads no hook file and has -no role file: an adapter's roles and hooks stay files of its copy, which AGENTS.md names with what to do with them -(hermit.hosts.inputs.resources), as it names the other files. dsh's own settings ($DSH_HOME) are never changed. +dsh reads, without any trust prompt, the instruction files (AGENTS.md, CLAUDE.md and their .local overlays) of every +folder from the project's root down to the working directory, and the skills of the root's .dsh/skills and +.agents/skills; the root is the nearest folder above with a `.git`, else the working directory. It has no project +file for MCP servers or for more skill folders: they are rows of its plugin tree, so the patch layer holds the +adapter's tool server (an `insert` row) and names the workspace's .dsh/skills (the `skill-filesystem` row's +customSkillDirs, so that a workspace below a repository's root keeps its skills), and a session loads it with `dsh +--patch .dsh/hermit.patch.yml` from the workspace (what the `dsh` acting agent passes). dsh cuts a chain of +instructions past 65,536 bytes (its agent-instructions maxBytes), so a longer AGENTS.md is refused. dsh runs no hook +file of its own (its experimental bridge to Claude Code's hooks is not used) and has no role file: an adapter's roles +and hooks stay files of its copy, which AGENTS.md names with what to do with them (hermit.hosts.inputs.resources), as +it names the other files. dsh's own settings ($DSH_HOME) are never changed. """ import argparse import json @@ -19,13 +22,14 @@ import sys from hermit.adapter.loader import agent_tools, hook_definitions, load_manifest, not_portable +from hermit.agent.base.cli import DSH_INSTRUCTION_BYTES from hermit.hosts.context import PROJECT_CONTEXT, START, append_context from hermit.hosts.inputs import (MARKER, PACKAGE_ROOT, _adapter_sandbox_env, adapter_files, adapter_folder, check_workspace, copy_adapter, copy_skills, folder_note, render_exports, resources, server_name, source_label) NATIVE = {"skills"} -PATCH = Path(".dsh") / "hermit.patch.yml" # the workspace's patch layer: the adapter's tool server +PATCH = Path(".dsh") / "hermit.patch.yml" # the workspace's patch layer: the adapter's tool server and skills SERVER_NAME = re.compile(r"[A-Za-z0-9_-]{1,32}") # what dsh takes as an MCP server's name @@ -58,6 +62,11 @@ def prepare(adapter: Path, output: Path, python: str, *, learning_material: Path exports["context"] if context is None else context, listed) if t), source=source_label(manifest), version=manifest["version"], adapter_id=manifest["id"]) + size = len(context_text.encode("utf-8")) + if size > DSH_INSTRUCTION_BYTES: + raise ValueError(f"AGENTS.md would be {size} bytes, past the {DSH_INSTRUCTION_BYTES} bytes dsh reads of a " + "project's instructions (its agent-instructions maxBytes): dsh would cut the rest, telling " + "only the model. Shorten the adapter's context") audit = output / "tool-calls.jsonl" server = None if has_tools: @@ -67,18 +76,20 @@ def prepare(adapter: Path, output: Path, python: str, *, learning_material: Path "toolCallTimeoutMs": 120_000} # Codex's tool_timeout_sec, not dsh's 60 s if learning_material is not None: server["args"].extend(["--learning-material", str(Path(learning_material).resolve())]) - rows = [{"insert": [{"id": f"mcp-{server_name(manifest)}", "name": "@deepseek-ai/dsh-mcp-client", - "config": server}]}] if has_tools else [] + skills = copy_skills(adapter, manifest, output / ".dsh" / "skills") + rows = [*([{"insert": [{"id": f"mcp-{server_name(manifest)}", "name": "@deepseek-ai/dsh-mcp-client", + "config": server}]}] if has_tools else []), + # the workspace's own skills wherever it lies (dsh's other roots, the user's own, stay as they are) + *([{"id": "skill-filesystem", "config": {"customSkillDirs": [".dsh/skills"]}}] if skills else [])] (output / PATCH.parent).mkdir(exist_ok=True) # JSON is YAML: dsh reads the layer as YAML, and nothing in it is a `!!js` expression (output / PATCH).write_text(json.dumps(rows, indent=2) + "\n") context_path.write_text(context_text) - skills = copy_skills(adapter, manifest, output / ".dsh" / "skills") copy_adapter(adapter, output, manifest, files, previous) result = {"host": "dsh", "adapter_id": manifest["id"], "version": manifest["version"], "adapter": str(adapter), "workspace": str(output), "context_injected_via": "AGENTS.md", "server_name": server_name(manifest) if has_tools else None, "server": server, - "patch": str(output / PATCH) if has_tools else None, "roles": {}, "role_name_map": {}, + "patch": str(output / PATCH) if rows else None, "roles": {}, "role_name_map": {}, "skills_dir": str(output / ".dsh" / "skills") if skills else None, "skills": skills, "hooks": None, "audit_log": str(audit) if has_tools else None, "adapter_dir": str(adapter_folder(output, manifest)), "files": files, "placed": placed} diff --git a/tests/core/test_context_tool.py b/tests/core/test_context_tool.py index 5cfcddbd..d08001d3 100644 --- a/tests/core/test_context_tool.py +++ b/tests/core/test_context_tool.py @@ -121,13 +121,13 @@ def test_the_codex_agent_gives_each_task_its_own_context(adapter, tmp_path): # ---------- plugin-portable: whether a template's adapters can be deployed or converted ---------- def test_a_template_that_is_not_plugin_portable_is_refused_by_deploy_and_convert(adapter, tmp_path): - from hermit.hosts import claude_code, codex, pi + from hermit.hosts import claude_code, codex, dsh, pi from hermit.plugins import TARGETS, Unsupported, build for target in TARGETS: with pytest.raises(Unsupported, match="plugin-portable: false"): build(target, adapter, tmp_path / f"plugin-{target}", sys.executable) assert not (tmp_path / f"plugin-{target}").exists() - for host in (codex, claude_code, pi): + for host in (codex, claude_code, pi, dsh): workspace = tmp_path / f"ws-{host.__name__.rsplit('.', 1)[-1]}" with pytest.raises(ValueError, match="plugin-portable: false"): host.prepare(adapter, workspace, sys.executable) diff --git a/tests/core/test_deploy.py b/tests/core/test_deploy.py index e0b3f189..469e308c 100644 --- a/tests/core/test_deploy.py +++ b/tests/core/test_deploy.py @@ -53,11 +53,15 @@ def test_pi(tmp_path): def test_dsh(tmp_path): - """dsh has no project file for MCP servers: the tools are an `insert` row of the workspace's patch layer, which a - session loads with --patch; the skills are dsh's project skills; roles stay files named in AGENTS.md.""" + """dsh has no project file for MCP servers or skill folders: the tools are an `insert` row of the workspace's + patch layer, which a session loads with --patch, and the layer names the workspace's .dsh/skills (dsh reads a + project's skills only at the nearest folder with a .git); roles stay files named in AGENTS.md; a context longer + than dsh reads of a project's instructions is refused.""" result = dsh_prepare(LEAN, tmp_path / "d", sys.executable) - [layer] = json.loads((tmp_path / "d" / ".dsh" / "hermit.patch.yml").read_text()) - [row] = layer["insert"] + layer = json.loads((tmp_path / "d" / ".dsh" / "hermit.patch.yml").read_text()) + [server] = [r for r in layer if "insert" in r] + assert {"id": "skill-filesystem", "config": {"customSkillDirs": [".dsh/skills"]}} in layer + [row] = server["insert"] assert row["name"] == "@deepseek-ai/dsh-mcp-client" and row["config"]["serverName"] == "lean_collab" assert row["config"]["transport"] == "stdio" and row["config"]["args"][:3] == ["-m", "hermit.runtime.mcp_server", "--adapter"] @@ -75,6 +79,11 @@ def test_dsh(tmp_path): assert result["placed"] == {"agents": "files"} and result["patch"] is None # no tools: no server row assert json.loads((tmp_path / "s" / ".dsh" / "hermit.patch.yml").read_text()) == [] assert "`reviewer`: Check a result against the task requirements." in (tmp_path / "s" / "AGENTS.md").read_text() + big = subagent_adapter(tmp_path / "big") + (big / "ADAPTER.md").write_text("Check the units.\n" * 4000) # 68,000 bytes + with pytest.raises(ValueError, match="past the 65536 bytes dsh reads of a project's instructions"): + dsh_prepare(big, tmp_path / "b", sys.executable) + assert not (tmp_path / "b" / "AGENTS.md").exists() def subagent_adapter(tmp_path): diff --git a/tests/core/test_dsh_agent.py b/tests/core/test_dsh_agent.py index 5acdd89d..ce4ce9af 100644 --- a/tests/core/test_dsh_agent.py +++ b/tests/core/test_dsh_agent.py @@ -278,24 +278,88 @@ def test_system_instructions_go_at_the_top_of_the_prompt(fake): assert call["args"][-1] == "-" and call["stdin"] == prompt -def test_the_workspaces_skills_are_the_sessions_own_and_the_last_calls_copy_is_replaced(fake): - """dsh reads a project's skills only at the project's root (the nearest folder above with a .git), so the - workspace's are copied to the session's own ($DSH_AGENTS_HOME/skills), which it reads wherever it runs; a skill - the last call copied is gone, and a link left in place of the folder is replaced, never written through.""" - home, work = fake / "home", fake / "work" - (work / ".dsh" / "skills" / "check").mkdir(parents=True) - (work / ".dsh" / "skills" / "check" / "SKILL.md").write_text("---\nname: check\ndescription: Check.\n---\n") - (home / ".agents" / "skills" / "old").mkdir(parents=True) - agent = make("dsh", {**bound(), "web_search": False, "home": str(home)}) - agent.run("go", workspace=work) - assert [p.name for p in (home / ".agents" / "skills").iterdir()] == ["check"] - outside = fake / "outside" - outside.mkdir() - shutil.rmtree(home / ".agents") - (home / ".agents").symlink_to(outside) - agent.run("go", workspace=work) - assert not (home / ".agents").is_symlink() and list(outside.iterdir()) == [] - assert (home / ".agents" / "skills" / "check" / "SKILL.md").is_file() +def test_the_workspace_is_the_sessions_project_wherever_it_lies(fake): + """dsh takes the nearest folder above with a .git for the project: a run's workspace inside a repository would + get the repository's skills (.dsh/skills, .agents/skills) and instructions, which outrank the adapter's, and lose + its own skills. The session's layer gives dsh the workspace's .dsh/skills alone and its instructions from the + deployment's root marker down, as the run's sandbox does by showing nothing above the workspace.""" + work = fake / "work" + work.mkdir() + make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}).run("go", workspace=work) + layer = rows(seen(fake)[0], fake / "home") + assert row(layer, "skill-filesystem")["config"] == {"includeDefaultRoots": False, "customSkillDirs": [".dsh/skills"]} + assert row(layer, "agent-instructions")["config"] == {"maxBytes": 65536, + "projectRootMarkers": [".hermit-deploy.json"]} + + +def test_what_dsh_cannot_use_is_refused_rather_than_dropped(fake, monkeypatch, tmp_path): + """An effort without a model reaches no model (it is the default model's reasoningEffort); a writable path dsh's + sandbox cannot take (a link, a missing folder) failed every command and write of the session: dsh requires an + existing directory by its real path, which the session is given. The user's own dsh home is where its .env is + copied from.""" + work = fake / "work" + work.mkdir() + with pytest.raises(ValueError, match="an effort needs its model"): + make("dsh", {"effort": "high", "web_search": False, "home": str(fake / "home")}).run("go", workspace=work) + real = tmp_path / "data" + real.mkdir() + (tmp_path / "link").symlink_to(real) + agent = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}) + agent.configured(writable=[str(tmp_path / "link")]).run("go", workspace=work) + args = seen(fake)[-1]["args"] + assert args[args.index("--add-dir") + 1] == os.path.realpath(real) + with pytest.raises(ValueError, match="a writable path is an existing directory"): + agent.configured(writable=[str(tmp_path / "missing")]).run("go", workspace=work) + monkeypatch.setenv("DSH_HOME", str(tmp_path / "users-dsh")) + assert base_cli.Dsh.user_home() == tmp_path / "users-dsh" + + +def test_a_run_whose_tools_did_not_start_has_failed_although_dsh_completed_it(fake, monkeypatch): + """dsh reports an MCP row it could not start on stderr and runs on without its tools (recorded: tools_down.jsonl, + a server dsh could not reach, exit 0): the session ran without what it was given, so it has failed.""" + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "tools_down.jsonl")) + work = fake / "work" + work.mkdir() + result = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}).run("go", workspace=work, + tools=[echo()]) + assert result.exit_code == 0 and result.extra["served"] and result.extra["failed"] + assert result.extra["failure"].startswith("mcp-hermit_tools did not start, so the session ran without its tools") + + +def test_a_run_dsh_refused_to_start_is_no_session(fake, monkeypatch): + """A usage error is an `error` event before any session (recorded: bad_option.jsonl), and a .env in the working + directory that sets a name dsh keeps for its launch stops it before any event (dotenv.jsonl): neither started + a session, so the acting agent reports it rather than taking every task for a wrong answer.""" + from hermit.agent.acting import workspace as acting + monkeypatch.setenv("FAKE_DSH_EXIT", "1") + work = fake / "work" + work.mkdir() + agent = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}) + for stream, says in (("bad_option.jsonl", "unknown option"), ("dotenv.jsonl", 'sets "PYTHONPATH"')): + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / stream)) + result = agent.run("go", workspace=work) + assert result.exit_code == 1 and not result.extra["started"] and says in result.extra["output_tail"] + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "bad_option.jsonl")) + with pytest.raises(RuntimeError, match="before starting a session"): + acting.run({"id": "q1"}, None, params={**bound(), "backend": "dsh", "web_search": False, "prompt": "go", + "workspaces": "ws"}, base=fake) + + +def test_dsh_permission_mode_reaches_the_acting_agents_session_or_is_refused(fake): + """agent.dsh.permission_mode picks dsh's own sandbox for the acting agent (it was dropped, so the session always + ran workspace-write); a mode dsh has not is refused when the call is checked.""" + from hermit.agent.acting import workspace as acting + from hermit.config import layered + data = {"agent": {"base": "dsh", "model": "fake/m-1", "dsh": {"permission_mode": "danger-full-access"}, + "params": {"prompt": "go"}}, + "evolver": {"name": "dsh", "base": "dsh", "model": "fake/m-1"}, "llm": None} + layered(data) + params = {**data["agent"]["params"], **bound(), "backend": "dsh", "web_search": False, "workspaces": "ws"} + acting.run({"id": "q1"}, None, params=params, base=fake) + args = seen(fake)[-1]["args"] + assert args[args.index("--permission-mode") + 1] == "danger-full-access" + with pytest.raises(ValueError, match="permission_mode is one of read-only, workspace-write"): + acting._check("dsh", {**params, "permission_mode": "bypassPermissions"}) def test_a_dsh_stream_is_read_back_by_its_name_or_by_its_events(tmp_path): @@ -327,6 +391,42 @@ def test_a_bash_result_is_read_as_dsh_marks_its_end(): ("sleep 5", None, "(no output)\n[timed out after 1000ms]\n[killed by signal: SIGTERM]")] +def test_a_command_whose_end_dsh_did_not_report_has_no_exit_code(): + """dsh's --json keeps 8 KiB of a string: a long result loses its end, where the marker is (recorded: + big_bash.jsonl, 1,500 lines then exit 2), and read as exit 0 before; a command started in the background + (background.jsonl) has no status yet. A command that printed a marker of its own and exited 0 (fake_marker.jsonl) + is a clean exit, as dsh reads it: the marker counts only at the very end.""" + def commands(name): + stream = [json.loads(line) for line in (STREAMS / name).read_text().splitlines() if line.startswith("{")] + return [s for s in trajectory.dsh_steps(stream) if s.kind == "command"] + [cut] = commands("big_bash.jsonl") + assert cut.exit_code is None and cut.output.startswith("line 1 of a long build log\n") + assert cut.output.endswith(trajectory.DSH_CUT) and "[exit code" not in cut.output + [background] = commands("background.jsonl") + assert background.exit_code is None and background.output.startswith("started background job ") + [printed] = commands("fake_marker.jsonl") + assert (printed.exit_code, printed.output) == (0, "built\n[exit code: 7]\n") + + +def test_a_step_counts_as_a_call_whenever_it_made_one(tmp_path): + """A step whose requests all failed reports no tokens and is no call (failure.jsonl); one that answered is a call + even when the provider reported no usage, which dsh reports as zeros (no_usage.jsonl), and so is a step dsh + reports no usage for at all (one of its attempts had none: dsh's json-stream.ts omits the partial sum, here taken + off a recorded step). parse_log reads a dsh stream as dsh's (it read every stream but Codex's as Claude Code's).""" + from hermit.agent.base.records import parse_dsh, parse_log + assert parse_dsh(STREAMS / "failure.jsonl", "m")[1] == [] + _, [answered] = parse_dsh(STREAMS / "no_usage.jsonl", "m") + assert (answered.calls, answered.input_tokens, answered.output_tokens) == (1, 0, 0) + session, [resumed] = parse_log(STREAMS / "resume.jsonl", "dsh", "m") + assert session == SESSION and (resumed.calls, resumed.input_tokens) == (1, 1000) + partial = tmp_path / "partial.jsonl" + partial.write_text("".join(json.dumps({k: v for k, v in json.loads(line).items() if k != "usage"}) + "\n" + if '"step_end"' in line else line + "\n" + for line in (STREAMS / "resume.jsonl").read_text().splitlines())) + [unknown] = parse_dsh(partial, "m")[1] + assert (unknown.calls, unknown.input_tokens) == (1, 0) + + def test_a_run_that_ends_at_the_token_limit_is_an_error_the_learner_can_read(): """dsh exits 1 for a run that ends `max-tokens` and writes nothing on stderr: the run is no success and no outage, and its end is a step of the trajectory and a notice (only errors and aborts were, before).""" @@ -360,13 +460,12 @@ def test_the_dsh_acting_agent_deploys_the_adapter_and_loads_its_patch_layer(fake patches = [call["args"][i + 1] for i, a in enumerate(call["args"]) if a == "--patch"] assert patches == [str(work / ".dsh" / "hermit.patch.yml"), str(work.parent / "home" / base_cli.DSH_PATCH)] assert call["args"].index("--json") > call["args"].index(patches[-1]) # the launcher's options come first - [layer] = json.loads((work / ".dsh" / "hermit.patch.yml").read_text()) - [server] = layer["insert"] + layer = json.loads((work / ".dsh" / "hermit.patch.yml").read_text()) + [server] = [r["insert"][0] for r in layer if "insert" in r] assert server["config"]["transport"] == "stdio" and server["config"]["args"][:2] == ["-m", "hermit.runtime.mcp_server"] - skill = work / ".dsh" / "skills" / "verify-output" / "SKILL.md" - assert (work.parent / "home" / ".agents" / "skills" / "verify-output" / "SKILL.md").read_text() == \ - skill.read_text() # also the session's own: _dsh_skills + assert (work / ".dsh" / "skills" / "verify-output" / "SKILL.md").is_file() + assert (work / ".hermit-deploy.json").is_file() # the instructions' project root (the marker) assert "do it yourself" in (work / "AGENTS.md").read_text() # roles stay files: none is passed to dsh assert out.output == "the answer" and out.extra["backend"] == "dsh" assert (out.log / "agent-dsh.jsonl").is_file() @@ -488,18 +587,56 @@ def script(body): assert {"DSH_HOME", "HOME", "MOCK_KEY"} <= set(data["environment"]) +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +def test_with_the_dsh_cli_a_command_runs_in_dshs_own_sandbox_inside_the_runs(tmp_path, monkeypatch): + """The acting agent keeps dsh's own sandbox (workspace-write) inside the run's: dsh starts it nested in the run's + bubblewrap sandbox (bubblewrap, else Landlock), and a command of the session writes its workspace there. Where + neither can start, dsh refuses every command (SANDBOX_UNAVAILABLE), which this would show.""" + from hermit.sandbox import Sandbox + if not _bwrap_works(): + pytest.skip("needs a working bubblewrap") + + def script(body): + done = sum(1 for m in body.get("messages") or [] if m.get("role") == "tool") + return ("bash", {"command": "echo made > made.txt && cat made.txt", "description": "Write a file"}) \ + if done == 0 else None + + server, url = serve_chat(script) + monkeypatch.setenv("MOCK_KEY", "sk-mock") + params = bind_agent_params({"model": "mock/m-1"}, "dsh", + registry({"mock": {"base_url": url, "api_key_env": "MOCK_KEY", "apis": ["chat"]}})) + work = tmp_path / "work" + work.mkdir() + box = Sandbox(network="isolated", env_allow=[], writable=[work], role="agent", record_dir=tmp_path / "records") + try: + result = make("dsh", {**params, "web_search": False, "home": str(tmp_path / "home"), "isolation": "sandbox", + "sandbox": box, "timeout_s": 300}).run("Write the file.", workspace=work) + finally: + server.shutdown() + assert result.exit_code == 0 and result.extra["served"] and not result.extra["failed"] + [command] = [s for s in result.trajectory if s.kind == "command"] + assert (command.exit_code, command.output) == (0, "made\n") and (work / "made.txt").read_text() == "made\n" + + @pytest.mark.integration @pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") @pytest.mark.skipif(not shutil.which("git"), reason="needs git") def test_with_the_dsh_cli_a_deployed_adapter_brings_its_skills_and_tools_into_a_git_repository(tmp_path, monkeypatch): - """The acting agent with the installed dsh, its workspaces inside a git repository (a config kept in a project). - dsh takes the repository for the project and reads project skills only from its .dsh/skills, so the deployed - skills reach it as the session's own (_dsh_skills; before, they were not offered there). The adapter's own tool - server (the deployment's stdio server, loaded from its patch layer) answers a call.""" + """The acting agent with the installed dsh, its workspaces inside a git repository (a config kept in a project, + run without the sandbox). dsh takes the repository for the project: before, the deployed skill was not offered, + and the repository's own skills and AGENTS.md were, ahead of the adapter's. Now the session's layer makes the + workspace the project. The adapter's own tool server (the deployment's stdio server, loaded from its patch layer) + answers a call.""" from hermit.agent.acting import workspace as acting repo = tmp_path / "repo" repo.mkdir() subprocess.run(["git", "init", "-q", str(repo)], check=True) + (repo / "AGENTS.md").write_text("REPO-INSTRUCTIONS-MARKER\n") + for folder in (".dsh", ".agents"): + (repo / folder / "skills" / "repo-skill").mkdir(parents=True) + (repo / folder / "skills" / "repo-skill" / "SKILL.md").write_text( + f"---\nname: repo-skill{folder}\ndescription: REPO-SKILL-MARKER\n---\nbody\n") requests = [] def script(body): @@ -519,6 +656,8 @@ def script(body): server.shutdown() assert out.output == "the answer" and len(requests) == 2 assert "Check a completed result against the task requirements." in json.dumps(requests[0]) # the skill + assert "REPO-SKILL-MARKER" not in json.dumps(requests[0]) and "REPO-INSTRUCTIONS-MARKER" not in json.dumps( + requests[0]) # nothing of the repository the run lies in assert "mcp__file_example__count_words" in [t["function"]["name"] for t in requests[0].get("tools") or []] [answer] = [m for m in requests[1]["messages"] if m.get("role") == "tool"] assert json.loads(answer["content"] if isinstance(answer["content"], str) else diff --git a/tests/core/test_isolate.py b/tests/core/test_isolate.py index c321cef1..f8a9f9d4 100644 --- a/tests/core/test_isolate.py +++ b/tests/core/test_isolate.py @@ -686,6 +686,10 @@ def test_a_cli_deploy_renders_the_adapter_in_a_sandbox(tmp_path, monkeypatch): assert not planted.exists() # the module could not read or write outside toml = (tmp_path / "ws" / ".codex" / "config.toml") assert "HA_ADAPTER_SANDBOX" in toml.read_text() # the tool server's Session is sandboxed too + from hermit.hosts.dsh import prepare as prepare_dsh # dsh's server takes it in its patch layer + prepare_dsh(adapter, tmp_path / "ws-dsh", sys.executable) + assert not planted.exists() + assert "HA_ADAPTER_SANDBOX" in (tmp_path / "ws-dsh" / ".dsh" / "hermit.patch.yml").read_text() def test_a_rollout_that_replaced_its_home_by_a_link_deletes_no_login_outside(tmp_path, monkeypatch, shared_login): diff --git a/tests/fixtures/dsh/README.md b/tests/fixtures/dsh/README.md index 6929400d..c11dfd01 100644 --- a/tests/fixtures/dsh/README.md +++ b/tests/fixtures/dsh/README.md @@ -22,6 +22,16 @@ layer like the Base Agent's) against a scripted chat-completions model: than moving it to a background job. - `max_tokens.jsonl`: the model's answer ends at its token limit (`finish_reason: length`): the run ends `max-tokens` and dsh exits 1, writing nothing on stderr. +- `big_bash.jsonl`: bash prints 1,500 lines (about 42 KB) and exits 2; dsh's `--json` keeps 8 KiB of the result and + marks the event `truncated`, so the exit marker is cut off. +- `background.jsonl`: bash with `run_in_background: true` returns `started background job `, no exit marker. +- `fake_marker.jsonl`: bash prints `built` and a line `[exit code: 7]` of its own, then exits 0. +- `no_usage.jsonl`: the model sends no usage; dsh reports the step's usage as zeros. +- `tools_down.jsonl`: the session's MCP row (`failOnStartupError: true`) points at a port nothing listens on: dsh + writes `warning: 1 entry did not activate` and the row's error on stderr, then completes the run and exits 0. +- `bad_option.jsonl`: an unknown option after `--json`: one `error` event, no session, exit 1. +- `dotenv.jsonl`: the working directory's `.env` sets `PYTHONPATH`: dsh stops before it writes an event (its + error on stderr) and exits 1. -The streams are dsh's output unchanged; the run's directory was `/run` (`/run/bash` and `/run/max_tokens` for the -last two). +The streams are dsh's stdout unchanged, followed by its stderr's lines where it wrote any, as the Base Agent keeps +them in one stream; the run's directory was `/run` (`/run/` for the streams recorded directly). diff --git a/tests/fixtures/dsh/background.jsonl b/tests/fixtures/dsh/background.jsonl new file mode 100644 index 00000000..65afd5d3 --- /dev/null +++ b/tests/fixtures/dsh/background.jsonl @@ -0,0 +1,11 @@ +{"type":"session","sessionId":"session-ad7761cf-4b99-4881-a5b7-d309a5d0ceb0","cwd":"/run/background/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"tool_call","callId":"call_0","tool":"bash","input":{"command":"sleep 2; echo done","run_in_background":true,"description":"edge"}} +{"type":"tool_result","callId":"call_0","status":"completed","result":"started background job bash-b055b99ea4af4d91"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":2} +{"type":"text","text":"done"} +{"type":"status","phase":"step_end","turn":1,"step":2,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-ad7761cf-4b99-4881-a5b7-d309a5d0ceb0","reason":{"kind":"completed"},"text":"done"} diff --git a/tests/fixtures/dsh/bad_option.jsonl b/tests/fixtures/dsh/bad_option.jsonl new file mode 100644 index 00000000..63172c20 --- /dev/null +++ b/tests/fixtures/dsh/bad_option.jsonl @@ -0,0 +1 @@ +{"type":"error","message":"unknown option '--no-such-option'"} diff --git a/tests/fixtures/dsh/big_bash.jsonl b/tests/fixtures/dsh/big_bash.jsonl new file mode 100644 index 00000000..b44c9894 --- /dev/null +++ b/tests/fixtures/dsh/big_bash.jsonl @@ -0,0 +1,11 @@ +{"type":"session","sessionId":"session-2055df01-dada-4a0a-a7ee-13ae0996ff9a","cwd":"/run/bigbash/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"tool_call","callId":"call_0","tool":"bash","input":{"command":"for i in $(seq 1 1500); do echo \"line $i of a long build log\"; done; exit 2","description":"edge"}} +{"type":"tool_result","callId":"call_0","status":"completed","result":"line 1 of a long build log\nline 2 of a long build log\nline 3 of a long build log\nline 4 of a long build log\nline 5 of a long build log\nline 6 of a long build log\nline 7 of a long build log\nline 8 of a long build log\nline 9 of a long build log\nline 10 of a long build log\nline 11 of a long build log\nline 12 of a long build log\nline 13 of a long build log\nline 14 of a long build log\nline 15 of a long build log\nline 16 of a long build log\nline 17 of a long build log\nline 18 of a long build log\nline 19 of a long build log\nline 20 of a long build log\nline 21 of a long build log\nline 22 of a long build log\nline 23 of a long build log\nline 24 of a long build log\nline 25 of a long build log\nline 26 of a long build log\nline 27 of a long build log\nline 28 of a long build log\nline 29 of a long build log\nline 30 of a long build log\nline 31 of a long build log\nline 32 of a long build log\nline 33 of a long build log\nline 34 of a long build log\nline 35 of a long build log\nline 36 of a long build log\nline 37 of a long build log\nline 38 of a long build log\nline 39 of a long build log\nline 40 of a long build log\nline 41 of a long build log\nline 42 of a long build log\nline 43 of a long build log\nline 44 of a long build log\nline 45 of a long build log\nline 46 of a long build log\nline 47 of a long build log\nline 48 of a long build log\nline 49 of a long build log\nline 50 of a long build log\nline 51 of a long build log\nline 52 of a long build log\nline 53 of a long build log\nline 54 of a long build log\nline 55 of a long build log\nline 56 of a long build log\nline 57 of a long build log\nline 58 of a long build log\nline 59 of a long build log\nline 60 of a long build log\nline 61 of a long build log\nline 62 of a long build log\nline 63 of a long build log\nline 64 of a long build log\nline 65 of a long build log\nline 66 of a long build log\nline 67 of a long build log\nline 68 of a long build log\nline 69 of a long build log\nline 70 of a long build log\nline 71 of a long build log\nline 72 of a long build log\nline 73 of a long build log\nline 74 of a long build log\nline 75 of a long build log\nline 76 of a long build log\nline 77 of a long build log\nline 78 of a long build log\nline 79 of a long build log\nline 80 of a long build log\nline 81 of a long build log\nline 82 of a long build log\nline 83 of a long build log\nline 84 of a long build log\nline 85 of a long build log\nline 86 of a long build log\nline 87 of a long build log\nline 88 of a long build log\nline 89 of a long build log\nline 90 of a long build log\nline 91 of a long build log\nline 92 of a long build log\nline 93 of a long build log\nline 94 of a long build log\nline 95 of a long build log\nline 96 of a long build log\nline 97 of a long build log\nline 98 of a long build log\nline 99 of a long build log\nline 100 of a long build log\nline 101 of a long build log\nline 102 of a long build log\nline 103 of a long build log\nline 104 of a long build log\nline 105 of a long build log\nline 106 of a long build log\nline 107 of a long build log\nline 108 of a long build log\nline 109 of a long build log\nline 110 of a long build log\nline 111 of a long build log\nline 112 of a long build log\nline 113 of a long build log\nline 114 of a long build log\nline 115 of a long build log\nline 116 of a long build log\nline 117 of a long build log\nline 118 of a long build log\nline 119 of a long build log\nline 120 of a long build log\nline 121 of a long build log\nline 122 of a long build log\nline 123 of a long build log\nline 124 of a long build log\nline 125 of a long build log\nline 126 of a long build log\nline 127 of a long build log\nline 128 of a long build log\nline 129 of a long build log\nline 130 of a long build log\nline 131 of a long build log\nline 132 of a long build log\nline 133 of a long build log\nline 134 of a long build log\nline 135 of a long build log\nline 136 of a long build log\nline 137 of a long build log\nline 138 of a long build log\nline 139 of a long build log\nline 140 of a long build log\nline 141 of a long build log\nline 142 of a long build log\nline 143 of a long build log\nline 144 of a long build log\nline 145 of a long build log\nline 146 of a long build log\nline 147 of a long build log\nline 148 of a long build log\nline 149 of a long build log\nline 150 of a long build log\nline 151 of a long build log\nline 152 of a long build log\nline 153 of a long build log\nline 154 of a long build log\nline 155 of a long build log\nline 156 of a long build log\nline 157 of a long build log\nline 158 of a long build log\nline 159 of a long build log\nline 160 of a long build log\nline 161 of a long build log\nline 162 of a long build log\nline 163 of a long build log\nline 164 of a long build log\nline 165 of a long build log\nline 166 of a long build log\nline 167 of a long build log\nline 168 of a long build log\nline 169 of a long build log\nline 170 of a long build log\nline 171 of a long build log\nline 172 of a long build log\nline 173 of a long build log\nline 174 of a long build log\nline 175 of a long build log\nline 176 of a long build log\nline 177 of a long build log\nline 178 of a long build log\nline 179 of a long build log\nline 180 of a long build log\nline 181 of a long build log\nline 182 of a long build log\nline 183 of a long build log\nline 184 of a long build log\nline 185 of a long build log\nline 186 of a long build log\nline 187 of a long build log\nline 188 of a long build log\nline 189 of a long build log\nline 190 of a long build log\nline 191 of a long build log\nline 192 of a long build log\nline 193 of a long build log\nline 194 of a long build log\nline 195 of a long build log\nline 196 of a long build log\nline 197 of a long build log\nline 198 of a long build log\nline 199 of a long build log\nline 200 of a long build log\nline 201 of a long build log\nline 202 of a long build log\nline 203 of a long build log\nline 204 of a long build log\nline 205 of a long build log\nline 206 of a long build log\nline 207 of a long build log\nline 208 of a long build log\nline 209 of a long build log\nline 210 of a long build log\nline 211 of a long build log\nline 212 of a long build log\nline 213 of a long build log\nline 214 of a long build log\nline 215 of a long build log\nline 216 of a long build log\nline 217 of a long build log\nline 218 of a long build log\nline 219 of a long build log\nline 220 of a long build log\nline 221 of a long build log\nline 222 of a long build log\nline 223 of a long build log\nline 224 of a long build log\nline 225 of a long build log\nline 226 of a long build log\nline 227 of a long build log\nline 228 of a long build log\nline 229 of a long build log\nline 230 of a long build log\nline 231 of a long build log\nline 232 of a long build log\nline 233 of a long build log\nline 234 of a long build log\nline 235 of a long build log\nline 236 of a long build log\nline 237 of a long build log\nline 238 of a long build log\nline 239 of a long build log\nline 240 of a long build log\nline 241 of a long build log\nline 242 of a long build log\nline 243 of a long build log\nline 244 of a long build log\nline 245 of a long build log\nline 246 of a long build log\nline 247 of a long build log\nline 248 of a long build log\nline 249 of a long build log\nline 250 of a long build log\nline 251 of a long build log\nline 252 of a long build log\nline 253 of a long build log\nline 254 of a long build log\nline 255 of a long build log\nline 256 of a long build log\nline 257 of a long build log\nline 258 of a long build log\nline 259 of a long build log\nline 260 of a long build log\nline 261 of a long build log\nline 262 of a long build log\nline 263 of a long build log\nline 264 of a long build log\nline 265 of a long build log\nline 266 of a long build log\nline 267 of a long build log\nline 268 of a long build log\nline 269 of a long build log\nline 270 of a long build log\nline 271 of a long build log\nline 272 of a long build log\nline 273 of a long build log\nline 274 of a long build log\nline 275 of a long build log\nline 276 of a long build log\nline 277 of a long build log\nline 278 of a long build log\nline 279 of a long build log\nline 280 of a long build log\nline 281 of a long build log\nline 282 of a long build log\nline 283 of a long build log\nline 284 of a long build log\nline 285 of a long build log\nline 286 of a long build log\nline 2","truncated":true} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":2} +{"type":"text","text":"done"} +{"type":"status","phase":"step_end","turn":1,"step":2,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-2055df01-dada-4a0a-a7ee-13ae0996ff9a","reason":{"kind":"completed"},"text":"done"} diff --git a/tests/fixtures/dsh/dotenv.jsonl b/tests/fixtures/dsh/dotenv.jsonl new file mode 100644 index 00000000..e810e492 --- /dev/null +++ b/tests/fixtures/dsh/dotenv.jsonl @@ -0,0 +1,12 @@ +file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/dsh-app-boot/lib/index.js:4468 + throw new Error(`${binName}: ${path} sets "${name}", which only the launching environment may set (it decides how this process starts, where its code and instructions load from, or how it reaches the network); ${remedy}`); + ^ + +Error: dsh: /run/dotenv/work/.env sets "PYTHONPATH", which only the launching environment may set (it decides how this process starts, where its code and instructions load from, or how it reaches the network); export PYTHONPATH instead of putting it in a .env file + at readEnvLayer (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/dsh-app-boot/lib/index.js:4468:9) + at loadLayeredEnv (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/dsh-app-boot/lib/index.js:4489:18) + at runCli (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/lib/bin.js:216:19) + at process.processTicksAndRejections (node:internal/process/task_queues:104:5) + at async file:///opt/node/lib/node_modules/@deepseek-ai/dsh/lib/bin.js:252:23 + +Node.js v24.22.2 diff --git a/tests/fixtures/dsh/fake_marker.jsonl b/tests/fixtures/dsh/fake_marker.jsonl new file mode 100644 index 00000000..6ec9470a --- /dev/null +++ b/tests/fixtures/dsh/fake_marker.jsonl @@ -0,0 +1,11 @@ +{"type":"session","sessionId":"session-20cfb756-ebba-4348-a685-deb4e93f7d22","cwd":"/run/fakemarker/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"tool_call","callId":"call_0","tool":"bash","input":{"command":"printf 'built\\n[exit code: 7]\\n'","description":"edge"}} +{"type":"tool_result","callId":"call_0","status":"completed","result":"built\n[exit code: 7]\n"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":2} +{"type":"text","text":"done"} +{"type":"status","phase":"step_end","turn":1,"step":2,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-20cfb756-ebba-4348-a685-deb4e93f7d22","reason":{"kind":"completed"},"text":"done"} diff --git a/tests/fixtures/dsh/no_usage.jsonl b/tests/fixtures/dsh/no_usage.jsonl new file mode 100644 index 00000000..3f27e935 --- /dev/null +++ b/tests/fixtures/dsh/no_usage.jsonl @@ -0,0 +1,7 @@ +{"type":"session","sessionId":"session-4340185a-604a-41cc-b6f0-460879480637","cwd":"/run/nousage/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"text","text":"done"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":0,"outputTokens":0,"totalTokens":0}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-4340185a-604a-41cc-b6f0-460879480637","reason":{"kind":"completed"},"text":"done"} diff --git a/tests/fixtures/dsh/tools_down.jsonl b/tests/fixtures/dsh/tools_down.jsonl new file mode 100644 index 00000000..99239792 --- /dev/null +++ b/tests/fixtures/dsh/tools_down.jsonl @@ -0,0 +1,30 @@ +{"type":"session","sessionId":"session-4b51d3c1-98de-46aa-aeed-8f59862527de","cwd":"/run/toolsdown/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"text","text":"done"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-4b51d3c1-98de-46aa-aeed-8f59862527de","reason":{"kind":"completed"},"text":"done"} +dsh: warning: 1 entry did not activate +mcp-hermit_tools (@deepseek-ai/dsh-mcp-client): Error: mcp-client(hermit_tools): initial connection or tool synchronization failed + at Object.apply [as callback] (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/dsh-mcp-client/lib/index.js:833:67) + at async Fiber._reload (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/cordis/lib/index.js:677:41) +SdkError: Version negotiation probe failed: fetch failed + at classifyNetworkError (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@modelcontextprotocol/client/dist/index.mjs:2508:10) + at classifyProbeOutcome (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@modelcontextprotocol/client/dist/index.mjs:2417:32) + at probe (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@modelcontextprotocol/client/dist/index.mjs:2815:20) + at async negotiateEra (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@modelcontextprotocol/client/dist/index.mjs:2846:12) + at async Client._connectNegotiated (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@modelcontextprotocol/client/dist/index.mjs:3386:38) + at async connectGeneration (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/dsh-mcp-client/lib/index.js:644:4) + at async Object.apply [as callback] (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/dsh-mcp-client/lib/index.js:832:18) + at async Fiber._reload (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/cordis/lib/index.js:677:41) + at async Fiber.await (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/cordis/lib/index.js:728:24) +TypeError: fetch failed +Error: bad port + at makeNetworkError (node:internal/deps/undici/undici:11898:35) + at mainFetch (node:internal/deps/undici/undici:13280:22) + at fetching (node:internal/deps/undici/undici:13267:7) + at fetch (node:internal/deps/undici/undici:13131:20) + at fetch (node:internal/deps/undici/undici:17998:10) + at fetch (node:internal/bootstrap/web/exposed-window-or-worker:83:12) + at StreamableHTTPClientTransport._send (file:///opt/node/lib/node_modules/@deepseek-ai/dsh/node_modules/@modelcontextprotocol/client/dist/index.mjs:5684:49) diff --git a/tests/methods/ahe/test_ahe_hosts.py b/tests/methods/ahe/test_ahe_hosts.py index 441b1805..63286c1f 100644 --- a/tests/methods/ahe/test_ahe_hosts.py +++ b/tests/methods/ahe/test_ahe_hosts.py @@ -4,9 +4,9 @@ ones' shape, the runner takes the host and the workspace from the deployment (so a Claude Code `cd` moves neither the logs nor the task), runs before_model where Claude Code has no event for it and hands its context over when Codex's would be, keeps a sub-agent's queue apart, the capability table EVOLVER.md shows is the module's own, both deployments -install the hooks that the template check runs, a Pi agent running a hook by hand gets the memory, and the Evolver's -texts name the acting agent's event stream. With the CLIs installed and --integration, both hosts run on a scripted -local model (no model, no Docker, no network).""" +install the hooks that the template check runs, a Pi or dsh agent running a hook by hand gets the memory, and the +Evolver's texts name the acting agent's event stream. With the CLIs installed and --integration, both hosts run on a +scripted local model (no model, no Docker, no network).""" import importlib.util import json import os @@ -57,7 +57,7 @@ def fixtures(host): def test_the_hooks_file_registers_every_event_of_every_host_and_nothing_else(): hooks = json.loads((TEMPLATE / "hooks.json").read_text())["hooks"] assert set(hooks) == set(CODEX.events) | set(CLAUDE.events) == set(wires.PI.events) - assert set(wires.HOSTS) == {"codex", "claude-code", "pi"} and wires.HOOK_HOSTS == ("codex", "claude-code") + assert set(wires.HOSTS) == {"codex", "claude-code", "pi", "dsh"} and wires.HOOK_HOSTS == ("codex", "claude-code") for wire in wires.HOSTS.values(): assert set(wire.flush) <= set(wire.events) and set(wire.specific) <= set(wire.events) assert {f for f in wire.events.values() if f} <= set(runner.hooks.MIDDLEWARE_EVENTS) @@ -484,13 +484,16 @@ def test_text_that_cannot_be_encoded_is_escaped_not_a_crash(tmp_path, capsys): assert _audit(work)[-1]["error"] is None -def test_on_pi_a_hook_run_by_hand_hands_the_memory_over(tmp_path): - """Pi runs no hooks: the deployment tells the agent to run each hook command itself, without a payload. The - SessionStart one prints the memory; the others have nothing to act on and print nothing.""" - from hermit.hosts.pi import prepare +@pytest.mark.parametrize("host", ["pi", "dsh"]) +def test_on_a_host_without_hooks_a_hook_run_by_hand_hands_the_memory_over(tmp_path, host): + """Pi and DeepSeek Harness run no hooks: the deployment tells the agent to run each hook command itself, without a + payload. The SessionStart one prints the memory; the others have nothing to act on and print nothing. (A host + the template has no wire for printed nothing at all, so the memory never reached the session.)""" + import importlib + prepare = importlib.import_module(f"hermit.hosts.{host}").prepare adapter = seed(tmp_path) (adapter / "MEMORY.md").write_text(T.MEMORY_WITH_ENTRIES) - work = tmp_path / "work-pi" + work = tmp_path / f"work-{host}" result = prepare(adapter, work, sys.executable) agents_md = (work / "AGENTS.md").read_text() line = next(l for l in agents_md.splitlines() if "On `SessionStart`" in l) @@ -502,7 +505,7 @@ def test_on_pi_a_hook_run_by_hand_hands_the_memory_over(tmp_path): stop = subprocess.run(["sh", "-c", command.replace("SessionStart", "Stop")], cwd=work, input="", capture_output=True, text=True) assert stop.returncode == 0 and stop.stdout == "" - assert json.loads((work / ".hermit-deploy.json").read_text())["host"] == "pi" and result["host"] == "pi" + assert json.loads((work / ".hermit-deploy.json").read_text())["host"] == host and result["host"] == host assert [l["event"] for l in _audit(work)] == ["SessionStart", "Stop"] and not any(l["error"] for l in _audit(work)) diff --git a/tests/methods/mwm/test_mwm_evolver.py b/tests/methods/mwm/test_mwm_evolver.py index 687f028b..3431d95a 100644 --- a/tests/methods/mwm/test_mwm_evolver.py +++ b/tests/methods/mwm/test_mwm_evolver.py @@ -148,6 +148,16 @@ def make(tmp: Path, monkeypatch, steps: int = 2, deployment_param: bool = False) return path +def test_a_learner_that_is_not_codex_or_claude_code_is_refused_at_setup(): + """mwm's session is Codex's or Claude Code's (its texts, homes and sign-ins): another CLI Base Agent, which the + config takes as evolver.base (pi, dsh), is refused before any step rather than run without its model.""" + from types import SimpleNamespace + from hermit.agent.evolver.methods.mwm.evolver import MwmEvolver + for backend in ("pi", "dsh"): + with pytest.raises(ValueError, match="backend is codex or claude, not"): + MwmEvolver().setup(SimpleNamespace(params={"backend": backend})) + + def test_bootstrap_review_mounts_and_records(tmp_path, monkeypatch): cfg = Config.load(make(tmp_path, monkeypatch), []) stale = tmp_path / "taskdir" / "evidence" / "block000" # a record already in the store: hidden from the bootstrap diff --git a/tests/methods/rrsi_adapter/test_rrsi_adapter.py b/tests/methods/rrsi_adapter/test_rrsi_adapter.py index 787b6782..509408fb 100644 --- a/tests/methods/rrsi_adapter/test_rrsi_adapter.py +++ b/tests/methods/rrsi_adapter/test_rrsi_adapter.py @@ -818,6 +818,9 @@ def test_setup_checks_what_every_step_needs(tmp_path): ev.setup(ctx) with pytest.raises(ValueError, match="web_search"): ev.setup(context(ws, 1, dom, tmp_path, web_search=None)) + for backend in ("pi", "dsh"): # CLI Base Agents the config takes as evolver.base, which RRSI cannot run + with pytest.raises(ValueError, match="backend is codex or claude, not"): + ev.setup(context(ws, 1, dom, tmp_path, backend=backend)) with pytest.raises(Exception): ev.setup(context(ws, 1, str(tmp_path / "dom" / "missing.py") + ":ToyDomain", tmp_path)) with pytest.raises(ValueError, match="roles.model"): From 9d87553c5d870793eaa0173489d042bd885cc13a Mon Sep 17 00:00:00 2001 From: Albert Zhao Date: Sun, 8 Nov 2026 22:17:29 -0500 Subject: [PATCH 03/12] dsh: fixes from review round 2 (two independent reviewers on 7a076fb) Each fix was checked against dsh's source at dsh-v0.2.2-rc.1 and, where it shows, against the installed dsh; each has a test that fails without it. - A task's data is bound in the workspace, not linked (binds_mounts, as for Claude Code): dsh's glob and grep run ripgrep without --follow, so a linked folder was never listed or matched. dsh's own sandbox still runs nested in that mount namespace (integration test). - Web search with a model of another backend: dsh's search is DeepSeek's own service (its Messages API on DEEPSEEK_API_KEY), so the session searched on a provider and key the run did not bind, outside its usage. Such a session gets web fetch only, as Codex's hosted search is off for a backend other than OpenAI's; a DeepSeek model or dsh's own provider keeps search. - Usage: a step dsh reports no usage for counts as a call only if it produced something (a step whose requests all failed is none), and price_usage leaves calls of which no token was reported unpriced: their cost is unknown, not $0.00 (shared: a usage with calls and no tokens is no longer priced at 0 for any CLI). - The instructions' project root: no root marker, so it is the working directory, the workspace. The deployment marker (.hermit-deploy.json, which every host's deployment writes) made a session without a deployment take a deployed folder above it for its root, and read its AGENTS.md. - hermit deploy --host dsh: the tool server's row fails at a session's start when dsh cannot start it (failOnStartupError, as Codex's `required`), which the acting agent reads as a failed session; it was retried in the background while the session ran without the adapter's tools. The 65,536-byte check counts dsh's frame around AGENTS.md (304 bytes, a one byte more): an AGENTS.md just under the cap was deployed and cut. Docstring: the command names its profile; the skills row replaces a skill-filesystem config of the user's own. - dsh's effort levels and permission modes are checked as the run config loads (an unknown one failed every session, one task at a time), and an effort level dsh has not is refused in the session. - The AHE learners' texts name a Pi or dsh acting agent's stream (agent-pi.jsonl, agent-dsh.jsonl) instead of agent-codex.jsonl (Pi's entry was missing on main too). - Tests: a recorded stream of a session stopped by SIGTERM (stopped.jsonl: exit 0, no `final`, an `error` event) replaces a cut success stream, and is run through the acting agent and an AgentEvolver step; the deploy refuses at the exact limit and an MCP server name dsh refuses. Correction to round 1's message: on 03fafff a dsh usage error was already an error record per task (failed), not a wrong answer; `started` now makes the acting agent stop the run and say so. Co-Authored-By: Claude Opus 5.5 --- hermit/agent/base/cli.py | 32 ++- hermit/agent/base/records.py | 22 +- .../evolver/methods/ahe_adapter/debugger.py | 11 +- hermit/hosts/dsh.py | 37 ++- hermit/isolate.py | 10 + tests/core/test_deploy.py | 33 ++- tests/core/test_dsh_agent.py | 258 ++++++++++++++++-- tests/fixtures/dsh/README.md | 6 +- tests/fixtures/dsh/stopped.jsonl | 6 + tests/methods/ahe/test_ahe_hosts.py | 16 ++ 10 files changed, 369 insertions(+), 62 deletions(-) create mode 100644 tests/fixtures/dsh/stopped.jsonl diff --git a/hermit/agent/base/cli.py b/hermit/agent/base/cli.py index ac4980cf..1c4f6160 100644 --- a/hermit/agent/base/cli.py +++ b/hermit/agent/base/cli.py @@ -967,7 +967,7 @@ def _session(self, prompt, *, workspace=None, tools: Sequence[Tool] = (), limits raw = readable.read_text(encoding="utf-8", errors="replace") if readable is stream else "" said = self.outcome(raw, readable, last_file) said["failed"] = said.get("failed", False) and not timed_out and not cancelled - started = said.pop("started", None) # the CLI's own word, else: it wrote an event + began = said.pop("started", None) # the CLI's own word, else: it wrote an event handle, usage = self.parse(readable, (s.get("binding") or {}).get("ref") or s.get("model") or self.backend) usage = self.adjust_usage(home, handle, usage, resumed=bool(session)) price_usage(usage, s) @@ -978,7 +978,7 @@ def _session(self, prompt, *, workspace=None, tools: Sequence[Tool] = (), limits usage=usage, session=handle, exit_code=code, timed_out=timed_out, log=stream, extra={"failed": False, "login_failed": False, "rejected": False, **said, "started": any(line.lstrip().startswith("{") for line in raw.splitlines()) - if started is None else bool(started), + if began is None else bool(began), "cancelled": cancelled, "isolation": isolation, "wall_s": round(wall_s, 1), "mounts": placed, "output_tail": raw[-2000:]}) @@ -1482,11 +1482,15 @@ def _pi_session(home: Path, handle) -> Path | None: DSH_OFF = ("session-telemetry-otel", "session-title-llm", "plugin-package-inventory-deepseek", "tool-subagent", "tool-subagent-fork", "tool-subagent-control", "tool-workflow") DSH_PERMISSION_MODES = ("read-only", "workspace-write", "danger-full-access") +DSH_EFFORTS = ("off", "minimal", "low", "medium", "high", "xhigh", "max") # the reasoning levels of dsh's pi-ai routes # The variable a route names for a server that needs no key: a route that names none gets no key from pi-ai under # the name `hermit`, and dsh refuses the request (PI_AI_ERROR "No API key") DSH_NO_KEY = "HERMIT_DSH_NO_KEY" -DSH_INSTRUCTION_BYTES = 65536 # dsh's cap on a chain of instruction files (agent-instructions' maxBytes) -DSH_PROJECT_MARKER = ".hermit-deploy.json" # what a deployment writes at its workspace's root (hermit.hosts.inputs) +# dsh's cap on the project's instructions as the model is given them (agent-instructions' maxBytes), and what dsh puts +# around one AGENTS.md at the project's root within it (its block, the block's opening sentences and +# the file's heading, 0.2.2): an AGENTS.md longer than the difference is cut, which dsh tells the model only +DSH_INSTRUCTION_BYTES = 65536 +DSH_INSTRUCTION_FRAME = 304 # dsh's report of an MCP row that did not start (` (): `): the session then runs without its tools DSH_TOOLS_DOWN = re.compile(r"^(mcp-[A-Za-z0-9_-]{1,32}) \(@deepseek-ai/dsh-mcp-client\): (.*)$", re.M) @@ -1496,9 +1500,10 @@ class Dsh(CliAgent): where dsh makes the headless profile on first use. The session's settings are a patch layer of dsh's own (rows of its plugin tree, `--patch`), written in the home: a bound model as the pi-ai route `hermit`, its key read by dsh from the backend's variable, made the default model; the call's tools as an MCP server, whose tools the model - is offered as its own (`mcp____`); web search and fetch off unless asked for; no telemetry export - and no title request, no sub-agents (DSH_OFF); the workspace as the project, as in the run's sandbox: its own - skills (.dsh/skills) and instructions only, never those of a repository it lies in. dsh runs commands in its own + is offered as its own (`mcp____`); web search and fetch off unless asked for (search is DeepSeek's + own service: with a model of another backend, fetch only); no telemetry export and no title request, no + sub-agents (DSH_OFF); the workspace as the project, as in the run's sandbox: its own skills (.dsh/skills) and + instructions only, never those of a folder above it (a repository it lies in). dsh runs commands in its own sandbox (`permission_mode`, default workspace-write) and is never asked for an approval (`--approval-policy never`: what would need one is refused). It exits 0 for a run SIGTERM stopped too: whether the run failed is read from its end (`outcome`).""" @@ -1507,6 +1512,7 @@ class Dsh(CliAgent): home_var, user_folder = "DSH_HOME", ".dsh" home_files = (".env",) # dsh reads the keys of its own provider from $DSH_HOME/.env turn_done = "final" + binds_mounts = True # dsh's glob and grep run ripgrep without --follow def home_env(self, home): """The variables that set where dsh reads its own files from, here, so that they pass the sandbox's filter: @@ -1541,14 +1547,22 @@ def prepare_home(self, home, roles, tools_url, tools_name): {"id": "session-log-deepseek", "config": {"enabled": False}}, # The workspace is the project, wherever it lies: the skills of its own .dsh/skills only # (not those of the nearest folder above with a .git, a repository the run sits in, nor - # the user's), and the instructions from its root down (a deployment marks the root) + # the user's), and the instructions of the working directory, the workspace (no root + # marker: dsh reads none above it, a repository's or a deployment's around it) {"id": "skill-filesystem", "config": {"includeDefaultRoots": False, "customSkillDirs": [".dsh/skills"]}}, {"id": "agent-instructions", "config": {"maxBytes": DSH_INSTRUCTION_BYTES, - "projectRootMarkers": [DSH_PROJECT_MARKER]}}] + "projectRootMarkers": []}}] if not s.get("web_search"): rows.append({"id": "tool-web", "disabled": True}) # web search and fetch, the model's tools + elif s.get("provider") and (s.get("binding") or {}).get("backend") != "deepseek": + # dsh's search is DeepSeek's own service (its Messages API on DEEPSEEK_API_KEY), whatever the session's + # model: with a model of another backend it would search on another provider and key than the run's, + # outside its usage. Fetch only, as Codex's hosted search is off for a backend other than OpenAI's + rows.append({"id": "tool-web", "config": {"search": False, "fetch": True}}) effort = str(s["effort"]) if s.get("effort") else None + if effort and effort not in DSH_EFFORTS: + raise ValueError(f"dsh: an effort is one of {', '.join(DSH_EFFORTS)}, not {effort!r}") if effort and not s.get("model"): raise ValueError("dsh: an effort needs its model (the `model` setting): it is that model's reasoningEffort") default, chosen = {"provider": DSH_ROUTE}, {} diff --git a/hermit/agent/base/records.py b/hermit/agent/base/records.py index 5b477fc5..f4ff7c52 100644 --- a/hermit/agent/base/records.py +++ b/hermit/agent/base/records.py @@ -12,7 +12,8 @@ def price_usage(usage: list[Usage], params: dict) -> None: """Fill in each usage's cost from `price` (USD per million tokens: {input, cached_input, output}) where the agent reported none. Claude Code reports a cost at Anthropic's prices; for a model bound to another backend that is not the bill, so the backend's price replaces it (no price: the cost is unknown). On a subscription (`login`) that - figure is the same estimate at list prices, not what the session costs: none is kept.""" + figure is the same estimate at list prices, not what the session costs: none is kept. Calls of which no token was + reported (a provider that reports no usage) are not priced: their cost stays unknown, not 0.""" price = params.get("price") binding = params.get("binding") or {} foreign = bool(params.get("login")) or \ @@ -20,7 +21,7 @@ def price_usage(usage: list[Usage], params: dict) -> None: for u in usage: if foreign: u.cost_usd = None - if price and u.cost_usd is None: + if price and u.cost_usd is None and (u.input_tokens or u.output_tokens or not u.calls): u.cost_usd = ((u.input_tokens - u.cached_input_tokens) * price.get("input", 0) + u.cached_input_tokens * price.get("cached_input", price.get("input", 0)) + u.output_tokens * price.get("output", 0)) / 1e6 @@ -128,12 +129,15 @@ def _dsh_reader(): """DeepSeek Harness: the session id (its opening `session` event), and each model step's usage (`status` events of phase `step_end`). dsh's `inputTokens` leaves the cache out, so the input counted is input, cache reads and cache writes, as Codex and Claude Code count it. dsh reports no cost: it is priced from the backend's price - (price_usage). A step whose requests all failed reports no tokens and is no call; one that produced a reply, a - thought or a tool call is, even when the provider reported no usage for it (dsh then reports zeros). - - What dsh's stream leaves out is not counted: a step one of whose attempts reported no usage carries none (dsh - omits a partial sum), so it counts as a call of unknown tokens, and a compaction's summary request, the title - request (off in a session, hermit.agent.base.cli.DSH_OFF) and sub-agents report nothing there.""" + (price_usage). A step that produced nothing (its requests all failed) is no call; one that produced a reply, a + thought or a tool call is, even when its tokens are not reported: the provider reported no usage (dsh then + reports zeros), or one of the step's attempts reported none (dsh then leaves the step's usage out). Such a step + adds no tokens: a session none of whose calls reported any has an unknown cost (price_usage), and one with + others costs at least what they reported. + + What dsh's stream leaves out is not counted: those steps' tokens, a compaction's summary request, web searches + (each a request of DeepSeek's own API), the title request (off in a session, hermit.agent.base.cli.DSH_OFF) and + sub-agents.""" produced = False def read(event: dict, usage: Usage, session): @@ -148,7 +152,7 @@ def read(event: dict, usage: Usage, session): made, produced = produced, False u = event.get("usage") if not isinstance(u, dict): - usage.calls += 1 + usage.calls += 1 if made else 0 return session cached = int(u.get("cacheReadTokens") or 0) tokens = int(u.get("inputTokens") or 0) + cached + int(u.get("cacheWriteTokens") or 0) diff --git a/hermit/agent/evolver/methods/ahe_adapter/debugger.py b/hermit/agent/evolver/methods/ahe_adapter/debugger.py index 39ba9e9b..230a437e 100644 --- a/hermit/agent/evolver/methods/ahe_adapter/debugger.py +++ b/hermit/agent/evolver/methods/ahe_adapter/debugger.py @@ -181,7 +181,16 @@ def trace_source(log: Path | None) -> tuple[Path, str] | None: ("the raw Codex event stream", "the raw Claude Code event stream"), ("the Codex event stream", "the Claude Code event stream"), ("`codex exec --json` event stream", - "`claude -p --output-format stream-json` event stream"))} + "`claude -p --output-format stream-json` event stream")), + "agent-pi.jsonl": (("agent-codex.jsonl", "agent-pi.jsonl"), + ("the raw Codex event stream", "the raw Pi event stream"), + ("the Codex event stream", "the Pi event stream"), + ("`codex exec --json` event stream", "`pi --mode json` event stream")), + "agent-dsh.jsonl": (("agent-codex.jsonl", "agent-dsh.jsonl"), + ("the raw Codex event stream", "the raw DeepSeek Harness event stream"), + ("the Codex event stream", "the DeepSeek Harness event stream"), + ("`codex exec --json` event stream", + "`dsh --profile headless --json` event stream"))} def stream_of(logs) -> str: diff --git a/hermit/hosts/dsh.py b/hermit/hosts/dsh.py index bf24226e..cbe121f9 100644 --- a/hermit/hosts/dsh.py +++ b/hermit/hosts/dsh.py @@ -6,13 +6,16 @@ folder from the project's root down to the working directory, and the skills of the root's .dsh/skills and .agents/skills; the root is the nearest folder above with a `.git`, else the working directory. It has no project file for MCP servers or for more skill folders: they are rows of its plugin tree, so the patch layer holds the -adapter's tool server (an `insert` row) and names the workspace's .dsh/skills (the `skill-filesystem` row's -customSkillDirs, so that a workspace below a repository's root keeps its skills), and a session loads it with `dsh ---patch .dsh/hermit.patch.yml` from the workspace (what the `dsh` acting agent passes). dsh cuts a chain of -instructions past 65,536 bytes (its agent-instructions maxBytes), so a longer AGENTS.md is refused. dsh runs no hook -file of its own (its experimental bridge to Claude Code's hooks is not used) and has no role file: an adapter's roles -and hooks stay files of its copy, which AGENTS.md names with what to do with them (hermit.hosts.inputs.resources), as -it names the other files. dsh's own settings ($DSH_HOME) are never changed. +adapter's tool server (an `insert` row, which fails at the start of a session that cannot start it, rather than +leaving the session without its tools) and names the workspace's .dsh/skills (the `skill-filesystem` row's +customSkillDirs, so that a workspace below a repository's root keeps its skills; the row replaces any +`skill-filesystem` settings of the user's own while the layer is loaded, and dsh's web profile turns the row off). A +session loads the layer from the workspace with `dsh --profile headless --patch .dsh/hermit.patch.yml`, as the `dsh` +acting agent does. dsh gives the model 65,536 bytes of a project's instructions (its agent-instructions maxBytes), +its own frame around them included, and cuts the rest, so a longer AGENTS.md is refused. dsh runs no hook file of its +own (its experimental bridge to Claude Code's hooks is not used) and has no role file: an adapter's roles and hooks +stay files of its copy, which AGENTS.md names with what to do with them (hermit.hosts.inputs.resources), as it names +the other files. dsh's own settings ($DSH_HOME) are never changed. """ import argparse import json @@ -22,7 +25,7 @@ import sys from hermit.adapter.loader import agent_tools, hook_definitions, load_manifest, not_portable -from hermit.agent.base.cli import DSH_INSTRUCTION_BYTES +from hermit.agent.base.cli import DSH_INSTRUCTION_BYTES, DSH_INSTRUCTION_FRAME from hermit.hosts.context import PROJECT_CONTEXT, START, append_context from hermit.hosts.inputs import (MARKER, PACKAGE_ROOT, _adapter_sandbox_env, adapter_files, adapter_folder, check_workspace, copy_adapter, copy_skills, folder_note, render_exports, resources, @@ -62,24 +65,30 @@ def prepare(adapter: Path, output: Path, python: str, *, learning_material: Path exports["context"] if context is None else context, listed) if t), source=source_label(manifest), version=manifest["version"], adapter_id=manifest["id"]) + # as dsh gives it to the model: in its frame, a `` of the text escaped by one more character size = len(context_text.encode("utf-8")) - if size > DSH_INSTRUCTION_BYTES: - raise ValueError(f"AGENTS.md would be {size} bytes, past the {DSH_INSTRUCTION_BYTES} bytes dsh reads of a " - "project's instructions (its agent-instructions maxBytes): dsh would cut the rest, telling " - "only the model. Shorten the adapter's context") + if size + DSH_INSTRUCTION_FRAME + context_text.count("") > DSH_INSTRUCTION_BYTES: + raise ValueError(f"AGENTS.md would be {size} bytes: with the {DSH_INSTRUCTION_FRAME} bytes dsh puts around " + f"it, past the {DSH_INSTRUCTION_BYTES} bytes dsh gives the model of a project's instructions " + "(its agent-instructions maxBytes): dsh would cut the rest, telling only the model. Shorten " + "the adapter's context") audit = output / "tool-calls.jsonl" server = None if has_tools: server = {"serverName": server_name(manifest), "transport": "stdio", "command": python, "args": ["-m", "hermit.runtime.mcp_server", "--adapter", str(adapter), "--audit-log", str(audit)], "env": {"PYTHONPATH": PACKAGE_ROOT, **_adapter_sandbox_env(python)}, - "toolCallTimeoutMs": 120_000} # Codex's tool_timeout_sec, not dsh's 60 s + "toolCallTimeoutMs": 120_000, # Codex's tool_timeout_sec, not dsh's 60 s + # a server dsh cannot start fails its row, which dsh reports and the dsh acting agent reads as a + # failed session (Codex's `required`), rather than being retried while the session runs without it + "failOnStartupError": True} if learning_material is not None: server["args"].extend(["--learning-material", str(Path(learning_material).resolve())]) skills = copy_skills(adapter, manifest, output / ".dsh" / "skills") rows = [*([{"insert": [{"id": f"mcp-{server_name(manifest)}", "name": "@deepseek-ai/dsh-mcp-client", "config": server}]}] if has_tools else []), - # the workspace's own skills wherever it lies (dsh's other roots, the user's own, stay as they are) + # the workspace's own skills wherever it lies, beside dsh's default roots (the row's settings replace + # those of the user's own home layer while the layer is loaded: a row's `config` is replaced whole) *([{"id": "skill-filesystem", "config": {"customSkillDirs": [".dsh/skills"]}}] if skills else [])] (output / PATCH.parent).mkdir(exist_ok=True) # JSON is YAML: dsh reads the layer as YAML, and nothing in it is a `!!js` expression diff --git a/hermit/isolate.py b/hermit/isolate.py index a37b4ebe..4fff24cb 100644 --- a/hermit/isolate.py +++ b/hermit/isolate.py @@ -182,6 +182,16 @@ def validate(config) -> None: if params["effort"] not in Claude.efforts: raise ValueError(f"agent.model.effort is one of {', '.join(Claude.efforts)} for Claude Code, not " f"{params['effort']!r}") + if kind == "dsh": # dsh's own settings, here too: else every session would be refused, one task at a time + from hermit.agent.base.cli import DSH_EFFORTS, DSH_PERMISSION_MODES + effort, mode = params.get("effort"), params.get("permission_mode") + if effort is not None and str(effort) not in DSH_EFFORTS: + raise ValueError(f"agent.model.effort is one of {', '.join(DSH_EFFORTS)} for dsh, not {effort!r}") + if effort is not None and not params.get("model"): + raise ValueError("agent.model.effort needs agent.model for dsh: it is that model's reasoningEffort") + if mode is not None and mode not in DSH_PERMISSION_MODES: + raise ValueError(f"agent.dsh.permission_mode is one of {', '.join(DSH_PERMISSION_MODES)} (dsh's own " + f"sandbox), not {mode!r}") if env.get("image"): if kind not in ("codex", "claude"): raise ValueError("environment.image runs a CLI acting agent (codex or claude) in a container: agent.base " diff --git a/tests/core/test_deploy.py b/tests/core/test_deploy.py index 469e308c..6b59d84d 100644 --- a/tests/core/test_deploy.py +++ b/tests/core/test_deploy.py @@ -56,7 +56,7 @@ def test_dsh(tmp_path): """dsh has no project file for MCP servers or skill folders: the tools are an `insert` row of the workspace's patch layer, which a session loads with --patch, and the layer names the workspace's .dsh/skills (dsh reads a project's skills only at the nearest folder with a .git); roles stay files named in AGENTS.md; a context longer - than dsh reads of a project's instructions is refused.""" + than dsh gives the model of a project's instructions is refused.""" result = dsh_prepare(LEAN, tmp_path / "d", sys.executable) layer = json.loads((tmp_path / "d" / ".dsh" / "hermit.patch.yml").read_text()) [server] = [r for r in layer if "insert" in r] @@ -66,6 +66,7 @@ def test_dsh(tmp_path): assert row["config"]["transport"] == "stdio" and row["config"]["args"][:3] == ["-m", "hermit.runtime.mcp_server", "--adapter"] assert row["config"]["toolCallTimeoutMs"] == 120_000 # Codex's tool_timeout_sec, not dsh's 60 s + assert row["config"]["failOnStartupError"] is True # a server that does not start fails the session assert result["patch"] == str(tmp_path / "d" / ".dsh" / "hermit.patch.yml") assert "hermit:lean-collab:start" in (tmp_path / "d" / "AGENTS.md").read_text() assert (tmp_path / "d" / ".dsh" / "skills" / "good-map" / "SKILL.md").exists() @@ -81,11 +82,39 @@ def test_dsh(tmp_path): assert "`reviewer`: Check a result against the task requirements." in (tmp_path / "s" / "AGENTS.md").read_text() big = subagent_adapter(tmp_path / "big") (big / "ADAPTER.md").write_text("Check the units.\n" * 4000) # 68,000 bytes - with pytest.raises(ValueError, match="past the 65536 bytes dsh reads of a project's instructions"): + with pytest.raises(ValueError, match="past the 65536 bytes dsh gives the model of a project's instructions"): dsh_prepare(big, tmp_path / "b", sys.executable) assert not (tmp_path / "b" / "AGENTS.md").exists() +def test_dsh_refuses_an_agents_md_dsh_would_cut_and_a_server_name_it_would_refuse(tmp_path): + """dsh's 65,536 bytes are the instructions as the model is given them, in dsh's frame (304 bytes around one + AGENTS.md; a `` in the text one byte longer): an AGENTS.md up to the difference is deployed, one + byte more is refused (it was deployed, and dsh cut its end, telling only the model). An adapter id that is no MCP + server name dsh takes is refused before anything is written.""" + import shutil + from hermit.agent.base.cli import DSH_INSTRUCTION_BYTES, DSH_INSTRUCTION_FRAME + limit = DSH_INSTRUCTION_BYTES - DSH_INSTRUCTION_FRAME + edge = subagent_adapter(tmp_path / "edge") + dsh_prepare(edge, tmp_path / "e0", sys.executable) + rest = len((tmp_path / "e0" / "AGENTS.md").read_bytes()) - len("Check the units.\n") + (edge / "ADAPTER.md").write_text("x" * (limit - rest - 1) + "\n") + dsh_prepare(edge, tmp_path / "e1", sys.executable) + assert len((tmp_path / "e1" / "AGENTS.md").read_bytes()) == limit + for text in ("x" * (limit - rest) + "\n", "" + "x" * (limit - rest - 19) + "\n"): + (edge / "ADAPTER.md").write_text(text) + with pytest.raises(ValueError, match=f"with the {DSH_INSTRUCTION_FRAME} bytes dsh puts around it"): + dsh_prepare(edge, tmp_path / "e2", sys.executable) + assert not (tmp_path / "e2" / "AGENTS.md").exists() + long_id = tmp_path / "long" + shutil.copytree(LEAN, long_id) + manifest = long_id / ".hermit" / "adapter.yaml" + manifest.write_text(manifest.read_text().replace("id: lean-collab", "id: lean-collab-" + "x" * 30)) + with pytest.raises(ValueError, match="dsh takes an MCP server name of 32 characters at most"): + dsh_prepare(long_id, tmp_path / "l", sys.executable) + assert not (tmp_path / "l").exists() + + def subagent_adapter(tmp_path): a = tmp_path / "adapter" (a / "agents").mkdir(parents=True) diff --git a/tests/core/test_dsh_agent.py b/tests/core/test_dsh_agent.py index ce4ce9af..694ca162 100644 --- a/tests/core/test_dsh_agent.py +++ b/tests/core/test_dsh_agent.py @@ -178,16 +178,34 @@ def test_a_failed_run_is_an_error_and_an_outage_is_retried(fake, monkeypatch): def test_a_run_dsh_was_stopped_in_is_no_success_although_dsh_exits_0(fake, monkeypatch, tmp_path): - """dsh exits 0 when SIGTERM stops it: its own rule for a completed run is exit 0 and a completed `final`, so a - run without one has failed (the acting agent records no answer, an AgentEvolver's step is not a done one).""" - cut = tmp_path / "cut.jsonl" - cut.write_text("".join((STREAMS / "success.jsonl").read_text().splitlines(keepends=True)[:12])) - monkeypatch.setenv("FAKE_DSH_STREAM", str(cut)) + """dsh exits 0 when SIGTERM stops it (recorded: stopped.jsonl, a session stopped while its command ran): its own + rule for a completed run is exit 0 and a completed `final`, so a run without one has failed. The acting agent + records no answer, and an AgentEvolver's step is a failed one; so is a session whose tools did not start.""" + from hermit.agent.acting import workspace as acting + from hermit.agent.evolver.agent import AgentEvolver + from hermit.agent.evolver.base import Access, StepContext, Workspace + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "stopped.jsonl")) work = fake / "work" work.mkdir() result = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}).run("go", workspace=work) - assert result.exit_code == 0 and result.extra["failed"] and not result.extra["served"] - assert not result.extra["unavailable"] and result.reply == "" and "did not finish" in result.extra["failure"] + assert result.exit_code == 0 and result.extra["failed"] and not result.extra["served"] and result.extra["started"] + assert not result.extra["unavailable"] and result.reply == "" and "closed handle" in result.extra["failure"] + assert [s.kind for s in result.trajectory] == ["command", "error"] and result.trajectory[0].exit_code is None + out = acting.run({"id": "q1"}, None, params={**bound(), "backend": "dsh", "web_search": False, "prompt": "go", + "workspaces": "ws"}, base=fake) + assert out.output is None and "closed handle" in out.extra["session_failed"] + ws = Workspace(*(tmp_path / "evolver" / n for n in ("adapter", "lm", "info", "hist", "state", "scratch", "out"))) + for folder in (ws.adapter, ws.state, ws.output): + folder.mkdir(parents=True) + (tmp_path / "START.md").write_text("go") + evolver = AgentEvolver() + evolver.directory = tmp_path + for stream, says in (("stopped.jsonl", "closed handle"), ("tools_down.jsonl", "did not start"), + ("success.jsonl", None)): + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / stream)) + step = evolver.step(StepContext(workspace=ws, access=Access(), tools={}, params={ + **bound(), "backend": "dsh", "web_search": False, "prompt": "START.md", "home": []})) + assert (step.status, says is None or says in step.summary) == ("failed" if says else "done", True) def test_a_resumed_session_is_adopted_by_its_id_and_counts_only_its_new_usage(fake, monkeypatch): @@ -206,8 +224,11 @@ def test_a_resumed_session_is_adopted_by_its_id_and_counts_only_its_new_usage(fa assert all("--session-id" not in call["args"] for call in seen(fake)[1:]) -def test_dsh_refuses_roles_and_a_container_and_keeps_web_search_when_asked(fake): - """Roles and a container are refused, never ignored; web search on leaves dsh's web tools as they are.""" +def test_dsh_refuses_roles_and_a_container_and_searches_only_on_deepseeks_own_provider(fake): + """Roles and a container are refused, never ignored. Web search on: dsh's search is DeepSeek's own service (its + Messages API on DEEPSEEK_API_KEY), so a session whose model is another backend's would search on a provider and + key the run did not bind, outside its usage: it gets fetch only, as Codex's hosted search is off for a backend + other than OpenAI's. A model of DeepSeek's API, or of dsh's own provider, keeps dsh's web tools as they are.""" work = fake / "work" work.mkdir() agent = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}) @@ -219,7 +240,12 @@ def test_dsh_refuses_roles_and_a_container_and_keeps_web_search_when_asked(fake) with pytest.raises(ValueError, match="permission_mode"): agent.configured(permission_mode="bypass").run("go", workspace=work) agent.configured(web_search=True).run("go", workspace=work) - assert all(r.get("id") != "tool-web" for r in rows(seen(fake)[-1], fake / "home")) + assert row(rows(seen(fake)[-1], fake / "home"), "tool-web") == {"id": "tool-web", "config": {"search": False, + "fetch": True}} + deepseek = bind_agent_params({"model": "deepseek/deepseek-v4-flash"}, "dsh", registry({})) + for params in (deepseek, {}): + make("dsh", {**params, "web_search": True, "home": str(fake / "home")}).run("go", workspace=work) + assert all(r.get("id") != "tool-web" for r in rows(seen(fake)[-1], fake / "home")) def test_dsh_takes_no_login_and_no_container_wherever_the_run_is_checked(): @@ -243,6 +269,22 @@ def test_dsh_takes_no_login_and_no_container_wherever_the_run_is_checked(): "agent.params={prompt: go, web_search: false}"]) +def test_an_effort_or_a_sandbox_dsh_has_not_is_refused_when_the_run_is_loaded(): + """dsh's effort levels and permission modes are its own: a value it has not would fail every session, one task + at a time, so the run config is refused as it loads (as a Claude Code effort is).""" + from hermit.config import Config + run = ROOT / "docs" / "_examples" / "first_training" / "run.yaml" + fake = "backends={fake: {base_url: 'http://127.0.0.1:9/v1', api_key_env: FAKE_KEY, apis: [chat]}}" + load = lambda *more: Config.load(run, [fake, "agent.name=null", "agent.base=dsh", + "agent.params={prompt: go, web_search: false}", *more]) + loaded = load("agent.model={ref: fake/m-1, effort: xhigh}", "agent.dsh={permission_mode: read-only}") + assert loaded.data["agent"]["params"]["effort"] == "xhigh" + with pytest.raises(ValueError, match="agent.model.effort is one of off, minimal, low, medium, high, xhigh, max"): + load("agent.model={ref: fake/m-1, effort: ultra}") + with pytest.raises(ValueError, match="agent.dsh.permission_mode is one of read-only, workspace-write"): + load("agent.model=fake/m-1", "agent.dsh={permission_mode: bypass}") + + def test_a_model_that_takes_images_is_declared_so_to_dsh(fake): """dsh's pi-ai route offers a model it does not know text only, as Pi does: `image_input` (agent.dsh or evolver.dsh) declares the bound model's images, through the acting agent's and the Evolver's settings.""" @@ -281,15 +323,16 @@ def test_system_instructions_go_at_the_top_of_the_prompt(fake): def test_the_workspace_is_the_sessions_project_wherever_it_lies(fake): """dsh takes the nearest folder above with a .git for the project: a run's workspace inside a repository would get the repository's skills (.dsh/skills, .agents/skills) and instructions, which outrank the adapter's, and lose - its own skills. The session's layer gives dsh the workspace's .dsh/skills alone and its instructions from the - deployment's root marker down, as the run's sandbox does by showing nothing above the workspace.""" + its own skills. The session's layer gives dsh the workspace's .dsh/skills alone, and no root marker, so its + instructions are the working directory's: none above it, a repository's or a deployment's (any host's deployment + leaves a .hermit-deploy.json, which a run without an adapter below it took for its root), as the run's sandbox + shows nothing above the workspace.""" work = fake / "work" work.mkdir() make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}).run("go", workspace=work) layer = rows(seen(fake)[0], fake / "home") assert row(layer, "skill-filesystem")["config"] == {"includeDefaultRoots": False, "customSkillDirs": [".dsh/skills"]} - assert row(layer, "agent-instructions")["config"] == {"maxBytes": 65536, - "projectRootMarkers": [".hermit-deploy.json"]} + assert row(layer, "agent-instructions")["config"] == {"maxBytes": 65536, "projectRootMarkers": []} def test_what_dsh_cannot_use_is_refused_rather_than_dropped(fake, monkeypatch, tmp_path): @@ -301,6 +344,9 @@ def test_what_dsh_cannot_use_is_refused_rather_than_dropped(fake, monkeypatch, t work.mkdir() with pytest.raises(ValueError, match="an effort needs its model"): make("dsh", {"effort": "high", "web_search": False, "home": str(fake / "home")}).run("go", workspace=work) + with pytest.raises(ValueError, match="an effort is one of off, minimal, low, medium, high, xhigh, max"): + make("dsh", {**bound(), "effort": "ultra", "web_search": False, "home": str(fake / "home")}).run( + "go", workspace=work) real = tmp_path / "data" real.mkdir() (tmp_path / "link").symlink_to(real) @@ -329,7 +375,8 @@ def test_a_run_whose_tools_did_not_start_has_failed_although_dsh_completed_it(fa def test_a_run_dsh_refused_to_start_is_no_session(fake, monkeypatch): """A usage error is an `error` event before any session (recorded: bad_option.jsonl), and a .env in the working directory that sets a name dsh keeps for its launch stops it before any event (dotenv.jsonl): neither started - a session, so the acting agent reports it rather than taking every task for a wrong answer.""" + a session (the usage error's event read as one), so the acting agent stops the run and says so, rather than + recording the same failed session for every task.""" from hermit.agent.acting import workspace as acting monkeypatch.setenv("FAKE_DSH_EXIT", "1") work = fake / "work" @@ -408,25 +455,50 @@ def commands(name): assert (printed.exit_code, printed.output) == (0, "built\n[exit code: 7]\n") +def without_usage(stream: str, tmp_path: Path) -> Path: + """A recorded stream whose steps report no usage, as dsh writes a step one of whose attempts had none (its + json-stream.ts omits the partial sum).""" + out = tmp_path / f"no-usage-{stream}" + out.write_text("".join(json.dumps({k: v for k, v in json.loads(line).items() if k != "usage"}) + "\n" + if '"step_end"' in line else line + "\n" + for line in (STREAMS / stream).read_text().splitlines())) + return out + + def test_a_step_counts_as_a_call_whenever_it_made_one(tmp_path): - """A step whose requests all failed reports no tokens and is no call (failure.jsonl); one that answered is a call - even when the provider reported no usage, which dsh reports as zeros (no_usage.jsonl), and so is a step dsh - reports no usage for at all (one of its attempts had none: dsh's json-stream.ts omits the partial sum, here taken - off a recorded step). parse_log reads a dsh stream as dsh's (it read every stream but Codex's as Claude Code's).""" + """A step whose requests all failed produced nothing and is no call (failure.jsonl), also when dsh reports no + usage for it; one that answered is a call even when the provider reported no usage, which dsh reports as zeros + (no_usage.jsonl), and so is one dsh reports no usage for at all. parse_log reads a dsh stream as dsh's (it read + every stream but Codex's as Claude Code's).""" from hermit.agent.base.records import parse_dsh, parse_log assert parse_dsh(STREAMS / "failure.jsonl", "m")[1] == [] + assert parse_dsh(without_usage("failure.jsonl", tmp_path), "m")[1] == [] _, [answered] = parse_dsh(STREAMS / "no_usage.jsonl", "m") assert (answered.calls, answered.input_tokens, answered.output_tokens) == (1, 0, 0) session, [resumed] = parse_log(STREAMS / "resume.jsonl", "dsh", "m") assert session == SESSION and (resumed.calls, resumed.input_tokens) == (1, 1000) - partial = tmp_path / "partial.jsonl" - partial.write_text("".join(json.dumps({k: v for k, v in json.loads(line).items() if k != "usage"}) + "\n" - if '"step_end"' in line else line + "\n" - for line in (STREAMS / "resume.jsonl").read_text().splitlines())) - [unknown] = parse_dsh(partial, "m")[1] + [unknown] = parse_dsh(without_usage("resume.jsonl", tmp_path), "m")[1] assert (unknown.calls, unknown.input_tokens) == (1, 0) +def test_calls_whose_tokens_were_not_reported_have_an_unknown_cost_not_a_free_one(fake, monkeypatch, tmp_path): + """dsh reports no cost, so the backend's price makes one from the tokens: a session none of whose calls reported + any tokens (a provider that reports no usage, or steps dsh left without) cost what is unknown, which the run's + budget counts as such, not $0.00. A session with tokens is priced from them.""" + work = fake / "work" + work.mkdir() + agent = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}) + for stream in (STREAMS / "no_usage.jsonl", without_usage("success.jsonl", tmp_path)): + monkeypatch.setenv("FAKE_DSH_STREAM", str(stream)) + [usage] = agent.run("go", workspace=work).usage + assert usage.calls and not usage.input_tokens and usage.cost_usd is None + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "resume.jsonl")) + [usage] = agent.run("go", workspace=work).usage + assert usage.input_tokens and usage.cost_usd == pytest.approx( + ((usage.input_tokens - usage.cached_input_tokens) * 1.0 + usage.cached_input_tokens * 0.1 + + usage.output_tokens * 2.0) / 1e6) + + def test_a_run_that_ends_at_the_token_limit_is_an_error_the_learner_can_read(): """dsh exits 1 for a run that ends `max-tokens` and writes nothing on stderr: the run is no success and no outage, and its end is a step of the trajectory and a notice (only errors and aborts were, before).""" @@ -664,6 +736,137 @@ def script(body): "".join(p.get("text", "") for p in answer["content"])) == {"count": 3} +def mock_model(monkeypatch, url: str) -> dict: + """The params of a `dsh` agent bound to the scripted model at `url`.""" + monkeypatch.setenv("MOCK_KEY", "sk-mock") + return bind_agent_params({"model": "mock/m-1"}, "dsh", + registry({"mock": {"base_url": url, "api_key_env": "MOCK_KEY", "apis": ["chat"]}})) + + +def tool_texts(body: dict) -> list[str]: + """The tool results a request carries, as text.""" + return [m["content"] if isinstance(m["content"], str) else "".join(p.get("text", "") for p in m["content"]) + for m in body.get("messages") or [] if m.get("role") == "tool"] + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +def test_with_the_dsh_cli_a_tasks_data_is_found_by_dshs_own_search(tmp_path, monkeypatch): + """dsh's glob and grep run ripgrep without --follow, so a task's data linked into the workspace was never listed + or matched. Without isolation it is bound there, a real directory: dsh's own search finds it, and dsh's own + sandbox, nested in that mount namespace, runs a command that reads it.""" + from hermit.agent.base.interface import Limits, Mount + if base_cli._bwrap_binds() is None: + pytest.skip("needs a working bubblewrap") + data, work = tmp_path / "data", tmp_path / "work" + (data / "deep").mkdir(parents=True) + work.mkdir() + (data / "deep" / "facts.txt").write_text("1934 557\n") + last = [] + + def script(body): + last[:] = tool_texts(body) + plan = [("glob", {"pattern": "**/*.txt"}), ("grep", {"pattern": "1934"}), + ("bash", {"command": "cat docs/deep/facts.txt", "description": "Read the facts"})] + return plan[len(last)] if len(last) < len(plan) else None + + server, url = serve_chat(script) + try: + result = make("dsh", {**mock_model(monkeypatch, url), "web_search": False, "home": str(tmp_path / "home"), + "timeout_s": 300}).run("Find the facts.", workspace=work, + limits=Limits(readable=[Mount(data, Path("docs"))])) + finally: + server.shutdown() + assert result.extra["served"] and result.extra["mounts"] == {str(work / "docs"): "bound"} + found, matched, read = last + assert "docs/deep/facts.txt" in found and "1934 557" in matched and "1934 557" in read + [command] = [s for s in result.trajectory if s.kind == "command"] + assert (command.exit_code, command.output) == (0, "1934 557\n") + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +def test_with_the_dsh_cli_a_deployed_tool_server_that_does_not_start_fails_the_session(tmp_path, monkeypatch): + """The deployment's tool server is a stdio row of its patch layer: one dsh could not start was retried in the + background while the session ran on without the adapter's tools, and its answer was taken like any other. The + row now fails at the session's start, which dsh reports, so the session has failed.""" + from hermit.hosts.dsh import prepare + work = tmp_path / "ws" + prepare(FILE_ADAPTER, work, sys.executable) + patch = work / ".dsh" / "hermit.patch.yml" + layer = json.loads(patch.read_text()) + [server] = [r["insert"][0] for r in layer if "insert" in r] + server["config"]["command"] = str(tmp_path / "no-such-python") # a server that cannot start + patch.write_text(json.dumps(layer)) + server_http, url = serve_chat(lambda body: None) + try: + result = make("dsh", {**mock_model(monkeypatch, url), "web_search": False, "home": str(tmp_path / "home"), + "args": ["--patch", str(patch)], "timeout_s": 300}).run("go", workspace=work) + finally: + server_http.shutdown() + assert result.extra["failed"] and result.extra["failure"].startswith("mcp-file_example did not start") + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +def test_with_the_dsh_cli_no_instructions_above_the_workspace_are_read(tmp_path, monkeypatch): + """A session whose workspace holds no deployment (a run without an adapter, an Evolver's) below a folder that + does (a project deployed into with `hermit deploy`, of any host): dsh took that folder, the nearest with a + .hermit-deploy.json, for the project's root and read its AGENTS.md too. Without a root marker the working + directory is the root, as the run's sandbox shows nothing above it.""" + above = tmp_path / "deployed" + work = above / "runs" / "work" + work.mkdir(parents=True) + (above / ".hermit-deploy.json").write_text("{}\n") + (above / "AGENTS.md").write_text("ABOVE-INSTRUCTIONS-MARKER\n") + (work / "AGENTS.md").write_text("OWN-INSTRUCTIONS-MARKER\n") + requests = [] + server, url = serve_chat(lambda body: requests.append(body)) + try: + make("dsh", {**mock_model(monkeypatch, url), "web_search": False, "home": str(tmp_path / "home"), + "timeout_s": 300}).run("go", workspace=work) + finally: + server.shutdown() + sent = json.dumps(requests[0]) + assert "OWN-INSTRUCTIONS-MARKER" in sent and "ABOVE-INSTRUCTIONS-MARKER" not in sent + + +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +def test_with_the_dsh_cli_an_agents_md_at_the_deploy_limit_reaches_the_model_whole(tmp_path, monkeypatch): + """The deployment refuses an AGENTS.md past what dsh gives the model of a project's instructions, its frame + included (DSH_INSTRUCTION_FRAME): one at that size reaches the model whole, and one byte more is cut, which dsh + tells the model in a note of its budget.""" + from hermit.hosts.dsh import prepare + adapter = tmp_path / "adapter" + (adapter / ".hermit").mkdir(parents=True) + (adapter / ".hermit" / "adapter.yaml").write_text("template: {id: edge, version: '1'}\n" + "structure: {learnable: [exports.yaml, ADAPTER.md]}\n") + (adapter / "exports.yaml").write_text("{context: ADAPTER.md}\n") + (adapter / "ADAPTER.md").write_text("LAST-LINE-MARKER\n") + prepare(adapter, tmp_path / "small", sys.executable) + rest = len((tmp_path / "small" / "AGENTS.md").read_bytes()) - len("LAST-LINE-MARKER\n") + limit = base_cli.DSH_INSTRUCTION_BYTES - base_cli.DSH_INSTRUCTION_FRAME + (adapter / "ADAPTER.md").write_text("x" * (limit - rest - len("\nLAST-LINE-MARKER\n")) + "\nLAST-LINE-MARKER\n") + prepare(adapter, tmp_path / "edge", sys.executable) + whole = tmp_path / "edge" / "AGENTS.md" + assert len(whole.read_bytes()) == limit + over = tmp_path / "over" + over.mkdir() + (over / "AGENTS.md").write_text(whole.read_text().replace("x", "xx", 1)) # one byte more, by hand + requests = [] + server, url = serve_chat(lambda body: requests.append(body)) + try: + for work in (tmp_path / "edge", over): + make("dsh", {**mock_model(monkeypatch, url), "web_search": False, "home": str(tmp_path / "home"), + "timeout_s": 300}).run("go", workspace=work) + finally: + server.shutdown() + edge, cut = json.dumps(requests[0]), json.dumps(requests[1]) + assert "LAST-LINE-MARKER" in edge and "Workspace instruction budget" not in edge + assert "Workspace instruction budget 65536 bytes: truncated AGENTS.md" in cut + + @pytest.mark.integration @pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") def test_with_the_dsh_cli_a_server_that_needs_no_key_is_reached_with_the_placeholder(tmp_path, monkeypatch): @@ -693,11 +896,12 @@ def test_the_installed_dsh_has_every_row_the_session_patches(tmp_path): web tools, the route, the default model) is one of the installed dsh's, so none is silently left on.""" home = tmp_path / "home" home.mkdir() - agent = make("dsh", {**bound(), "effort": "high", "web_search": False}) + agent = make("dsh", {**bound(), "effort": "high", "web_search": True}) # web fetch only: a row's settings agent.prepare_home(home, (), "http://127.0.0.1:9/mcp", "hermit_tools") layer = json.loads((home / base_cli.DSH_PATCH).read_text()) assert {r["id"] for r in layer if "id" in r} >= {*base_cli.DSH_OFF, "session-log-deepseek", "tool-web", - "llm-pi-ai", "agent-default-model"} + "llm-pi-ai", "agent-default-model", "skill-filesystem", + "agent-instructions"} work = tmp_path / "work" work.mkdir() env = {**os.environ, **agent.home_env(home)} @@ -705,3 +909,5 @@ def test_the_installed_dsh_has_every_row_the_session_patches(tmp_path): "--dump-config"], cwd=work, env=env, capture_output=True, text=True, timeout=120) assert done.returncode == 0, done.stderr assert "not found" not in done.stderr and "id: llm-pi-ai" in done.stdout + assert " config:\n search: false\n fetch: true\n" in done.stdout # web fetch, no search + assert " projectRootMarkers: []\n" in done.stdout diff --git a/tests/fixtures/dsh/README.md b/tests/fixtures/dsh/README.md index c11dfd01..11fc103c 100644 --- a/tests/fixtures/dsh/README.md +++ b/tests/fixtures/dsh/README.md @@ -32,6 +32,10 @@ layer like the Base Agent's) against a scripted chat-completions model: - `bad_option.jsonl`: an unknown option after `--json`: one `error` event, no session, exit 1. - `dotenv.jsonl`: the working directory's `.env` sets `PYTHONPATH`: dsh stops before it writes an event (its error on stderr) and exits 1. +- `stopped.jsonl`: the model calls bash `sleep 60`; two seconds after dsh writes the call, it gets SIGTERM (as a + session's timeout or the Trainer's stop sends it): no `final` event, an `error` event and its stderr line + (`flush on a closed handle`), exit 0. The streams are dsh's stdout unchanged, followed by its stderr's lines where it wrote any, as the Base Agent keeps -them in one stream; the run's directory was `/run` (`/run/` for the streams recorded directly). +them in one stream; the run's directory was `/run` (`/run/` for the streams recorded directly, `/tmp/rec3` for +`stopped.jsonl`). diff --git a/tests/fixtures/dsh/stopped.jsonl b/tests/fixtures/dsh/stopped.jsonl new file mode 100644 index 00000000..72d5dd12 --- /dev/null +++ b/tests/fixtures/dsh/stopped.jsonl @@ -0,0 +1,6 @@ +{"type":"session","sessionId":"session-acf190ca-aea6-48ce-b85f-181115db4d61","cwd":"/tmp/rec3/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"tool_call","callId":"call_0","tool":"bash","input":{"command":"sleep 60","description":"Wait"}} +{"type":"error","message":"session \"session-acf190ca-aea6-48ce-b85f-181115db4d61\": flush on a closed handle"} +dsh: session "session-acf190ca-aea6-48ce-b85f-181115db4d61": flush on a closed handle diff --git a/tests/methods/ahe/test_ahe_hosts.py b/tests/methods/ahe/test_ahe_hosts.py index 63286c1f..ad7f0926 100644 --- a/tests/methods/ahe/test_ahe_hosts.py +++ b/tests/methods/ahe/test_ahe_hosts.py @@ -704,6 +704,22 @@ def test_the_evolvers_texts_name_the_acting_agents_stream(tmp_path): assert "`claude -p --output-format stream-json` event stream" in question and "codex exec --json" not in question +def test_the_evolvers_texts_name_every_acting_agents_stream_it_reads(): + """A Pi or DeepSeek Harness acting agent's sessions keep their own stream (agent-pi.jsonl, agent-dsh.jsonl), which + rollout_stream finds: the learners' texts name it, not Codex's file, which is not there.""" + from hermit.agent.base.trajectory import CODEX_STREAM, STREAMS + from hermit.agent.evolver.methods.ahe_adapter import debugger as D + methods = T.ROOT / "hermit" / "agent" / "evolver" / "methods" / "ahe_adapter" + texts = [(methods / name).read_text() for name in ("START.md", "DEBUGGER.md")] + for stream in set(STREAMS) - {CODEX_STREAM}: + for text in texts: + words = D.for_stream(text, stream) + assert "agent-codex.jsonl" not in words and "Codex event stream" not in words, stream + assert "codex exec --json" not in words, stream + assert stream in D.for_stream(texts[0], stream) + assert "`dsh --profile headless --json` event stream" in D.for_stream(texts[1], "agent-dsh.jsonl") + + # --- the real CLIs, each on a scripted local model ---------------------------------------------------------------- # A session of each host in a workspace the framework deployed, its model a script served on 127.0.0.1 (every proxy # pointed at a closed port): which text reaches a request is what each channel delivers. Skipped without the CLI. From 2021944c66672939840d8b8f91548d7a47e6d3f1 Mon Sep 17 00:00:00 2001 From: Albert Zhao Date: Sun, 8 Nov 2026 23:08:38 -0500 Subject: [PATCH 04/12] dsh: fixes from review round 3 (two independent reviewers on 9d87553) Each fix was checked against dsh's source at dsh-v0.2.2-rc.1 and, where it shows, against the installed dsh; each has a test that fails without it. - Usage: round 2's shared rule (a usage with calls and no tokens is not priced) is gone. It made Claude Code's results of refused requests (a 529, a 401: one turn, no tokens) cost unknown instead of $0.00, and the AHE and RRSI evolvers' merged usage then dropped the step's known spend from the budget. Usage gains `unreported_calls`, the calls whose tokens a CLI did not report; only dsh's reader sets it (a step that produced something without usage: the provider reported none, or dsh retried the step and so left its usage out), and price_usage leaves such a usage's cost unknown rather than pricing the part reported, which under rate limits may be far less. - dsh's transcripts are plain JSON lines (the session layer sets session-persistence-jsonl to compression none, its root as dsh's own `!!js dshHomePath('sessions')`): they were zstd, which hermit.redact cannot scrub a key from. A scrubbed transcript still resumes. - The dsh Evolver's sessions take no instruction file (instruction_files false: the agent-instructions budget 0): dsh adds a folder's AGENTS.md when its file tools touch it, so the learner was given the adapter's AGENTS.md, the acting agent's policy, as instructions. Codex's learner is given none. - dsh's settings are checked as the run loads for the dsh Evolver too (an effort it has not failed every learner step, after its rollouts); evolver.dsh.permission_mode is refused (the Evolver runs dsh's own sandbox off), and YAML's bare `off`, a boolean, is `off`. - Web search is kept only for a model reached at api.deepseek.com on DEEPSEEK_API_KEY, where dsh's search goes (a `deepseek` backend moved by DEEPSEEK_BASE_URL searched elsewhere). - A run that ends CONTEXT_WINDOW_EXCEEDED with a backend that declares no context_window says to declare it (recorded: context_window.jsonl, vLLM's refusal of dsh's 128,000 output tokens; with the window declared dsh sizes the request and completes). - A unit test with the fake dsh for the task data's binding, which CI runs where bubblewrap can bind (the only test needed the dsh CLI). - Wording: the deployed tool server's row makes dsh report a server it cannot start and the acting agent fail the session (dsh itself goes on); a run dsh never started is each task's error record, not a stopped run (round 2's message said so); the deploy's size check is exact for a session at the workspace's root; the linked-data warning names dsh's glob and grep. Co-Authored-By: Claude Opus 5.5 --- hermit/agent/base/cli.py | 61 +++++--- hermit/agent/base/records.py | 27 ++-- hermit/agent/evolver/agent.py | 5 +- hermit/agent/model/usage.py | 1 + hermit/hosts/dsh.py | 27 ++-- hermit/isolate.py | 21 ++- tests/core/test_dsh_agent.py | 178 +++++++++++++++++++++--- tests/fixtures/dsh/README.md | 8 +- tests/fixtures/dsh/context_window.jsonl | 7 + 9 files changed, 265 insertions(+), 70 deletions(-) create mode 100644 tests/fixtures/dsh/context_window.jsonl diff --git a/hermit/agent/base/cli.py b/hermit/agent/base/cli.py index 1c4f6160..be4b11b5 100644 --- a/hermit/agent/base/cli.py +++ b/hermit/agent/base/cli.py @@ -71,9 +71,11 @@ model it does not know), append_system_prompt, effort (its --thinking level). Pi takes no roles, has no web search and runs under isolation none or sandbox. DeepSeek Harness (dsh) only: provider (its pi-ai route for a bound model, from the binding: Binding.dsh_provider), - image_input (as Pi's), effort (the default model's reasoningEffort), permission_mode (its sandbox: read-only, - workspace-write, the default, or danger-full-access; under workspace-write the `writable` paths are granted to it - with --add-dir). dsh takes no roles here and runs under isolation none or sandbox; `args` may hold its own options. + image_input (as Pi's), effort (the default model's reasoningEffort; YAML's bare `off`, a boolean, is `off`), + permission_mode (its sandbox: read-only, workspace-write, the default, or danger-full-access; under workspace-write + the `writable` paths are granted to it with --add-dir), instruction_files (false: no AGENTS.md or CLAUDE.md of the + working directory or below it is given to the model; default true). dsh takes no roles here and runs under + isolation none or sandbox; `args` may hold its own options. Limits are enforced by the isolation, or the call is refused (Unsupported): @@ -98,6 +100,7 @@ import threading import time from typing import Any, Sequence +import urllib.parse from hermit.agent.base import container, login as codex_login from hermit.agent.base.interface import BaseAgent, Limits, Mount, Result, Unsupported @@ -559,8 +562,8 @@ def adjust_usage(self, home: Path, handle, usage: list, resumed: bool) -> list: return usage # Without isolation, a mount inside the workspace is a link to its source (`_link`); a CLI whose searches do not - # follow links (Claude Code's Glob and Grep run ripgrep without --follow; `grep -r` and `find` in its Bash neither) - # gets it bound there instead, a real directory, as a container gives it (`_bound`). + # follow links (Claude Code's Glob and Grep and dsh's glob and grep run ripgrep without --follow; `grep -r` and + # `find` in a shell neither) gets it bound there instead, a real directory, as a container gives it (`_bound`). binds_mounts = False # ---------- what the call may do ---------- @@ -1493,6 +1496,13 @@ def _pi_session(home: Path, handle) -> Path | None: DSH_INSTRUCTION_FRAME = 304 # dsh's report of an MCP row that did not start (` (): `): the session then runs without its tools DSH_TOOLS_DOWN = re.compile(r"^(mcp-[A-Za-z0-9_-]{1,32}) \(@deepseek-ai/dsh-mcp-client\): (.*)$", re.M) +# The session's transcripts as plain JSON lines (dsh compresses them with zstd), so that hermit.redact scrubs a key a +# session wrote from its home as from every record; the row's root is dsh's own, an expression (`!!js`) of the home +# as dsh sees it, which a sandbox shows at another path than this process's +DSH_TRANSCRIPTS = ('{"id": "session-persistence-jsonl", "config": {"root": !!js "dshHomePath(\'sessions\')", ' + '"compression": "none"}}') +# Where dsh's web search goes, whatever the session's model: DeepSeek's own API, on this key +DSH_SEARCH_HOST, DSH_SEARCH_KEY = "api.deepseek.com", "DEEPSEEK_API_KEY" class Dsh(CliAgent): @@ -1536,7 +1546,8 @@ def dropped_env(self): def prepare_home(self, home, roles, tools_url, tools_name): """The patch layer (DSH_PATCH): dsh's own rows, by their ids. A row's `config` replaces the row's, so a row is given whole (the instructions' maxBytes is dsh's own). No host path is written in it: a sandbox translates - only the command, the environment and the prompt; the skills' folder is the working directory's.""" + only the command, the environment and the prompt; the skills' folder is the working directory's, the + transcripts' root dsh's own expression (DSH_TRANSCRIPTS).""" if roles: raise Unsupported("dsh: roles are not passed to DeepSeek Harness; give it none") if tools_url and not re.fullmatch(r"[A-Za-z0-9_-]{1,32}", tools_name): @@ -1548,19 +1559,25 @@ def prepare_home(self, home, roles, tools_url, tools_name): # The workspace is the project, wherever it lies: the skills of its own .dsh/skills only # (not those of the nearest folder above with a .git, a repository the run sits in, nor # the user's), and the instructions of the working directory, the workspace (no root - # marker: dsh reads none above it, a repository's or a deployment's around it) + # marker: dsh reads none above it, a repository's or a deployment's around it); none with + # instruction_files false (dsh also adds a folder's AGENTS.md below the working directory + # when its file tools touch the folder) {"id": "skill-filesystem", "config": {"includeDefaultRoots": False, "customSkillDirs": [".dsh/skills"]}}, - {"id": "agent-instructions", "config": {"maxBytes": DSH_INSTRUCTION_BYTES, - "projectRootMarkers": []}}] + {"id": "agent-instructions", "config": { + "maxBytes": DSH_INSTRUCTION_BYTES if s.get("instruction_files", True) else 0, + "projectRootMarkers": []}}] + binding = s.get("binding") or {} if not s.get("web_search"): rows.append({"id": "tool-web", "disabled": True}) # web search and fetch, the model's tools - elif s.get("provider") and (s.get("binding") or {}).get("backend") != "deepseek": + elif s.get("provider") and (urllib.parse.urlparse(str(binding.get("base_url") or "")).hostname, + binding.get("api_key_env")) != (DSH_SEARCH_HOST, DSH_SEARCH_KEY): # dsh's search is DeepSeek's own service (its Messages API on DEEPSEEK_API_KEY), whatever the session's - # model: with a model of another backend it would search on another provider and key than the run's, - # outside its usage. Fetch only, as Codex's hosted search is off for a backend other than OpenAI's + # model: with a model reached elsewhere or on another key it would search on another provider and key + # than the run's, outside its usage. Fetch only, as Codex's hosted search is off for a backend other than + # OpenAI's rows.append({"id": "tool-web", "config": {"search": False, "fetch": True}}) - effort = str(s["effort"]) if s.get("effort") else None + effort = "off" if s.get("effort") is False else str(s["effort"]) if s.get("effort") else None if effort and effort not in DSH_EFFORTS: raise ValueError(f"dsh: an effort is one of {', '.join(DSH_EFFORTS)}, not {effort!r}") if effort and not s.get("model"): @@ -1586,14 +1603,16 @@ def prepare_home(self, home, roles, tools_url, tools_name): elif s.get("provider"): raise ValueError("dsh: a provider route needs its model (the `model` setting)") if tools_url: - # failOnStartupError: a server dsh cannot reach fails its row at once (dsh says so, `outcome` reads it) - # rather than being retried while the session runs without the tools + # failOnStartupError: a server dsh cannot reach fails its row at once (dsh says so on stderr and goes on, + # and `outcome` fails the session) rather than being retried while the session runs without the tools rows.append({"insert": [{"id": f"mcp-{tools_name}", "name": "@deepseek-ai/dsh-mcp-client", "config": {"serverName": tools_name, "transport": "streamable-http", "url": tools_url, "toolCallTimeoutMs": 1_800_000, "failOnStartupError": True}}]}) _clear(home / DSH_PATCH) - (home / DSH_PATCH).write_text(json.dumps(rows, indent=1) + "\n") # JSON is YAML: dsh reads it as such + # JSON is YAML, which dsh reads: a flow sequence, to which the transcripts' row adds its `!!js` root + text = json.dumps(rows, indent=1) + (home / DSH_PATCH).write_text(f"{text[:text.rindex(']')].rstrip()},\n {DSH_TRANSCRIPTS}\n]\n") @classmethod def web_search_args(cls, enabled: bool) -> list[str]: @@ -1627,6 +1646,12 @@ def outcome(self, raw, stream, last_file): down = DSH_TOOLS_DOWN.findall(notes) failure = f"{down[0][0]} did not start, so the session ran without its tools: {down[0][1]}" if down else \ notes or "the run did not finish (no completed `final` event)" + if "CONTEXT_WINDOW_EXCEEDED" in notes and self.settings.get("binding") and \ + not self.settings["binding"].get("context_window"): # dsh's error first, then what to do + failure = failure[:300] + ("; the backend declares no context_window, so dsh sized the model by its own " + "catalog or, for a model it does not know, as 1,000,000 tokens with 128,000 " + "output tokens a request, which a smaller server refuses: declare the " + "backend's context_window") return {"reply": self.last_message(stream, last_file), "served": done, "unavailable": bool(UNAVAILABLE.search(notes)), "refused": False, "failed": not done or bool(down), "failure": failure[-600:], "started": any(e.get("type") == "session" for e in _objects(raw)[0])} @@ -1789,8 +1814,8 @@ def _bound(command: list[str], mounts: list[Mount], workspace: Path, hidden: lis if inside and "linked" not in _WARNED: _WARNED.add("linked") print("hermit: bubblewrap cannot make a mount namespace here, so a task's data in the workspace is linked, " - "not bound: a search that walks the tree (`grep -r`, `find`, Claude Code's Glob and Grep) does not " - "follow a link", + "not bound: a search that walks the tree (`grep -r`, `find`, Claude Code's Glob and Grep, dsh's glob " + "and grep) does not follow a link", file=sys.stderr, flush=True) if hidden and "plan" not in _WARNED: _WARNED.add("plan") diff --git a/hermit/agent/base/records.py b/hermit/agent/base/records.py index f4ff7c52..46a364f9 100644 --- a/hermit/agent/base/records.py +++ b/hermit/agent/base/records.py @@ -12,8 +12,9 @@ def price_usage(usage: list[Usage], params: dict) -> None: """Fill in each usage's cost from `price` (USD per million tokens: {input, cached_input, output}) where the agent reported none. Claude Code reports a cost at Anthropic's prices; for a model bound to another backend that is not the bill, so the backend's price replaces it (no price: the cost is unknown). On a subscription (`login`) that - figure is the same estimate at list prices, not what the session costs: none is kept. Calls of which no token was - reported (a provider that reports no usage) are not priced: their cost stays unknown, not 0.""" + figure is the same estimate at list prices, not what the session costs: none is kept. A usage that holds calls + whose tokens the CLI did not report (`unreported_calls`) keeps an unknown cost: its reported tokens' price would + be too little.""" price = params.get("price") binding = params.get("binding") or {} foreign = bool(params.get("login")) or \ @@ -21,7 +22,7 @@ def price_usage(usage: list[Usage], params: dict) -> None: for u in usage: if foreign: u.cost_usd = None - if price and u.cost_usd is None and (u.input_tokens or u.output_tokens or not u.calls): + if price and u.cost_usd is None and not u.unreported_calls: u.cost_usd = ((u.input_tokens - u.cached_input_tokens) * price.get("input", 0) + u.cached_input_tokens * price.get("cached_input", price.get("input", 0)) + u.output_tokens * price.get("output", 0)) / 1e6 @@ -131,13 +132,12 @@ def _dsh_reader(): cache writes, as Codex and Claude Code count it. dsh reports no cost: it is priced from the backend's price (price_usage). A step that produced nothing (its requests all failed) is no call; one that produced a reply, a thought or a tool call is, even when its tokens are not reported: the provider reported no usage (dsh then - reports zeros), or one of the step's attempts reported none (dsh then leaves the step's usage out). Such a step - adds no tokens: a session none of whose calls reported any has an unknown cost (price_usage), and one with - others costs at least what they reported. + reports zeros), or one of the step's attempts reported none (dsh then leaves the step's usage out, the + successful attempt's included, as for any step a rate limit made dsh retry). Such a step is an unreported call: + it adds no tokens, and the session's cost is unknown (price_usage), not the price of what was reported. - What dsh's stream leaves out is not counted: those steps' tokens, a compaction's summary request, web searches - (each a request of DeepSeek's own API), the title request (off in a session, hermit.agent.base.cli.DSH_OFF) and - sub-agents.""" + What dsh's stream leaves out is not counted: a compaction's summary request, web searches (each a request of + DeepSeek's own API), the title request (off in a session, hermit.agent.base.cli.DSH_OFF) and sub-agents.""" produced = False def read(event: dict, usage: Usage, session): @@ -150,13 +150,12 @@ def read(event: dict, usage: Usage, session): if kind != "status" or event.get("phase") != "step_end": return session made, produced = produced, False - u = event.get("usage") - if not isinstance(u, dict): - usage.calls += 1 if made else 0 - return session + u = event.get("usage") if isinstance(event.get("usage"), dict) else {} cached = int(u.get("cacheReadTokens") or 0) tokens = int(u.get("inputTokens") or 0) + cached + int(u.get("cacheWriteTokens") or 0) - if not tokens and not u.get("outputTokens") and not made: + if not tokens and not u.get("outputTokens"): + usage.calls += 1 if made else 0 + usage.unreported_calls += 1 if made else 0 return session usage.input_tokens += tokens usage.cached_input_tokens += cached diff --git a/hermit/agent/evolver/agent.py b/hermit/agent/evolver/agent.py index 33080088..79e20051 100644 --- a/hermit/agent/evolver/agent.py +++ b/hermit/agent/evolver/agent.py @@ -132,7 +132,10 @@ def settings(self, ctx: Context, backend: str) -> dict: "web_search": params["web_search"], "env": env, "isolation": "sandbox" if ctx.sandbox is not None else "none", "sandbox": ctx.sandbox, "codex_sandbox": "danger-full-access", "shell_env": "all", - **({"permission_mode": "danger-full-access"} if backend == "dsh" else {}), # dsh's own sandbox + # dsh's own sandbox off, as Codex's; and no instruction file of the evolver workspace: dsh adds a + # folder's AGENTS.md when its file tools touch it (an adapter's is the acting agent's policy), which + # Codex's learner is never given + **({"permission_mode": "danger-full-access", "instruction_files": False} if backend == "dsh" else {}), "home": ctx.workspace.state.parent / "agent_home", "home_files": params.get("home"), "log": ctx.workspace.output / f"agent-{backend}.jsonl", "timeout_s": None, "cancel": ctx.cancel, "tools_name": "evolver_tools"} diff --git a/hermit/agent/model/usage.py b/hermit/agent/model/usage.py index 096498a2..7845f44c 100644 --- a/hermit/agent/model/usage.py +++ b/hermit/agent/model/usage.py @@ -12,3 +12,4 @@ class Usage: output_tokens: int = 0 calls: int = 0 cost_usd: float | None = None # None when no price applies (for example a subscription) + unreported_calls: int = 0 # calls whose tokens the CLI did not report: no price makes a cost of them diff --git a/hermit/hosts/dsh.py b/hermit/hosts/dsh.py index cbe121f9..1007f438 100644 --- a/hermit/hosts/dsh.py +++ b/hermit/hosts/dsh.py @@ -6,16 +6,18 @@ folder from the project's root down to the working directory, and the skills of the root's .dsh/skills and .agents/skills; the root is the nearest folder above with a `.git`, else the working directory. It has no project file for MCP servers or for more skill folders: they are rows of its plugin tree, so the patch layer holds the -adapter's tool server (an `insert` row, which fails at the start of a session that cannot start it, rather than -leaving the session without its tools) and names the workspace's .dsh/skills (the `skill-filesystem` row's -customSkillDirs, so that a workspace below a repository's root keeps its skills; the row replaces any -`skill-filesystem` settings of the user's own while the layer is loaded, and dsh's web profile turns the row off). A -session loads the layer from the workspace with `dsh --profile headless --patch .dsh/hermit.patch.yml`, as the `dsh` -acting agent does. dsh gives the model 65,536 bytes of a project's instructions (its agent-instructions maxBytes), -its own frame around them included, and cuts the rest, so a longer AGENTS.md is refused. dsh runs no hook file of its -own (its experimental bridge to Claude Code's hooks is not used) and has no role file: an adapter's roles and hooks -stay files of its copy, which AGENTS.md names with what to do with them (hermit.hosts.inputs.resources), as it names -the other files. dsh's own settings ($DSH_HOME) are never changed. +adapter's tool server (an `insert` row: a server dsh cannot start is reported on its stderr, not retried in the +background, and the `dsh` acting agent then fails the session; dsh itself goes on without the tools) and names the +workspace's .dsh/skills (the `skill-filesystem` row's customSkillDirs, so that a workspace below a repository's root +keeps its skills; the row replaces any `skill-filesystem` settings of the user's own while the layer is loaded, and +dsh's web profile turns the row off). A session loads the layer from the workspace with `dsh --profile headless +--patch .dsh/hermit.patch.yml`, as the `dsh` acting agent does. dsh gives the model 65,536 bytes of a project's +instructions (its agent-instructions maxBytes), its own frame around them included, and cuts the rest, so a longer +AGENTS.md is refused: exactly for a session whose project root is the workspace and which holds no other instruction +file (the acting agent's); a session below a repository's root also counts that repository's AGENTS.md files. dsh +runs no hook file of its own (its experimental bridge to Claude Code's hooks is not used) and has no role file: an +adapter's roles and hooks stay files of its copy, which AGENTS.md names with what to do with them +(hermit.hosts.inputs.resources), as it names the other files. dsh's own settings ($DSH_HOME) are never changed. """ import argparse import json @@ -79,8 +81,9 @@ def prepare(adapter: Path, output: Path, python: str, *, learning_material: Path "args": ["-m", "hermit.runtime.mcp_server", "--adapter", str(adapter), "--audit-log", str(audit)], "env": {"PYTHONPATH": PACKAGE_ROOT, **_adapter_sandbox_env(python)}, "toolCallTimeoutMs": 120_000, # Codex's tool_timeout_sec, not dsh's 60 s - # a server dsh cannot start fails its row, which dsh reports and the dsh acting agent reads as a - # failed session (Codex's `required`), rather than being retried while the session runs without it + # a server dsh cannot start fails its row, which dsh reports on stderr and the dsh acting agent + # reads as a failed session, rather than retrying it in the background while the session runs + # without it (dsh itself goes on either way) "failOnStartupError": True} if learning_material is not None: server["args"].extend(["--learning-material", str(Path(learning_material).resolve())]) diff --git a/hermit/isolate.py b/hermit/isolate.py index 4fff24cb..e311cbc9 100644 --- a/hermit/isolate.py +++ b/hermit/isolate.py @@ -182,16 +182,25 @@ def validate(config) -> None: if params["effort"] not in Claude.efforts: raise ValueError(f"agent.model.effort is one of {', '.join(Claude.efforts)} for Claude Code, not " f"{params['effort']!r}") - if kind == "dsh": # dsh's own settings, here too: else every session would be refused, one task at a time + # dsh's own settings, here too: else every session would be refused, one task (or one learner step) at a time + learner = d["evolver"].get("params") or {} + for where, own in (("agent", params if kind == "dsh" else None), + ("evolver", learner if learner.get("backend") == "dsh" else None)): + if own is None: + continue from hermit.agent.base.cli import DSH_EFFORTS, DSH_PERMISSION_MODES - effort, mode = params.get("effort"), params.get("permission_mode") + effort, mode = own.get("effort"), own.get("permission_mode") + effort = "off" if effort is False else effort # YAML reads a bare `off` as false if effort is not None and str(effort) not in DSH_EFFORTS: - raise ValueError(f"agent.model.effort is one of {', '.join(DSH_EFFORTS)} for dsh, not {effort!r}") - if effort is not None and not params.get("model"): - raise ValueError("agent.model.effort needs agent.model for dsh: it is that model's reasoningEffort") - if mode is not None and mode not in DSH_PERMISSION_MODES: + raise ValueError(f"{where}.model.effort is one of {', '.join(DSH_EFFORTS)} for dsh, not {effort!r}") + if effort is not None and not own.get("model"): + raise ValueError(f"{where}.model.effort needs {where}.model for dsh: it is that model's reasoningEffort") + if where == "agent" and mode is not None and mode not in DSH_PERMISSION_MODES: raise ValueError(f"agent.dsh.permission_mode is one of {', '.join(DSH_PERMISSION_MODES)} (dsh's own " f"sandbox), not {mode!r}") + if where == "evolver" and mode is not None: + raise ValueError("evolver.dsh.permission_mode is not taken: the dsh Evolver runs dsh's own sandbox off " + "(danger-full-access), the evolver sandbox being the boundary, as for Codex") if env.get("image"): if kind not in ("codex", "claude"): raise ValueError("environment.image runs a CLI acting agent (codex or claude) in a container: agent.base " diff --git a/tests/core/test_dsh_agent.py b/tests/core/test_dsh_agent.py index 694ca162..747e45fa 100644 --- a/tests/core/test_dsh_agent.py +++ b/tests/core/test_dsh_agent.py @@ -30,8 +30,8 @@ FAKE_DSH = f'''#!{sys.executable} """A stand-in for `dsh --profile headless ...`: records its arguments, dsh's variables, the patch layers it is given -and its stdin, then prints FAKE_DSH_STREAM and exits FAKE_DSH_EXIT.""" -import json, os, sys +and its stdin, runs FAKE_DSH_RUN in its working directory, then prints FAKE_DSH_STREAM and exits FAKE_DSH_EXIT.""" +import json, os, subprocess, sys args = sys.argv[1:] patches = {{args[i + 1]: open(args[i + 1]).read() for i, a in enumerate(args[:-1]) if a == "--patch"}} names = ("DSH_HOME", "HOME", "DSH_AGENTS_HOME", "DSH_TELEMETRY_DISABLED", "{base_cli.DSH_NO_KEY}", "FAKE_KEY", @@ -39,6 +39,8 @@ seen = {{"args": args, "env": {{k: os.environ.get(k) for k in names}}, "patches": patches, "stdin": sys.stdin.read()}} with open(os.environ["FAKE_DSH_SEEN"], "a") as f: f.write(json.dumps(seen) + "\\n") +if os.environ.get("FAKE_DSH_RUN"): + subprocess.run(os.environ["FAKE_DSH_RUN"], shell=True) sys.stdout.write(open(os.environ["FAKE_DSH_STREAM"]).read()) sys.exit(int(os.environ.get("FAKE_DSH_EXIT", "0"))) ''' @@ -62,9 +64,19 @@ def seen(tmp_path) -> list[dict]: return [json.loads(line) for line in (tmp_path / "seen.jsonl").read_text().splitlines()] +def layer_rows(text: str) -> list[dict]: + """A patch layer's rows, read as dsh reads it (YAML), a `!!js` expression as {"!!js": its text}.""" + import yaml + + class Loader(yaml.SafeLoader): + pass + Loader.add_constructor("tag:yaml.org,2002:js", lambda loader, node: {"!!js": loader.construct_scalar(node)}) + return yaml.load(text, Loader=Loader) + + def rows(call: dict, home: Path) -> list[dict]: """The session's own patch layer, as the fake read it.""" - return json.loads(call["patches"][str(home / base_cli.DSH_PATCH)]) + return layer_rows(call["patches"][str(home / base_cli.DSH_PATCH)]) def row(layer: list[dict], row_id: str) -> dict: @@ -243,9 +255,14 @@ def test_dsh_refuses_roles_and_a_container_and_searches_only_on_deepseeks_own_pr assert row(rows(seen(fake)[-1], fake / "home"), "tool-web") == {"id": "tool-web", "config": {"search": False, "fetch": True}} deepseek = bind_agent_params({"model": "deepseek/deepseek-v4-flash"}, "dsh", registry({})) - for params in (deepseek, {}): + named = bind_agent_params({"model": "ds/deepseek-v4-flash"}, "dsh", registry({"ds": {"from": "deepseek"}})) + for params in (deepseek, named, {}): # DeepSeek's API on its key, under any name; dsh's own make("dsh", {**params, "web_search": True, "home": str(fake / "home")}).run("go", workspace=work) assert all(r.get("id") != "tool-web" for r in rows(seen(fake)[-1], fake / "home")) + relayed = bind_agent_params({"model": "deepseek/deepseek-v4-flash"}, "dsh", registry( + {}, environ={"DEEPSEEK_BASE_URL": "http://127.0.0.1:9/v1"})) # DEEPSEEK_BASE_URL moves the backend + make("dsh", {**relayed, "web_search": True, "home": str(fake / "home")}).run("go", workspace=work) + assert row(rows(seen(fake)[-1], fake / "home"), "tool-web")["config"] == {"search": False, "fetch": True} def test_dsh_takes_no_login_and_no_container_wherever_the_run_is_checked(): @@ -271,7 +288,9 @@ def test_dsh_takes_no_login_and_no_container_wherever_the_run_is_checked(): def test_an_effort_or_a_sandbox_dsh_has_not_is_refused_when_the_run_is_loaded(): """dsh's effort levels and permission modes are its own: a value it has not would fail every session, one task - at a time, so the run config is refused as it loads (as a Claude Code effort is).""" + (or one learner step, after its rollouts were paid for) at a time, so the run config is refused as it loads (as a + Claude Code effort is), for the acting agent and the dsh Evolver. YAML reads a bare `off` as false, which is + `off`. The Evolver runs dsh's own sandbox off: a permission mode for it is refused, not dropped.""" from hermit.config import Config run = ROOT / "docs" / "_examples" / "first_training" / "run.yaml" fake = "backends={fake: {base_url: 'http://127.0.0.1:9/v1', api_key_env: FAKE_KEY, apis: [chat]}}" @@ -279,10 +298,17 @@ def test_an_effort_or_a_sandbox_dsh_has_not_is_refused_when_the_run_is_loaded(): "agent.params={prompt: go, web_search: false}", *more]) loaded = load("agent.model={ref: fake/m-1, effort: xhigh}", "agent.dsh={permission_mode: read-only}") assert loaded.data["agent"]["params"]["effort"] == "xhigh" + assert load("agent.model={ref: fake/m-1, effort: off}").data["agent"]["params"]["effort"] is False with pytest.raises(ValueError, match="agent.model.effort is one of off, minimal, low, medium, high, xhigh, max"): load("agent.model={ref: fake/m-1, effort: ultra}") with pytest.raises(ValueError, match="agent.dsh.permission_mode is one of read-only, workspace-write"): load("agent.model=fake/m-1", "agent.dsh={permission_mode: bypass}") + learner = ("agent.model=fake/m-1", "evolver.name=dsh", "evolver.base=dsh") + assert load(*learner, "evolver.model={ref: fake/m-1, effort: off}").data["evolver"]["params"]["effort"] is False + with pytest.raises(ValueError, match="evolver.model.effort is one of off, minimal, low, medium, high, xhigh"): + load(*learner, "evolver.model={ref: fake/m-1, effort: ultra}") + with pytest.raises(ValueError, match="evolver.dsh.permission_mode is not taken"): + load(*learner, "evolver.model=fake/m-1", "evolver.dsh={permission_mode: workspace-write}") def test_a_model_that_takes_images_is_declared_so_to_dsh(fake): @@ -320,6 +346,30 @@ def test_system_instructions_go_at_the_top_of_the_prompt(fake): assert call["args"][-1] == "-" and call["stdin"] == prompt +def test_the_learners_sessions_take_no_instruction_file_and_every_transcript_is_plain_text(fake, tmp_path): + """dsh adds a folder's AGENTS.md (or CLAUDE.md) when its file tools touch a file there: the learner, which reads + and edits an adapter whose AGENTS.md is the acting agent's policy, would take that policy, or its own text, for + instructions (Codex's learner is given no such file). The dsh Evolver's sessions take none. dsh keeps its + transcripts zstd-compressed, which hermit.redact cannot scrub a key from: every session keeps them in plain JSON + lines, under dsh's own root for them (its home's sessions folder, as dsh sees it in a sandbox too).""" + from hermit.agent.evolver.agent import AgentEvolver + + class Ctx: + params = {**bound(), "web_search": False} + sandbox = cancel = None + workspace = type("W", (), {"state": tmp_path / "state", "output": tmp_path / "output"}) + settings = AgentEvolver().settings(Ctx(), "dsh") + assert settings["instruction_files"] is False and "instruction_files" not in AgentEvolver().settings(Ctx(), "pi") + work = fake / "work" + work.mkdir() + for given, budget in (({}, 65536), ({"instruction_files": False}, 0)): + make("dsh", {**bound(), "web_search": False, "home": str(fake / "home"), **given}).run("go", workspace=work) + layer = rows(seen(fake)[-1], fake / "home") + assert row(layer, "agent-instructions")["config"]["maxBytes"] == budget + assert row(layer, "session-persistence-jsonl")["config"] == {"root": {"!!js": "dshHomePath('sessions')"}, + "compression": "none"} + + def test_the_workspace_is_the_sessions_project_wherever_it_lies(fake): """dsh takes the nearest folder above with a .git for the project: a run's workspace inside a repository would get the repository's skills (.dsh/skills, .agents/skills) and instructions, which outrank the adapter's, and lose @@ -347,6 +397,11 @@ def test_what_dsh_cannot_use_is_refused_rather_than_dropped(fake, monkeypatch, t with pytest.raises(ValueError, match="an effort is one of off, minimal, low, medium, high, xhigh, max"): make("dsh", {**bound(), "effort": "ultra", "web_search": False, "home": str(fake / "home")}).run( "go", workspace=work) + make("dsh", {**bound(), "effort": False, "web_search": False, "home": str(fake / "home")}).run( + "go", workspace=work) # YAML's bare `off` + [model] = row(rows(seen(fake)[-1], fake / "home"), "llm-pi-ai")["config"]["providers"][base_cli.DSH_PROVIDER][ + "models"] + assert model["reasoningEfforts"] is False real = tmp_path / "data" real.mkdir() (tmp_path / "link").symlink_to(real) @@ -360,6 +415,24 @@ def test_what_dsh_cannot_use_is_refused_rather_than_dropped(fake, monkeypatch, t assert base_cli.Dsh.user_home() == tmp_path / "users-dsh" +def test_a_tasks_data_is_bound_in_the_workspace_where_bubblewrap_can_bind_it(fake, monkeypatch, tmp_path): + """dsh's glob and grep run ripgrep without --follow: without isolation a task's data inside the workspace is + bound there, a real read-only directory that a search walks into (as for Claude Code), not linked.""" + from hermit.agent.base.interface import Limits, Mount + if base_cli._bwrap_binds() is None: + pytest.skip("needs a working bubblewrap") + data = tmp_path / "data" + (data / "deep").mkdir(parents=True) + (data / "deep" / "a.txt").write_text("1934 557\n") + work = fake / "work" + work.mkdir() + monkeypatch.setenv("FAKE_DSH_RUN", 'find docs -name "*.txt" > found.txt; [ -L docs ] && echo link >> found.txt') + result = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}).run( + "go", workspace=work, limits=Limits(readable=[Mount(data, Path("docs"))])) + assert result.extra["mounts"] == {str(work / "docs"): "bound"} + assert (work / "found.txt").read_text().split() == ["docs/deep/a.txt"] + + def test_a_run_whose_tools_did_not_start_has_failed_although_dsh_completed_it(fake, monkeypatch): """dsh reports an MCP row it could not start on stderr and runs on without its tools (recorded: tools_down.jsonl, a server dsh could not reach, exit 0): the session ran without what it was given, so it has failed.""" @@ -375,8 +448,8 @@ def test_a_run_whose_tools_did_not_start_has_failed_although_dsh_completed_it(fa def test_a_run_dsh_refused_to_start_is_no_session(fake, monkeypatch): """A usage error is an `error` event before any session (recorded: bad_option.jsonl), and a .env in the working directory that sets a name dsh keeps for its launch stops it before any event (dotenv.jsonl): neither started - a session (the usage error's event read as one), so the acting agent stops the run and says so, rather than - recording the same failed session for every task.""" + a session (the usage error's event read as one), so the acting agent reports a CLI that exited before starting + one (each task's error record says so), rather than a session that failed.""" from hermit.agent.acting import workspace as acting monkeypatch.setenv("FAKE_DSH_EXIT", "1") work = fake / "work" @@ -474,24 +547,37 @@ def test_a_step_counts_as_a_call_whenever_it_made_one(tmp_path): assert parse_dsh(STREAMS / "failure.jsonl", "m")[1] == [] assert parse_dsh(without_usage("failure.jsonl", tmp_path), "m")[1] == [] _, [answered] = parse_dsh(STREAMS / "no_usage.jsonl", "m") - assert (answered.calls, answered.input_tokens, answered.output_tokens) == (1, 0, 0) + assert (answered.calls, answered.unreported_calls, answered.input_tokens, answered.output_tokens) == (1, 1, 0, 0) session, [resumed] = parse_log(STREAMS / "resume.jsonl", "dsh", "m") - assert session == SESSION and (resumed.calls, resumed.input_tokens) == (1, 1000) + assert session == SESSION and (resumed.calls, resumed.unreported_calls, resumed.input_tokens) == (1, 0, 1000) [unknown] = parse_dsh(without_usage("resume.jsonl", tmp_path), "m")[1] - assert (unknown.calls, unknown.input_tokens) == (1, 0) + assert (unknown.calls, unknown.unreported_calls, unknown.input_tokens) == (1, 1, 0) def test_calls_whose_tokens_were_not_reported_have_an_unknown_cost_not_a_free_one(fake, monkeypatch, tmp_path): - """dsh reports no cost, so the backend's price makes one from the tokens: a session none of whose calls reported - any tokens (a provider that reports no usage, or steps dsh left without) cost what is unknown, which the run's - budget counts as such, not $0.00. A session with tokens is priced from them.""" + """dsh reports no cost, so the backend's price makes one from the tokens: a session with a call whose tokens it + did not report (a provider that reports no usage, a step dsh retried) cost what is unknown, which the run's + budget counts as such, not $0.00 nor the price of the rest, which may be far less (a retried step's successful + attempt goes unreported too). A session whose calls all reported tokens is priced from them. Only dsh's reader + says so: another CLI's call of no tokens (Claude Code's result of a refused request) keeps its price, $0.00.""" + from hermit.agent.base.records import price_usage + from hermit.agent.model.usage import Usage work = fake / "work" work.mkdir() agent = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}) - for stream in (STREAMS / "no_usage.jsonl", without_usage("success.jsonl", tmp_path)): + partial = tmp_path / "partial.jsonl" + lines = (STREAMS / "success.jsonl").read_text().splitlines() + first = next(i for i, line in enumerate(lines) if '"step_end"' in line) + lines[first] = json.dumps({k: v for k, v in json.loads(lines[first]).items() if k != "usage"}) + partial.write_text("\n".join(lines) + "\n") # one of six steps without usage + for stream, tokens in ((STREAMS / "no_usage.jsonl", 0), (without_usage("success.jsonl", tmp_path), 0), + (partial, 5000)): monkeypatch.setenv("FAKE_DSH_STREAM", str(stream)) [usage] = agent.run("go", workspace=work).usage - assert usage.calls and not usage.input_tokens and usage.cost_usd is None + assert usage.unreported_calls and usage.input_tokens == tokens and usage.cost_usd is None + refused = [Usage(model="claude-x", calls=1)] + price_usage(refused, {"price": {"input": 1.0, "output": 2.0}}) + assert refused[0].cost_usd == 0.0 monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "resume.jsonl")) [usage] = agent.run("go", workspace=work).usage assert usage.input_tokens and usage.cost_usd == pytest.approx( @@ -499,6 +585,23 @@ def test_calls_whose_tokens_were_not_reported_have_an_unknown_cost_not_a_free_on + usage.output_tokens * 2.0) / 1e6) +def test_a_model_whose_backend_declares_no_context_window_fails_with_what_to_declare(fake, monkeypatch): + """A model dsh's catalog does not know is taken for 1,000,000 tokens, asked for up to 128,000 output tokens at a + time: a server sized below that (vLLM refuses a request past its model's length) fails every request (recorded: + context_window.jsonl). The failure says to declare the backend's context_window, which dsh then sizes requests + by; a backend that declared one gets dsh's error alone.""" + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "context_window.jsonl")) + monkeypatch.setenv("FAKE_DSH_EXIT", "1") + work = fake / "work" + work.mkdir() + for backend, hinted in (({}, True), ({"context_window": 32768}, False)): + result = make("dsh", {**bound(**backend), "web_search": False, "home": str(fake / "home")}).run( + "go", workspace=work) + assert result.extra["failed"] and "CONTEXT_WINDOW_EXCEEDED" in result.extra["failure"] + assert ("declare the backend's context_window" in result.extra["failure"]) is hinted + assert result.usage == [] # the refused requests used nothing + + def test_a_run_that_ends_at_the_token_limit_is_an_error_the_learner_can_read(): """dsh exits 1 for a run that ends `max-tokens` and writes nothing on stderr: the run is no success and no outage, and its end is a step of the trajectory and a notice (only errors and aborts were, before).""" @@ -831,6 +934,46 @@ def test_with_the_dsh_cli_no_instructions_above_the_workspace_are_read(tmp_path, assert "OWN-INSTRUCTIONS-MARKER" in sent and "ABOVE-INSTRUCTIONS-MARKER" not in sent +@pytest.mark.integration +@pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") +def test_with_the_dsh_cli_a_learners_session_takes_no_instruction_file_and_its_transcript_is_scrubbed(tmp_path, + monkeypatch): + """The installed dsh adds a folder's AGENTS.md when its read tool touches a file there; a session with + instruction_files false (the dsh Evolver's) is given none. Every session's transcript is plain JSON lines in its + home, so a key a session wrote there is scrubbed (hermit.redact), and the session is resumed from it.""" + from hermit.redact import redact_path + work = tmp_path / "work" + (work / "adapter").mkdir(parents=True) + (work / "adapter" / "AGENTS.md").write_text("NESTED-INSTRUCTIONS-MARKER\n") + (work / "adapter" / "notes.txt").write_text("the notes\n") + secret = "sk-mock-0123456789abcdef" + requests = [] + + def script(body): + requests.append(body) + return ("read", {"file_path": "adapter/notes.txt"}) if not tool_texts(body) else None + + server, url = serve_chat(script) + params = mock_model(monkeypatch, url) + monkeypatch.setenv("MOCK_KEY", secret) + try: + for given in ({}, {"instruction_files": False}): + home = tmp_path / f"home-{len(given)}" + agent = make("dsh", {**params, "web_search": False, "home": str(home), "timeout_s": 300, **given}) + first = len(requests) + result = agent.run(f"Read the notes; the key is {secret}.", workspace=work) + sent = json.dumps(requests[first:]) + assert result.extra["served"] and ("NESTED-INSTRUCTIONS-MARKER" in sent) is not bool(given) + [transcript] = (home / "sessions").rglob("session.v4.jsonl") + assert secret in transcript.read_text() and not list((home / "sessions").rglob("*.zstd")) + redact_path(home) + assert secret not in transcript.read_text() and "" in transcript.read_text() + again = agent.run("Again.", workspace=work, session=result.session) + finally: + server.shutdown() + assert again.extra["served"] and again.session == result.session + + @pytest.mark.integration @pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") def test_with_the_dsh_cli_an_agents_md_at_the_deploy_limit_reaches_the_model_whole(tmp_path, monkeypatch): @@ -898,10 +1041,10 @@ def test_the_installed_dsh_has_every_row_the_session_patches(tmp_path): home.mkdir() agent = make("dsh", {**bound(), "effort": "high", "web_search": True}) # web fetch only: a row's settings agent.prepare_home(home, (), "http://127.0.0.1:9/mcp", "hermit_tools") - layer = json.loads((home / base_cli.DSH_PATCH).read_text()) + layer = layer_rows((home / base_cli.DSH_PATCH).read_text()) assert {r["id"] for r in layer if "id" in r} >= {*base_cli.DSH_OFF, "session-log-deepseek", "tool-web", "llm-pi-ai", "agent-default-model", "skill-filesystem", - "agent-instructions"} + "agent-instructions", "session-persistence-jsonl"} work = tmp_path / "work" work.mkdir() env = {**os.environ, **agent.home_env(home)} @@ -911,3 +1054,4 @@ def test_the_installed_dsh_has_every_row_the_session_patches(tmp_path): assert "not found" not in done.stderr and "id: llm-pi-ai" in done.stdout assert " config:\n search: false\n fetch: true\n" in done.stdout # web fetch, no search assert " projectRootMarkers: []\n" in done.stdout + assert " root: !!js dshHomePath('sessions')\n compression: none\n" in done.stdout # dsh's own root, plain diff --git a/tests/fixtures/dsh/README.md b/tests/fixtures/dsh/README.md index 11fc103c..7892c723 100644 --- a/tests/fixtures/dsh/README.md +++ b/tests/fixtures/dsh/README.md @@ -34,8 +34,12 @@ layer like the Base Agent's) against a scripted chat-completions model: error on stderr) and exits 1. - `stopped.jsonl`: the model calls bash `sleep 60`; two seconds after dsh writes the call, it gets SIGTERM (as a session's timeout or the Trainer's stop sends it): no `final` event, an `error` event and its stderr line - (`flush on a closed handle`), exit 0. + (`flush on a closed handle`), exit 0. (With a task's data bound in the workspace, the binding's bubblewrap is what + the signal ends, and dsh with it: no event at all.) +- `context_window.jsonl`: the route declares no `contextWindow`, and the server refuses a request whose prompt and + output tokens pass its model's 32,768, with vLLM's error: dsh asks for 128,000 output tokens, then 65,536, and ends + the run `CONTEXT_WINDOW_EXCEEDED` (exit 1). With `contextWindow: 32768` declared it asked for 22,503 and completed. The streams are dsh's stdout unchanged, followed by its stderr's lines where it wrote any, as the Base Agent keeps them in one stream; the run's directory was `/run` (`/run/` for the streams recorded directly, `/tmp/rec3` for -`stopped.jsonl`). +`stopped.jsonl`, `/tmp/r4` for `context_window.jsonl`). diff --git a/tests/fixtures/dsh/context_window.jsonl b/tests/fixtures/dsh/context_window.jsonl new file mode 100644 index 00000000..f86c7987 --- /dev/null +++ b/tests/fixtures/dsh/context_window.jsonl @@ -0,0 +1,7 @@ +{"type":"session","sessionId":"session-483b5af4-c1a2-46d9-b2d7-68a9b00a6a7a","cwd":"/tmp/r4/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":0,"outputTokens":0,"totalTokens":0}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"error","error":{"message":"400: {\"object\":\"error\",\"message\":\"This model's maximum context length is 32768 tokens. However, you requested 128743 tokens (743 in the messages, 128000 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}","code":"CONTEXT_WINDOW_EXCEEDED"}}} +{"type":"final","sessionId":"session-483b5af4-c1a2-46d9-b2d7-68a9b00a6a7a","reason":{"kind":"error","error":{"message":"400: {\"object\":\"error\",\"message\":\"This model's maximum context length is 32768 tokens. However, you requested 128743 tokens (743 in the messages, 128000 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}","code":"CONTEXT_WINDOW_EXCEEDED"}},"text":""} +dsh: CONTEXT_WINDOW_EXCEEDED: 400: {"object":"error","message":"This model's maximum context length is 32768 tokens. However, you requested 128743 tokens (743 in the messages, 128000 in the completion). Please reduce the length of the messages or completion.","type":"BadRequestError","param":null,"code":400} From f4b7b09fb5e82abf455462de383529dd3972eec7 Mon Sep 17 00:00:00 2001 From: Albert Zhao Date: Mon, 9 Nov 2026 00:14:19 -0500 Subject: [PATCH 05/12] dsh: fixes from review round 4 (two independent reviewers on 2021944) Each fix was checked against dsh's source at dsh-v0.2.2-rc.1 and against the installed dsh; each has a test that fails without it. - A step dsh retried after the provider refused a request lost its cache counts: the refused attempt reports zeros, pi-ai reports a count of 0 as none, and dsh sums a count into the step's only when every attempt reported it, while the step's totalTokens still holds them. The reader counts what the total holds beyond the counts reported as input, and the call as unreported (its cost unknown: those tokens were read from the cache or written to it, which are priced apart). Recorded: retried.jsonl (a 503, then 1000 prompt tokens, 900 cached: dsh reports inputTokens 100, totalTokens 1010). Round 3 said such a step's usage is left out; dsh does that only for an attempt that reports no usage at all (a stream that ended before its last event, an attempt a stop cut short). - The dsh Evolver's settings are checked from its params as the Trainer has them (its method's own under the run's, the model bound), in login.evolver_problems, which train runs before its first step and hermit check now reports for an Evolver without a login. Round 3's check, as the run loaded, missed the dsh method named with its params (`backend: dsh` comes from the method's manifest) and refused an effort given with the run's own model. The acting agent's are still checked as the run loads; one function (cli.dsh_problems) holds both, and isolate.py's change is that call. - dsh's own provider (a model of no binding, DeepSeek's API) takes the efforts off, low, high and max; another level failed every session (UNSUPPORTED_REASONING_EFFORT). - evolver.dsh.instruction_files is refused, as evolver.dsh.permission_mode is: the Evolver dropped it. - The caller's DEEPSEEK_SEARCH_BASE_URL is dropped with its DSH_* variables: without the sandbox it moved dsh's web search off DeepSeek's API. - The context_window hint follows dsh's error whole (once), not its first 300 characters. Co-Authored-By: Claude Opus 5.5 --- hermit/agent/base/cli.py | 52 ++++++++++++--- hermit/agent/base/login.py | 13 ++-- hermit/agent/base/records.py | 37 +++++++---- hermit/agent/model/usage.py | 2 +- hermit/isolate.py | 24 ++----- hermit/settings_check.py | 5 +- tests/core/test_dsh_agent.py | 107 +++++++++++++++++++++++-------- tests/fixtures/dsh/README.md | 5 +- tests/fixtures/dsh/retried.jsonl | 7 ++ 9 files changed, 177 insertions(+), 75 deletions(-) create mode 100644 tests/fixtures/dsh/retried.jsonl diff --git a/hermit/agent/base/cli.py b/hermit/agent/base/cli.py index be4b11b5..9a5accb4 100644 --- a/hermit/agent/base/cli.py +++ b/hermit/agent/base/cli.py @@ -1486,6 +1486,7 @@ def _pi_session(home: Path, handle) -> Path | None: "tool-subagent-fork", "tool-subagent-control", "tool-workflow") DSH_PERMISSION_MODES = ("read-only", "workspace-write", "danger-full-access") DSH_EFFORTS = ("off", "minimal", "low", "medium", "high", "xhigh", "max") # the reasoning levels of dsh's pi-ai routes +DSH_OWN_EFFORTS = ("off", "low", "high", "max") # those of its own route (DSH_ROUTE, DeepSeek's API): no binding # The variable a route names for a server that needs no key: a route that names none gets no key from pi-ai under # the name `hermit`, and dsh refuses the request (PI_AI_ERROR "No API key") DSH_NO_KEY = "HERMIT_DSH_NO_KEY" @@ -1505,6 +1506,35 @@ def _pi_session(home: Path, handle) -> Path | None: DSH_SEARCH_HOST, DSH_SEARCH_KEY = "api.deepseek.com", "DEEPSEEK_API_KEY" +def dsh_problems(settings: dict, who: str) -> list[str]: + """What every session of a dsh agent would refuse of its settings, found before the first one rather than at each + session, after its task's or its learner step's rollouts: an effort its model's route has not (dsh's own, for a + model of no binding, has fewer levels than a bound model's pi-ai route) or one with no model to take it; a sandbox + of dsh's own it has not. The Evolver runs dsh's sandbox off and gives its sessions no instruction file + (AgentEvolver.settings): a setting of either is refused rather than dropped. `settings` are the agent's params as + bound (the run's default model in; the Evolver's method's own params under the run's); `who` is its section of + the run config, agent or evolver.""" + effort, mode = settings.get("effort"), settings.get("permission_mode") + effort = "off" if effort is False else effort # YAML reads a bare `off` as false + levels, route = (DSH_EFFORTS, "a bound model") if settings.get("provider") else \ + (DSH_OWN_EFFORTS, "a model of dsh's own provider (DeepSeek's API)") + out = [] + if effort is not None and str(effort) not in levels: + out.append(f"{who}.model.effort is one of {', '.join(levels)} for {route}, not {effort!r}") + if effort is not None and not settings.get("model"): + out.append(f"{who}.model.effort needs {who}.model for dsh: it is that model's reasoningEffort") + if who == "agent" and mode is not None and mode not in DSH_PERMISSION_MODES: + out.append(f"agent.dsh.permission_mode is one of {', '.join(DSH_PERMISSION_MODES)} (dsh's own sandbox), " + f"not {mode!r}") + if who == "evolver" and mode is not None: + out.append("evolver.dsh.permission_mode is not taken: the dsh Evolver runs dsh's own sandbox off " + "(danger-full-access), the evolver sandbox being the boundary, as for Codex") + if who == "evolver" and settings.get("instruction_files") is not None: + out.append("evolver.dsh.instruction_files is not taken: the dsh Evolver's sessions read no AGENTS.md (an " + "adapter's is the acting agent's policy)") + return out + + class Dsh(CliAgent): """DeepSeek Harness (`dsh --profile headless --json`). Its home is DSH_HOME and its HOME (dsh reads ~/.agents), where dsh makes the headless profile on first use. The session's settings are a patch layer of dsh's own (rows @@ -1540,8 +1570,9 @@ def user_home(cls) -> Path: def dropped_env(self): """The caller's own dsh variables (`DSH_*`: another permission mode, a folder of skills, a telemetry endpoint, - a web provider, ...), which would change the session without the sandbox: its own are home_env's.""" - return tuple(name for name in os.environ if name.startswith("DSH_")) + a web provider, ...; and DEEPSEEK_SEARCH_BASE_URL, where its web search goes), which would change the session + without the sandbox: its own are home_env's.""" + return tuple(name for name in os.environ if name.startswith("DSH_") or name == "DEEPSEEK_SEARCH_BASE_URL") def prepare_home(self, home, roles, tools_url, tools_name): """The patch layer (DSH_PATCH): dsh's own rows, by their ids. A row's `config` replaces the row's, so a row is @@ -1578,8 +1609,10 @@ def prepare_home(self, home, roles, tools_url, tools_name): # OpenAI's rows.append({"id": "tool-web", "config": {"search": False, "fetch": True}}) effort = "off" if s.get("effort") is False else str(s["effort"]) if s.get("effort") else None - if effort and effort not in DSH_EFFORTS: - raise ValueError(f"dsh: an effort is one of {', '.join(DSH_EFFORTS)}, not {effort!r}") + levels = DSH_EFFORTS if s.get("provider") else DSH_OWN_EFFORTS # a bound model's route, or dsh's own + if effort and effort not in levels: + raise ValueError(f"dsh: an effort is one of {', '.join(levels)}" + f"{'' if s.get('provider') else ' for a model of its own provider'}, not {effort!r}") if effort and not s.get("model"): raise ValueError("dsh: an effort needs its model (the `model` setting): it is that model's reasoningEffort") default, chosen = {"provider": DSH_ROUTE}, {} @@ -1647,11 +1680,12 @@ def outcome(self, raw, stream, last_file): failure = f"{down[0][0]} did not start, so the session ran without its tools: {down[0][1]}" if down else \ notes or "the run did not finish (no completed `final` event)" if "CONTEXT_WINDOW_EXCEEDED" in notes and self.settings.get("binding") and \ - not self.settings["binding"].get("context_window"): # dsh's error first, then what to do - failure = failure[:300] + ("; the backend declares no context_window, so dsh sized the model by its own " - "catalog or, for a model it does not know, as 1,000,000 tokens with 128,000 " - "output tokens a request, which a smaller server refuses: declare the " - "backend's context_window") + not self.settings["binding"].get("context_window"): # dsh's error (once), then what to do + hint = ("; the backend declares no context_window, so dsh sized the model by its own catalog or, for a " + "model it does not know, as 1,000,000 tokens with 128,000 output tokens a request, which a smaller " + "server refuses: declare the backend's context_window") + said = next(line for line in notes.splitlines() if "CONTEXT_WINDOW_EXCEEDED" in line) + failure = said[:600 - len(hint)] + hint return {"reply": self.last_message(stream, last_file), "served": done, "unavailable": bool(UNAVAILABLE.search(notes)), "refused": False, "failed": not done or bool(down), "failure": failure[-600:], "started": any(e.get("type") == "session" for e in _objects(raw)[0])} diff --git a/hermit/agent/base/login.py b/hermit/agent/base/login.py index 1d11e5ed..8778f465 100644 --- a/hermit/agent/base/login.py +++ b/hermit/agent/base/login.py @@ -603,15 +603,20 @@ def logins_of(data: dict, evolver_params: dict | None = None) -> list[tuple[str, def evolver_problems(params: dict | None) -> list[str]: - """What an agent Evolver's own sign-in cannot run, found before the first step rather than at each step's session - (after its rollouts): a login that is not its CLI's (`backend`), a Claude login with no model name to run.""" + """What an agent Evolver's own sign-in or settings cannot run, found before the first step rather than at each + step's session (after its rollouts): a login that is not its CLI's (`backend`), a Claude login with no model name + to run, a dsh setting dsh refuses (cli.dsh_problems). `params` are the Evolver's as the Trainer gives them: its + method's own under the run's, the model bound.""" params = params or {} login = params.get("login") + out = [] + if params.get("backend") == "dsh": + from hermit.agent.base.cli import dsh_problems + out += dsh_problems(params, "evolver") if not login: - return [] + return out own = LOGINS.get(login) backend = {"claude-code": "claude"}.get(params.get("backend"), params.get("backend")) - out = [] if backend in ("pi", "dsh"): out.append(f"login: {login} is a Codex or Claude Code sign-in; the {backend} Evolver signs in through its " "model reference (evolver.model: /)") diff --git a/hermit/agent/base/records.py b/hermit/agent/base/records.py index 46a364f9..0b689072 100644 --- a/hermit/agent/base/records.py +++ b/hermit/agent/base/records.py @@ -13,8 +13,8 @@ def price_usage(usage: list[Usage], params: dict) -> None: reported none. Claude Code reports a cost at Anthropic's prices; for a model bound to another backend that is not the bill, so the backend's price replaces it (no price: the cost is unknown). On a subscription (`login`) that figure is the same estimate at list prices, not what the session costs: none is kept. A usage that holds calls - whose tokens the CLI did not report (`unreported_calls`) keeps an unknown cost: its reported tokens' price would - be too little.""" + whose tokens the CLI did not report, or not in full (`unreported_calls`), keeps an unknown cost: the price of + what was reported would not be the bill.""" price = params.get("price") binding = params.get("binding") or {} foreign = bool(params.get("login")) or \ @@ -128,16 +128,23 @@ def parse_pi(log: Path, model: str) -> tuple[str | None, list[Usage]]: def _dsh_reader(): """DeepSeek Harness: the session id (its opening `session` event), and each model step's usage (`status` events - of phase `step_end`). dsh's `inputTokens` leaves the cache out, so the input counted is input, cache reads and - cache writes, as Codex and Claude Code count it. dsh reports no cost: it is priced from the backend's price - (price_usage). A step that produced nothing (its requests all failed) is no call; one that produced a reply, a - thought or a tool call is, even when its tokens are not reported: the provider reported no usage (dsh then - reports zeros), or one of the step's attempts reported none (dsh then leaves the step's usage out, the - successful attempt's included, as for any step a rate limit made dsh retry). Such a step is an unreported call: - it adds no tokens, and the session's cost is unknown (price_usage), not the price of what was reported. + of phase `step_end`): the sum of its attempts' (a request dsh retried is an attempt). dsh's `inputTokens` leaves + the cache out, so the input counted is input, cache reads and cache writes, as Codex and Claude Code count it. dsh + reports no cost: it is priced from the backend's price (price_usage). A step that produced nothing (its requests + all failed) is no call; one that produced a reply, a thought or a tool call is, even when its tokens are not + reported in full. Such a step is an unreported call, and the session's cost is unknown (price_usage), not the + price of what was reported: + - the provider reported no usage: dsh reports zeros, and the call adds no tokens; + - an attempt reported no usage at all (a stream that ended before its last event, an attempt a stop cut short, a + failed request of dsh's own provider): dsh leaves the step's usage out, and the call adds no tokens; + - an attempt reported no cache count (a request the provider refused, as for a rate limit dsh retried, reports + zeros, and a count of 0 is not reported): dsh leaves the step's cache counts out, which its `totalTokens` + still holds. What the total holds beyond the counts reported is counted as input; whether it was read from the + cache or written to it, which are priced apart, is unknown. What dsh's stream leaves out is not counted: a compaction's summary request, web searches (each a request of - DeepSeek's own API), the title request (off in a session, hermit.agent.base.cli.DSH_OFF) and sub-agents.""" + DeepSeek's own API), the title request (off in a session, hermit.agent.base.cli.DSH_OFF), sub-agents, and what a + provider billed for an attempt it reported as zeros (one cut off mid-stream, before its usage).""" produced = False def read(event: dict, usage: Usage, session): @@ -151,16 +158,18 @@ def read(event: dict, usage: Usage, session): return session made, produced = produced, False u = event.get("usage") if isinstance(event.get("usage"), dict) else {} - cached = int(u.get("cacheReadTokens") or 0) + cached, output = int(u.get("cacheReadTokens") or 0), int(u.get("outputTokens") or 0) tokens = int(u.get("inputTokens") or 0) + cached + int(u.get("cacheWriteTokens") or 0) - if not tokens and not u.get("outputTokens"): + dropped = max(int(u.get("totalTokens") or 0) - tokens - output, 0) # cache counts dsh left out + if not tokens and not output and not dropped: usage.calls += 1 if made else 0 usage.unreported_calls += 1 if made else 0 return session - usage.input_tokens += tokens + usage.input_tokens += tokens + dropped usage.cached_input_tokens += cached - usage.output_tokens += int(u.get("outputTokens") or 0) + usage.output_tokens += output usage.calls += 1 + usage.unreported_calls += 1 if dropped else 0 return session return read diff --git a/hermit/agent/model/usage.py b/hermit/agent/model/usage.py index 7845f44c..97a34f5a 100644 --- a/hermit/agent/model/usage.py +++ b/hermit/agent/model/usage.py @@ -12,4 +12,4 @@ class Usage: output_tokens: int = 0 calls: int = 0 cost_usd: float | None = None # None when no price applies (for example a subscription) - unreported_calls: int = 0 # calls whose tokens the CLI did not report: no price makes a cost of them + unreported_calls: int = 0 # calls whose tokens the CLI did not report (in full): no price makes their cost diff --git a/hermit/isolate.py b/hermit/isolate.py index e311cbc9..b32e8fb0 100644 --- a/hermit/isolate.py +++ b/hermit/isolate.py @@ -182,25 +182,11 @@ def validate(config) -> None: if params["effort"] not in Claude.efforts: raise ValueError(f"agent.model.effort is one of {', '.join(Claude.efforts)} for Claude Code, not " f"{params['effort']!r}") - # dsh's own settings, here too: else every session would be refused, one task (or one learner step) at a time - learner = d["evolver"].get("params") or {} - for where, own in (("agent", params if kind == "dsh" else None), - ("evolver", learner if learner.get("backend") == "dsh" else None)): - if own is None: - continue - from hermit.agent.base.cli import DSH_EFFORTS, DSH_PERMISSION_MODES - effort, mode = own.get("effort"), own.get("permission_mode") - effort = "off" if effort is False else effort # YAML reads a bare `off` as false - if effort is not None and str(effort) not in DSH_EFFORTS: - raise ValueError(f"{where}.model.effort is one of {', '.join(DSH_EFFORTS)} for dsh, not {effort!r}") - if effort is not None and not own.get("model"): - raise ValueError(f"{where}.model.effort needs {where}.model for dsh: it is that model's reasoningEffort") - if where == "agent" and mode is not None and mode not in DSH_PERMISSION_MODES: - raise ValueError(f"agent.dsh.permission_mode is one of {', '.join(DSH_PERMISSION_MODES)} (dsh's own " - f"sandbox), not {mode!r}") - if where == "evolver" and mode is not None: - raise ValueError("evolver.dsh.permission_mode is not taken: the dsh Evolver runs dsh's own sandbox off " - "(danger-full-access), the evolver sandbox being the boundary, as for Codex") + if kind == "dsh": # dsh's own settings, here too: else every session would be refused, one task at a time + from hermit.agent.base.cli import dsh_problems + problems = dsh_problems(params, "agent") # the dsh Evolver's: login.evolver_problems, before its steps + if problems: + raise ValueError("; ".join(problems)) if env.get("image"): if kind not in ("codex", "claude"): raise ValueError("environment.image runs a CLI acting agent (codex or claude) in a container: agent.base " diff --git a/hermit/settings_check.py b/hermit/settings_check.py index cd31a021..f2787aa0 100644 --- a/hermit/settings_check.py +++ b/hermit/settings_check.py @@ -57,7 +57,10 @@ def report(item: str, problems: list[str], note: str = ""): cls = load_class(spec) _, evolver_params = instantiate(spec, d["evolver"]["params"], backends=registry(d.get("backends")), default_model=d.get("model")) - report("evolver", needs_errors(config, spec.needs, cls.supports_resume), f"{spec.name} ({spec.class_path})") + # an Evolver's own settings (with a login, its problems are the login's, below) + report("evolver", needs_errors(config, spec.needs, cls.supports_resume) + + ([] if evolver_params.get("login") else subscription.evolver_problems(evolver_params)), + f"{spec.name} ({spec.class_path})") except Exception as exc: report("evolver", [str(exc)]) # Models: every model the run calls, the backend each is bound to, and whether that backend speaks the caller's diff --git a/tests/core/test_dsh_agent.py b/tests/core/test_dsh_agent.py index 747e45fa..7e76be3c 100644 --- a/tests/core/test_dsh_agent.py +++ b/tests/core/test_dsh_agent.py @@ -35,7 +35,7 @@ args = sys.argv[1:] patches = {{args[i + 1]: open(args[i + 1]).read() for i, a in enumerate(args[:-1]) if a == "--patch"}} names = ("DSH_HOME", "HOME", "DSH_AGENTS_HOME", "DSH_TELEMETRY_DISABLED", "{base_cli.DSH_NO_KEY}", "FAKE_KEY", - "DSH_BUNDLED_SKILL_DIR") + "DSH_BUNDLED_SKILL_DIR", "DEEPSEEK_SEARCH_BASE_URL") seen = {{"args": args, "env": {{k: os.environ.get(k) for k in names}}, "patches": patches, "stdin": sys.stdin.read()}} with open(os.environ["FAKE_DSH_SEEN"], "a") as f: f.write(json.dumps(seen) + "\\n") @@ -113,9 +113,10 @@ def test_a_bound_model_is_a_dsh_route_with_its_key_variable_in_the_dialect_the_b def test_dsh_runs_headless_in_its_home_with_its_model_and_the_tools_as_its_own(fake, monkeypatch): """Without the sandbox a session gets this process's environment: a dsh variable of the caller's (here a folder - of skills) does not reach it, so the session's settings are its own home's.""" + of skills, and where its web search goes) does not reach it, so the session's settings are its own home's.""" tmp_path = fake monkeypatch.setenv("DSH_BUNDLED_SKILL_DIR", str(tmp_path / "callers-skills")) + monkeypatch.setenv("DEEPSEEK_SEARCH_BASE_URL", "http://127.0.0.1:9") home, work = tmp_path / "home", tmp_path / "work" work.mkdir() agent = make("dsh", {**bound(), "effort": "high", "web_search": False, "home": str(home), @@ -127,7 +128,8 @@ def test_dsh_runs_headless_in_its_home_with_its_model_and_the_tools_as_its_own(f assert call["args"][-2:] == ["--", "-a prompt that starts with a dash"] assert call["env"] == {"DSH_HOME": str(home), "HOME": str(home), "DSH_AGENTS_HOME": str(home / ".agents"), "DSH_TELEMETRY_DISABLED": "1", base_cli.DSH_NO_KEY: "local", - "FAKE_KEY": "sk-fake", "DSH_BUNDLED_SKILL_DIR": None} # none of the user's own skills + "FAKE_KEY": "sk-fake", "DSH_BUNDLED_SKILL_DIR": None, # none of the user's own skills + "DEEPSEEK_SEARCH_BASE_URL": None} layer = rows(call, home) assert "sk-fake" not in json.dumps(layer) assert {r["id"] for r in layer if r.get("disabled")} == {*base_cli.DSH_OFF, "tool-web"} @@ -286,29 +288,66 @@ def test_dsh_takes_no_login_and_no_container_wherever_the_run_is_checked(): "agent.params={prompt: go, web_search: false}"]) -def test_an_effort_or_a_sandbox_dsh_has_not_is_refused_when_the_run_is_loaded(): +def test_an_effort_or_a_sandbox_dsh_has_not_is_refused_before_any_session(tmp_path, monkeypatch): """dsh's effort levels and permission modes are its own: a value it has not would fail every session, one task - (or one learner step, after its rollouts were paid for) at a time, so the run config is refused as it loads (as a - Claude Code effort is), for the acting agent and the dsh Evolver. YAML reads a bare `off` as false, which is - `off`. The Evolver runs dsh's own sandbox off: a permission mode for it is refused, not dropped.""" + (or one learner step, after its rollouts were paid for) at a time. The acting agent's are refused as the run + config loads (as a Claude Code effort is); the dsh Evolver's by `hermit check` and before `train`'s first step, + from its params as the Trainer has them, whichever way the config names it (evolver.base: dsh, or the dsh method + with its params) and with the run's model as its own. dsh's own provider (a model of no binding) has fewer + levels than a bound model's route. YAML reads a bare `off` as false, which is `off`. The Evolver runs dsh's own + sandbox off and gives its sessions no AGENTS.md: a setting of either is refused, not dropped.""" + from hermit.agent.base.login import evolver_problems + from hermit.agent.evolver.loader import find, instantiate from hermit.config import Config + from hermit.settings_check import run_check + from hermit.trainer.trainer import Trainer + monkeypatch.setenv("HA_BACKENDS", str(tmp_path / "no-user-file.yaml")) + monkeypatch.setenv("FAKE_KEY", "sk-fake") run = ROOT / "docs" / "_examples" / "first_training" / "run.yaml" fake = "backends={fake: {base_url: 'http://127.0.0.1:9/v1', api_key_env: FAKE_KEY, apis: [chat]}}" - load = lambda *more: Config.load(run, [fake, "agent.name=null", "agent.base=dsh", - "agent.params={prompt: go, web_search: false}", *more]) + load = lambda *more: Config.load(run, [fake, f"trainer.output_dir={tmp_path / 'runs'}", "agent.name=null", + "agent.base=dsh", "agent.params={prompt: go, web_search: false}", *more]) loaded = load("agent.model={ref: fake/m-1, effort: xhigh}", "agent.dsh={permission_mode: read-only}") assert loaded.data["agent"]["params"]["effort"] == "xhigh" assert load("agent.model={ref: fake/m-1, effort: off}").data["agent"]["params"]["effort"] is False with pytest.raises(ValueError, match="agent.model.effort is one of off, minimal, low, medium, high, xhigh, max"): load("agent.model={ref: fake/m-1, effort: ultra}") + with pytest.raises(ValueError, match="one of off, low, high, max for a model of dsh's own provider"): + load("agent.model={name: deepseek-v4-flash, effort: medium}", "sandbox.enabled=false") + assert load("agent.model={name: deepseek-v4-flash, effort: max}", "sandbox.enabled=false") with pytest.raises(ValueError, match="agent.dsh.permission_mode is one of read-only, workspace-write"): load("agent.model=fake/m-1", "agent.dsh={permission_mode: bypass}") - learner = ("agent.model=fake/m-1", "evolver.name=dsh", "evolver.base=dsh") - assert load(*learner, "evolver.model={ref: fake/m-1, effort: off}").data["evolver"]["params"]["effort"] is False - with pytest.raises(ValueError, match="evolver.model.effort is one of off, minimal, low, medium, high, xhigh"): - load(*learner, "evolver.model={ref: fake/m-1, effort: ultra}") - with pytest.raises(ValueError, match="evolver.dsh.permission_mode is not taken"): - load(*learner, "evolver.model=fake/m-1", "evolver.dsh={permission_mode: workspace-write}") + + def problems(*more) -> list[str]: # the Evolver's, from its params as the Trainer and the check have them + config = load("agent.model=fake/m-1", *more) + d = config.data + _, params = instantiate(find(d["evolver"]["name"], config.base), d["evolver"]["params"], + backends=registry(d.get("backends")), default_model=d.get("model")) + return evolver_problems(params) + + layered, flat = ("evolver.name=dsh", "evolver.base=dsh"), ("evolver.name=dsh",) + assert problems(*layered, "evolver.model={ref: fake/m-1, effort: off}") == [] + assert problems(*flat, "evolver.params={model: fake/m-1, effort: xhigh}") == [] + assert problems("model=fake/m-1", *layered, "evolver.dsh={effort: high}") == [] # the run's model, its own + for more, said in (((*layered, "evolver.model={ref: fake/m-1, effort: ultra}"), "is one of off, minimal, low"), + ((*flat, "evolver.params={model: fake/m-1, effort: ultra}"), "is one of off, minimal, low"), + ((*flat, "evolver.params={effort: high}"), "evolver.model.effort needs evolver.model"), + ((*layered, "evolver.model=fake/m-1", "evolver.dsh={permission_mode: workspace-write}"), + "evolver.dsh.permission_mode is not taken"), + ((*flat, "evolver.params={model: fake/m-1, instruction_files: true}"), + "evolver.dsh.instruction_files is not taken")): + assert any(said in problem for problem in problems(*more)), more + flat_ultra = load("agent.model=fake/m-1", *flat, "evolver.params={model: fake/m-1, effort: ultra}") + lines = [] + assert run_check(flat_ultra, probe=False, log=lines.append) is False + at = lines.index("[PROBLEM] evolver") + assert "evolver.model.effort is one of off, minimal, low, medium, high, xhigh, max" in lines[at + 1] + with pytest.raises(SystemExit, match="train refuses to start: evolver: evolver.model.effort is one of off, min"): + Trainer(flat_ultra, log=lambda *_: None)._prepare() + lines.clear() + run_check(load("agent.model=fake/m-1", "model=fake/m-1", *layered, "evolver.dsh={effort: high}"), probe=False, + log=lines.append) + assert any(line.startswith("[ok] evolver — dsh") for line in lines) def test_a_model_that_takes_images_is_declared_so_to_dsh(fake): @@ -386,7 +425,8 @@ def test_the_workspace_is_the_sessions_project_wherever_it_lies(fake): def test_what_dsh_cannot_use_is_refused_rather_than_dropped(fake, monkeypatch, tmp_path): - """An effort without a model reaches no model (it is the default model's reasoningEffort); a writable path dsh's + """An effort without a model reaches no model (it is the default model's reasoningEffort), and dsh's own provider + (a model of no binding, DeepSeek's API) takes fewer levels than a bound model's route; a writable path dsh's sandbox cannot take (a link, a missing folder) failed every command and write of the session: dsh requires an existing directory by its real path, which the session is given. The user's own dsh home is where its .env is copied from.""" @@ -397,6 +437,13 @@ def test_what_dsh_cannot_use_is_refused_rather_than_dropped(fake, monkeypatch, t with pytest.raises(ValueError, match="an effort is one of off, minimal, low, medium, high, xhigh, max"): make("dsh", {**bound(), "effort": "ultra", "web_search": False, "home": str(fake / "home")}).run( "go", workspace=work) + with pytest.raises(ValueError, match="an effort is one of off, low, high, max for a model of its own provider"): + make("dsh", {"model": "deepseek-v4-flash", "effort": "medium", "web_search": False, + "home": str(fake / "home")}).run("go", workspace=work) + make("dsh", {"model": "deepseek-v4-flash", "effort": "max", "web_search": False, "home": str(fake / "home")}).run( + "go", workspace=work) + assert row(rows(seen(fake)[-1], fake / "home"), "agent-default-model")["config"] == { + "provider": base_cli.DSH_ROUTE, "model": "deepseek-v4-flash", "reasoningEffort": "max"} make("dsh", {**bound(), "effort": False, "web_search": False, "home": str(fake / "home")}).run( "go", workspace=work) # YAML's bare `off` [model] = row(rows(seen(fake)[-1], fake / "home"), "llm-pi-ai")["config"]["providers"][base_cli.DSH_PROVIDER][ @@ -529,8 +576,8 @@ def commands(name): def without_usage(stream: str, tmp_path: Path) -> Path: - """A recorded stream whose steps report no usage, as dsh writes a step one of whose attempts had none (its - json-stream.ts omits the partial sum).""" + """A recorded stream whose steps report no usage, as dsh writes a step one of whose attempts reported none (its + json-stream.ts omits the partial sum): a stream that ended before its last event, an attempt a stop cut short.""" out = tmp_path / f"no-usage-{stream}" out.write_text("".join(json.dumps({k: v for k, v in json.loads(line).items() if k != "usage"}) + "\n" if '"step_end"' in line else line + "\n" @@ -541,8 +588,10 @@ def without_usage(stream: str, tmp_path: Path) -> Path: def test_a_step_counts_as_a_call_whenever_it_made_one(tmp_path): """A step whose requests all failed produced nothing and is no call (failure.jsonl), also when dsh reports no usage for it; one that answered is a call even when the provider reported no usage, which dsh reports as zeros - (no_usage.jsonl), and so is one dsh reports no usage for at all. parse_log reads a dsh stream as dsh's (it read - every stream but Codex's as Claude Code's).""" + (no_usage.jsonl), and so is one dsh reports no usage for at all. A step dsh retried after the provider refused a + request (retried.jsonl: a 503, then 1000 prompt tokens, 900 of them cached) reports its input and output but not + its cache counts, which its total holds: those tokens count as input, read from the cache or written to it, which + is unknown. parse_log reads a dsh stream as dsh's (it read every stream but Codex's as Claude Code's).""" from hermit.agent.base.records import parse_dsh, parse_log assert parse_dsh(STREAMS / "failure.jsonl", "m")[1] == [] assert parse_dsh(without_usage("failure.jsonl", tmp_path), "m")[1] == [] @@ -552,14 +601,18 @@ def test_a_step_counts_as_a_call_whenever_it_made_one(tmp_path): assert session == SESSION and (resumed.calls, resumed.unreported_calls, resumed.input_tokens) == (1, 0, 1000) [unknown] = parse_dsh(without_usage("resume.jsonl", tmp_path), "m")[1] assert (unknown.calls, unknown.unreported_calls, unknown.input_tokens) == (1, 1, 0) + [retried] = parse_dsh(STREAMS / "retried.jsonl", "m")[1] + assert (retried.calls, retried.unreported_calls, retried.input_tokens, retried.cached_input_tokens, + retried.output_tokens) == (1, 1, 1000, 0, 10) def test_calls_whose_tokens_were_not_reported_have_an_unknown_cost_not_a_free_one(fake, monkeypatch, tmp_path): """dsh reports no cost, so the backend's price makes one from the tokens: a session with a call whose tokens it - did not report (a provider that reports no usage, a step dsh retried) cost what is unknown, which the run's - budget counts as such, not $0.00 nor the price of the rest, which may be far less (a retried step's successful - attempt goes unreported too). A session whose calls all reported tokens is priced from them. Only dsh's reader - says so: another CLI's call of no tokens (Claude Code's result of a refused request) keeps its price, $0.00.""" + did not report in full (a provider that reports no usage, an attempt that reported none, a step whose cache + counts dsh left out after a retry) cost what is unknown, which the run's budget counts as such, not $0.00 nor + the price of the rest, which may be far less (the successful attempt of a step whose usage dsh left out goes + unreported too). A session whose calls all reported tokens is priced from them. Only dsh's reader says so: + another CLI's call of no tokens (Claude Code's result of a refused request) keeps its price, $0.00.""" from hermit.agent.base.records import price_usage from hermit.agent.model.usage import Usage work = fake / "work" @@ -571,7 +624,7 @@ def test_calls_whose_tokens_were_not_reported_have_an_unknown_cost_not_a_free_on lines[first] = json.dumps({k: v for k, v in json.loads(lines[first]).items() if k != "usage"}) partial.write_text("\n".join(lines) + "\n") # one of six steps without usage for stream, tokens in ((STREAMS / "no_usage.jsonl", 0), (without_usage("success.jsonl", tmp_path), 0), - (partial, 5000)): + (partial, 5000), (STREAMS / "retried.jsonl", 1000)): monkeypatch.setenv("FAKE_DSH_STREAM", str(stream)) [usage] = agent.run("go", workspace=work).usage assert usage.unreported_calls and usage.input_tokens == tokens and usage.cost_usd is None @@ -594,10 +647,12 @@ def test_a_model_whose_backend_declares_no_context_window_fails_with_what_to_dec monkeypatch.setenv("FAKE_DSH_EXIT", "1") work = fake / "work" work.mkdir() + said = base_cli.notices("dsh", (STREAMS / "context_window.jsonl").read_text()) # its error, twice for backend, hinted in (({}, True), ({"context_window": 32768}, False)): result = make("dsh", {**bound(**backend), "web_search": False, "home": str(fake / "home")}).run( "go", workspace=work) assert result.extra["failed"] and "CONTEXT_WINDOW_EXCEEDED" in result.extra["failure"] + assert said.splitlines()[0] in result.extra["failure"] # dsh's error, whole assert ("declare the backend's context_window" in result.extra["failure"]) is hinted assert result.usage == [] # the refused requests used nothing @@ -640,7 +695,7 @@ def test_the_dsh_acting_agent_deploys_the_adapter_and_loads_its_patch_layer(fake assert server["config"]["transport"] == "stdio" and server["config"]["args"][:2] == ["-m", "hermit.runtime.mcp_server"] assert (work / ".dsh" / "skills" / "verify-output" / "SKILL.md").is_file() - assert (work / ".hermit-deploy.json").is_file() # the instructions' project root (the marker) + assert (work / ".hermit-deploy.json").is_file() # the deployment's record assert "do it yourself" in (work / "AGENTS.md").read_text() # roles stay files: none is passed to dsh assert out.output == "the answer" and out.extra["backend"] == "dsh" assert (out.log / "agent-dsh.jsonl").is_file() diff --git a/tests/fixtures/dsh/README.md b/tests/fixtures/dsh/README.md index 7892c723..a3b04325 100644 --- a/tests/fixtures/dsh/README.md +++ b/tests/fixtures/dsh/README.md @@ -39,7 +39,10 @@ layer like the Base Agent's) against a scripted chat-completions model: - `context_window.jsonl`: the route declares no `contextWindow`, and the server refuses a request whose prompt and output tokens pass its model's 32,768, with vLLM's error: dsh asks for 128,000 output tokens, then 65,536, and ends the run `CONTEXT_WINDOW_EXCEEDED` (exit 1). With `contextWindow: 32768` declared it asked for 22,503 and completed. +- `retried.jsonl`: the server answers the first request 503, then the retry with 1000 prompt tokens (900 of them + cached) and 10 output tokens: dsh sums the two attempts' usage into the step's, and leaves out the cache count the + refused attempt did not report (its zeros), so the step reports `inputTokens` 100 and `totalTokens` 1010. The streams are dsh's stdout unchanged, followed by its stderr's lines where it wrote any, as the Base Agent keeps them in one stream; the run's directory was `/run` (`/run/` for the streams recorded directly, `/tmp/rec3` for -`stopped.jsonl`, `/tmp/r4` for `context_window.jsonl`). +`stopped.jsonl`, `/tmp/r4` for `context_window.jsonl`, `/tmp/r5` for `retried.jsonl`). diff --git a/tests/fixtures/dsh/retried.jsonl b/tests/fixtures/dsh/retried.jsonl new file mode 100644 index 00000000..0a15113f --- /dev/null +++ b/tests/fixtures/dsh/retried.jsonl @@ -0,0 +1,7 @@ +{"type":"session","sessionId":"session-4a46d7a7-7e83-4f43-b5c9-ab2c48c086b8","cwd":"/tmp/r5/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"text","text":"the answer"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":100,"outputTokens":10,"totalTokens":1010}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-4a46d7a7-7e83-4f43-b5c9-ab2c48c086b8","reason":{"kind":"completed"},"text":"the answer"} From f1a92eab6e25da6664e553f591cd63e0f1cf66a6 Mon Sep 17 00:00:00 2001 From: Albert Zhao Date: Mon, 9 Nov 2026 00:56:27 -0500 Subject: [PATCH 06/12] dsh: fixes from review round 5 (two independent reviewers on f4b7b09) Each fix was checked against dsh's source at dsh-v0.2.2-rc.1 and against the installed dsh; each has a test that fails without it. - A step dsh retried after a refused request is priced again, its dropped cache tokens counted as input. Round 4 made such a session's cost unknown, and a USD budget then counted none of its spend (rollouts and learner steps add no dollars for an unknown cost), under the very load that causes retries. Its tokens are known, and hermit prices a cache write as input: the price is the step's with the counts reported, or more if those tokens were read from the cache, never less. A call stays unreported, and its session's cost unknown, only when dsh reported none of its tokens. - The caller's DEEPSEEK_BASE_URL is dropped as DEEPSEEK_SEARCH_BASE_URL is: without the sandbox it sent the requests of dsh's own provider, with DEEPSEEK_API_KEY, elsewhere. A variable the session is given (`env`) still passes. - dsh's settings check takes what each agent uses: the Evolver's danger-full-access and instruction_files false are accepted (round 4 refused them); the acting agent's instruction_files false is refused (it was dropped; it would cut the adapter's context, which reaches dsh as its AGENTS.md). - Tests pin the session's tool row (failOnStartupError, the 30-minute call timeout), every row DSH_OFF turns off, all six outage codes, and dsh's own provider's levels for the Evolver; the outage test is named for what it checks. - Wording: effort off asks a bound model for no reasoning level, which leaves its server's default (dsh's own provider turns thinking off); the context-window hint keeps dsh's error line as far as it fits. Co-Authored-By: Claude Opus 5.5 --- hermit/agent/base/cli.py | 42 +++++++++++---------- hermit/agent/base/records.py | 29 ++++++++------- hermit/agent/model/usage.py | 2 +- tests/core/test_dsh_agent.py | 71 ++++++++++++++++++++++++------------ 4 files changed, 87 insertions(+), 57 deletions(-) diff --git a/hermit/agent/base/cli.py b/hermit/agent/base/cli.py index 9a5accb4..41ec95b0 100644 --- a/hermit/agent/base/cli.py +++ b/hermit/agent/base/cli.py @@ -71,11 +71,12 @@ model it does not know), append_system_prompt, effort (its --thinking level). Pi takes no roles, has no web search and runs under isolation none or sandbox. DeepSeek Harness (dsh) only: provider (its pi-ai route for a bound model, from the binding: Binding.dsh_provider), - image_input (as Pi's), effort (the default model's reasoningEffort; YAML's bare `off`, a boolean, is `off`), - permission_mode (its sandbox: read-only, workspace-write, the default, or danger-full-access; under workspace-write - the `writable` paths are granted to it with --add-dir), instruction_files (false: no AGENTS.md or CLAUDE.md of the - working directory or below it is given to the model; default true). dsh takes no roles here and runs under - isolation none or sandbox; `args` may hold its own options. + image_input (as Pi's), effort (the default model's reasoningEffort; YAML's bare `off`, a boolean, is `off`: dsh + then asks a bound model for no reasoning level, which leaves its server's default, and turns thinking off on its + own provider), permission_mode (its sandbox: read-only, workspace-write, the default, or danger-full-access; under + workspace-write the `writable` paths are granted to it with --add-dir), instruction_files (false: no AGENTS.md or + CLAUDE.md of the working directory or below it is given to the model; default true). dsh takes no roles here and + runs under isolation none or sandbox; `args` may hold its own options. Limits are enforced by the isolation, or the call is refused (Unsupported): @@ -1510,10 +1511,10 @@ def dsh_problems(settings: dict, who: str) -> list[str]: """What every session of a dsh agent would refuse of its settings, found before the first one rather than at each session, after its task's or its learner step's rollouts: an effort its model's route has not (dsh's own, for a model of no binding, has fewer levels than a bound model's pi-ai route) or one with no model to take it; a sandbox - of dsh's own it has not. The Evolver runs dsh's sandbox off and gives its sessions no instruction file - (AgentEvolver.settings): a setting of either is refused rather than dropped. `settings` are the agent's params as - bound (the run's default model in; the Evolver's method's own params under the run's); `who` is its section of - the run config, agent or evolver.""" + of dsh's own it has not. A setting the agent does not take is refused rather than dropped: the Evolver runs dsh's + sandbox off and gives its sessions no instruction file (AgentEvolver.settings), and the adapter's context reaches + the acting agent as its AGENTS.md. `settings` are the agent's params as bound (the run's default model in; the + Evolver's method's own params under the run's); `who` is its section of the run config, agent or evolver.""" effort, mode = settings.get("effort"), settings.get("permission_mode") effort = "off" if effort is False else effort # YAML reads a bare `off` as false levels, route = (DSH_EFFORTS, "a bound model") if settings.get("provider") else \ @@ -1526,12 +1527,13 @@ def dsh_problems(settings: dict, who: str) -> list[str]: if who == "agent" and mode is not None and mode not in DSH_PERMISSION_MODES: out.append(f"agent.dsh.permission_mode is one of {', '.join(DSH_PERMISSION_MODES)} (dsh's own sandbox), " f"not {mode!r}") - if who == "evolver" and mode is not None: + if who == "evolver" and mode not in (None, "danger-full-access"): out.append("evolver.dsh.permission_mode is not taken: the dsh Evolver runs dsh's own sandbox off " "(danger-full-access), the evolver sandbox being the boundary, as for Codex") - if who == "evolver" and settings.get("instruction_files") is not None: - out.append("evolver.dsh.instruction_files is not taken: the dsh Evolver's sessions read no AGENTS.md (an " - "adapter's is the acting agent's policy)") + if settings.get("instruction_files") not in (None, who == "agent"): # what each takes: its default + out.append(f"{who}.dsh.instruction_files is not taken: " + ( + "the adapter's context reaches the acting agent as its AGENTS.md" if who == "agent" else + "the dsh Evolver's sessions read no AGENTS.md (an adapter's is the acting agent's policy)")) return out @@ -1570,9 +1572,11 @@ def user_home(cls) -> Path: def dropped_env(self): """The caller's own dsh variables (`DSH_*`: another permission mode, a folder of skills, a telemetry endpoint, - a web provider, ...; and DEEPSEEK_SEARCH_BASE_URL, where its web search goes), which would change the session - without the sandbox: its own are home_env's.""" - return tuple(name for name in os.environ if name.startswith("DSH_") or name == "DEEPSEEK_SEARCH_BASE_URL") + a web provider, ...; and the hosts of DeepSeek's API it reaches, DEEPSEEK_BASE_URL for its own provider's + requests and DEEPSEEK_SEARCH_BASE_URL for its web search), which would change the session without the + sandbox: its own are home_env's, and a variable the session is given (`env`) still passes.""" + return tuple(name for name in os.environ + if name.startswith("DSH_") or name in ("DEEPSEEK_BASE_URL", "DEEPSEEK_SEARCH_BASE_URL")) def prepare_home(self, home, roles, tools_url, tools_name): """The patch layer (DSH_PATCH): dsh's own rows, by their ids. A row's `config` replaces the row's, so a row is @@ -1620,7 +1624,7 @@ def prepare_home(self, home, roles, tools_url, tools_name): provider = json.loads(json.dumps(s["provider"])) provider.setdefault("apiKeyEnv", DSH_NO_KEY) # a server that needs no key: the placeholder for model in provider.get("models") or []: - if effort: # a declared model reasons only at the levels it lists; off: it does not reason + if effort: # a declared model is asked only the levels it lists; off: none, its server's default model.setdefault("reasoningEfforts", False if effort == "off" else {effort: effort}) if s.get("image_input"): # else dsh sends the model no image (a text-only model refuses them) model["input"] = ["text", "image"] @@ -1630,7 +1634,7 @@ def prepare_home(self, home, roles, tools_url, tools_name): default["provider"] = DSH_PROVIDER if s.get("model"): default["model"] = str(s["model"]) - if effort and chosen.get("reasoningEfforts") is not False: # a model that does not reason takes none + if effort and chosen.get("reasoningEfforts") is not False: # a model declared with no levels takes none default["reasoningEffort"] = effort rows.append({"id": "agent-default-model", "config": default}) elif s.get("provider"): @@ -1680,7 +1684,7 @@ def outcome(self, raw, stream, last_file): failure = f"{down[0][0]} did not start, so the session ran without its tools: {down[0][1]}" if down else \ notes or "the run did not finish (no completed `final` event)" if "CONTEXT_WINDOW_EXCEEDED" in notes and self.settings.get("binding") and \ - not self.settings["binding"].get("context_window"): # dsh's error (once), then what to do + not self.settings["binding"].get("context_window"): # dsh's error line, then what to do hint = ("; the backend declares no context_window, so dsh sized the model by its own catalog or, for a " "model it does not know, as 1,000,000 tokens with 128,000 output tokens a request, which a smaller " "server refuses: declare the backend's context_window") diff --git a/hermit/agent/base/records.py b/hermit/agent/base/records.py index 0b689072..a6c89f97 100644 --- a/hermit/agent/base/records.py +++ b/hermit/agent/base/records.py @@ -13,8 +13,8 @@ def price_usage(usage: list[Usage], params: dict) -> None: reported none. Claude Code reports a cost at Anthropic's prices; for a model bound to another backend that is not the bill, so the backend's price replaces it (no price: the cost is unknown). On a subscription (`login`) that figure is the same estimate at list prices, not what the session costs: none is kept. A usage that holds calls - whose tokens the CLI did not report, or not in full (`unreported_calls`), keeps an unknown cost: the price of - what was reported would not be the bill.""" + whose tokens the CLI did not report (`unreported_calls`) keeps an unknown cost: its reported tokens' price would + be too little.""" price = params.get("price") binding = params.get("binding") or {} foreign = bool(params.get("login")) or \ @@ -130,17 +130,19 @@ def _dsh_reader(): """DeepSeek Harness: the session id (its opening `session` event), and each model step's usage (`status` events of phase `step_end`): the sum of its attempts' (a request dsh retried is an attempt). dsh's `inputTokens` leaves the cache out, so the input counted is input, cache reads and cache writes, as Codex and Claude Code count it. dsh - reports no cost: it is priced from the backend's price (price_usage). A step that produced nothing (its requests - all failed) is no call; one that produced a reply, a thought or a tool call is, even when its tokens are not - reported in full. Such a step is an unreported call, and the session's cost is unknown (price_usage), not the - price of what was reported: - - the provider reported no usage: dsh reports zeros, and the call adds no tokens; - - an attempt reported no usage at all (a stream that ended before its last event, an attempt a stop cut short, a - failed request of dsh's own provider): dsh leaves the step's usage out, and the call adds no tokens; - - an attempt reported no cache count (a request the provider refused, as for a rate limit dsh retried, reports - zeros, and a count of 0 is not reported): dsh leaves the step's cache counts out, which its `totalTokens` - still holds. What the total holds beyond the counts reported is counted as input; whether it was read from the - cache or written to it, which are priced apart, is unknown. + reports no cost: it is priced from the backend's price (price_usage), which prices a cache write as input. + + A step that produced nothing (its requests all failed) is no call; one that produced a reply, a thought or a tool + call is, even when its tokens are not reported: the provider reported no usage (dsh then reports zeros), or an + attempt reported none at all (a stream that ended before its last event, an attempt a stop cut short, a failed + request of dsh's own provider: dsh then leaves the step's usage out). Such a step is an unreported call: it adds + no tokens, and the session's cost is unknown (price_usage), not the price of what was reported. + + A step one of whose attempts reported no cache count (a request the provider refused, as for a rate limit dsh + retried, reports zeros, and pi-ai reports a count of 0 as none) has its cache counts left out by dsh, which its + `totalTokens` still holds. What the total holds beyond the counts reported is counted as input, priced as a cache + write is: the step's price is what it would be with the counts reported, or more if those tokens were read from + the cache, never less. What dsh's stream leaves out is not counted: a compaction's summary request, web searches (each a request of DeepSeek's own API), the title request (off in a session, hermit.agent.base.cli.DSH_OFF), sub-agents, and what a @@ -169,7 +171,6 @@ def read(event: dict, usage: Usage, session): usage.cached_input_tokens += cached usage.output_tokens += output usage.calls += 1 - usage.unreported_calls += 1 if dropped else 0 return session return read diff --git a/hermit/agent/model/usage.py b/hermit/agent/model/usage.py index 97a34f5a..7845f44c 100644 --- a/hermit/agent/model/usage.py +++ b/hermit/agent/model/usage.py @@ -12,4 +12,4 @@ class Usage: output_tokens: int = 0 calls: int = 0 cost_usd: float | None = None # None when no price applies (for example a subscription) - unreported_calls: int = 0 # calls whose tokens the CLI did not report (in full): no price makes their cost + unreported_calls: int = 0 # calls whose tokens the CLI did not report: no price makes a cost of them diff --git a/tests/core/test_dsh_agent.py b/tests/core/test_dsh_agent.py index 7e76be3c..29b51fad 100644 --- a/tests/core/test_dsh_agent.py +++ b/tests/core/test_dsh_agent.py @@ -35,7 +35,7 @@ args = sys.argv[1:] patches = {{args[i + 1]: open(args[i + 1]).read() for i, a in enumerate(args[:-1]) if a == "--patch"}} names = ("DSH_HOME", "HOME", "DSH_AGENTS_HOME", "DSH_TELEMETRY_DISABLED", "{base_cli.DSH_NO_KEY}", "FAKE_KEY", - "DSH_BUNDLED_SKILL_DIR", "DEEPSEEK_SEARCH_BASE_URL") + "DSH_BUNDLED_SKILL_DIR", "DEEPSEEK_BASE_URL", "DEEPSEEK_SEARCH_BASE_URL") seen = {{"args": args, "env": {{k: os.environ.get(k) for k in names}}, "patches": patches, "stdin": sys.stdin.read()}} with open(os.environ["FAKE_DSH_SEEN"], "a") as f: f.write(json.dumps(seen) + "\\n") @@ -113,9 +113,12 @@ def test_a_bound_model_is_a_dsh_route_with_its_key_variable_in_the_dialect_the_b def test_dsh_runs_headless_in_its_home_with_its_model_and_the_tools_as_its_own(fake, monkeypatch): """Without the sandbox a session gets this process's environment: a dsh variable of the caller's (here a folder - of skills, and where its web search goes) does not reach it, so the session's settings are its own home's.""" + of skills, and where DeepSeek's API is reached, for dsh's own provider and for its web search) does not reach it, + so the session's settings are its own home's. The call's tools are an MCP row whose server, if dsh cannot start + it, fails the session (failOnStartupError), and whose calls may take as long as a Trainer tool's rollouts.""" tmp_path = fake monkeypatch.setenv("DSH_BUNDLED_SKILL_DIR", str(tmp_path / "callers-skills")) + monkeypatch.setenv("DEEPSEEK_BASE_URL", "http://127.0.0.1:9") monkeypatch.setenv("DEEPSEEK_SEARCH_BASE_URL", "http://127.0.0.1:9") home, work = tmp_path / "home", tmp_path / "work" work.mkdir() @@ -129,11 +132,13 @@ def test_dsh_runs_headless_in_its_home_with_its_model_and_the_tools_as_its_own(f assert call["env"] == {"DSH_HOME": str(home), "HOME": str(home), "DSH_AGENTS_HOME": str(home / ".agents"), "DSH_TELEMETRY_DISABLED": "1", base_cli.DSH_NO_KEY: "local", "FAKE_KEY": "sk-fake", "DSH_BUNDLED_SKILL_DIR": None, # none of the user's own skills - "DEEPSEEK_SEARCH_BASE_URL": None} + "DEEPSEEK_BASE_URL": None, "DEEPSEEK_SEARCH_BASE_URL": None} layer = rows(call, home) assert "sk-fake" not in json.dumps(layer) - assert {r["id"] for r in layer if r.get("disabled")} == {*base_cli.DSH_OFF, "tool-web"} - assert {"tool-subagent", "session-telemetry-otel", "session-title-llm"} <= set(base_cli.DSH_OFF) + assert {r["id"] for r in layer if r.get("disabled")} == {"session-telemetry-otel", "session-title-llm", + "plugin-package-inventory-deepseek", "tool-subagent", + "tool-subagent-fork", "tool-subagent-control", + "tool-workflow", "tool-web"} assert row(layer, "session-log-deepseek")["config"] == {"enabled": False} route = row(layer, "llm-pi-ai")["config"]["providers"][base_cli.DSH_PROVIDER] assert route["apiKeyEnv"] == "FAKE_KEY" and route["models"] == [{"id": "m-1", "reasoningEfforts": {"high": "high"}}] @@ -143,6 +148,7 @@ def test_dsh_runs_headless_in_its_home_with_its_model_and_the_tools_as_its_own(f [entry] = server["insert"] assert entry["name"] == "@deepseek-ai/dsh-mcp-client" and entry["config"]["serverName"] == "hermit_tools" assert entry["config"]["transport"] == "streamable-http" and entry["config"]["url"].startswith("http://127.0.0.1:") + assert entry["config"]["failOnStartupError"] is True and entry["config"]["toolCallTimeoutMs"] == 1_800_000 assert result.reply == "the answer" and result.exit_code == 0 and result.extra["served"] assert not result.extra["failed"] and result.session == SESSION [usage] = result.usage @@ -158,9 +164,10 @@ def test_dsh_runs_headless_in_its_home_with_its_model_and_the_tools_as_its_own(f assert result.trajectory[0] == trajectory.Step("reasoning", "Thinking it over.") -def test_an_effort_is_a_level_the_bound_model_lists_and_off_is_a_model_that_does_not_reason(fake): +def test_an_effort_is_a_level_the_bound_model_lists_and_off_asks_it_for_none(fake): """dsh refuses an effort its route does not list for the model (UNSUPPORTED_REASONING_EFFORT), and a list of - `off` alone (INVALID_CONFIG): effort off declares a model that does not reason, and gives no effort.""" + `off` alone (INVALID_CONFIG): effort off declares the model with no levels and gives no effort, so dsh asks for + no reasoning level, which leaves the server's default (only dsh's own provider has a way to turn thinking off).""" work = fake / "work" work.mkdir() agent = make("dsh", {**bound(), "web_search": False, "home": str(fake / "home")}) @@ -172,9 +179,10 @@ def test_an_effort_is_a_level_the_bound_model_lists_and_off_is_a_model_that_does assert row(layer, "agent-default-model")["config"].get("reasoningEffort") == default -def test_a_failed_run_is_an_error_and_an_outage_is_retried(fake, monkeypatch): - """dsh exits 1 for a run that did not complete; its error code tells an outage (SERVER, RATE_LIMIT, ...) from - a refusal, so the acting agent runs the task again only when the provider could not serve it.""" +def test_a_failed_run_is_an_error_and_only_an_outage_is_unavailable(fake, monkeypatch): + """dsh exits 1 for a run that did not complete; its error code tells an outage (the codes it retries, and a + stream the provider closed) from a refusal, and only an outage is `unavailable`: the acting agent runs the task + again only when the provider could not serve it.""" monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "failure.jsonl")) monkeypatch.setenv("FAKE_DSH_EXIT", "1") work = fake / "work" @@ -187,8 +195,9 @@ def test_a_failed_run_is_an_error_and_an_outage_is_retried(fake, monkeypatch): refused = (STREAMS / "failure.jsonl").read_text().replace('"SERVER"', '"AUTH"').replace("dsh: SERVER:", "dsh: AUTH:") assert not base_cli.UNAVAILABLE.search(base_cli.notices("dsh", refused)) # a bad key is no outage - closed = (STREAMS / "failure.jsonl").read_text().replace("SERVER", "STREAM_CLOSED") # dsh does not retry it - assert base_cli.UNAVAILABLE.search(base_cli.notices("dsh", closed)) + for code in ("RATE_LIMIT", "SERVER", "TIMEOUT", "TRANSPORT", "EMPTY_RESPONSE", "STREAM_CLOSED"): + outage = (STREAMS / "failure.jsonl").read_text().replace("SERVER", code) + assert base_cli.UNAVAILABLE.search(base_cli.notices("dsh", outage)), code def test_a_run_dsh_was_stopped_in_is_no_success_although_dsh_exits_0(fake, monkeypatch, tmp_path): @@ -317,6 +326,9 @@ def test_an_effort_or_a_sandbox_dsh_has_not_is_refused_before_any_session(tmp_pa assert load("agent.model={name: deepseek-v4-flash, effort: max}", "sandbox.enabled=false") with pytest.raises(ValueError, match="agent.dsh.permission_mode is one of read-only, workspace-write"): load("agent.model=fake/m-1", "agent.dsh={permission_mode: bypass}") + assert load("agent.model=fake/m-1", "agent.dsh={instruction_files: true}") + with pytest.raises(ValueError, match="agent.dsh.instruction_files is not taken: the adapter's context reaches"): + load("agent.model=fake/m-1", "agent.dsh={instruction_files: false}") def problems(*more) -> list[str]: # the Evolver's, from its params as the Trainer and the check have them config = load("agent.model=fake/m-1", *more) @@ -329,9 +341,13 @@ def problems(*more) -> list[str]: # the Evolver's, from its params as t assert problems(*layered, "evolver.model={ref: fake/m-1, effort: off}") == [] assert problems(*flat, "evolver.params={model: fake/m-1, effort: xhigh}") == [] assert problems("model=fake/m-1", *layered, "evolver.dsh={effort: high}") == [] # the run's model, its own + assert problems(*layered, "evolver.model=fake/m-1", # what the Evolver uses + "evolver.dsh={permission_mode: danger-full-access, instruction_files: false}") == [] for more, said in (((*layered, "evolver.model={ref: fake/m-1, effort: ultra}"), "is one of off, minimal, low"), ((*flat, "evolver.params={model: fake/m-1, effort: ultra}"), "is one of off, minimal, low"), ((*flat, "evolver.params={effort: high}"), "evolver.model.effort needs evolver.model"), + ((*flat, "evolver.params={model: deepseek-v4-flash, effort: medium}"), + "is one of off, low, high, max for a model of dsh's own provider"), ((*layered, "evolver.model=fake/m-1", "evolver.dsh={permission_mode: workspace-write}"), "evolver.dsh.permission_mode is not taken"), ((*flat, "evolver.params={model: fake/m-1, instruction_files: true}"), @@ -590,8 +606,8 @@ def test_a_step_counts_as_a_call_whenever_it_made_one(tmp_path): usage for it; one that answered is a call even when the provider reported no usage, which dsh reports as zeros (no_usage.jsonl), and so is one dsh reports no usage for at all. A step dsh retried after the provider refused a request (retried.jsonl: a 503, then 1000 prompt tokens, 900 of them cached) reports its input and output but not - its cache counts, which its total holds: those tokens count as input, read from the cache or written to it, which - is unknown. parse_log reads a dsh stream as dsh's (it read every stream but Codex's as Claude Code's).""" + its cache counts, which its total holds: those tokens count as input, and the call as one whose tokens are known. + parse_log reads a dsh stream as dsh's (it read every stream but Codex's as Claude Code's).""" from hermit.agent.base.records import parse_dsh, parse_log assert parse_dsh(STREAMS / "failure.jsonl", "m")[1] == [] assert parse_dsh(without_usage("failure.jsonl", tmp_path), "m")[1] == [] @@ -603,16 +619,19 @@ def test_a_step_counts_as_a_call_whenever_it_made_one(tmp_path): assert (unknown.calls, unknown.unreported_calls, unknown.input_tokens) == (1, 1, 0) [retried] = parse_dsh(STREAMS / "retried.jsonl", "m")[1] assert (retried.calls, retried.unreported_calls, retried.input_tokens, retried.cached_input_tokens, - retried.output_tokens) == (1, 1, 1000, 0, 10) + retried.output_tokens) == (1, 0, 1000, 0, 10) def test_calls_whose_tokens_were_not_reported_have_an_unknown_cost_not_a_free_one(fake, monkeypatch, tmp_path): - """dsh reports no cost, so the backend's price makes one from the tokens: a session with a call whose tokens it - did not report in full (a provider that reports no usage, an attempt that reported none, a step whose cache - counts dsh left out after a retry) cost what is unknown, which the run's budget counts as such, not $0.00 nor - the price of the rest, which may be far less (the successful attempt of a step whose usage dsh left out goes - unreported too). A session whose calls all reported tokens is priced from them. Only dsh's reader says so: - another CLI's call of no tokens (Claude Code's result of a refused request) keeps its price, $0.00.""" + """dsh reports no cost, so the backend's price makes one from the tokens. A session with a call whose tokens it + did not report (a provider that reports no usage, an attempt that reported none) has an unknown cost (None), not + $0.00 nor the price of the rest, which may be far less (the successful attempt of a step whose usage dsh left out + goes unreported too): the run's budget counts its calls as of unknown cost, and none of its dollars. A session + whose tokens dsh reported is priced from them; a step whose cache counts dsh left out after a retry is priced + with those tokens as input, as a cache write is, never less than with the counts, so that a USD budget counts + the session. Only dsh's reader marks a call unreported: another CLI's call of no tokens (Claude Code's result of + a refused request) keeps its price, $0.00.""" + from hermit.trainer.budget import Budget from hermit.agent.base.records import price_usage from hermit.agent.model.usage import Usage work = fake / "work" @@ -624,13 +643,19 @@ def test_calls_whose_tokens_were_not_reported_have_an_unknown_cost_not_a_free_on lines[first] = json.dumps({k: v for k, v in json.loads(lines[first]).items() if k != "usage"}) partial.write_text("\n".join(lines) + "\n") # one of six steps without usage for stream, tokens in ((STREAMS / "no_usage.jsonl", 0), (without_usage("success.jsonl", tmp_path), 0), - (partial, 5000), (STREAMS / "retried.jsonl", 1000)): + (partial, 5000)): monkeypatch.setenv("FAKE_DSH_STREAM", str(stream)) [usage] = agent.run("go", workspace=work).usage assert usage.unreported_calls and usage.input_tokens == tokens and usage.cost_usd is None refused = [Usage(model="claude-x", calls=1)] price_usage(refused, {"price": {"input": 1.0, "output": 2.0}}) assert refused[0].cost_usd == 0.0 + monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "retried.jsonl")) + [retried] = agent.run("go", workspace=work).usage + assert not retried.unreported_calls and retried.cost_usd == pytest.approx((1000 * 1.0 + 10 * 2.0) / 1e6) + budget = Budget(usd=0.001) + budget.add("rollout", retried) + assert budget.exceeded() and budget.unknown_cost_calls == 0 monkeypatch.setenv("FAKE_DSH_STREAM", str(STREAMS / "resume.jsonl")) [usage] = agent.run("go", workspace=work).usage assert usage.input_tokens and usage.cost_usd == pytest.approx( @@ -652,7 +677,7 @@ def test_a_model_whose_backend_declares_no_context_window_fails_with_what_to_dec result = make("dsh", {**bound(**backend), "web_search": False, "home": str(fake / "home")}).run( "go", workspace=work) assert result.extra["failed"] and "CONTEXT_WINDOW_EXCEEDED" in result.extra["failure"] - assert said.splitlines()[0] in result.extra["failure"] # dsh's error, whole + assert said.splitlines()[0] in result.extra["failure"] # dsh's error line (this one fits whole) assert ("declare the backend's context_window" in result.extra["failure"]) is hinted assert result.usage == [] # the refused requests used nothing From 37aee8c2fa105f8a650879fcd81f42c1b76de30e Mon Sep 17 00:00:00 2001 From: Albert Zhao Date: Mon, 9 Nov 2026 01:55:22 -0500 Subject: [PATCH 07/12] dsh: fixes from review round 6 (two independent reviewers on f1a92ea) Each finding was checked against dsh's source at dsh-v0.2.2-rc.1 and the installed dsh; each test added fails without what it checks. No behaviour changes: the fixes are tests and claims. - Tests: a session's own `env` reaches dsh while the caller's DEEPSEEK_BASE_URL is dropped; the integration test of dsh's own sandbox shows a command's write outside the workspace refused (it passed with the sandbox off); a bash call that failed to run (a result dsh marks `error`) has no exit code; a linked session folder, or a session in a linked folder, is not resumed; a stopped session's usage is pinned (none). - The reader's account: the last step of a session a stop cut short is not counted (dsh writes a step's usage at its end, after its tool calls ran, and a stopped session writes nothing more); the docstring said such a step was a call of unknown tokens. dsh's own adapter, not pi-ai, drops a cache count of 0. A retried step's price is never less than with its counts for a price whose cached input is no dearer than its input; on the Responses API the total is the server's own `total_tokens`, taken as given. - dropped_env says why a sandbox's grant of DEEPSEEK_BASE_URL does not reach dsh (hermit's own is its deepseek backend's chat completions root, which dsh reads as its own provider's Messages root) and that `env` gives dsh its own; the Settings paragraph says so too. - A stale test docstring: the Evolver's own permission mode and instruction_files are taken. Not changed, disclosed: the acting agent's re-run after an outage keeps only its last attempt's usage, for every CLI (shared code, a separate change). Co-Authored-By: Claude Opus 5.5 --- hermit/agent/base/cli.py | 9 +++- hermit/agent/base/records.py | 22 ++++++---- tests/core/test_dsh_agent.py | 80 +++++++++++++++++++++++++++--------- 3 files changed, 81 insertions(+), 30 deletions(-) diff --git a/hermit/agent/base/cli.py b/hermit/agent/base/cli.py index 41ec95b0..1aba60eb 100644 --- a/hermit/agent/base/cli.py +++ b/hermit/agent/base/cli.py @@ -76,7 +76,9 @@ own provider), permission_mode (its sandbox: read-only, workspace-write, the default, or danger-full-access; under workspace-write the `writable` paths are granted to it with --add-dir), instruction_files (false: no AGENTS.md or CLAUDE.md of the working directory or below it is given to the model; default true). dsh takes no roles here and - runs under isolation none or sandbox; `args` may hold its own options. + runs under isolation none or sandbox; `args` may hold its own options. The caller's DSH_* variables, + DEEPSEEK_BASE_URL and DEEPSEEK_SEARCH_BASE_URL never reach a session, whatever a sandbox lets through: `env` + gives them (Dsh.dropped_env). Limits are enforced by the isolation, or the call is refused (Unsupported): @@ -1574,7 +1576,10 @@ def dropped_env(self): """The caller's own dsh variables (`DSH_*`: another permission mode, a folder of skills, a telemetry endpoint, a web provider, ...; and the hosts of DeepSeek's API it reaches, DEEPSEEK_BASE_URL for its own provider's requests and DEEPSEEK_SEARCH_BASE_URL for its web search), which would change the session without the - sandbox: its own are home_env's, and a variable the session is given (`env`) still passes.""" + sandbox: its own are home_env's. They are dropped whatever a sandbox lets through (sandbox.evolver.env, + environment.env), which may be for the run's other processes: hermit's own DEEPSEEK_BASE_URL moves its + `deepseek` backend, a chat completions root, and dsh reads it as its own provider's Messages root. A variable + the session is given (`env`, where `$DEEPSEEK_BASE_URL` is this process's own) still passes.""" return tuple(name for name in os.environ if name.startswith("DSH_") or name in ("DEEPSEEK_BASE_URL", "DEEPSEEK_SEARCH_BASE_URL")) diff --git a/hermit/agent/base/records.py b/hermit/agent/base/records.py index a6c89f97..2b2d2d11 100644 --- a/hermit/agent/base/records.py +++ b/hermit/agent/base/records.py @@ -134,19 +134,23 @@ def _dsh_reader(): A step that produced nothing (its requests all failed) is no call; one that produced a reply, a thought or a tool call is, even when its tokens are not reported: the provider reported no usage (dsh then reports zeros), or an - attempt reported none at all (a stream that ended before its last event, an attempt a stop cut short, a failed - request of dsh's own provider: dsh then leaves the step's usage out). Such a step is an unreported call: it adds - no tokens, and the session's cost is unknown (price_usage), not the price of what was reported. + attempt reported none at all (a stream that ended before its last event, a failed request of dsh's own provider: + dsh then leaves the step's usage out). Such a step is an unreported call: it adds no tokens, and the session's + cost is unknown (price_usage), not the price of what was reported. A step one of whose attempts reported no cache count (a request the provider refused, as for a rate limit dsh - retried, reports zeros, and pi-ai reports a count of 0 as none) has its cache counts left out by dsh, which its - `totalTokens` still holds. What the total holds beyond the counts reported is counted as input, priced as a cache - write is: the step's price is what it would be with the counts reported, or more if those tokens were read from - the cache, never less. + retried, reports zeros, and dsh keeps a cache count only when it is not 0) has its cache counts left out by dsh, + which its `totalTokens` still holds. What the total holds beyond the counts reported is counted as input, priced + as a cache write is: the step's price is what it would be with the counts reported, or more if those tokens were + read from the cache (a price whose cached input is no dearer than its input), never less. The total is the sum + of the counts on every route but the Responses API's, where it is the server's own `total_tokens`, trusted: what + it holds beyond the counts is priced as input, and a server that reports none leaves those tokens out. What dsh's stream leaves out is not counted: a compaction's summary request, web searches (each a request of - DeepSeek's own API), the title request (off in a session, hermit.agent.base.cli.DSH_OFF), sub-agents, and what a - provider billed for an attempt it reported as zeros (one cut off mid-stream, before its usage).""" + DeepSeek's own API), the title request (off in a session, hermit.agent.base.cli.DSH_OFF), sub-agents, what a + provider billed for an attempt it reported as zeros (one cut off mid-stream, before its usage), and the last + step of a session a stop (SIGTERM: a timeout, a cancelled call) cut short before that step's end: dsh writes a + step's usage at its end, after its tool calls ran, and a stopped session writes nothing more.""" produced = False def read(event: dict, usage: Usage, session): diff --git a/tests/core/test_dsh_agent.py b/tests/core/test_dsh_agent.py index 29b51fad..cb58c205 100644 --- a/tests/core/test_dsh_agent.py +++ b/tests/core/test_dsh_agent.py @@ -13,6 +13,7 @@ import stat import subprocess import sys +import tempfile import threading import pytest @@ -114,8 +115,9 @@ def test_a_bound_model_is_a_dsh_route_with_its_key_variable_in_the_dialect_the_b def test_dsh_runs_headless_in_its_home_with_its_model_and_the_tools_as_its_own(fake, monkeypatch): """Without the sandbox a session gets this process's environment: a dsh variable of the caller's (here a folder of skills, and where DeepSeek's API is reached, for dsh's own provider and for its web search) does not reach it, - so the session's settings are its own home's. The call's tools are an MCP row whose server, if dsh cannot start - it, fails the session (failOnStartupError), and whose calls may take as long as a Trainer tool's rollouts.""" + so the session's settings are its own home's; one the session is given (`env`, here the caller's by name) does. + The call's tools are an MCP row whose server, if dsh cannot start it, fails the session (failOnStartupError), and + whose calls may take as long as a Trainer tool's rollouts.""" tmp_path = fake monkeypatch.setenv("DSH_BUNDLED_SKILL_DIR", str(tmp_path / "callers-skills")) monkeypatch.setenv("DEEPSEEK_BASE_URL", "http://127.0.0.1:9") @@ -162,6 +164,8 @@ def test_dsh_runs_headless_in_its_home_with_its_model_and_the_tools_as_its_own(f assert tool == {"arguments": {"text": "from dsh"}, "server": "hermit_tools", "tool": "echo"} assert json.loads(result.trajectory[5].output) == {"echoed": "from dsh"} assert result.trajectory[0] == trajectory.Step("reasoning", "Thinking it over.") + agent.configured(env={"DEEPSEEK_BASE_URL": "$DEEPSEEK_BASE_URL"}).run("go", workspace=work) + assert seen(tmp_path)[-1]["env"]["DEEPSEEK_BASE_URL"] == "http://127.0.0.1:9" def test_an_effort_is_a_level_the_bound_model_lists_and_off_asks_it_for_none(fake): @@ -203,7 +207,9 @@ def test_a_failed_run_is_an_error_and_only_an_outage_is_unavailable(fake, monkey def test_a_run_dsh_was_stopped_in_is_no_success_although_dsh_exits_0(fake, monkeypatch, tmp_path): """dsh exits 0 when SIGTERM stops it (recorded: stopped.jsonl, a session stopped while its command ran): its own rule for a completed run is exit 0 and a completed `final`, so a run without one has failed. The acting agent - records no answer, and an AgentEvolver's step is a failed one; so is a session whose tools did not start.""" + records no answer, and an AgentEvolver's step is a failed one; so is a session whose tools did not start. The + stopped step's request is not counted (dsh writes a step's usage at its end, which a stopped session never + writes): the session has no usage.""" from hermit.agent.acting import workspace as acting from hermit.agent.evolver.agent import AgentEvolver from hermit.agent.evolver.base import Access, StepContext, Workspace @@ -214,6 +220,7 @@ def test_a_run_dsh_was_stopped_in_is_no_success_although_dsh_exits_0(fake, monke assert result.exit_code == 0 and result.extra["failed"] and not result.extra["served"] and result.extra["started"] assert not result.extra["unavailable"] and result.reply == "" and "closed handle" in result.extra["failure"] assert [s.kind for s in result.trajectory] == ["command", "error"] and result.trajectory[0].exit_code is None + assert result.usage == [] out = acting.run({"id": "q1"}, None, params={**bound(), "backend": "dsh", "web_search": False, "prompt": "go", "workspaces": "ws"}, base=fake) assert out.output is None and "closed handle" in out.extra["session_failed"] @@ -303,8 +310,10 @@ def test_an_effort_or_a_sandbox_dsh_has_not_is_refused_before_any_session(tmp_pa config loads (as a Claude Code effort is); the dsh Evolver's by `hermit check` and before `train`'s first step, from its params as the Trainer has them, whichever way the config names it (evolver.base: dsh, or the dsh method with its params) and with the run's model as its own. dsh's own provider (a model of no binding) has fewer - levels than a bound model's route. YAML reads a bare `off` as false, which is `off`. The Evolver runs dsh's own - sandbox off and gives its sessions no AGENTS.md: a setting of either is refused, not dropped.""" + levels than a bound model's route. YAML reads a bare `off` as false, which is `off`. A setting an agent does not + take is refused, not dropped: the Evolver runs dsh's own sandbox off and gives its sessions no AGENTS.md (its own + danger-full-access and instruction_files false are taken, another value refused), and the acting agent's + sessions read the adapter's context as their AGENTS.md (instruction_files false is refused).""" from hermit.agent.base.login import evolver_problems from hermit.agent.evolver.loader import find, instantiate from hermit.config import Config @@ -578,7 +587,8 @@ def test_a_command_whose_end_dsh_did_not_report_has_no_exit_code(): """dsh's --json keeps 8 KiB of a string: a long result loses its end, where the marker is (recorded: big_bash.jsonl, 1,500 lines then exit 2), and read as exit 0 before; a command started in the background (background.jsonl) has no status yet. A command that printed a marker of its own and exited 0 (fake_marker.jsonl) - is a clean exit, as dsh reads it: the marker counts only at the very end.""" + is a clean exit, as dsh reads it: the marker counts only at the very end. A call that failed to run (a result + dsh marks `error`) has no exit status either, as dsh's own view of a bash result has none for it.""" def commands(name): stream = [json.loads(line) for line in (STREAMS / name).read_text().splitlines() if line.startswith("{")] return [s for s in trajectory.dsh_steps(stream) if s.kind == "command"] @@ -589,11 +599,35 @@ def commands(name): assert background.exit_code is None and background.output.startswith("started background job ") [printed] = commands("fake_marker.jsonl") assert (printed.exit_code, printed.output) == (0, "built\n[exit code: 7]\n") + [unrun] = [s for s in trajectory.dsh_steps([ + {"type": "tool_call", "callId": "call_0", "tool": "bash", "input": {"command": "make"}}, + {"type": "tool_result", "callId": "call_0", "status": "error", "result": "bash could not run"}])] + assert (unrun.kind, unrun.exit_code, unrun.output) == ("command", None, "bash could not run") + + +def test_a_session_is_resumed_only_from_a_directory_of_the_home_itself(tmp_path): + """dsh keeps a session at sessions/// in its home, which the session's commands may write: a + link there (in place of the session's folder, or of the folder it lies in) is no session to resume, so a call + starts a new one; so is a handle that is no session id.""" + home, elsewhere = tmp_path / "home", tmp_path / "elsewhere" + (home / "sessions" / "project" / SESSION).mkdir(parents=True) + (elsewhere / SESSION).mkdir(parents=True) + assert base_cli._dsh_session(home, SESSION) == SESSION + assert base_cli._dsh_session(home, f"../{SESSION}") is None + linked = tmp_path / "linked" + (linked / "sessions" / "project").mkdir(parents=True) + (linked / "sessions" / "project" / SESSION).symlink_to(elsewhere / SESSION) + assert base_cli._dsh_session(linked, SESSION) is None + parent = tmp_path / "parent" + (parent / "sessions").mkdir(parents=True) + (parent / "sessions" / "project").symlink_to(elsewhere) + assert base_cli._dsh_session(parent, SESSION) is None def without_usage(stream: str, tmp_path: Path) -> Path: """A recorded stream whose steps report no usage, as dsh writes a step one of whose attempts reported none (its - json-stream.ts omits the partial sum): a stream that ended before its last event, an attempt a stop cut short.""" + json-stream.ts omits the partial sum): a stream that ended before its last event, a failed request of dsh's own + provider.""" out = tmp_path / f"no-usage-{stream}" out.write_text("".join(json.dumps({k: v for k, v in json.loads(line).items() if k != "usage"}) + "\n" if '"step_end"' in line else line + "\n" @@ -628,8 +662,8 @@ def test_calls_whose_tokens_were_not_reported_have_an_unknown_cost_not_a_free_on $0.00 nor the price of the rest, which may be far less (the successful attempt of a step whose usage dsh left out goes unreported too): the run's budget counts its calls as of unknown cost, and none of its dollars. A session whose tokens dsh reported is priced from them; a step whose cache counts dsh left out after a retry is priced - with those tokens as input, as a cache write is, never less than with the counts, so that a USD budget counts - the session. Only dsh's reader marks a call unreported: another CLI's call of no tokens (Claude Code's result of + with those tokens as input, as a cache write is, never less than with the counts (the cached input being no + dearer than the input), so that a USD budget counts the session. Only dsh's reader marks a call unreported: another CLI's call of no tokens (Claude Code's result of a refused request) keeps its price, $0.00.""" from hermit.trainer.budget import Budget from hermit.agent.base.records import price_usage @@ -846,16 +880,21 @@ def script(body): @pytest.mark.skipif(not shutil.which("dsh"), reason="needs the dsh CLI") def test_with_the_dsh_cli_a_command_runs_in_dshs_own_sandbox_inside_the_runs(tmp_path, monkeypatch): """The acting agent keeps dsh's own sandbox (workspace-write) inside the run's: dsh starts it nested in the run's - bubblewrap sandbox (bubblewrap, else Landlock), and a command of the session writes its workspace there. Where - neither can start, dsh refuses every command (SANDBOX_UNAVAILABLE), which this would show.""" + bubblewrap sandbox (bubblewrap, else Landlock), and a command of the session writes its workspace there, not the + session's home, which the run's sandbox lets the session write (the home lies outside /tmp, which workspace-write + lets a command write). Where neither can start, dsh refuses every command (SANDBOX_UNAVAILABLE), which this would + show.""" from hermit.sandbox import Sandbox - if not _bwrap_works(): - pytest.skip("needs a working bubblewrap") + if not _bwrap_works() or not os.path.isdir("/var/tmp"): + pytest.skip("needs a working bubblewrap and /var/tmp") + home = Path(tempfile.mkdtemp(prefix="hermit-dsh-home-", dir="/var/tmp")) + outside = home / "outside.txt" + commands = [{"command": "echo made > made.txt && cat made.txt", "description": "Write a file"}, + {"command": f"echo out > {outside}", "description": "Write the home"}] def script(body): done = sum(1 for m in body.get("messages") or [] if m.get("role") == "tool") - return ("bash", {"command": "echo made > made.txt && cat made.txt", "description": "Write a file"}) \ - if done == 0 else None + return ("bash", commands[done]) if done < len(commands) else None server, url = serve_chat(script) monkeypatch.setenv("MOCK_KEY", "sk-mock") @@ -865,13 +904,16 @@ def script(body): work.mkdir() box = Sandbox(network="isolated", env_allow=[], writable=[work], role="agent", record_dir=tmp_path / "records") try: - result = make("dsh", {**params, "web_search": False, "home": str(tmp_path / "home"), "isolation": "sandbox", - "sandbox": box, "timeout_s": 300}).run("Write the file.", workspace=work) + result = make("dsh", {**params, "web_search": False, "home": str(home), "isolation": "sandbox", + "sandbox": box, "timeout_s": 300}).run("Write the files.", workspace=work) + escaped = outside.exists() finally: server.shutdown() + shutil.rmtree(home, ignore_errors=True) assert result.exit_code == 0 and result.extra["served"] and not result.extra["failed"] - [command] = [s for s in result.trajectory if s.kind == "command"] - assert (command.exit_code, command.output) == (0, "made\n") and (work / "made.txt").read_text() == "made\n" + [inside, written] = [s for s in result.trajectory if s.kind == "command"] + assert (inside.exit_code, inside.output) == (0, "made\n") and (work / "made.txt").read_text() == "made\n" + assert written.exit_code != 0 and not escaped @pytest.mark.integration From 207ea8856879f7fc556b529567f81de5a3758962 Mon Sep 17 00:00:00 2001 From: Albert Zhao Date: Mon, 9 Nov 2026 03:04:56 -0500 Subject: [PATCH 08/12] dsh: leave the Pi fixes to their own PRs The Pi fixes this branch carried change Pi's behaviour on main, not dsh's: parse_log's Pi reader, the AHE Evolver's texts for a Pi acting agent's stream, and mwm and RRSI refusing a Pi learner. They go to their own PRs (fix/pi-stream-readers, fix/mwm-rrsi-backend-check). This branch keeps dsh's entries only: parse_log reads a dsh stream as dsh's, the AHE texts name a dsh acting agent's stream, and mwm and RRSI refuse dsh (the other PR's general check covers it too, whichever merges first). The integration test of dsh's own sandbox keeps the session's home under ~/.cache/hermit-tests, as the test checklist asks of a check under bubblewrap (it was under /var/tmp). Co-Authored-By: Claude Opus 5.5 --- hermit/agent/base/records.py | 5 ++--- .../evolver/methods/ahe_adapter/debugger.py | 4 ---- hermit/agent/evolver/methods/mwm/evolver.py | 4 ++-- .../evolver/methods/rrsi_adapter/evolver.py | 4 ++-- tests/core/test_dsh_agent.py | 8 ++++--- tests/methods/ahe/test_ahe_hosts.py | 21 +++++++++---------- tests/methods/mwm/test_mwm_evolver.py | 12 +++++------ .../methods/rrsi_adapter/test_rrsi_adapter.py | 5 ++--- 8 files changed, 29 insertions(+), 34 deletions(-) diff --git a/hermit/agent/base/records.py b/hermit/agent/base/records.py index 2b2d2d11..d2657989 100644 --- a/hermit/agent/base/records.py +++ b/hermit/agent/base/records.py @@ -195,6 +195,5 @@ def parse_claude(log: Path, model: str) -> tuple[str | None, list[Usage]]: def parse_log(log: Path, backend: str, model: str) -> tuple[str | None, list[Usage]]: - """Session handle and usage from the agent's JSON event stream, read as its CLI writes it (Claude Code's for a - backend it does not know).""" - return {"codex": parse_codex, "pi": parse_pi, "dsh": parse_dsh}.get(backend, parse_claude)(log, model) + """Session handle and usage from the agent's JSON event stream: Codex's, dsh's, else Claude Code's.""" + return {"codex": parse_codex, "dsh": parse_dsh}.get(backend, parse_claude)(log, model) diff --git a/hermit/agent/evolver/methods/ahe_adapter/debugger.py b/hermit/agent/evolver/methods/ahe_adapter/debugger.py index 230a437e..b457cec4 100644 --- a/hermit/agent/evolver/methods/ahe_adapter/debugger.py +++ b/hermit/agent/evolver/methods/ahe_adapter/debugger.py @@ -182,10 +182,6 @@ def trace_source(log: Path | None) -> tuple[Path, str] | None: ("the Codex event stream", "the Claude Code event stream"), ("`codex exec --json` event stream", "`claude -p --output-format stream-json` event stream")), - "agent-pi.jsonl": (("agent-codex.jsonl", "agent-pi.jsonl"), - ("the raw Codex event stream", "the raw Pi event stream"), - ("the Codex event stream", "the Pi event stream"), - ("`codex exec --json` event stream", "`pi --mode json` event stream")), "agent-dsh.jsonl": (("agent-codex.jsonl", "agent-dsh.jsonl"), ("the raw Codex event stream", "the raw DeepSeek Harness event stream"), ("the Codex event stream", "the DeepSeek Harness event stream"), diff --git a/hermit/agent/evolver/methods/mwm/evolver.py b/hermit/agent/evolver/methods/mwm/evolver.py index 9b3f473b..2d87393c 100644 --- a/hermit/agent/evolver/methods/mwm/evolver.py +++ b/hermit/agent/evolver/methods/mwm/evolver.py @@ -456,9 +456,9 @@ def _notes(admission: dict | None) -> str: # ------------------------------------------------------------------ steps def setup(self, ctx: Context) -> None: - if ctx.params.get("backend", "codex") not in ("codex", "claude"): # its session texts, homes and sign-ins + if ctx.params.get("backend") == "dsh": # its session texts, homes and sign-ins are Codex's or Claude Code's raise ValueError("the mwm Evolver's learner is a Codex or Claude Code session: evolver.params.backend is " - f"codex or claude, not {ctx.params.get('backend')!r}") + "codex or claude, not 'dsh'") if "network_isolation" in ctx.params: raise ValueError("evolver.params.network_isolation was removed: the learner runs in the run's evolver " "sandbox, whose network reaches only its models, the tool endpoint and " diff --git a/hermit/agent/evolver/methods/rrsi_adapter/evolver.py b/hermit/agent/evolver/methods/rrsi_adapter/evolver.py index 31add202..ed26de5f 100644 --- a/hermit/agent/evolver/methods/rrsi_adapter/evolver.py +++ b/hermit/agent/evolver/methods/rrsi_adapter/evolver.py @@ -417,9 +417,9 @@ def setup(self, ctx) -> None: """What every step needs, checked once: a broken hyperparameter file or domain, or an unset web_search, stops the run here rather than failing each step.""" _require_rrsi() - if ctx.params.get("backend", "codex") not in ("codex", "claude"): # its roles' settings are those CLIs' + if ctx.params.get("backend") == "dsh": # its roles' settings are Codex's or Claude Code's raise ValueError("the RRSI Adapter's proposer and roles are Codex or Claude Code sessions: " - f"evolver.params.backend is codex or claude, not {ctx.params.get('backend')!r}") + "evolver.params.backend is codex or claude, not 'dsh'") if not isinstance(ctx.params.get("web_search"), bool): raise ValueError("evolver.params.web_search must be set, true or false") role = ctx.params.get("roles") or {} diff --git a/tests/core/test_dsh_agent.py b/tests/core/test_dsh_agent.py index cb58c205..66a4a389 100644 --- a/tests/core/test_dsh_agent.py +++ b/tests/core/test_dsh_agent.py @@ -885,9 +885,11 @@ def test_with_the_dsh_cli_a_command_runs_in_dshs_own_sandbox_inside_the_runs(tmp lets a command write). Where neither can start, dsh refuses every command (SANDBOX_UNAVAILABLE), which this would show.""" from hermit.sandbox import Sandbox - if not _bwrap_works() or not os.path.isdir("/var/tmp"): - pytest.skip("needs a working bubblewrap and /var/tmp") - home = Path(tempfile.mkdtemp(prefix="hermit-dsh-home-", dir="/var/tmp")) + if not _bwrap_works(): + pytest.skip("needs a working bubblewrap") + base = Path.home() / ".cache" / "hermit-tests" # outside /tmp, an empty tmpfs under bubblewrap + base.mkdir(parents=True, exist_ok=True) + home = Path(tempfile.mkdtemp(prefix="dsh-home-", dir=base)) outside = home / "outside.txt" commands = [{"command": "echo made > made.txt && cat made.txt", "description": "Write a file"}, {"command": f"echo out > {outside}", "description": "Write the home"}] diff --git a/tests/methods/ahe/test_ahe_hosts.py b/tests/methods/ahe/test_ahe_hosts.py index ad7f0926..3c4ab97a 100644 --- a/tests/methods/ahe/test_ahe_hosts.py +++ b/tests/methods/ahe/test_ahe_hosts.py @@ -704,20 +704,19 @@ def test_the_evolvers_texts_name_the_acting_agents_stream(tmp_path): assert "`claude -p --output-format stream-json` event stream" in question and "codex exec --json" not in question -def test_the_evolvers_texts_name_every_acting_agents_stream_it_reads(): - """A Pi or DeepSeek Harness acting agent's sessions keep their own stream (agent-pi.jsonl, agent-dsh.jsonl), which - rollout_stream finds: the learners' texts name it, not Codex's file, which is not there.""" - from hermit.agent.base.trajectory import CODEX_STREAM, STREAMS +def test_the_evolvers_texts_name_a_dsh_acting_agents_stream(): + """A DeepSeek Harness acting agent's sessions keep their own stream (agent-dsh.jsonl), which rollout_stream finds: + the learners' texts name it, not Codex's file, which is not there.""" + from hermit.agent.base.trajectory import DSH_STREAM from hermit.agent.evolver.methods.ahe_adapter import debugger as D methods = T.ROOT / "hermit" / "agent" / "evolver" / "methods" / "ahe_adapter" texts = [(methods / name).read_text() for name in ("START.md", "DEBUGGER.md")] - for stream in set(STREAMS) - {CODEX_STREAM}: - for text in texts: - words = D.for_stream(text, stream) - assert "agent-codex.jsonl" not in words and "Codex event stream" not in words, stream - assert "codex exec --json" not in words, stream - assert stream in D.for_stream(texts[0], stream) - assert "`dsh --profile headless --json` event stream" in D.for_stream(texts[1], "agent-dsh.jsonl") + for text in texts: + words = D.for_stream(text, DSH_STREAM) + assert "agent-codex.jsonl" not in words and "Codex event stream" not in words + assert "codex exec --json" not in words + assert DSH_STREAM in D.for_stream(texts[0], DSH_STREAM) + assert "`dsh --profile headless --json` event stream" in D.for_stream(texts[1], DSH_STREAM) # --- the real CLIs, each on a scripted local model ---------------------------------------------------------------- diff --git a/tests/methods/mwm/test_mwm_evolver.py b/tests/methods/mwm/test_mwm_evolver.py index 3431d95a..670c39de 100644 --- a/tests/methods/mwm/test_mwm_evolver.py +++ b/tests/methods/mwm/test_mwm_evolver.py @@ -148,14 +148,14 @@ def make(tmp: Path, monkeypatch, steps: int = 2, deployment_param: bool = False) return path -def test_a_learner_that_is_not_codex_or_claude_code_is_refused_at_setup(): - """mwm's session is Codex's or Claude Code's (its texts, homes and sign-ins): another CLI Base Agent, which the - config takes as evolver.base (pi, dsh), is refused before any step rather than run without its model.""" +def test_a_dsh_learner_is_refused_at_setup(): + """mwm's session is Codex's or Claude Code's (its texts, homes and sign-ins): DeepSeek Harness, which the config + takes as evolver.base, is refused before any step rather than run on dsh's own provider instead of its bound + model (mwm's session settings carry no `provider`).""" from types import SimpleNamespace from hermit.agent.evolver.methods.mwm.evolver import MwmEvolver - for backend in ("pi", "dsh"): - with pytest.raises(ValueError, match="backend is codex or claude, not"): - MwmEvolver().setup(SimpleNamespace(params={"backend": backend})) + with pytest.raises(ValueError, match="backend is codex or claude, not 'dsh'"): + MwmEvolver().setup(SimpleNamespace(params={"backend": "dsh"})) def test_bootstrap_review_mounts_and_records(tmp_path, monkeypatch): diff --git a/tests/methods/rrsi_adapter/test_rrsi_adapter.py b/tests/methods/rrsi_adapter/test_rrsi_adapter.py index 509408fb..eed28427 100644 --- a/tests/methods/rrsi_adapter/test_rrsi_adapter.py +++ b/tests/methods/rrsi_adapter/test_rrsi_adapter.py @@ -818,9 +818,8 @@ def test_setup_checks_what_every_step_needs(tmp_path): ev.setup(ctx) with pytest.raises(ValueError, match="web_search"): ev.setup(context(ws, 1, dom, tmp_path, web_search=None)) - for backend in ("pi", "dsh"): # CLI Base Agents the config takes as evolver.base, which RRSI cannot run - with pytest.raises(ValueError, match="backend is codex or claude, not"): - ev.setup(context(ws, 1, dom, tmp_path, backend=backend)) + with pytest.raises(ValueError, match="backend is codex or claude, not 'dsh'"): # its roles' settings are not dsh's + ev.setup(context(ws, 1, dom, tmp_path, backend="dsh")) with pytest.raises(Exception): ev.setup(context(ws, 1, str(tmp_path / "dom" / "missing.py") + ":ToyDomain", tmp_path)) with pytest.raises(ValueError, match="roles.model"): From 1bbd57c4ae312357859077581841fb95177001ef Mon Sep 17 00:00:00 2001 From: Albert Zhao Date: Mon, 9 Nov 2026 03:32:07 -0500 Subject: [PATCH 09/12] dsh: test which steps of dsh's stream count as calls Review round 7 (five reviewers on 37aee8c, one lens each, every finding then reproduced by a separate agent told to refute it) found 34 issues, all confirmed: one of medium severity, the rest low. The medium one: no test guarded the reader's three rules for a step whose tokens dsh did not report (a step that only called a tool is a call, so is one that only thought, and what a step produced does not carry over to the next); each, undone, left every test passing while it misstated a session's cost. The new test fails under each. The low ones are listed in the PR description, not changed. Also: a round-6 test docstring rewrapped. Co-Authored-By: Claude Opus 5.5 --- tests/core/test_dsh_agent.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/core/test_dsh_agent.py b/tests/core/test_dsh_agent.py index 66a4a389..cfa7fbff 100644 --- a/tests/core/test_dsh_agent.py +++ b/tests/core/test_dsh_agent.py @@ -656,6 +656,34 @@ def test_a_step_counts_as_a_call_whenever_it_made_one(tmp_path): retried.output_tokens) == (1, 0, 1000, 0, 10) + + +def test_a_step_is_a_call_when_it_called_a_tool_or_thought_and_none_when_its_requests_then_failed(tmp_path): + """Which steps whose tokens dsh did not report are calls, from the recorded streams: a step whose only output is a + tool call (bash.jsonl's eight, recorded without reasoning) is one, and so is a step whose only output is a + thought (resume.jsonl's step without its reply); a step whose requests all failed (failure.jsonl's) after a step + that produced something is none: what a step produced does not carry over to the next.""" + from hermit.agent.base.records import parse_dsh + + def stream(name: str, events: list[dict]) -> Path: + path = tmp_path / name + path.write_text("".join(json.dumps(e) + "\n" for e in events)) + return path + + def read(name: str) -> list[dict]: + return [json.loads(line) for line in (STREAMS / name).read_text().splitlines() if line.startswith("{")] + [tools] = parse_dsh(without_usage("bash.jsonl", tmp_path), "m")[1] + assert (tools.calls, tools.unreported_calls) == (9, 9) + resume = read("resume.jsonl") + thought = [{k: v for k, v in e.items() if k != "usage"} for e in resume if e.get("type") != "text"] + [thinking] = parse_dsh(stream("thought.jsonl", thought), "m")[1] + assert (thinking.calls, thinking.unreported_calls) == (1, 1) + end = next(i for i, e in enumerate(resume) if e.get("phase") == "step_end") + failed = resume[:end + 1] + [e for e in read("failure.jsonl") if e.get("phase") in ("step_start", "step_end")] + [after] = parse_dsh(stream("failed_after.jsonl", failed), "m")[1] + assert (after.calls, after.unreported_calls, after.input_tokens) == (1, 0, 1000) + + def test_calls_whose_tokens_were_not_reported_have_an_unknown_cost_not_a_free_one(fake, monkeypatch, tmp_path): """dsh reports no cost, so the backend's price makes one from the tokens. A session with a call whose tokens it did not report (a provider that reports no usage, an attempt that reported none) has an unknown cost (None), not @@ -663,8 +691,8 @@ def test_calls_whose_tokens_were_not_reported_have_an_unknown_cost_not_a_free_on goes unreported too): the run's budget counts its calls as of unknown cost, and none of its dollars. A session whose tokens dsh reported is priced from them; a step whose cache counts dsh left out after a retry is priced with those tokens as input, as a cache write is, never less than with the counts (the cached input being no - dearer than the input), so that a USD budget counts the session. Only dsh's reader marks a call unreported: another CLI's call of no tokens (Claude Code's result of - a refused request) keeps its price, $0.00.""" + dearer than the input), so that a USD budget counts the session. Only dsh's reader marks a call unreported: + another CLI's call of no tokens (Claude Code's result of a refused request) keeps its price, $0.00.""" from hermit.trainer.budget import Budget from hermit.agent.base.records import price_usage from hermit.agent.model.usage import Usage From f4792e07f2f3ea69e5bb1974143a194f006994ff Mon Sep 17 00:00:00 2001 From: Albert Zhao Date: Mon, 9 Nov 2026 13:17:54 -0500 Subject: [PATCH 10/12] dsh: test a command dsh moves to the background at its timeout In a hermit dsh session a command still running at its timeout (dsh's default is 60 s) is moved to a background job, unless dsh's job limit is reached: the session's layer leaves tool-bash's promoteOnTimeout at its default, true. Its result ends `[still running after ms; moved to background job ]` with no exit marker, and only the DSH_BACKGROUND check keeps it from reading as exit 0. No test held that case: bash.jsonl was recorded with promotion off. Review round 7 found this (rated medium by its reviewer, low by the agent that reproduced it, as the code was right); the author asked for it fixed. promoted.jsonl is recorded through make("dsh").run, as the other Base Agent streams are, and the test now reads it, and also a command stopped from outside the call (`[stopped: ]`, which no headless session writes, so written as dsh's tool-bash renders it), with and without a marker after it. Each check undone (the background check, the `stopped:` alternative, the markers after a stop) fails the test. Co-Authored-By: Claude Opus 5.5 --- tests/core/test_dsh_agent.py | 27 +++++++++++++++++++++------ tests/fixtures/dsh/README.md | 4 ++++ tests/fixtures/dsh/promoted.jsonl | 13 +++++++++++++ 3 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/dsh/promoted.jsonl diff --git a/tests/core/test_dsh_agent.py b/tests/core/test_dsh_agent.py index cfa7fbff..71b69843 100644 --- a/tests/core/test_dsh_agent.py +++ b/tests/core/test_dsh_agent.py @@ -586,22 +586,37 @@ def test_a_bash_result_is_read_as_dsh_marks_its_end(): def test_a_command_whose_end_dsh_did_not_report_has_no_exit_code(): """dsh's --json keeps 8 KiB of a string: a long result loses its end, where the marker is (recorded: big_bash.jsonl, 1,500 lines then exit 2), and read as exit 0 before; a command started in the background - (background.jsonl) has no status yet. A command that printed a marker of its own and exited 0 (fake_marker.jsonl) - is a clean exit, as dsh reads it: the marker counts only at the very end. A call that failed to run (a result - dsh marks `error`) has no exit status either, as dsh's own view of a bash result has none for it.""" + (background.jsonl) has no status yet, nor has one still running at its timeout (60 s by default), which dsh moves + to a background job in a session unless its job limit is reached (promoted.jsonl): only DSH_BACKGROUND keeps it + from reading as exit 0. A command stopped from outside the call (the job stopped in dsh's UI) is marked + `[stopped: ]` before any other marker, and may have trapped the signal and exited 0: no headless session + writes one, so its results are written here as dsh's bash tool renders them (tool-bash's render.ts, + dsh 0.2.2-rc.1). A command that printed a marker of its own and exited 0 (fake_marker.jsonl) is a clean exit, as + dsh reads it: the marker counts only at the very end. A call that failed to run (a result dsh marks `error`) has + no exit status either, as dsh's own view of a bash result has none for it.""" def commands(name): stream = [json.loads(line) for line in (STREAMS / name).read_text().splitlines() if line.startswith("{")] return [s for s in trajectory.dsh_steps(stream) if s.kind == "command"] + + def command(result, status="completed"): + [step] = trajectory.dsh_steps([ + {"type": "tool_call", "callId": "call_0", "tool": "bash", "input": {"command": "make"}}, + {"type": "tool_result", "callId": "call_0", "status": status, "result": result}]) + return step [cut] = commands("big_bash.jsonl") assert cut.exit_code is None and cut.output.startswith("line 1 of a long build log\n") assert cut.output.endswith(trajectory.DSH_CUT) and "[exit code" not in cut.output [background] = commands("background.jsonl") assert background.exit_code is None and background.output.startswith("started background job ") + [promoted] = commands("promoted.jsonl") + assert promoted.exit_code is None and \ + promoted.output.startswith("building\n[still running after 1000ms; moved to background job bash-") + for stopped in ("(no output)\n[stopped: cancelled by the user]", + "cleaning up\n[stopped: cancelled by the user]\n[exit code: 143]"): + assert command(stopped).exit_code is None, stopped [printed] = commands("fake_marker.jsonl") assert (printed.exit_code, printed.output) == (0, "built\n[exit code: 7]\n") - [unrun] = [s for s in trajectory.dsh_steps([ - {"type": "tool_call", "callId": "call_0", "tool": "bash", "input": {"command": "make"}}, - {"type": "tool_result", "callId": "call_0", "status": "error", "result": "bash could not run"}])] + unrun = command("bash could not run", status="error") assert (unrun.kind, unrun.exit_code, unrun.output) == ("command", None, "bash could not run") diff --git a/tests/fixtures/dsh/README.md b/tests/fixtures/dsh/README.md index a3b04325..e2da1bc2 100644 --- a/tests/fixtures/dsh/README.md +++ b/tests/fixtures/dsh/README.md @@ -11,6 +11,10 @@ delta before each answer when it is asked for a reasoning effort. - `resume.jsonl`: the same session resumed (`--session-id`): one request. - `failure.jsonl`: every request answered 503; dsh retries five times, ends the run with the error `SERVER`, writes `dsh: SERVER: ...` on stderr (the last line, which the Base Agent keeps in the stream) and exits 1. +- `promoted.jsonl`: bash `echo building; sleep 5` with `timeoutMs: 1000`, then the answer. The Base Agent's layer + leaves the `tool-bash` row's `promoteOnTimeout` at dsh's default, true, so at its timeout dsh moves the command to + a background job: the result is the output so far, then `[still running after 1000ms; moved to background job + ]`, with no exit marker. Recorded by running dsh directly (`dsh --profile headless --json --permission-mode danger-full-access`, with a patch layer like the Base Agent's) against a scripted chat-completions model: diff --git a/tests/fixtures/dsh/promoted.jsonl b/tests/fixtures/dsh/promoted.jsonl new file mode 100644 index 00000000..41c8f168 --- /dev/null +++ b/tests/fixtures/dsh/promoted.jsonl @@ -0,0 +1,13 @@ +{"type":"session","sessionId":"session-f5415a1e-694e-4496-a644-8cbc8f0f266f","cwd":"/run/work"} +{"type":"status","phase":"turn_start","turn":1} +{"type":"status","phase":"step_start","turn":1,"step":1} +{"type":"thinking","text":"Thinking it over."} +{"type":"tool_call","callId":"call_1","tool":"bash","input":{"command":"echo building; sleep 5","description":"Build","timeoutMs":1000}} +{"type":"tool_result","callId":"call_1","status":"completed","result":"building\n[still running after 1000ms; moved to background job bash-1b164353b005474d]"} +{"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"step_start","turn":1,"step":2} +{"type":"thinking","text":"Thinking it over."} +{"type":"text","text":"the answer"} +{"type":"status","phase":"step_end","turn":1,"step":2,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} +{"type":"final","sessionId":"session-f5415a1e-694e-4496-a644-8cbc8f0f266f","reason":{"kind":"completed"},"text":"the answer"} From d25958a0c680607c94fc350c70a53e4823fc6b90 Mon Sep 17 00:00:00 2001 From: Albert Zhao Date: Mon, 9 Nov 2026 14:13:19 -0500 Subject: [PATCH 11/12] dsh: read a timeout marker at any timeout dsh prints (review round 8) Review round 8 (two reviewers on f4792e0, one on the test and its fixture, one on the claims, each finding then reproduced by a separate agent told to refute it) found: - dsh prints a command's timeout as the model gave it, a JavaScript number (`1000.5`, `0.5`), and both timeout markers were matched only at a whole number of milliseconds: a command still running at such a timeout, moved to a background job, read as exit 0, and one killed at it that trapped the signal read as its trapped exit (rated medium). Both markers now match any number dsh prints. Recorded: promoted.jsonl's third call (1000.5 ms, moved to the background) and bash.jsonl's ninth (1000.5 ms, killed, exit 0 trapped). - The test of a promoted command held only one with output before its marker; the marker alone, what a command that printed nothing yet gets, rests on DSH_BACKGROUND matching without a newline before it, which nothing held (rated medium). promoted.jsonl's second call is `sleep 5`. - Wording (low, in f4792e0's own docstring): the stop marker comes before the signal or exit marker, not before every other marker (a sandbox denial's come first). Each fix undone fails its test: the marker with a newline required before it, and either marker at whole milliseconds only. bash.jsonl is recorded again by the same script, its first eight results unchanged; its steps' count in the usage test follows (10). Co-Authored-By: Claude Opus 5.5 --- hermit/agent/base/trajectory.py | 8 +++++--- tests/core/test_dsh_agent.py | 31 ++++++++++++++++++------------- tests/fixtures/dsh/README.md | 18 ++++++++++-------- tests/fixtures/dsh/bash.jsonl | 10 +++++++--- tests/fixtures/dsh/promoted.jsonl | 18 ++++++++++++++---- 5 files changed, 54 insertions(+), 31 deletions(-) diff --git a/hermit/agent/base/trajectory.py b/hermit/agent/base/trajectory.py index 841b176b..be846482 100644 --- a/hermit/agent/base/trajectory.py +++ b/hermit/agent/base/trajectory.py @@ -220,11 +220,13 @@ def pi_steps(stream: list[dict]) -> list[Step]: # How dsh's bash tool ends a command's result (its own parseExitStatus reads it so, anchored at the very end, as # JavaScript's `$` is: \Z here): a nonzero exit or a signal is marked last, a clean exit is not; a command stopped by # its timeout or from outside is marked before that (a command that traps the signal may still exit 0), and one still -# running was moved to the background, unfinished +# running was moved to the background, unfinished (its marker is the whole result when it printed nothing yet). A +# timeout is printed as the model gave it, a JavaScript number: `1000`, `1000.5`, `1e-7` DSH_EXIT = re.compile(r"\n\[exit code: (\d+)\]\Z") DSH_SIGNAL = re.compile(r"\n\[killed by signal: [^\]\n]+\]\Z") -DSH_STOPPED = re.compile(r"\n\[(?:timed out after \d+ms|stopped: [^\]\n]*)\](?:\n\[[^\]\n]*\])*\Z") -DSH_BACKGROUND = re.compile(r"\[still running after \d+ms; moved to background job [^\]\n]+\]\Z") +DSH_STOPPED = re.compile( + r"\n\[(?:timed out after \d+(?:\.\d+)?(?:e[+-]\d+)?ms|stopped: [^\]\n]*)\](?:\n\[[^\]\n]*\])*\Z") +DSH_BACKGROUND = re.compile(r"\[still running after \d+(?:\.\d+)?(?:e[+-]\d+)?ms; moved to background job [^\]\n]+\]\Z") DSH_NO_OUTPUT = "(no output)" # what the result holds before its marker when the command printed nothing # dsh's --json keeps 8 KiB of each string and marks the event `truncated`: the end, where the marker was, is gone DSH_CUT = "\n[the rest was cut: dsh's --json keeps 8 KiB of a string]" diff --git a/tests/core/test_dsh_agent.py b/tests/core/test_dsh_agent.py index 71b69843..ed418539 100644 --- a/tests/core/test_dsh_agent.py +++ b/tests/core/test_dsh_agent.py @@ -573,14 +573,16 @@ def test_a_bash_result_is_read_as_dsh_marks_its_end(): """dsh marks a nonzero exit (`[exit code: N]`) or a signal after a command's output and nothing after a clean exit; `(no output)` stands for an empty output, also before a marker. A command stopped by its timeout has no exit code to trust, even one that trapped the signal and exited 0 (it read as a clean exit before), as a - signal's has none: unknown, as Codex reports it.""" + signal's has none: unknown, as Codex reports it. dsh prints the timeout as the model gave it, which need not be + a whole number of milliseconds (`1000.5ms`, read as a clean exit before).""" stream = [json.loads(line) for line in (STREAMS / "bash.jsonl").read_text().splitlines()] commands = [(s.command, s.exit_code, s.output) for s in trajectory.dsh_steps(stream) if s.kind == "command"] assert commands == [("exit 3", 3, ""), ("printf ''", 0, ""), ("true", 0, ""), ("echo out; exit 4", 4, "out"), ("kill -9 $$", None, "(no output)\n[killed by signal: SIGKILL]"), ("printf 'no newline'; exit 5", 5, "no newline"), ("trap 'exit 0' TERM; sleep 5 & wait", None, "(no output)\n[timed out after 1000ms]"), - ("sleep 5", None, "(no output)\n[timed out after 1000ms]\n[killed by signal: SIGTERM]")] + ("sleep 5", None, "(no output)\n[timed out after 1000ms]\n[killed by signal: SIGTERM]"), + ("trap 'exit 0' TERM; sleep 5 & wait", None, "(no output)\n[timed out after 1000.5ms]")] def test_a_command_whose_end_dsh_did_not_report_has_no_exit_code(): @@ -588,12 +590,14 @@ def test_a_command_whose_end_dsh_did_not_report_has_no_exit_code(): big_bash.jsonl, 1,500 lines then exit 2), and read as exit 0 before; a command started in the background (background.jsonl) has no status yet, nor has one still running at its timeout (60 s by default), which dsh moves to a background job in a session unless its job limit is reached (promoted.jsonl): only DSH_BACKGROUND keeps it - from reading as exit 0. A command stopped from outside the call (the job stopped in dsh's UI) is marked - `[stopped: ]` before any other marker, and may have trapped the signal and exited 0: no headless session - writes one, so its results are written here as dsh's bash tool renders them (tool-bash's render.ts, - dsh 0.2.2-rc.1). A command that printed a marker of its own and exited 0 (fake_marker.jsonl) is a clean exit, as - dsh reads it: the marker counts only at the very end. A call that failed to run (a result dsh marks `error`) has - no exit status either, as dsh's own view of a bash result has none for it.""" + from reading as exit 0, also when its marker is the whole result (the command printed nothing yet) and when the + timeout is no whole number of milliseconds (`1000.5ms`, read as exit 0 before). A command stopped from outside + the call (the job stopped in dsh's UI) is marked `[stopped: ]` before its signal or exit marker, and may + have trapped the signal and exited 0: no headless session writes one, so its results are written here as dsh's + bash tool renders them (tool-bash's render.ts, dsh 0.2.2-rc.1). A command that printed a marker of its own and + exited 0 (fake_marker.jsonl) is a clean exit, as dsh reads it: the marker counts only at the very end. A call + that failed to run (a result dsh marks `error`) has no exit status either, as dsh's own view of a bash result has + none for it.""" def commands(name): stream = [json.loads(line) for line in (STREAMS / name).read_text().splitlines() if line.startswith("{")] return [s for s in trajectory.dsh_steps(stream) if s.kind == "command"] @@ -608,9 +612,10 @@ def command(result, status="completed"): assert cut.output.endswith(trajectory.DSH_CUT) and "[exit code" not in cut.output [background] = commands("background.jsonl") assert background.exit_code is None and background.output.startswith("started background job ") - [promoted] = commands("promoted.jsonl") - assert promoted.exit_code is None and \ - promoted.output.startswith("building\n[still running after 1000ms; moved to background job bash-") + [building, silent, fraction] = commands("promoted.jsonl") + for step, start in ((building, "building\n[still running after 1000ms"), (silent, "[still running after 1000ms"), + (fraction, "building\n[still running after 1000.5ms")): + assert step.exit_code is None and step.output.startswith(f"{start}; moved to background job bash-"), start for stopped in ("(no output)\n[stopped: cancelled by the user]", "cleaning up\n[stopped: cancelled by the user]\n[exit code: 143]"): assert command(stopped).exit_code is None, stopped @@ -675,7 +680,7 @@ def test_a_step_counts_as_a_call_whenever_it_made_one(tmp_path): def test_a_step_is_a_call_when_it_called_a_tool_or_thought_and_none_when_its_requests_then_failed(tmp_path): """Which steps whose tokens dsh did not report are calls, from the recorded streams: a step whose only output is a - tool call (bash.jsonl's eight, recorded without reasoning) is one, and so is a step whose only output is a + tool call (bash.jsonl's nine, recorded without reasoning) is one, and so is a step whose only output is a thought (resume.jsonl's step without its reply); a step whose requests all failed (failure.jsonl's) after a step that produced something is none: what a step produced does not carry over to the next.""" from hermit.agent.base.records import parse_dsh @@ -688,7 +693,7 @@ def stream(name: str, events: list[dict]) -> Path: def read(name: str) -> list[dict]: return [json.loads(line) for line in (STREAMS / name).read_text().splitlines() if line.startswith("{")] [tools] = parse_dsh(without_usage("bash.jsonl", tmp_path), "m")[1] - assert (tools.calls, tools.unreported_calls) == (9, 9) + assert (tools.calls, tools.unreported_calls) == (10, 10) resume = read("resume.jsonl") thought = [{k: v for k, v in e.items() if k != "usage"} for e in resume if e.get("type") != "text"] [thinking] = parse_dsh(stream("thought.jsonl", thought), "m")[1] diff --git a/tests/fixtures/dsh/README.md b/tests/fixtures/dsh/README.md index e2da1bc2..e73937bd 100644 --- a/tests/fixtures/dsh/README.md +++ b/tests/fixtures/dsh/README.md @@ -11,19 +11,21 @@ delta before each answer when it is asked for a reasoning effort. - `resume.jsonl`: the same session resumed (`--session-id`): one request. - `failure.jsonl`: every request answered 503; dsh retries five times, ends the run with the error `SERVER`, writes `dsh: SERVER: ...` on stderr (the last line, which the Base Agent keeps in the stream) and exits 1. -- `promoted.jsonl`: bash `echo building; sleep 5` with `timeoutMs: 1000`, then the answer. The Base Agent's layer - leaves the `tool-bash` row's `promoteOnTimeout` at dsh's default, true, so at its timeout dsh moves the command to - a background job: the result is the output so far, then `[still running after 1000ms; moved to background job - ]`, with no exit marker. +- `promoted.jsonl`: bash `echo building; sleep 5` and `sleep 5` with `timeoutMs: 1000`, and the first again with + `timeoutMs: 1000.5`, then the answer. The Base Agent's layer leaves the `tool-bash` row's `promoteOnTimeout` at + dsh's default, true, so at its timeout dsh moves each command to a background job: the result is the output so + far, then `[still running after ms; moved to background job ]` (the marker alone for `sleep 5`, + which printed nothing), with no exit marker. Recorded by running dsh directly (`dsh --profile headless --json --permission-mode danger-full-access`, with a patch layer like the Base Agent's) against a scripted chat-completions model: -- `bash.jsonl`: how dsh's bash tool marks a command's end. The model calls bash eight times: `exit 3`, `printf ''`, +- `bash.jsonl`: how dsh's bash tool marks a command's end. The model calls bash nine times: `exit 3`, `printf ''`, `true`, `echo out; exit 4`, `kill -9 $$`, `printf 'no newline'; exit 5`, then with `timeoutMs: 1000` - `trap 'exit 0' TERM; sleep 5 & wait` (stopped by its timeout, it exits 0) and `sleep 5` (killed), then answers - `done`. The layer also sets the `tool-bash` row's `promoteOnTimeout: false`, so a timeout stops the command rather - than moving it to a background job. + `trap 'exit 0' TERM; sleep 5 & wait` (stopped by its timeout, it exits 0) and `sleep 5` (killed), and the first of + these again with `timeoutMs: 1000.5` (dsh prints the timeout as given), then answers `done`. The layer also sets + the `tool-bash` row's `promoteOnTimeout: false`, so a timeout stops the command rather than moving it to a + background job. - `max_tokens.jsonl`: the model's answer ends at its token limit (`finish_reason: length`): the run ends `max-tokens` and dsh exits 1, writing nothing on stderr. - `big_bash.jsonl`: bash prints 1,500 lines (about 42 KB) and exits 2; dsh's `--json` keeps 8 KiB of the result and diff --git a/tests/fixtures/dsh/bash.jsonl b/tests/fixtures/dsh/bash.jsonl index ca3b051d..a05f2d20 100644 --- a/tests/fixtures/dsh/bash.jsonl +++ b/tests/fixtures/dsh/bash.jsonl @@ -1,4 +1,4 @@ -{"type":"session","sessionId":"session-b0463c76-d042-46aa-89d0-92ad134f0e29","cwd":"/run/bash/work"} +{"type":"session","sessionId":"session-9fc3acf0-ab99-4d1f-8770-d448a699fe26","cwd":"/run/bash/work"} {"type":"status","phase":"turn_start","turn":1} {"type":"status","phase":"step_start","turn":1,"step":1} {"type":"tool_call","callId":"call_0","tool":"bash","input":{"command":"exit 3","description":"edge"}} @@ -33,7 +33,11 @@ {"type":"tool_result","callId":"call_7","status":"completed","result":"(no output)\n[timed out after 1000ms]\n[killed by signal: SIGTERM]"} {"type":"status","phase":"step_end","turn":1,"step":8,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} {"type":"status","phase":"step_start","turn":1,"step":9} -{"type":"text","text":"done"} +{"type":"tool_call","callId":"call_8","tool":"bash","input":{"command":"trap 'exit 0' TERM; sleep 5 & wait","timeoutMs":1000.5,"description":"edge"}} +{"type":"tool_result","callId":"call_8","status":"completed","result":"(no output)\n[timed out after 1000.5ms]"} {"type":"status","phase":"step_end","turn":1,"step":9,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} +{"type":"status","phase":"step_start","turn":1,"step":10} +{"type":"text","text":"done"} +{"type":"status","phase":"step_end","turn":1,"step":10,"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}} {"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} -{"type":"final","sessionId":"session-b0463c76-d042-46aa-89d0-92ad134f0e29","reason":{"kind":"completed"},"text":"done"} +{"type":"final","sessionId":"session-9fc3acf0-ab99-4d1f-8770-d448a699fe26","reason":{"kind":"completed"},"text":"done"} diff --git a/tests/fixtures/dsh/promoted.jsonl b/tests/fixtures/dsh/promoted.jsonl index 41c8f168..c5a0cb3e 100644 --- a/tests/fixtures/dsh/promoted.jsonl +++ b/tests/fixtures/dsh/promoted.jsonl @@ -1,13 +1,23 @@ -{"type":"session","sessionId":"session-f5415a1e-694e-4496-a644-8cbc8f0f266f","cwd":"/run/work"} +{"type":"session","sessionId":"session-89dbecd8-5170-42c8-85d6-b721d96c3364","cwd":"/run/work"} {"type":"status","phase":"turn_start","turn":1} {"type":"status","phase":"step_start","turn":1,"step":1} {"type":"thinking","text":"Thinking it over."} {"type":"tool_call","callId":"call_1","tool":"bash","input":{"command":"echo building; sleep 5","description":"Build","timeoutMs":1000}} -{"type":"tool_result","callId":"call_1","status":"completed","result":"building\n[still running after 1000ms; moved to background job bash-1b164353b005474d]"} +{"type":"tool_result","callId":"call_1","status":"completed","result":"building\n[still running after 1000ms; moved to background job bash-215d1b678a6b4016]"} {"type":"status","phase":"step_end","turn":1,"step":1,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} {"type":"status","phase":"step_start","turn":1,"step":2} {"type":"thinking","text":"Thinking it over."} -{"type":"text","text":"the answer"} +{"type":"tool_call","callId":"call_2","tool":"bash","input":{"command":"sleep 5","description":"Wait","timeoutMs":1000}} +{"type":"tool_result","callId":"call_2","status":"completed","result":"[still running after 1000ms; moved to background job bash-05097e6c1e3f423d]"} {"type":"status","phase":"step_end","turn":1,"step":2,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"step_start","turn":1,"step":3} +{"type":"thinking","text":"Thinking it over."} +{"type":"tool_call","callId":"call_3","tool":"bash","input":{"command":"echo building; sleep 5","description":"Build","timeoutMs":1000.5}} +{"type":"tool_result","callId":"call_3","status":"completed","result":"building\n[still running after 1000.5ms; moved to background job bash-edd1522dc3ff48cf]"} +{"type":"status","phase":"step_end","turn":1,"step":3,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} +{"type":"status","phase":"step_start","turn":1,"step":4} +{"type":"thinking","text":"Thinking it over."} +{"type":"text","text":"the answer"} +{"type":"status","phase":"step_end","turn":1,"step":4,"usage":{"inputTokens":600,"outputTokens":20,"totalTokens":1020,"cacheReadTokens":400}} {"type":"status","phase":"turn_end","turn":1,"reason":{"kind":"completed"}} -{"type":"final","sessionId":"session-f5415a1e-694e-4496-a644-8cbc8f0f266f","reason":{"kind":"completed"},"text":"the answer"} +{"type":"final","sessionId":"session-89dbecd8-5170-42c8-85d6-b721d96c3364","reason":{"kind":"completed"},"text":"the answer"} From a43698db513cd61851f9cac51193754b9f691dcc Mon Sep 17 00:00:00 2001 From: Albert Zhao Date: Mon, 9 Nov 2026 15:01:21 -0500 Subject: [PATCH 12/12] dsh: wording from review round 9 Review round 9 (two reviewers on d25958a, one on the code and its tests, one on the claims, each finding then reproduced by a separate agent told to refute it) found 6 issues (4 distinct), all low. Three are wording this change's own text got wrong, corrected here (comments, a docstring and the fixture README; no code or test changes): - dsh prints the timeout it applied, not always the model's: its default (60 s) when the model gives none, and at most its cap (600 s). - DSH_NO_OUTPUT's comment said `(no output)` comes before the marker of a command that printed nothing, which a command moved to the background does not get (its marker stands alone). The fourth, the exponent form both timeout patterns match (`1e-7`, a timeout under 0.000001 ms) held by no test, is listed in the PR description, not changed. Co-Authored-By: Claude Opus 5.5 --- hermit/agent/base/trajectory.py | 4 ++-- tests/core/test_dsh_agent.py | 4 ++-- tests/fixtures/dsh/README.md | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/hermit/agent/base/trajectory.py b/hermit/agent/base/trajectory.py index be846482..91a4a321 100644 --- a/hermit/agent/base/trajectory.py +++ b/hermit/agent/base/trajectory.py @@ -221,13 +221,13 @@ def pi_steps(stream: list[dict]) -> list[Step]: # JavaScript's `$` is: \Z here): a nonzero exit or a signal is marked last, a clean exit is not; a command stopped by # its timeout or from outside is marked before that (a command that traps the signal may still exit 0), and one still # running was moved to the background, unfinished (its marker is the whole result when it printed nothing yet). A -# timeout is printed as the model gave it, a JavaScript number: `1000`, `1000.5`, `1e-7` +# timeout is printed as dsh applied it, a JavaScript number: the model's (`1000.5`, `1e-7`), else 60000, at most 600000 DSH_EXIT = re.compile(r"\n\[exit code: (\d+)\]\Z") DSH_SIGNAL = re.compile(r"\n\[killed by signal: [^\]\n]+\]\Z") DSH_STOPPED = re.compile( r"\n\[(?:timed out after \d+(?:\.\d+)?(?:e[+-]\d+)?ms|stopped: [^\]\n]*)\](?:\n\[[^\]\n]*\])*\Z") DSH_BACKGROUND = re.compile(r"\[still running after \d+(?:\.\d+)?(?:e[+-]\d+)?ms; moved to background job [^\]\n]+\]\Z") -DSH_NO_OUTPUT = "(no output)" # what the result holds before its marker when the command printed nothing +DSH_NO_OUTPUT = "(no output)" # a finished command's empty output, alone or before its markers # dsh's --json keeps 8 KiB of each string and marks the event `truncated`: the end, where the marker was, is gone DSH_CUT = "\n[the rest was cut: dsh's --json keeps 8 KiB of a string]" diff --git a/tests/core/test_dsh_agent.py b/tests/core/test_dsh_agent.py index ed418539..ffe42d57 100644 --- a/tests/core/test_dsh_agent.py +++ b/tests/core/test_dsh_agent.py @@ -573,8 +573,8 @@ def test_a_bash_result_is_read_as_dsh_marks_its_end(): """dsh marks a nonzero exit (`[exit code: N]`) or a signal after a command's output and nothing after a clean exit; `(no output)` stands for an empty output, also before a marker. A command stopped by its timeout has no exit code to trust, even one that trapped the signal and exited 0 (it read as a clean exit before), as a - signal's has none: unknown, as Codex reports it. dsh prints the timeout as the model gave it, which need not be - a whole number of milliseconds (`1000.5ms`, read as a clean exit before).""" + signal's has none: unknown, as Codex reports it. dsh prints the timeout it applied (the model's, else 60 s, at + most 600 s), which need not be a whole number of milliseconds (`1000.5ms`, read as a clean exit before).""" stream = [json.loads(line) for line in (STREAMS / "bash.jsonl").read_text().splitlines()] commands = [(s.command, s.exit_code, s.output) for s in trajectory.dsh_steps(stream) if s.kind == "command"] assert commands == [("exit 3", 3, ""), ("printf ''", 0, ""), ("true", 0, ""), ("echo out; exit 4", 4, "out"), diff --git a/tests/fixtures/dsh/README.md b/tests/fixtures/dsh/README.md index e73937bd..695d77cb 100644 --- a/tests/fixtures/dsh/README.md +++ b/tests/fixtures/dsh/README.md @@ -23,9 +23,9 @@ layer like the Base Agent's) against a scripted chat-completions model: - `bash.jsonl`: how dsh's bash tool marks a command's end. The model calls bash nine times: `exit 3`, `printf ''`, `true`, `echo out; exit 4`, `kill -9 $$`, `printf 'no newline'; exit 5`, then with `timeoutMs: 1000` `trap 'exit 0' TERM; sleep 5 & wait` (stopped by its timeout, it exits 0) and `sleep 5` (killed), and the first of - these again with `timeoutMs: 1000.5` (dsh prints the timeout as given), then answers `done`. The layer also sets - the `tool-bash` row's `promoteOnTimeout: false`, so a timeout stops the command rather than moving it to a - background job. + these again with `timeoutMs: 1000.5` (dsh prints the timeout it applied, here the model's), then answers `done`. + The layer also sets the `tool-bash` row's `promoteOnTimeout: false`, so a timeout stops the command rather than + moving it to a background job. - `max_tokens.jsonl`: the model's answer ends at its token limit (`finish_reason: length`): the run ends `max-tokens` and dsh exits 1, writing nothing on stderr. - `big_bash.jsonl`: bash prints 1,500 lines (about 42 KB) and exits 2; dsh's `--json` keeps 8 KiB of the result and