diff --git a/mkdocs/docs/concepts/presets.md b/mkdocs/docs/concepts/presets.md index 12a06edbd..47f19a8ad 100644 --- a/mkdocs/docs/concepts/presets.md +++ b/mkdocs/docs/concepts/presets.md @@ -91,13 +91,15 @@ To stop a creation and its runs, use `dstack preset stop`. export DSTACK_AGENT_ANTHROPIC_API_KEY=... ``` - By default, the agent uses `claude-opus-4-8`. It doesn't set an effort level, so the `claude` CLI default applies. To override them, set: + By default, the agent sets neither a model nor an effort level, so the `claude` CLI's built-in defaults apply. To override them, set: ```shell - export DSTACK_AGENT_ANTHROPIC_MODEL=claude-opus-5 + export DSTACK_AGENT_ANTHROPIC_MODEL=claude-fable-5-1 export DSTACK_AGENT_CLAUDE_EFFORT=max ``` + See the [Models overview](https://platform.claude.com/docs/en/models/overview) for the available models and their IDs. + Supported effort levels are `low`, `medium`, `high`, `xhigh`, and `max`. ??? info "Presets directory" diff --git a/mkdocs/docs/reference/cli/dstack/preset.md b/mkdocs/docs/reference/cli/dstack/preset.md index 3d8bc9d2d..1520295cc 100644 --- a/mkdocs/docs/reference/cli/dstack/preset.md +++ b/mkdocs/docs/reference/cli/dstack/preset.md @@ -50,7 +50,7 @@ Preset creation uses the existing `claude` login unless | --- | --- | | `DSTACK_AGENT_ANTHROPIC_API_KEY` | Anthropic API key used by the agent. | | `DSTACK_AGENT_CLAUDE_PATH` | `claude` executable name or path. Defaults to `claude` from `PATH`. | -| `DSTACK_AGENT_ANTHROPIC_MODEL` | Claude model used by the agent. Defaults to `claude-opus-4-8`. | +| `DSTACK_AGENT_ANTHROPIC_MODEL` | Claude model used by the agent. If unset, the `claude` CLI's built-in default is used. | | `DSTACK_AGENT_CLAUDE_EFFORT` | Claude effort level: `low`, `medium`, `high`, `xhigh`, or `max`. If unset, the `claude` CLI default is used. | Agent progress is written to `agent.log` under `~/.dstack/presets//`, diff --git a/mkdocs/docs/reference/env.md b/mkdocs/docs/reference/env.md index 9b1c9f17d..64ced37ed 100644 --- a/mkdocs/docs/reference/env.md +++ b/mkdocs/docs/reference/env.md @@ -219,5 +219,5 @@ $ find ~/.dstack/logs/cli/ - `DSTACK_PROJECT`{ #DSTACK_PROJECT } – Has the same effect as `--project`. Defaults to `None`. - `DSTACK_AGENT_ANTHROPIC_API_KEY`{ #DSTACK_AGENT_ANTHROPIC_API_KEY } – The Anthropic API key used by the preset agent. If unset, the existing `claude` login is used. - `DSTACK_AGENT_CLAUDE_PATH`{ #DSTACK_AGENT_CLAUDE_PATH } – The `claude` executable name or path used by the preset agent. Defaults to `claude` from `PATH`. -- `DSTACK_AGENT_ANTHROPIC_MODEL`{ #DSTACK_AGENT_ANTHROPIC_MODEL } – The Claude model used by the preset agent. Defaults to `claude-opus-4-8`. +- `DSTACK_AGENT_ANTHROPIC_MODEL`{ #DSTACK_AGENT_ANTHROPIC_MODEL } – The Claude model used by the preset agent. If unset, the `claude` CLI's built-in default is used. - `DSTACK_AGENT_CLAUDE_EFFORT`{ #DSTACK_AGENT_CLAUDE_EFFORT } – The Claude effort level used by the preset agent. Can be `low`, `medium`, `high`, `xhigh`, or `max`. If unset, the `claude` CLI default is used. diff --git a/src/dstack/_internal/cli/models/preset_agent.py b/src/dstack/_internal/cli/models/preset_agent.py index 6bd4e622b..b975a135e 100644 --- a/src/dstack/_internal/cli/models/preset_agent.py +++ b/src/dstack/_internal/cli/models/preset_agent.py @@ -125,6 +125,18 @@ class PresetSessionState(CoreModel): run: Optional[PresetSessionRun] +class PresetAgentInfo(CoreModel): + """`agent.json`: how the claude agent was launched. A debug record.""" + + executable: str + version: Optional[str] + auth_status: str + # None is the claude CLI's default. + effort: Optional[str] + # Reported by claude on its init line; None until then. + model: Optional[str] + + class ClaudeStreamEvent(CoreModel): """One line of the claude CLI's `--output-format stream-json`. Not our format: unknown fields are dropped and omitted fields default.""" @@ -135,6 +147,8 @@ class ClaudeStreamEvent(CoreModel): # Identifies the claude conversation, so an interrupted creation can be # resumed with `claude --resume`. Not every line carries it. session_id: Optional[str] = None + # The model claude runs with, e.g. "claude-opus-5[1m]". Set on the init line only. + model: Optional[str] = None class ClaudeResultEvent(ClaudeStreamEvent): diff --git a/src/dstack/_internal/cli/services/presets/agent.py b/src/dstack/_internal/cli/services/presets/agent.py index 64cedcfa6..70d22be98 100644 --- a/src/dstack/_internal/cli/services/presets/agent.py +++ b/src/dstack/_internal/cli/services/presets/agent.py @@ -18,6 +18,7 @@ AnyClaudeStreamEvent, ClaudeResultEvent, PresetAgentFailure, + PresetAgentInfo, PresetAgentSuccess, PresetSessionProcess, ) @@ -90,9 +91,9 @@ class ClaudeAuth: api_key: Optional[str] executable: str - # None uses the claude CLI's own default. + # None means the flag is not passed and claude uses its own default. effort: Optional[ClaudeEffort] - model: str + model: Optional[str] @dataclass @@ -131,6 +132,16 @@ def _get_claude_auth_status(auth: "ClaudeAuth") -> str: return "unknown" +def get_agent_info(auth: ClaudeAuth) -> PresetAgentInfo: + return PresetAgentInfo( + executable=auth.executable, + version=_get_claude_version(auth), + auth_status=_get_claude_auth_status(auth), + effort=auth.effort, + model=None, + ) + + def get_claude_auth() -> ClaudeAuth: api_key = os.getenv("DSTACK_AGENT_ANTHROPIC_API_KEY") or None configured_path = os.getenv("DSTACK_AGENT_CLAUDE_PATH") or "claude" @@ -146,7 +157,7 @@ def get_claude_auth() -> ClaudeAuth: api_key=api_key, executable=executable, effort=effort, - model=os.getenv("DSTACK_AGENT_ANTHROPIC_MODEL", "claude-opus-4-8"), + model=os.getenv("DSTACK_AGENT_ANTHROPIC_MODEL") or None, ) @@ -351,8 +362,6 @@ def _build_claude_command(*, auth: ClaudeAuth, resume_session_id: Optional[str]) "Task,NotebookEdit", "--permission-mode", "bypassPermissions", - "--model", - auth.model, "--json-schema", json.dumps(_get_report_json_schema()), ] @@ -362,6 +371,8 @@ def _build_claude_command(*, auth: ClaudeAuth, resume_session_id: Optional[str]) command[2:2] = ["--bare"] if auth.effort is not None: command[2:2] = ["--effort", auth.effort] + if auth.model is not None: + command[2:2] = ["--model", auth.model] if resume_session_id is not None: command += ["--resume", resume_session_id] return command @@ -559,6 +570,8 @@ async def _read_process_stream( if output.session_id is None and event.session_id: output.session_id = event.session_id session.record_claude_session_id(event.session_id) + if event.model: + session.record_agent_model(event.model) if event.type == "assistant": output.made_progress = True if not isinstance(event, ClaudeResultEvent): diff --git a/src/dstack/_internal/cli/services/presets/create.py b/src/dstack/_internal/cli/services/presets/create.py index 5fb29b4e4..e2617098b 100644 --- a/src/dstack/_internal/cli/services/presets/create.py +++ b/src/dstack/_internal/cli/services/presets/create.py @@ -28,6 +28,7 @@ PresetAgentProcessOutput, attach_preset_agent, build_preset_agent_env, + get_agent_info, get_claude_auth, run_preset_agent, terminate_agent_process, @@ -636,8 +637,8 @@ async def _create_preset( # while the listing and `--previous` read constraints from the session dir. session.write_constraints(constraints_text) session.write_prompt(prompt) - if setup.auth is not None: - session.write_agent_info(setup.auth) + if setup.auth is not None: + session.write_agent_info(get_agent_info(setup.auth)) try: if mode == "attach": process_output = await attach_preset_agent( diff --git a/src/dstack/_internal/cli/services/presets/session.py b/src/dstack/_internal/cli/services/presets/session.py index 81df2c951..8e054cf11 100644 --- a/src/dstack/_internal/cli/services/presets/session.py +++ b/src/dstack/_internal/cli/services/presets/session.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path -from typing import TYPE_CHECKING, Any, Iterator, Optional, Sequence +from typing import Any, Iterator, Optional, Sequence import psutil import yaml @@ -18,6 +18,7 @@ from rich.text import Text from dstack._internal.cli.models.preset_agent import ( + PresetAgentInfo, PresetSessionFinalize, PresetSessionProcess, PresetSessionRun, @@ -28,14 +29,10 @@ from dstack._internal.cli.utils.common import console from dstack._internal.compat import IS_WINDOWS from dstack._internal.core.errors import CLIError -from dstack._internal.core.models.common import validate_extra_ignore +from dstack._internal.core.models.common import validate_extra_ignore, validate_json_extra_ignore from dstack._internal.core.models.configurations import PresetConfiguration from dstack._internal.utils.common import get_dstack_dir -if TYPE_CHECKING: - from dstack._internal.cli.services.presets.agent import ClaudeAuth - - _PROGRESS_FILENAME = "progress.jsonl" _RUNS_FILENAME = "runs.jsonl" TRIALS_DIRNAME = "trials" @@ -45,6 +42,7 @@ _CONSTRAINTS_FILENAME = "constraints.json" _FINAL_REPORT_FILENAME = "final_report.json" _SESSION_FILENAME = "session.json" +_AGENT_INFO_FILENAME = "agent.json" _USER_PROMPT_FILENAME = "user_prompt.md" @@ -114,21 +112,25 @@ def write_constraints(self, constraints_text: str) -> None: def write_final_report(self, report_text: str) -> None: _write_private_text(self.path / _FINAL_REPORT_FILENAME, report_text) - def write_agent_info(self, auth: "ClaudeAuth") -> None: - from dstack._internal.cli.services.presets.agent import ( - _get_claude_auth_status, - _get_claude_version, + def write_agent_info(self, info: PresetAgentInfo) -> None: + _write_private_text( + self.path / _AGENT_INFO_FILENAME, + info.model_dump_json(indent=2) + "\n", ) - # `agent.json`: a debug document written once and read by nothing, so it - # is a plain dump, not a model. - info = { - "executable": auth.executable, - "version": _get_claude_version(auth), - "model": {"name": auth.model, "effort": auth.effort or "default"}, - "auth_status": _get_claude_auth_status(auth), - } - _write_private_text(self.path / "agent.json", json.dumps(info, indent=2) + "\n") + def read_agent_info(self) -> Optional[PresetAgentInfo]: + try: + text = (self.path / _AGENT_INFO_FILENAME).read_text(encoding="utf-8") + return validate_json_extra_ignore(PresetAgentInfo, text) + except (OSError, ValidationError): + return None + + def record_agent_model(self, model: str) -> None: + info = self.read_agent_info() + if info is None: + return + info.model = model + self.write_agent_info(info) def append_log(self, line: str) -> None: if not self._log_enabled: diff --git a/src/tests/_internal/cli/services/presets/test_agent.py b/src/tests/_internal/cli/services/presets/test_agent.py index f101258c5..f36940e87 100644 --- a/src/tests/_internal/cli/services/presets/test_agent.py +++ b/src/tests/_internal/cli/services/presets/test_agent.py @@ -13,13 +13,14 @@ import pytest import yaml -from dstack._internal.cli.models.preset_agent import PresetSessionProcess +from dstack._internal.cli.models.preset_agent import PresetAgentInfo, PresetSessionProcess from dstack._internal.cli.services.presets.agent import ( ClaudeAuth, _build_claude_command, _prepare_subprocess_command, _terminate_process, build_preset_agent_env, + get_agent_info, get_claude_auth, run_preset_agent, ) @@ -63,12 +64,14 @@ def _record_run(session, workspace_record): pytestmark = pytest.mark.windows -def _claude_auth(*, api_key: str | None = "anthropic-secret", effort=None) -> ClaudeAuth: +def _claude_auth( + *, api_key: str | None = "anthropic-secret", effort=None, model: str | None = "claude-test" +) -> ClaudeAuth: return ClaudeAuth( api_key=api_key, executable="claude", effort=effort, - model="claude-test", + model=model, ) @@ -85,6 +88,18 @@ def test_uses_api_key_only_when_env_is_set(self, monkeypatch, api_key_env): assert auth.api_key == api_key_env + @pytest.mark.parametrize("model_env", ["claude-pinned", "", None]) + def test_leaves_model_to_claude_unless_env_is_set(self, monkeypatch, model_env): + if model_env is None: + monkeypatch.delenv("DSTACK_AGENT_ANTHROPIC_MODEL", raising=False) + else: + monkeypatch.setenv("DSTACK_AGENT_ANTHROPIC_MODEL", model_env) + monkeypatch.setattr(shutil, "which", lambda _: "/usr/bin/claude") + + auth = get_claude_auth() + + assert auth.model == (model_env or None) + @pytest.mark.parametrize("api_key", ["key", None]) def test_builds_command_for_selected_auth_mode(self, api_key): command = _build_claude_command( @@ -95,6 +110,15 @@ def test_builds_command_for_selected_auth_mode(self, api_key): assert ("--setting-sources" in command) is (api_key is None) assert command[command.index("--effort") + 1] == "high" + @pytest.mark.parametrize("model", ["claude-pinned", None]) + def test_passes_model_only_when_pinned(self, model): + command = _build_claude_command(auth=_claude_auth(model=model), resume_session_id=None) + + if model is None: + assert "--model" not in command + else: + assert command[command.index("--model") + 1] == model + @pytest.mark.windows_only def test_runs_windows_batch_launcher(self, tmp_path): script = tmp_path / "fake-claude.cmd" @@ -615,8 +639,8 @@ def test_skips_files_above_the_size_limit(self, tmp_path, monkeypatch): assert not (tmp_path / "session" / "trials" / "1" / "trial.json").exists() -class TestWriteAgentInfo: - def test_writes_model_params_and_auth(self, tmp_path, monkeypatch): +class TestGetAgentInfo: + def test_describes_the_launch_without_a_model(self, tmp_path, monkeypatch): monkeypatch.setattr( "dstack._internal.cli.services.presets.agent._get_claude_version", lambda auth: "2.1.0 (Claude Code)", @@ -629,18 +653,50 @@ def test_writes_model_params_and_auth(self, tmp_path, monkeypatch): session_dir.mkdir() session = PresetSession(path=session_dir, preset_id="ab12cd34") + # Even a pinned model is not recorded here: claude reports the model it + # runs on its init line, and that is what gets recorded. session.write_agent_info( - ClaudeAuth(api_key=None, executable="claude", effort=None, model="claude-opus-4-8") + get_agent_info( + ClaudeAuth(api_key=None, executable="claude", effort="high", model="claude-pinned") + ) ) assert json.loads((session_dir / "agent.json").read_text()) == { "executable": "claude", "version": "2.1.0 (Claude Code)", - "model": {"name": "claude-opus-4-8", "effort": "default"}, "auth_status": '{"authMethod": "claude.ai", "loggedIn": true}', + "effort": "high", + "model": None, } +class TestRecordAgentModel: + def test_records_the_model_claude_reports(self, tmp_path): + session_dir = tmp_path / "session" + session_dir.mkdir() + session = PresetSession(path=session_dir, preset_id="ab12cd34") + session.write_agent_info( + PresetAgentInfo( + executable="claude", version=None, auth_status="{}", effort=None, model=None + ) + ) + + session.record_agent_model("claude-opus-5[1m]") + + info = session.read_agent_info() + assert info is not None + assert info.model == "claude-opus-5[1m]" + + def test_does_not_record_a_model_without_launch_info(self, tmp_path): + session_dir = tmp_path / "session" + session_dir.mkdir() + session = PresetSession(path=session_dir, preset_id="ab12cd34") + + session.record_agent_model("claude-opus-5[1m]") + + assert not (session_dir / "agent.json").exists() + + def _offsets(tmp_path): session_dir = tmp_path / "offsets-session" session_dir.mkdir(exist_ok=True) @@ -702,7 +758,7 @@ async def test_resumes_after_connection_error(self, tmp_path, monkeypatch, capsy "structured_output": {"resumed": True}, })) else: - print(json.dumps({"type": "system", "subtype": "init", "session_id": "sid-123"})) + print(json.dumps({"type": "system", "session_id": "sid-123", "model": "claude-effective"})) print(json.dumps({ "type": "result", "is_error": True, @@ -716,17 +772,26 @@ async def test_resumes_after_connection_error(self, tmp_path, monkeypatch, capsy ) _patch_claude_command(monkeypatch, script) workspace, session = _agent_setup(tmp_path) + session.write_agent_info( + PresetAgentInfo( + executable="claude", version=None, auth_status="{}", effort=None, model=None + ) + ) output = await run_preset_agent( prompt="system prompt", env=_subprocess_env(), workspace=workspace, - auth=ClaudeAuth(api_key=None, executable="claude", effort=None, model="m"), + auth=ClaudeAuth(api_key=None, executable="claude", effort=None, model=None), redacted_values=(), session=session, ) assert output.report_data == {"resumed": True} + # The model comes from claude's init line, not from our configuration. + agent_info = session.read_agent_info() + assert agent_info is not None + assert agent_info.model == "claude-effective" calls = [json.loads(line) for line in (tmp_path / "calls.jsonl").read_text().splitlines()] assert len(calls) == 2 assert calls[0]["args"] == []