diff --git a/.env.example b/.env.example index 60a5a7a..06d645b 100644 --- a/.env.example +++ b/.env.example @@ -51,6 +51,9 @@ OLI_LOGS_DIR="" OLI_TRUNCATION_SMALL=4000 OLI_TRUNCATION_LARGE=100000 +# Max characters accepted by a single write_file call +OLI_MAX_WRITE_CHARS=100000 + # Dry-run: preview destructive actions without executing OLI_DRY_RUN=false diff --git a/AGENTS.md b/AGENTS.md index eaf2f98..9fb9b10 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,12 +11,13 @@ This file is the top-level orientation map. Depth lives under [docs/](docs/). | Path | Purpose | | ---------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [oli_bot/chat.py](oli_bot/chat.py) | Textual TUI app (`OliBot`). Owns command handling, session UI, the `#command-suggestions` autocomplete `ListView`, and (when pooling is enabled) the "Active Sub-Agents" `Tree`. Tracks cumulative session token usage (persisted per session) and renders it in the `#status-bar`. Slash-command names come from the module-level `COMMANDS` tuple. | +| [oli_bot/chat.py](oli_bot/chat.py) | Textual TUI app (`OliBot`). Owns command handling, session UI, the `#command-suggestions` autocomplete `ListView`, and (when pooling is enabled) the "Active Sub-Agents" `Tree`. Tracks live context size (tokenizer-backed, shown in the `#status-bar`) plus cumulative session usage (persisted per session). Slash-command names come from the module-level `COMMANDS` tuple. | | [oli_bot/api/](oli_bot/api/) | FastAPI package exposing the harness over an OpenAI-compatible REST API (`GET /v1/models`, `POST /v1/chat/completions` streaming + non-streaming, `GET /health`) plus a stateful `WS /v1/chat` WebSocket that relays every `AgentEvent` as a typed JSON envelope (`text_chunk`/`thinking`/`tool_call_executing`/`tool_call_result`/`assistant_response`/`usage`/`error`/`done`, plus `sub_agent_started`/`sub_agent_progress`/`sub_agent_completed` and `todo` frames) for real-time browser UIs. Sub-agent events are demuxed by `task_id`; `_wire_todo_relay()` pushes `builtin__todowrite` snapshots onto `mcp_manager.pending_todos`, drained by the socket loop. REST is stateless from the caller's POV; the WebSocket keeps per-connection history, backed by a server-persisted `ConversationStore` under the `"default"` server: turns are sent with an optional `session_id` (`{"content": "...", "session_id": "..."}`, `{"action": "clear", "session_id": "..."}` wipes it) and each completed turn is saved to disk, so browser and TUI share the same session files. Sessions are created/listed/loaded/renamed/deleted via `GET/POST /v1/sessions` and `GET/PUT/DELETE /v1/sessions/{id}`. A single process-private `Agent` is shared across requests and serialised with an `asyncio.Lock` held across `await` points (a `threading.RLock` would be per-thread reentrant and not serialize coroutines sharing the event loop); the WebSocket holds the lock for the duration of each run. Auto-approves permissions (no human), but offline/dry-run still apply. Also exposes `GET/POST /v1/mcp` plus `PUT/DELETE /v1/mcp/{name}` for the browser UI's MCP server configuration view (backed by `MCPClientManager` on `app.state.agent.mcp_manager`). `app.py` holds the `create_app()` factory + `init_state()`; `routers/` holds the route modules (`chat`, `ws`, `sessions`, `config`, `mcp`, `workspace`, `health`); `runner.py` implements the completion handoff, `harness.py` the agent construction + todo relay + `dispatch` handler, `session_service.py` the load/persist helpers, `errors.py` the OpenAI-style error handlers, `convert.py` the wire-message/event converters, `deps.py` the FastAPI dependencies, and `__main__.py` the `oli-server` entrypoint. [`oli_bot/api_server.py`](oli_bot/api_server.py) is a compat shim re-exporting the old single-module surface. See [docs/API_SERVER.md](docs/API_SERVER.md). | -| [oli_bot/agent.py](oli_bot/agent.py) | `Agent` — mode + system prompt owner; orchestrates the tool-calling loop and streams typed events (`TextChunk`, `ThinkingChunk`, `ToolCallChunk`, `ToolCallExecuting`, `ToolCallResult`, `StreamChunk`, `UsageEvent`, `Error`, `Done`). Aggregates per-call `UsageChunk`s from each backend round into a single per-run `UsageEvent`. Also hosts `sanitize_tool_history`, `_merge_usage`, `stream_sub_agent_run`, and `AgentPool` (built from `agents.yaml` when `--use-pool` is set; located via `$OLI_AGENTS_YAML`, the package dir, the repo root, the current working directory, or `~/.config/oli`). | -| [oli_bot/backends/](oli_bot/backends/) | Backend package — `ModelBackend` ABC, `OllamaBackend`, `OpenAIBackend`, `HuggingFaceBackend`, `TransformersBackend`, and the `create_model_backend()` factory. `OpenAIBackend` speaks both Chat Completions (default) and the Responses API, selected by the `OLI_OPENAI_RESPONSES_ENABLED` / `openai_responses_enabled` flag routed through `generate()` / `stream_generate()`. Also hosts the shared `_StreamingThinkParser` and per-backend message formatting (Ollama native `images`, OpenAI `image_url` or Bedrock-native blocks via `openai_vision_style`, Responses `input_text`/`input_image` items, textual placeholder for text-only backends). Every backend surfaces a trailing `UsageChunk`: exact counts from provider usage where available (OpenAI `usage`/`stream_options`, Responses `input_tokens`/`output_tokens`, Ollama `prompt_eval_count`/`eval_count`, HF `usage`), else a `~chars/4` estimate via `estimate_tokens`. See [docs/BACKENDS.md](docs/BACKENDS.md). | +| [oli_bot/agent.py](oli_bot/agent.py) | `Agent` — mode + system prompt owner; orchestrates the tool-calling loop and streams typed events (`TextChunk`, `ThinkingChunk`, `ToolCallChunk`, `ToolCallExecuting`, `ToolCallResult`, `StreamChunk`, `TokenCount`, `UsageEvent`, `Error`, `Done`). Emits a `TokenCount` snapshot at every checkpoint (`prompt`, `thinking`, `round`, `tool_result`, `final`) and aggregates per-call `UsageChunk`s from each backend round into a single per-run `UsageEvent`. Also hosts `sanitize_tool_history`, `_merge_usage`, `stream_sub_agent_run`, and `AgentPool` (built from `agents.yaml` when `--use-pool` is set; located via `$OLI_AGENTS_YAML`, the package dir, the repo root, the current working directory, or `~/.config/oli`). | +| [oli_bot/backends/](oli_bot/backends/) | Backend package — `ModelBackend` ABC, `OllamaBackend`, `OpenAIBackend`, `HuggingFaceBackend`, `TransformersBackend`, and the `create_model_backend()` factory. `OpenAIBackend` speaks both Chat Completions (default) and the Responses API, selected by the `OLI_OPENAI_RESPONSES_ENABLED` / `openai_responses_enabled` flag routed through `generate()` / `stream_generate()`. Also hosts the shared `_StreamingThinkParser` and per-backend message formatting (Ollama native `images`, OpenAI `image_url` or Bedrock-native blocks via `openai_vision_style`, Responses `input_text`/`input_image` items, textual placeholder for text-only backends). Every backend surfaces a trailing `UsageChunk`: exact counts from provider usage where available (OpenAI `usage`/`stream_options`, Responses `input_tokens`/`output_tokens`, Ollama `prompt_eval_count`/`eval_count`, HF `usage`), else a `~chars/4` estimate via `estimate_tokens`. Both `generate()` and `stream_generate()` accept a `max_tokens` cap; `Agent` threads the live `config.max_tokens` through streaming (and `_final_stream`) so a value set via `/config`/`settings.json` actually applies. Tool-call `arguments` JSON is decoded through `parse_tool_arguments()` / `tool_call_from_raw()` in `base.py`, which capture a truncated/malformed payload as `ToolCall.parse_error` instead of raising mid-stream. See [docs/BACKENDS.md](docs/BACKENDS.md). | | [oli_bot/screens/](oli_bot/screens/) | All `ModalScreen` subclasses: `PermissionScreen`, `ConfirmScreen`, `ModelPickerScreen`, `ServerListScreen`, `MCPSetupScreen`, `SessionListScreen`, `WorkspaceListScreen`, `SubAgentViewScreen`, `ConfigScreen`, `InputPromptScreen`, `QuestionScreen` (answers the `builtin__question` tool — one modal listing the posed questions with options + per-question freeform fields and a Confirm button), plus `taglines.py` / `todo_widget.py`. | -| [oli_bot/models.py](oli_bot/models.py) | Shared dataclasses: `Message`, `ToolCall`, `ModelResponse`, `HostConfig`, `MCPServerConfig`, `ProfileData`, `SubAgentRun`, `ImageAttachment`, `TodoItem` / `TodoListState`, plus `AgentEvent` variants and the `AgentRole` enum. Token accounting lives here too: `Usage` (prompt/completion/`estimated` flag), the per-call `UsageChunk` stream event, and the per-run `UsageEvent`. `Message.images` is in-memory only (dropped on session save); `ModelResponse.usage` is optionally set by backends. | +| [oli_bot/models.py](oli_bot/models.py) | Shared dataclasses: `Message`, `ToolCall`, `ModelResponse`, `HostConfig`, `MCPServerConfig`, `ProfileData`, `SubAgentRun`, `ImageAttachment`, `TodoItem` / `TodoListState`, plus `AgentEvent` variants and the `AgentRole` enum. Token accounting lives here too: `Usage` (prompt/completion/`estimated` flag), the per-call `UsageChunk` stream event, the per-run `UsageEvent`, and the live `TokenCount` snapshot. `Message.images` is in-memory only (dropped on session save); `ModelResponse.usage` is optionally set by backends. | +| [oli_bot/tokens.py](oli_bot/tokens.py) | Token counting shared by the agent, TUI, and API: `estimate_tokens` (`~chars/4` fallback), `count_text_tokens` / `count_message_tokens` (tiktoken, with generic BPE fallback for unknown models and an `estimated` flag), and `ContextTokenTracker` (incremental prompt + in-flight completion accounting for live `TokenCount` snapshots). | | [oli_bot/config.py](oli_bot/config.py) | `AppConfig` — `pydantic_settings.BaseSettings`. Env vars prefixed `OLI_`, plus `.env` support and `OLI_TRUNCATION_SMALL` / `_LARGE` aliases via `AliasChoices`. Module-level `configs = AppConfig()` singleton. See [docs/CONFIGURE.md](docs/CONFIGURE.md). | | [oli_bot/settings.py](oli_bot/settings.py) | `SettingsManager` — load/save/merge `~/.config/oli/settings.json`; precedence `settings.json` > `OLI_*` env > SDK-standard env (`OPENAI_API_KEY`, `OPENAI_BASE_URL`, `HUGGINGFACE_API_KEY`, `HF_TOKEN`) > declared defaults. Empty API-key strings in JSON fall through to env. | | [oli_bot/profiles/](oli_bot/profiles/) | `ProfileManifest` / `PermissionsManifest` (Pydantic) in `schema.py`; `ProfilePermissionEnforcer` (layered allow/deny glob patterns, base-profile inheritance, deny-overrides-allow) in `permissions.py`; `ProfileManager` (profile CRUD, manifest loading, circular-dependency detection) in `manager.py`. Built-in profiles ship as sibling directories. | @@ -70,8 +71,8 @@ See [docs/AGENT_POOLING.md](docs/AGENT_POOLING.md) for the `agents.yaml` schema, ## Key flows - **Session lifecycle** — new UUID by default; `-s`/`--load-session ` loads a specific one, `--resume-last` loads the newest for the active server (mutually exclusive). Auto-saved after every response; `/sessions` opens `SessionListScreen` for browse/switch/rename/delete. On exit `main()` prints the resume hint. Restored history is re-rendered by `chat._restore_history()` → module-level `_history_entries()`: prose goes through `Markdown`, each tool call is merged with its result into one `✓/✗` "Tool Result" line (name + args, result body never mounted), and every substituted fragment is `rich_escape`d — session text must never be parsed as Rich markup or file contents like `[/{color}]` blank the TUI with a `MarkupError` (`_add_message(body=...)` renders the safe body while `plain_text` keeps the raw copy payload). -- **Tool loop** — `Agent._tool_loop` calls `backend.stream_generate()`, streaming text and thinking to the TUI while accumulating tool calls; after each round it calls `MCPClientManager.drain_builtin_attachments()` so `view_image` results ride into the next turn as a synthetic user-role `Message(images=...)` (kept **after** all `role=tool` messages so the assistant→tool block stays contiguous for `sanitize_tool_history`). If iterations are exhausted, `_final_stream` does one text-only pass to guarantee a reply. -- **Token accounting** — each backend yields a trailing `UsageChunk` (exact where the provider reports counts, else `~chars/4` via `estimate_tokens`). The agent sums per-call chunks across a run and emits a single `UsageEvent` before `Done`; the TUI adds it to the session total in the `#status-bar` (`~` prefix when any value was estimated). The running total persists in session JSON (`total_tokens` / `total_tokens_estimated`) and is reset on `/clear`, `/home`, and new-session flows. +- **Tool loop** — `Agent._tool_loop` calls `backend.stream_generate(..., max_tokens=self.config.max_tokens)`, streaming text and thinking to the TUI while accumulating tool calls; after each round it calls `MCPClientManager.drain_builtin_attachments()` so `view_image` results ride into the next turn as a synthetic user-role `Message(images=...)` (kept **after** all `role=tool` messages so the assistant→tool block stays contiguous for `sanitize_tool_history`). A tool call whose streamed `arguments` failed to parse (`ToolCall.parse_error` set, e.g. the model hit the output-token limit mid-write) is **not** dispatched: the error is fed back as the tool result so the model can retry with a smaller payload. If iterations are exhausted, `_final_stream` does one text-only pass to guarantee a reply. +- **Token accounting** — two complementary metrics. **Live context size** is a `TokenCount` snapshot emitted by `Agent.process()` at every checkpoint: `prompt` (user submitted, system + history + message), `thinking` (a thinking block finished), `round` (provider usage for a completed model call), `tool_result` (a tool result was appended), and `final` (turn complete). Counts come from `oli_bot/tokens.py` (tiktoken with a generic-BPE fallback, else the `~chars/4` heuristic; `estimated=True` when heuristic/image overhead is involved). The TUI renders it in the `#status-bar` as `ctx N tok` (`~` prefix when estimated) and recomputes it on session load/switch/clear/profile/mode/model/server changes. **Cumulative billed usage** is still produced by each backend's trailing `UsageChunk` (exact where the provider reports counts); the agent sums per-call chunks across a run and emits a single `UsageEvent` before `Done`. The TUI adds that to the session total, which persists in session JSON (`total_tokens` / `total_tokens_estimated`) and resets on `/clear`, `/home`, and new-session flows. The REST API returns the summed usage in non-streaming responses, and the WebSocket relays `token_count` frames alongside `usage`. - **Error paths** — a failed round yields `Error` for the UI then `Done(full_text="")`; the empty `full_text` deliberately trips the TUI's "skip empty assistant append" branch so error text never enters `self.messages` (prevents the Bedrock/Anthropic 400 poisoning loop). All `stream_generate()` implementations re-raise instead of swallowing. - **Permission gating** — `BuiltinToolManager.call_tool()` runs the profile enforcer, then `Session.needs_permission()`, then (if needed) `confirm_callback(description)`; the TUI shows `PermissionScreen` and the callback returns `"once"`, `"session"`, or `"deny"`. Session grants persist per scope for the process lifetime. `glob`/`grep` targeting patterns like `.env*`, `*.pem`, `*secret*` trigger the `workspace_sensitive` scope even inside the workspace. - **Built-in tool naming** — registered as `builtin__` and dispatched by `MCPClientManager.call_tool`. @@ -102,7 +103,7 @@ Runtime deps are declared in [pyproject.toml](pyproject.toml) `[project.dependen - **TUI + rendering** — `textual`, `rich`, `art` - **Config** — `pydantic`, `pydantic-settings`, `python-dotenv` -- **Backends** — `ollama`, `openai`, `huggingface_hub`, `transformers`, `accelerate` +- **Backends** — `ollama`, `openai`, `huggingface_hub`, `transformers`, `accelerate`, `tiktoken` (live token counting; Docker pre-caches the `cl100k_base`/`o200k_base` encodings) - **MCP + API** — `mcp` (v2 SDK — pulls in `httpx2`, `mcp-types`, `opentelemetry-api`), `fastapi`, `uvicorn[standard]` - **Tools** — `Pillow` (image handling), `httpx` / `requests` / `aiohttp`, `beautifulsoup4`, `ddgs`, `wikipedia`, `arxiv`, `googlesearch_python`, `stackapi`, `search_engine_parser`, `gnews`, `newspaper4k` - **Voice (optional)** — `[project.optional-dependencies].voice`: `pyaudio` (needs the PortAudio system library — `brew install portaudio` on macOS), `webrtcvad-wheels`, `faster-whisper`, `piper-tts`, `simpleaudio`. Install via `pip install -e '.[voice]'`. diff --git a/Dockerfile b/Dockerfile index d1a80cf..d2326f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,8 @@ ENV PYTHONUNBUFFERED=1 \ # Tell HuggingFace where to cache models inside the image ENV TRANSFORMERS_CACHE=/app/.cache/huggingface \ - HF_HOME=/app/.cache/huggingface + HF_HOME=/app/.cache/huggingface \ + TIKTOKEN_CACHE_DIR=/app/.cache/tiktoken # Install system dependencies (single RUN layer to keep image smaller) RUN apt-get update && apt-get install -y \ @@ -43,6 +44,9 @@ RUN pip install --no-cache-dir transformers>=4.40 accelerate>=0.2 COPY . . RUN pip install --no-cache-dir . +# Pre-cache tiktoken BPE files so token counting works without runtime network. +RUN python -c "import tiktoken; [tiktoken.get_encoding(n) for n in ('cl100k_base', 'o200k_base')]" + EXPOSE 9734 # Launch API Server diff --git a/README.md b/README.md index 221669e..c0070e0 100644 --- a/README.md +++ b/README.md @@ -29,10 +29,11 @@ A lightweight, terminal-based AI agent harness for local-first, vendor-agnostic - **Declarative sub-agent pooling (optional)** — with `--use-pool`, the root agent can fan tasks out concurrently to vendor-agnostic sub-agents defined in an optional [agents.yaml](agents.yaml) file via a `dispatch` tool. Each pool entry binds a model _and_ a backend, so dispatch decisions are also compute-location decisions — a frontier model can plan while sensitive work stays on a local model, or a local root can fan out to faster remote SLMs for latency-sensitive tool calls. - **Agent profiles** — drop-in system prompts with permission manifests, base-profile inheritance, and auto-generated profiles via `/profile create`. Bundled profiles: `default`, `coder`, `reviewer`, `editor`, `writer`, `planner`, `researcher`, `analyst`. -- **Rich built-in tool set** — file ops, shell access (including PHP and read-only Docker inspection), web search/fetch, Wikipedia/GitHub/arXiv search, task tracking, reasoning scratchpad, notebook, interactive user-question modals, and more. Sandbox-locked with shell allowlists, SSRF protection, and sensitive-file gating. +- **Rich built-in tool set** — file ops, shell access, web search/fetch, Wikipedia/GitHub/arXiv search, task tracking, reasoning scratchpad, notebook, interactive user-question modals, and more. Sandbox-locked with shell allowlists, SSRF protection, and sensitive-file gating. - **Permission system** — write operations and sensitive reads require user approval. Session grants, workspace scoping, and profile-level allow/deny lists. - **OpenAI-compatible API server** — run the same agent harness behind `/v1/models` and `/v1/chat/completions` (streaming + non-streaming) so any workflow that speaks the OpenAI wire protocol (the `openai` Python SDK, curl, or plain REST) can drive the agent. - **Voice mode (optional, experimental)** — `/voice` toggles a hands-free mic → STT → LLM → TTS loop (faster-whisper, Piper TTS, WebRTC VAD) for the TUI. Fully local; requires the `voice` extras and a downloaded Piper model. +- **Live context counter** — the TUI status bar shows a tiktoken-backed context-size snapshot (`ctx N tok`) that updates on submit, after thinking blocks, after each tool result, and at the end of every turn, so you can see how close the conversation is to the model's window before the next request. - **Multi-backend support** — Ollama, OpenAI, HuggingFace (remote or local), and Transformers (local GPU/CPU). Switch at runtime. - **MCP integration** — add stdio or HTTP MCP servers at runtime for custom tools. HTTP servers can authenticate with a static API token or OAuth 2.1 (set `OLI_USE_OAUTH=true`, or toggle it under `/config`). @@ -153,16 +154,16 @@ Type `/voice` again (or `Ctrl+Q` to quit the app) to exit voice mode — the mic ## Documentation -| Document | Contents | -| ---------------------------------------------- | ------------------------------------------------------------------------------- | -| [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) | Architectural overview of the application, permission, configurations, and more | -| [docs/API_SERVER.md](docs/API_SERVER.md) | Running the `oli-server` REST/WebSocket API, endpoints, config, and examples | -| [docs/CONFIGURE.md](docs/CONFIGURE.md) | Full configuration reference, settings precedence, sessions, workspace | -| [docs/TOOLS.md](docs/TOOLS.md) | Built-in tools, permission system, security, dry-run/offline modes, truncation | -| [docs/BACKENDS.md](docs/BACKENDS.md) | Backend setup (Ollama, OpenAI, HuggingFace, Transformers), model tier switching | -| [docs/PROFILES.md](docs/PROFILES.md) | Profile structure, manifests, built-in profiles, creating and loading profiles | -| [docs/AGENT_POOLING.md](docs/AGENT_POOLING.md) | Agent pooling configuration, parsing, and usage | -| [docs/SECURITY.md](docs/SECURITY.md) | Security precedence and settings | +| Document | Contents | +| ---------------------------------------------- | ----------------------------------------------------------------------------------------- | +| [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) | Architectural overview of the application, permission, configurations, and more | +| [docs/API_SERVER.md](docs/API_SERVER.md) | Running the `oli-server` REST/WebSocket API, endpoints, config, and examples | +| [docs/CONFIGURE.md](docs/CONFIGURE.md) | Full configuration reference, settings precedence, sessions, workspace | +| [docs/TOOLS.md](docs/TOOLS.md) | Built-in tools, permission system, security, dry-run/offline modes, truncation, write cap | +| [docs/BACKENDS.md](docs/BACKENDS.md) | Backend setup (Ollama, OpenAI, HuggingFace, Transformers), model tier switching | +| [docs/PROFILES.md](docs/PROFILES.md) | Profile structure, manifests, built-in profiles, creating and loading profiles | +| [docs/AGENT_POOLING.md](docs/AGENT_POOLING.md) | Agent pooling configuration, parsing, and usage | +| [docs/SECURITY.md](docs/SECURITY.md) | Security precedence and settings | ## Docker diff --git a/docs/API_SERVER.md b/docs/API_SERVER.md index 903b871..c9b0eb1 100644 --- a/docs/API_SERVER.md +++ b/docs/API_SERVER.md @@ -166,15 +166,16 @@ Non-streaming response: } ], "usage": { - "prompt_tokens": 0, + "prompt_tokens": 18, "completion_tokens": 42, - "total_tokens": 42 + "total_tokens": 60 } } ``` -`usage` is an approximation (the agent tool loop does not surface exact prompt -counts); `completion_tokens` is a `~chars/4` estimate. +`usage` is the summed per-call usage reported by the backend for the whole +agent run (prompt + completion across every tool-loop round). When the backend +reports no counts, it falls back to a completion-only `~chars/4` estimate. **Errors** are returned in OpenAI format with HTTP 500: @@ -261,6 +262,7 @@ inbound message: | `tool_call_executing` | `{"name": "...", "parameters": {...}}` | | `tool_call_result` | `{"name": "...", "result": "..."}` | | `assistant_response` | `{"content": "..."}` — final assembled assistant text | +| `token_count` | `{"prompt_tokens", "completion_tokens", "total_tokens", "estimated", "phase"}` — live context snapshot (`phase` is `prompt`/`thinking`/`round`/`tool_result`/`final`) | | `usage` | `{"prompt_tokens": ..., "completion_tokens": ..., ...}` | | `error` | `{"message": "..."}` | | `done` | `{"full_text": "..."}` — run finished | diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ef4e64d..ffc8335 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -158,6 +158,45 @@ flowchart TD --- +## Live Token Accounting + +The agent emits a `TokenCount` snapshot (not a delta) at every point where the +request context changes, so a UI can show the current context size without +waiting for the run to finish. `oli_bot/tokens.py` uses `tiktoken` when it can +resolve an encoding for the active model, falls back to a generic BPE encoding +for unknown/local model names, and finally to the `~chars/4` heuristic; the +`estimated` flag drives the `~` prefix in the TUI. + +```mermaid +sequenceDiagram + participant U as User + participant T as TUI + participant A as Agent + participant B as Backend + U->>T: submit message + T->>A: process(messages) + A-->>T: TokenCount(phase="prompt") + A->>B: stream_generate() + B-->>A: ThinkingChunk* + B-->>A: TextChunk* + A-->>T: TokenCount(phase="thinking") + B-->>A: ToolCallChunk* + A-->>T: ToolCallExecuting / ToolCallResult + A-->>T: TokenCount(phase="tool_result") + B-->>A: UsageChunk (exact round usage) + A-->>T: TokenCount(phase="round") + A->>B: next round / final stream + A-->>T: TokenCount(phase="final") + A-->>T: UsageEvent (cumulative run) + Done +``` + +`UsageEvent` remains the cumulative billed total for the run (persisted in +session JSON and returned by the REST API); `TokenCount` is the live context +size (rendered in the TUI status bar and relayed as `token_count` WebSocket +frames). + +--- + ## Voice Mode Loop `/voice` toggles a background `@work(exclusive=False)` worker (`_run_voice_loop`) diff --git a/docs/BACKENDS.md b/docs/BACKENDS.md index 1ab0a10..28dfd13 100644 --- a/docs/BACKENDS.md +++ b/docs/BACKENDS.md @@ -9,6 +9,12 @@ oli supports four model backends, selected via the `backend` setting (`OLI_BACKE | `huggingface` | HuggingFace Inference API (remote) or TGI/vLLM (local) | | `transformers` | Local model inference via HuggingFace `transformers` | +## Output-token cap + +Every backend's `generate()` and `stream_generate()` accept a `max_tokens` cap. `Agent` passes the live `config.max_tokens` (set via `OLI_MAX_TOKENS`, `settings.json`, or `/config`) on each streaming round, so the value actually bounds a single generation — important for large tool calls like `write_file`, whose `content` argument is generated as output tokens. When `max_tokens` is omitted (`None`), Ollama/Transformers fall back to the import-time `MAX_TOKENS` default and OpenAI-compatible APIs use the provider default. + +If generation is cut off mid-tool-call, the `arguments` JSON is truncated. Backends parse it leniently (`tool_call_from_raw`) and set `ToolCall.parse_error`; the agent returns that error as the tool result so the model can retry with a smaller payload instead of the run failing. + ## Ollama Connects to a local Ollama server. Default URL: `http://localhost:11434`. diff --git a/docs/CONFIGURE.md b/docs/CONFIGURE.md index 5b3aa19..d400004 100644 --- a/docs/CONFIGURE.md +++ b/docs/CONFIGURE.md @@ -41,7 +41,7 @@ If `~/.config/oli/settings.json` does not exist, it is auto-created on first loa | `use_agent_pool` | `false` | `OLI_USE_AGENT_POOL` | Enable agent pooling (`--use-pool`); root agent can dispatch tasks to agents.yaml sub-agents | | `agent_pool_size` | `5` | `OLI_AGENT_POOL_SIZE` | Max agents allowed per pool in agents.yaml | | `agents_yaml` | `""` | `OLI_AGENTS_YAML` | Explicit path to agents.yaml for pooling (empty = auto-locate: package dir, repo root, `~/.config/oli`) | -| `max_tokens` | `2048` | `OLI_MAX_TOKENS` | Max tokens per generation | +| `max_tokens` | `2048` | `OLI_MAX_TOKENS` | Max tokens per generation, applied to streaming and non-streaming calls alike | | `temperature` | `0.7` | `OLI_TEMPERATURE` | Model temperature | | `max_retries` | `3` | `OLI_MAX_RETRIES` | Max retries on API failure | | `retry_delay` | `1.0` | `OLI_RETRY_DELAY` | Delay between retries (seconds) | @@ -52,6 +52,7 @@ If `~/.config/oli/settings.json` does not exist, it is auto-created on first loa | `model_filters` | `""` | `OLI_MODEL_FILTERS` | Comma-separated model name exclusion filters | | `truncation_max_chars_small` | `4000` | `OLI_TRUNCATION_SMALL` | Max chars for tool results under the small model tier | | `truncation_max_chars_large` | `100000` | `OLI_TRUNCATION_LARGE` | Max chars for tool results under the large model tier | +| `max_write_chars` | `100000` | `OLI_MAX_WRITE_CHARS` | Max characters accepted by the `write_file` tool (also advertised in its description) | | `dry_run` | `false` | `OLI_DRY_RUN` | Preview destructive actions without executing | | `offline_mode` | `true` | `OLI_OFFLINE_MODE` | Block network access for web tools and MCP servers | | `use_oauth` | `false` | `OLI_USE_OAUTH` | Use OAuth 2.1 (PKCE + dynamic client registration) for HTTP MCP servers added without an API token | diff --git a/docs/TOOLS.md b/docs/TOOLS.md index 262ba59..b174f39 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -126,3 +126,9 @@ Tool results are automatically truncated to conserve context window: - **Large tier** -- 100,000 characters (configurable via `OLI_TRUNCATION_LARGE`) Truncation preserves sentence and paragraph boundaries when possible, appending a `[... truncated: N characters remaining]` notice. + +## Write cap + +`write_file` rejects a single write larger than `max_write_chars` (default 100,000 characters; configurable via `OLI_MAX_WRITE_CHARS` or `/config`). The limit is also advertised in the tool's description so the model can self-limit. + +Note this is a *post-generation* guardrail: it fires only after the model has emitted the full `content` argument. The limit that actually truncates large writes is the per-round output-token budget (`max_tokens`). When generation hits that ceiling mid-call, the tool-call JSON is unterminated; the backend parses it leniently and the agent feeds back a retryable `parse_error` result ("...truncated JSON... split large content across multiple writes...") instead of aborting the run. diff --git a/oli_bot/agent.py b/oli_bot/agent.py index 355be9a..299b282 100644 --- a/oli_bot/agent.py +++ b/oli_bot/agent.py @@ -36,10 +36,12 @@ SubAgentRun, SubAgentStarted, ThinkingChunk, + TokenCount, Usage, UsageChunk, UsageEvent, ) +from .tokens import ContextTokenTracker, count_message_tokens from .tools.memory import _current_sub_run logger = logging.getLogger(__name__) @@ -51,6 +53,7 @@ | AssistantResponse | StreamChunk | ThinkingChunk + | TokenCount | UsageEvent | Error | Done @@ -137,6 +140,18 @@ def _merge_usage(acc: Usage, u: Usage) -> Usage: ) +def _token_count_event(tracker: ContextTokenTracker, phase: str) -> TokenCount: + """Build a ``TokenCount`` snapshot from the tracker's current state.""" + prompt, completion, total, estimated = tracker.snapshot() + return TokenCount( + prompt_tokens=prompt, + completion_tokens=completion, + total_tokens=total, + estimated=estimated, + phase=phase, + ) + + class Agent: def __init__( self, @@ -237,14 +252,17 @@ async def process( # with a length check made after control returns to us. base_len = len(messages) + tracker = ContextTokenTracker(working_messages, self.backend.model) + yield _token_count_event(tracker, "prompt") + if self._mode == "chat": - async for event in self._final_stream(working_messages, tools): + async for event in self._final_stream(working_messages, tools, tracker): yield event self._sync_back(messages, working_messages, base_len) return async for event in self._tool_loop( - working_messages, tools, confirm_callback + working_messages, tools, confirm_callback, tracker ): if isinstance(event, Done): # Sync tool-call turns back before yielding Done, since @@ -259,7 +277,7 @@ async def process( # Force a final text-only stream (no tools) so the user sees a # response instead of a silent hang. self._sync_back(messages, working_messages, base_len) - async for event in self._final_stream(working_messages, tools): + async for event in self._final_stream(working_messages, tools, tracker): yield event finally: self._generating = False @@ -289,6 +307,12 @@ def _compose_messages(self, messages: List[Message]) -> List[Message]: composed.insert(0, Message(role="system", content=header)) return composed + def count_context_tokens(self, messages: List[Message]) -> tuple[int, bool]: + """Count the prompt tokens for ``messages`` under the current profile.""" + return count_message_tokens( + self._compose_messages(messages), self.backend.model + ) + @staticmethod def _sync_back( original: List[Message], working: List[Message], base_len: int @@ -315,18 +339,22 @@ async def _tool_loop( messages: List[Message], tools: Optional[List[Dict[str, Any]]], confirm_callback: Optional[Callable[[str], Any]], + tracker: ContextTokenTracker, ) -> AsyncIterator[AgentEvent]: run_usage = Usage() activity = TurnActivity() for _ in range(self.config.max_tool_iterations): stream = self.backend.stream_generate( - messages, tools=tools if tools else [] + messages, + tools=tools if tools else [], + max_tokens=self.config.max_tokens, ) full_response = "" tool_calls = [] pending_images: List = [] pending_caption: str = "" + thinking_open = False try: # Use a longer deadline for the very first chunk: large # accumulated contexts (many tool-call rounds) legitimately @@ -351,12 +379,24 @@ async def _tool_loop( if isinstance(event, TextChunk): activity.close_thinking_segment() full_response += event.text + tracker.add_inflight(event.text) + if thinking_open: + thinking_open = False + yield _token_count_event(tracker, "thinking") yield StreamChunk(event.text) elif isinstance(event, ThinkingChunk): + thinking_open = True + tracker.add_inflight(event.text) activity.add_thinking(event.text) yield event elif isinstance(event, UsageChunk): run_usage = _merge_usage(run_usage, event.usage) + tracker.set_exact( + event.usage.prompt_tokens, + event.usage.completion_tokens, + event.usage.estimated, + ) + yield _token_count_event(tracker, "round") elif isinstance(event, ToolCallChunk): activity.close_thinking_segment() tool_calls = event.tool_calls @@ -367,6 +407,31 @@ async def _tool_loop( tc.name, tc.parameters, ) + if thinking_open: + thinking_open = False + try: + tracker.add_inflight( + json.dumps( + [ + { + "id": tc.id, + "type": "function", + "function": { + "name": tc.name, + "arguments": json.dumps( + tc.parameters + ), + }, + } + for tc in tool_calls + ] + ) + ) + except (TypeError, ValueError): + logger.debug( + "Failed to serialize tool calls for token counting" + ) + yield _token_count_event(tracker, "thinking") except asyncio.TimeoutError: which = "first chunk" if _first_chunk else "inter-chunk gap" deadline = ( @@ -408,6 +473,7 @@ async def _tool_loop( if not tool_calls: if run_usage.total_tokens: yield UsageEvent(usage=run_usage) + yield _token_count_event(tracker, "final") yield Done(full_text=full_response, activity=activity.to_dict()) return @@ -425,6 +491,9 @@ async def _tool_loop( } for tc in tool_calls ] + # The generated output is about to become history, so move it out + # of the in-flight count before adding the assistant message. + tracker.clear_inflight() messages.append( Message( role="assistant", @@ -432,13 +501,20 @@ async def _tool_loop( tool_calls=tool_calls_dicts, ) ) + tracker.add_message(messages[-1]) for tc in tool_calls: yield ToolCallExecuting(name=tc.name, parameters=tc.parameters) activity.start_tool_call(tc.id, tc.name, tc.parameters) call_started = time.perf_counter() try: - if tc.name == "builtin__dispatch": + if tc.parse_error: + # The model's streamed arguments JSON was truncated or + # malformed; surface it as a tool result so the model + # can retry (e.g. with a smaller write) instead of the + # whole run aborting. + result = tc.parse_error + elif tc.name == "builtin__dispatch": result_holder: List[str] = [""] async for _ev in self._run_dispatch_with_events( tc.name, @@ -466,6 +542,8 @@ async def _tool_loop( messages.append( Message(role="tool", content=result, tool_call_id=tc.id) ) + tracker.add_message(messages[-1]) + yield _token_count_event(tracker, "tool_result") # Buffer image attachments until after all tool results land so # the assistant->tool block stays contiguous for sanitize_tool_history. drain = getattr(self.mcp_manager, "drain_builtin_attachments", None) @@ -485,6 +563,8 @@ async def _tool_loop( images=pending_images, ) ) + tracker.add_message(messages[-1]) + yield _token_count_event(tracker, "tool_result") async def _run_dispatch_with_events( self, @@ -548,18 +628,24 @@ async def _final_stream( self, messages: List[Message], tools: Optional[List[Dict[str, Any]]] = None, + tracker: Optional[ContextTokenTracker] = None, ) -> AsyncIterator[AgentEvent]: full_response = "" errored = False usage = Usage() stream = None activity = TurnActivity() + if tracker is None: + tracker = ContextTokenTracker(messages, self.backend.model) + thinking_open = False try: # Forward tools so backends that require a matching tool schema # for any toolUse/toolResult blocks in history (e.g. Bedrock via # OpenAI-compatible proxies) don't 400. ToolCallChunks from this # pass are ignored below — only text/thinking is consumed. - stream = self.backend.stream_generate(messages, tools=tools) + stream = self.backend.stream_generate( + messages, tools=tools, max_tokens=self.config.max_tokens + ) _first_chunk = True while True: timeout = ( @@ -568,21 +654,31 @@ async def _final_stream( else self.config.stream_timeout ) try: - event = await asyncio.wait_for( - stream.__anext__(), timeout=timeout - ) + event = await asyncio.wait_for(stream.__anext__(), timeout=timeout) except StopAsyncIteration: break _first_chunk = False if isinstance(event, TextChunk): activity.close_thinking_segment() full_response += event.text + tracker.add_inflight(event.text) + if thinking_open: + thinking_open = False + yield _token_count_event(tracker, "thinking") yield StreamChunk(event.text) elif isinstance(event, ThinkingChunk): + thinking_open = True + tracker.add_inflight(event.text) activity.add_thinking(event.text) yield event elif isinstance(event, UsageChunk): usage = _merge_usage(usage, event.usage) + tracker.set_exact( + event.usage.prompt_tokens, + event.usage.completion_tokens, + event.usage.estimated, + ) + yield _token_count_event(tracker, "round") except asyncio.TimeoutError: which = "first chunk" if _first_chunk else "inter-chunk gap" deadline = ( @@ -618,6 +714,7 @@ async def _final_stream( full_response = "The model completed all tool calls but did not produce a final summary." if usage.total_tokens: yield UsageEvent(usage=usage) + yield _token_count_event(tracker, "final") yield Done(full_text=full_response, activity=activity.to_dict()) diff --git a/oli_bot/api/convert.py b/oli_bot/api/convert.py index 9817663..6a88c1c 100644 --- a/oli_bot/api/convert.py +++ b/oli_bot/api/convert.py @@ -17,6 +17,7 @@ SubAgentProgress, SubAgentStarted, ThinkingChunk, + TokenCount, ToolCallExecuting, ToolCallResult, UsageEvent, @@ -117,6 +118,17 @@ def _event_to_frame(event: "AgentEvent") -> Dict[str, Any]: } if isinstance(event, AssistantResponse): return {"type": "assistant_response", "data": {"content": event.content}} + if isinstance(event, TokenCount): + return { + "type": "token_count", + "data": { + "prompt_tokens": event.prompt_tokens, + "completion_tokens": event.completion_tokens, + "total_tokens": event.total_tokens, + "estimated": event.estimated, + "phase": event.phase, + }, + } if isinstance(event, UsageEvent): return {"type": "usage", "data": dataclasses.asdict(event.usage)} if isinstance(event, Error): diff --git a/oli_bot/api/routers/chat.py b/oli_bot/api/routers/chat.py index 7af8fcd..856227b 100644 --- a/oli_bot/api/routers/chat.py +++ b/oli_bot/api/routers/chat.py @@ -42,7 +42,7 @@ async def chat_completions( media_type="text/event-stream", headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, ) - completion_text = await _collect_response(request, agent, lock) + completion_text, usage = await _collect_response(request, agent, lock) return { "id": _completion_id(), "object": "chat.completion", @@ -55,5 +55,5 @@ async def chat_completions( "finish_reason": "stop", } ], - "usage": _usage(completion_text), + "usage": _usage(completion_text, usage), } diff --git a/oli_bot/api/routers/config.py b/oli_bot/api/routers/config.py index 46b7c99..d156e7b 100644 --- a/oli_bot/api/routers/config.py +++ b/oli_bot/api/routers/config.py @@ -52,6 +52,7 @@ "model_filters": ("model_params", "model_filters"), "truncation_max_chars_small": ("model_params", "truncation_max_chars_small"), "truncation_max_chars_large": ("model_params", "truncation_max_chars_large"), + "max_write_chars": ("model_params", "max_write_chars"), "dry_run": ("model_params", "dry_run"), "offline_mode": ("model_params", "offline_mode"), "use_oauth": ("model_params", "use_oauth"), diff --git a/oli_bot/api/runner.py b/oli_bot/api/runner.py index 892b6fc..b31931c 100644 --- a/oli_bot/api/runner.py +++ b/oli_bot/api/runner.py @@ -7,13 +7,15 @@ import uuid from typing import Any, AsyncIterator, Dict, List, Optional -from ..agent import Agent +from ..agent import Agent, _merge_usage from ..models import ( AssistantResponse, ChatCompletionRequest, Done, Error, StreamChunk, + Usage, + UsageEvent, ) from .convert import _to_message from .errors import AgentError @@ -37,9 +39,15 @@ def _completion_id() -> str: return f"chatcmpl-{uuid.uuid4().hex}" -def _usage(completion_text: str) -> Dict[str, int]: - # Rustic token estimate; real prompt usage depends on the backend. Kept - # approximate since the agent loop does not surface exact counts. +def _usage(completion_text: str, usage: Optional[Usage] = None) -> Dict[str, int]: + # Prefer the agent's summed per-call usage; fall back to a rustic + # completion-only estimate for backends that report nothing. + if usage is not None and usage.total_tokens: + return { + "prompt_tokens": usage.prompt_tokens, + "completion_tokens": usage.completion_tokens, + "total_tokens": usage.total_tokens, + } completion = max(1, (len(completion_text) + 3) // 4) return { "prompt_tokens": 0, @@ -50,10 +58,10 @@ def _usage(completion_text: str) -> Dict[str, int]: async def _collect_response( request: ChatCompletionRequest, agent: Agent, lock: Any -) -> str: +) -> tuple[str, Usage]: """Run the agent tool loop for a request and return the final assistant - text. Raises ``AgentError`` (formatted as an OpenAI error body) when the - run fails or produces no text.""" + text plus its summed usage. Raises ``AgentError`` (formatted as an OpenAI + error body) when the run fails or produces no text.""" async with lock: messages = [_to_message(m) for m in request.messages] try: @@ -64,6 +72,7 @@ async def _collect_response( full_text = "" error_text: Optional[str] = None + usage = Usage() try: async for event in agent.process( messages, tools=tools, confirm_callback=_api_confirm @@ -72,6 +81,8 @@ async def _collect_response( full_text += event.text elif isinstance(event, AssistantResponse): full_text += event.content + elif isinstance(event, UsageEvent): + usage = _merge_usage(usage, event.usage) elif isinstance(event, Error): error_text = event.message elif isinstance(event, Done): @@ -86,7 +97,7 @@ async def _collect_response( if not full_text: logger.warning("Agent produced empty response for request") raise AgentError("The agent produced no response.", code="empty_response") - return full_text + return full_text, usage async def _stream_response( diff --git a/oli_bot/backends/base.py b/oli_bot/backends/base.py index 3e83a92..351b8cf 100644 --- a/oli_bot/backends/base.py +++ b/oli_bot/backends/base.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import logging from abc import ABC, abstractmethod from typing import Any, AsyncIterator, Dict, List, Optional @@ -10,6 +11,7 @@ ModelResponse, TextChunk, ThinkingChunk, + ToolCall, ToolCallChunk, UsageChunk, ) @@ -33,6 +35,42 @@ def estimate_tokens(text: str) -> int: return max(1, (len(text) + 3) // 4) +TOOL_CALL_PARSE_ERROR = ( + "Error: the tool-call arguments could not be parsed (invalid or truncated " + "JSON). This usually means the model hit its output-token limit mid-call. " + "Retry with a smaller payload — e.g. split large content across multiple " + "writes — or raise the configured max_tokens." +) + + +def parse_tool_arguments(raw: Optional[str]) -> tuple[Dict[str, Any], Optional[str]]: + """Parse a tool call's ``arguments`` JSON. + + Returns ``(parameters, error)``. On failure ``parameters`` is an empty dict + and ``error`` holds an actionable message describing the truncation so the + caller can surface it as a tool result rather than raising or dropping the + call silently. + """ + try: + parsed = json.loads(raw or "{}") + except (json.JSONDecodeError, TypeError): + logger.warning("Failed to parse tool-call arguments JSON: %r", raw) + return {}, TOOL_CALL_PARSE_ERROR + if not isinstance(parsed, dict): + return {}, TOOL_CALL_PARSE_ERROR + return parsed, None + + +def tool_call_from_raw(call_id: str, name: str, raw: Optional[str]) -> "ToolCall": + """Build a ``ToolCall`` from raw ``arguments`` JSON, capturing parse + failures as ``parse_error`` so the agent can surface a retryable result + instead of raising mid-stream.""" + params, err = parse_tool_arguments(raw) + return ToolCall( + id=call_id, name=name, description="", parameters=params, parse_error=err + ) + + class ModelBackend(ABC): @abstractmethod async def generate( @@ -51,8 +89,14 @@ async def stream_generate( self, messages: List[Message], tools: Optional[List[Dict]] = None, + max_tokens: Optional[int] = None, ) -> AsyncIterator[StreamEvent]: - """Stream a response from the model, yielding TextChunk and ToolCallChunk events""" + """Stream a response from the model, yielding TextChunk and ToolCallChunk events. + + ``max_tokens`` caps the number of tokens generated this round. When + ``None`` each backend falls back to its import-time ``MAX_TOKENS`` + default (or the provider's own default, for OpenAI-compatible APIs). + """ pass @@ -62,6 +106,9 @@ async def stream_generate( "TEMPERATURE", "StreamEvent", "estimate_tokens", + "parse_tool_arguments", + "tool_call_from_raw", + "TOOL_CALL_PARSE_ERROR", "logger", "Any", "AsyncIterator", diff --git a/oli_bot/backends/huggingface.py b/oli_bot/backends/huggingface.py index 1d59521..61863cf 100644 --- a/oli_bot/backends/huggingface.py +++ b/oli_bot/backends/huggingface.py @@ -23,6 +23,7 @@ ModelBackend, StreamEvent, estimate_tokens, + tool_call_from_raw, ) from .messages import _format_messages, _format_tools from .streaming import _StreamingThinkParser @@ -128,6 +129,7 @@ async def stream_generate( self, messages: List[Message], tools: Optional[List[Dict]] = None, + max_tokens: Optional[int] = None, ) -> AsyncIterator[StreamEvent]: try: stream = await self.client.chat.completions.create( @@ -136,6 +138,7 @@ async def stream_generate( tools=_format_tools(tools or []), tool_choice="auto" if tools else "none", stream=True, + **({"max_tokens": max_tokens} if max_tokens else {}), ) tool_calls_acc: Dict[int, dict] = {} flushed = False @@ -190,12 +193,7 @@ async def stream_generate( data["arguments"], ) tool_calls = [ - ToolCall( - id=data["id"], - name=data["name"], - description="", - parameters=json.loads(data["arguments"] or "{}"), - ) + tool_call_from_raw(data["id"], data["name"], data["arguments"]) for data in tool_calls_acc.values() ] yield ToolCallChunk(tool_calls) @@ -214,12 +212,7 @@ async def stream_generate( data["arguments"], ) tool_calls = [ - ToolCall( - id=data["id"], - name=data["name"], - description="", - parameters=json.loads(data["arguments"] or "{}"), - ) + tool_call_from_raw(data["id"], data["name"], data["arguments"]) for data in tool_calls_acc.values() ] yield ToolCallChunk(tool_calls) diff --git a/oli_bot/backends/ollama.py b/oli_bot/backends/ollama.py index 06d6122..d012812 100644 --- a/oli_bot/backends/ollama.py +++ b/oli_bot/backends/ollama.py @@ -125,12 +125,16 @@ async def stream_generate( self, messages: List[Message], tools: Optional[List[Dict]] = None, + max_tokens: Optional[int] = None, ) -> AsyncIterator[StreamEvent]: try: kwargs: Dict[str, Any] = { "model": self.model, "messages": _format_messages(messages, image_style="ollama"), - "options": {"num_predict": MAX_TOKENS, "temperature": TEMPERATURE}, + "options": { + "num_predict": max_tokens or MAX_TOKENS, + "temperature": TEMPERATURE, + }, "stream": True, } if tools: diff --git a/oli_bot/backends/openai.py b/oli_bot/backends/openai.py index 529ce9b..a65152e 100644 --- a/oli_bot/backends/openai.py +++ b/oli_bot/backends/openai.py @@ -27,6 +27,7 @@ ModelBackend, StreamEvent, estimate_tokens, + tool_call_from_raw, ) from .messages import ( _format_messages, @@ -193,10 +194,13 @@ async def stream_generate( self, messages: List[Message], tools: Optional[List[Dict]] = None, + max_tokens: Optional[int] = None, ) -> AsyncIterator[StreamEvent]: try: if self.responses_enabled: - async for event in self._responses_stream_generate(messages, tools): + async for event in self._responses_stream_generate( + messages, tools, max_tokens=max_tokens + ): yield event return @@ -213,6 +217,7 @@ async def stream_generate( tool_choice="auto" if tools else "none", stream=True, stream_options={"include_usage": True}, + **({"max_tokens": max_tokens} if max_tokens else {}), ) except Exception as e: # Some OpenAI-compatible providers (Bedrock proxies, LM Studio, @@ -227,6 +232,7 @@ async def stream_generate( tools=_format_tools(tools) if tools else [], tool_choice="auto" if tools else "none", stream=True, + **({"max_tokens": max_tokens} if max_tokens else {}), ) tool_calls_acc: Dict[int, dict] = {} flushed = False @@ -282,12 +288,7 @@ async def stream_generate( data["arguments"], ) tool_calls = [ - ToolCall( - id=data["id"], - name=data["name"], - description="", - parameters=json.loads(data["arguments"] or "{}"), - ) + tool_call_from_raw(data["id"], data["name"], data["arguments"]) for data in tool_calls_acc.values() ] yield ToolCallChunk(tool_calls) @@ -307,12 +308,7 @@ async def stream_generate( data["arguments"], ) tool_calls = [ - ToolCall( - id=data["id"], - name=data["name"], - description="", - parameters=json.loads(data["arguments"] or "{}"), - ) + tool_call_from_raw(data["id"], data["name"], data["arguments"]) for data in tool_calls_acc.values() ] yield ToolCallChunk(tool_calls) @@ -353,11 +349,10 @@ def _parse_responses_output( parts.append(text) elif getattr(item, "type", "") == "function_call": tool_calls.append( - ToolCall( - id=getattr(item, "call_id", "") or "", - name=getattr(item, "name", ""), - description="", - parameters=json.loads(getattr(item, "arguments", None) or "{}"), + tool_call_from_raw( + getattr(item, "call_id", "") or "", + getattr(item, "name", ""), + getattr(item, "arguments", None), ) ) return "".join(parts), tool_calls @@ -366,6 +361,7 @@ async def _responses_stream_generate( self, messages: List[Message], tools: Optional[List[Dict]] = None, + max_tokens: Optional[int] = None, ) -> AsyncIterator[StreamEvent]: """Responses API streaming variant of ``stream_generate``. @@ -389,6 +385,7 @@ async def _responses_stream_generate( tool_choice="auto" if tools else "none", stream=True, stream_options={"include_usage": True}, + **({"max_output_tokens": max_tokens} if max_tokens else {}), ) except Exception as e: logger.debug( @@ -401,6 +398,7 @@ async def _responses_stream_generate( tools=responses_tools, tool_choice="auto" if tools else "none", stream=True, + **({"max_output_tokens": max_tokens} if max_tokens else {}), ) tool_calls_acc: Dict[str, dict] = {} @@ -459,12 +457,7 @@ async def _responses_stream_generate( if tool_calls_acc: tool_calls = [ - ToolCall( - id=data["call_id"], - name=data["name"], - description="", - parameters=json.loads(data["arguments"] or "{}"), - ) + tool_call_from_raw(data["call_id"], data["name"], data["arguments"]) for data in tool_calls_acc.values() ] logger.debug( diff --git a/oli_bot/backends/transformers.py b/oli_bot/backends/transformers.py index 0b8d4a0..468829e 100644 --- a/oli_bot/backends/transformers.py +++ b/oli_bot/backends/transformers.py @@ -17,7 +17,14 @@ UsageChunk, ) -from .base import MAX_TOKENS, TEMPERATURE, ModelBackend, StreamEvent, estimate_tokens +from .base import ( + MAX_TOKENS, + TEMPERATURE, + TOOL_CALL_PARSE_ERROR, + ModelBackend, + StreamEvent, + estimate_tokens, +) from .messages import _append_image_placeholder_text from .streaming import _StreamingThinkParser @@ -202,6 +209,20 @@ def _parse_tool_calls(self, text: str) -> List[ToolCall]: data, _ = decoder.raw_decode(text, start) except json.JSONDecodeError: logger.warning("Failed to parse tool call JSON at pos %d", start) + # Recover the tool name so the truncated call can still be + # surfaced to the model as a retryable error result rather than + # silently dropped. + name_match = re.search(r'"name"\s*:\s*"([^"]*)"', text[start:]) + if name_match: + tool_calls.append( + ToolCall( + id=f"call_{len(tool_calls)}", + name=name_match.group(1), + description="", + parameters={}, + parse_error=TOOL_CALL_PARSE_ERROR, + ) + ) continue tool_calls.append( ToolCall( @@ -281,6 +302,7 @@ async def stream_generate( self, messages: List[Message], tools: Optional[List[Dict]] = None, + max_tokens: Optional[int] = None, ) -> AsyncIterator[StreamEvent]: self._ensure_loaded() try: @@ -303,7 +325,7 @@ async def stream_generate( gen_kwargs = { "input_ids": encoded["input_ids"], "attention_mask": encoded["attention_mask"], - "max_new_tokens": MAX_TOKENS, + "max_new_tokens": max_tokens or MAX_TOKENS, "temperature": TEMPERATURE if TEMPERATURE > 0 else None, "do_sample": TEMPERATURE > 0, "top_p": 1.0 if TEMPERATURE <= 0 else None, diff --git a/oli_bot/chat.py b/oli_bot/chat.py index 7b74577..9a63426 100644 --- a/oli_bot/chat.py +++ b/oli_bot/chat.py @@ -70,7 +70,7 @@ ) from .config import configs -from .models import SubAgentRun, UsageEvent +from .models import SubAgentRun, TokenCount, UsageEvent from .tools.manager import BuiltinToolManager from .mcp_client import MCPClientManager from .backends.upstream_manager import UpstreamManager @@ -420,6 +420,8 @@ def __init__( self.messages: list[Message] = [] self.total_tokens = 0 self.tokens_estimated = False + self.context_tokens = 0 + self.context_estimated = False cwd = Path.cwd() if is_sensitive_path(cwd): self.session = Session(workspace=None) @@ -656,6 +658,7 @@ async def on_unmount(self) -> None: await self.mcp_manager.disconnect_all() def on_mount(self) -> None: + self._refresh_context_tokens(render=False) self.update_header() if self.agent_pool is not None: self.query_one("#sub-agent-tree", Tree).border_title = "Active Sub-Agents" @@ -729,6 +732,18 @@ def _save_session(self) -> None: ) self.current_session_id = new_id + def _refresh_context_tokens(self, render: bool = True) -> None: + """Recompute the live context count for the current message history.""" + try: + total, estimated = self.agent.count_context_tokens(self.messages) + except Exception: + logger.warning("Failed to count context tokens", exc_info=True) + return + self.context_tokens = total + self.context_estimated = estimated + if render: + self.update_header() + def update_header(self) -> None: active = self.upstream_manager.get_active() server_part = f" {active.name}" if active else "" @@ -745,9 +760,9 @@ def update_header(self) -> None: f"{self._status_badges()} {model_part} " f"[dim]:: {self.agent.profile_name}{server_part}[/dim]" ) - if self.total_tokens > 0: - est = "~" if self.tokens_estimated else "" - info += f" [dim]·[/dim] {est}{self.total_tokens:,} tok" + if self.context_tokens > 0: + est = "~" if self.context_estimated else "" + info += f" [dim]·[/dim] ctx {est}{self.context_tokens:,} tok" status.add_row(hints, info) self.query_one("#status-bar", Static).update(status) @@ -1083,7 +1098,7 @@ def _sessions_new(self) -> None: ) self.total_tokens = 0 self.tokens_estimated = False - self.update_header() + self._refresh_context_tokens() self.query_one("#chat-log").remove_children() self._add_message("System", "New session created.") @@ -1192,7 +1207,7 @@ def _sessions_switch(self, target: str) -> None: self.current_session_id = session_id self.total_tokens = data.get("total_tokens", 0) or 0 self.tokens_estimated = bool(data.get("total_tokens_estimated", False)) - self.update_header() + self._refresh_context_tokens() self.query_one("#chat-log").remove_children() self._clear_todo_widget() self._restore_todos(data.get("todos")) @@ -1224,7 +1239,7 @@ def _sessions_delete(self, target: str) -> None: ) self.total_tokens = 0 self.tokens_estimated = False - self.update_header() + self._refresh_context_tokens() self.query_one("#chat-log").remove_children() self._add_message( "System", "Current session was deleted. New session created." @@ -1282,7 +1297,7 @@ async def _sessions_purge(self, server: str | None = None) -> None: ) self.total_tokens = 0 self.tokens_estimated = False - self.update_header() + self._refresh_context_tokens() self.query_one("#chat-log").remove_children() self._add_message( "System", @@ -1453,7 +1468,7 @@ def _handle_mode(self, parts: list[str]) -> None: timeout=4, ) self.agent.set_mode(mode) - self.update_header() + self._refresh_context_tokens() self._add_message("System", f"Switched to [bold]{mode}[/bold] mode.") def _handle_profile(self, text: str) -> None: @@ -1493,7 +1508,7 @@ async def _profile_load(self, name: str) -> None: self.messages.clear() self.query_one("#chat-log").remove_children() self.messages.append(Message(role="system", content=self.agent.system_prompt)) - self.update_header() + self._refresh_context_tokens() self._save_session() self._add_message("System", f"Loaded profile: [bold]{name}[/bold]") @@ -1765,7 +1780,7 @@ def _server_switch(self, name: str) -> None: self.messages.append( Message(role="system", content=self.agent.system_prompt) ) - self.update_header() + self._refresh_context_tokens() self._add_message( "System", f"Switched to server: [bold]{name}[/bold] ({config.url})" ) @@ -1828,7 +1843,7 @@ async def _show_model_picker(self) -> None: def _switch_model(self, name: str) -> None: self.backend.model = name - self.update_header() + self._refresh_context_tokens() self._add_message("System", f"Switched to model: [bold]{name}[/bold]") def _get_large_model(self) -> str: @@ -2186,7 +2201,7 @@ def action_clear_chat(self) -> None: ) self.total_tokens = 0 self.tokens_estimated = False - self.update_header() + self._refresh_context_tokens() self.query_one("#chat-log").remove_children() self._clear_todo_widget() self._add_message("System", "Conversation cleared.") @@ -2199,7 +2214,7 @@ def action_go_home(self) -> None: self.messages.clear() self.total_tokens = 0 self.tokens_estimated = False - self.update_header() + self._refresh_context_tokens() chat_log = self.query_one("#chat-log") try: welcome = chat_log.query_one("#welcome") @@ -2391,6 +2406,7 @@ async def _run_voice_loop(self) -> None: def _handle_user_message(self, text: str) -> None: ts = datetime.now(timezone.utc).isoformat() self.messages.append(Message(role="user", content=text, timestamp=ts)) + self._refresh_context_tokens() chat_log = self.query_one("#chat-log") color = self._role_color("You") icon = self._role_icon("You") @@ -2639,6 +2655,10 @@ def stop_spinner() -> None: self.total_tokens += usage.total_tokens self.tokens_estimated = self.tokens_estimated or usage.estimated self.update_header() + case TokenCount(total_tokens=total, estimated=estimated): + self.context_tokens = total + self.context_estimated = estimated + self.update_header() case AgentDone(full_text, activity): stop_spinner() if think_widget is not None: @@ -2681,6 +2701,7 @@ def stop_spinner() -> None: timestamp=assistant_ts, ) ) + self._refresh_context_tokens() else: logger.debug( "Skipping empty assistant message append " @@ -2700,6 +2721,7 @@ def stop_spinner() -> None: start -= 1 pruned = system + rest[start:] self.messages = sanitize_tool_history(pruned) + self._refresh_context_tokens() finally: self._save_session() stop_spinner() diff --git a/oli_bot/config.py b/oli_bot/config.py index 34df3db..35207c1 100644 --- a/oli_bot/config.py +++ b/oli_bot/config.py @@ -129,6 +129,15 @@ class AppConfig(BaseSettings): "OLI_TRUNCATION_MAX_CHARS_LARGE", ), ) + # Maximum characters accepted by the write_file built-in tool. Bounds a + # single write so a runaway model can't emit an unbounded payload. + max_write_chars: int = Field( + default=100000, + validation_alias=AliasChoices( + "max_write_chars", + "OLI_MAX_WRITE_CHARS", + ), + ) # Dry-run mode: preview destructive actions without executing dry_run: bool = Field(default=False) diff --git a/oli_bot/models.py b/oli_bot/models.py index 3c5a2d4..657c015 100644 --- a/oli_bot/models.py +++ b/oli_bot/models.py @@ -42,6 +42,11 @@ class ToolCall: name: str = "" description: str = "" parameters: Dict[str, Any] = field(default_factory=dict) + # Set when the model's streamed ``arguments`` JSON could not be parsed + # (typically because generation hit the output-token limit mid-call). The + # agent surfaces this as a tool result instead of executing the call, so + # the model can retry with a smaller payload. + parse_error: Optional[str] = None @dataclass @@ -142,6 +147,23 @@ class UsageEvent: usage: Usage +@dataclass +class TokenCount: + """Live snapshot of the tokens the next/current request will send. + + ``phase`` identifies the checkpoint that produced the snapshot: + ``prompt`` (user submitted), ``thinking`` (a thinking block finished), + ``round`` (provider usage for a completed model call), ``tool_result`` + (a tool result was appended), or ``final`` (turn complete). + """ + + prompt_tokens: int + completion_tokens: int + total_tokens: int + estimated: bool + phase: str + + @dataclass class SubAgentStarted: """Emitted when a sub-agent run begins.""" diff --git a/oli_bot/profiles/default/AGENTS.md b/oli_bot/profiles/default/AGENTS.md index ff58145..a4df21f 100644 --- a/oli_bot/profiles/default/AGENTS.md +++ b/oli_bot/profiles/default/AGENTS.md @@ -10,7 +10,7 @@ All built-in tools are called via `builtin__`. | ------------------------------- | ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `builtin__read_file` | `file_path: string`, `offset?: number`, `length?: number` | always allowed in workspace; outside requires permission | Read a file from disk, optionally from an offset/length | | `builtin__view_image` | `file_path: string` (path or http(s) URL), `max_edge_px?: number` (default 1568), `question?: string` | same as read tools; URL branch subject to offline mode | Load an image and attach it as vision input. Ollama and OpenAI vision models see the actual image; other backends receive a text description of it. Use for screenshots, diagrams, or any image the user wants analyzed. | -| `builtin__write_file` | `file_path: string`, `content: string` | **always requires permission** | Create or overwrite a file (max 100k chars, creates parent dirs) | +| `builtin__write_file` | `file_path: string`, `content: string` | **always requires permission** | Create or overwrite a file (creates parent dirs). Max chars per call is configurable (`OLI_MAX_WRITE_CHARS`, default 100k). If a large write fails with a truncated-JSON error, split it into smaller writes. | | `builtin__edit_file` | `file_path: string`, `old_string: string`, `new_string: string` | **always requires permission** | Find-and-replace edit (first match only; include surrounding context if ambiguous) | | `builtin__download_file` | `url: string`, `file_path: string` | **always requires permission** | Download a file from a URL and save it locally | | `builtin__upload_file` | `url: string`, `file_path: string`, `method?: string`, `field_name?: string` | **always requires permission** | Upload a local file to an external server via HTTP PUT/POST | diff --git a/oli_bot/screens/config_screen.py b/oli_bot/screens/config_screen.py index 5bbb6e6..5c67c79 100644 --- a/oli_bot/screens/config_screen.py +++ b/oli_bot/screens/config_screen.py @@ -347,6 +347,12 @@ def compose(self) -> ComposeResult: classes="config-input", value=str(mp.get("truncation_max_chars_large", 100000)), ) + yield Input( + placeholder=f"Max write_file chars ({mp.get('max_write_chars', 100000)})", + id="cfg-max-write-chars", + classes="config-input", + value=str(mp.get("max_write_chars", 100000)), + ) yield Label("Offline Mode", classes="section-title") yield Checkbox( @@ -618,6 +624,7 @@ def _save(self) -> None: "truncation_max_chars_large": self._int( "#cfg-truncation-large", 100000 ), + "max_write_chars": self._int("#cfg-max-write-chars", 100000), "offline_mode": self._bool("#cfg-offline-mode"), "dry_run": self._bool("#cfg-dry-run"), "use_oauth": self._bool("#cfg-use-oauth"), diff --git a/oli_bot/settings.py b/oli_bot/settings.py index e24e780..c9b7b31 100644 --- a/oli_bot/settings.py +++ b/oli_bot/settings.py @@ -54,6 +54,7 @@ "model_filters": "", "truncation_max_chars_small": 4000, "truncation_max_chars_large": 100000, + "max_write_chars": 100000, "dry_run": False, "offline_mode": True, "use_oauth": False, @@ -130,6 +131,7 @@ "OLI_MODEL_FILTERS": "model_params.model_filters", "OLI_TRUNCATION_SMALL": "model_params.truncation_max_chars_small", "OLI_TRUNCATION_LARGE": "model_params.truncation_max_chars_large", + "OLI_MAX_WRITE_CHARS": "model_params.max_write_chars", "OLI_DRY_RUN": "model_params.dry_run", "OLI_OFFLINE_MODE": "model_params.offline_mode", "OLI_USE_OAUTH": "model_params.use_oauth", @@ -352,6 +354,7 @@ def to_appconfig(self, settings: dict) -> AppConfig: model_filters=mp.get("model_filters", ""), truncation_max_chars_small=mp.get("truncation_max_chars_small", 4000), truncation_max_chars_large=mp.get("truncation_max_chars_large", 100000), + max_write_chars=mp.get("max_write_chars", 100000), dry_run=mp.get("dry_run", False), offline_mode=mp.get("offline_mode", True), use_oauth=mp.get("use_oauth", False), @@ -420,6 +423,7 @@ def from_appconfig(self, config: AppConfig) -> dict: settings["model_params"][ "truncation_max_chars_large" ] = config.truncation_max_chars_large + settings["model_params"]["max_write_chars"] = config.max_write_chars settings["model_params"]["dry_run"] = config.dry_run settings["model_params"]["offline_mode"] = config.offline_mode settings["model_params"]["use_oauth"] = config.use_oauth diff --git a/oli_bot/tokens.py b/oli_bot/tokens.py new file mode 100644 index 0000000..5205266 --- /dev/null +++ b/oli_bot/tokens.py @@ -0,0 +1,181 @@ +"""Token counting helpers shared by the agent, TUI, and API. + +Uses ``tiktoken`` for models it recognises (with a generic BPE fallback for +unknown models) and falls back to the historical ~4 chars/token heuristic when +no encoding can be loaded. Counts are marked ``estimated`` so callers can show +a ``~`` prefix. +""" + +from __future__ import annotations + +import json +import logging +from functools import lru_cache +from typing import Iterable, Optional, Tuple + +from .models import Message + +logger = logging.getLogger(__name__) + +# Approximate framing overhead per chat message (role separators, etc.). +_MESSAGE_OVERHEAD = 4 +# Rough per-image cost for vision models when the tokenizer cannot see pixels. +_IMAGE_OVERHEAD = 256 +_GENERIC_ENCODINGS = ("o200k_base", "cl100k_base") + + +def estimate_tokens(text: str) -> int: + """Rough token estimate (~4 chars/token) used as a final fallback.""" + return max(1, (len(text) + 3) // 4) + + +@lru_cache(maxsize=16) +def _resolve_encoding(model: str): + """Return a tiktoken encoding for ``model``, or ``None`` if unavailable. + + The result (including ``None``) is cached so an unknown model or a failed + encoding download does not retry on every checkpoint. + """ + try: + import tiktoken + except Exception as e: # pragma: no cover - tiktoken is a declared dependency + logger.debug("tiktoken unavailable (%s); using heuristic token counts", e) + return None + + try: + return tiktoken.encoding_for_model(model) + except Exception as e: + logger.debug( + "tiktoken has no encoding for model %r (%s); using generic BPE", + model, + e, + ) + + for name in _GENERIC_ENCODINGS: + try: + return tiktoken.get_encoding(name) + except Exception as e: + logger.debug("tiktoken encoding %s unavailable: %s", name, e) + return None + + +def count_text_tokens(text: str, model: Optional[str] = None) -> Tuple[int, bool]: + """Count tokens in ``text``. + + Returns ``(count, estimated)``. ``estimated`` is ``False`` only when a real + tokenizer produced the count. + """ + if not text: + return 0, False + encoding = _resolve_encoding(model or "") + if encoding is not None: + try: + return len(encoding.encode(text, disallowed_special=())), False + except Exception as e: + logger.debug("tiktoken encode failed (%s); using heuristic", e) + return estimate_tokens(text), True + + +def _message_text(message: Message) -> str: + parts = [message.role or "", message.content or ""] + if message.name: + parts.append(message.name) + if message.tool_call_id: + parts.append(message.tool_call_id) + if message.tool_calls: + try: + parts.append( + json.dumps(message.tool_calls, ensure_ascii=False, default=str) + ) + except (TypeError, ValueError): + parts.append(str(message.tool_calls)) + return "\n".join(parts) + + +def count_message_tokens( + messages: Iterable[Message], model: Optional[str] = None +) -> Tuple[int, bool]: + """Count the prompt tokens for a list of messages. + + Adds a small per-message framing overhead (role separators etc.) and a + fixed allowance per image attachment. + """ + total = 0 + estimated = False + for message in messages: + count, est = count_text_tokens(_message_text(message), model) + total += count + _MESSAGE_OVERHEAD + if message.images: + total += _IMAGE_OVERHEAD * len(message.images) + estimated = estimated or est or bool(message.images) + return total, estimated + + +class ContextTokenTracker: + """Incremental count of the tokens the next/current request will send.""" + + def __init__(self, messages: Iterable[Message], model: Optional[str] = None): + self.model = model + self._prompt_tokens, self._estimated = count_message_tokens(messages, model) + self._inflight_text = "" + self._exact_completion: Optional[int] = None + + def add_message(self, message: Message) -> None: + """Add a message that was appended to the conversation history.""" + count, est = count_text_tokens(_message_text(message), self.model) + self._prompt_tokens += count + _MESSAGE_OVERHEAD + if message.images: + self._prompt_tokens += _IMAGE_OVERHEAD * len(message.images) + self._estimated = self._estimated or est or bool(message.images) + + def add_inflight(self, text: str) -> None: + """Accumulate model output that is not yet part of the history.""" + if not text: + return + self._inflight_text += text + self._exact_completion = None + + def clear_inflight(self) -> None: + """Drop in-flight output (it has been folded into the history).""" + self._inflight_text = "" + self._exact_completion = None + + def set_exact( + self, prompt_tokens: int, completion_tokens: int, estimated: bool + ) -> None: + """Adopt provider-reported counts for the completed model call. + + A provider that omits ``prompt_tokens`` (some OpenAI-compatible + servers report ``0`` while still reporting completions) must not wipe + out the locally counted context; keep the local count and flag the + snapshot as estimated in that case. + """ + if prompt_tokens > 0: + self._prompt_tokens = prompt_tokens + self._estimated = estimated + else: + self._estimated = True + self._exact_completion = completion_tokens + + def snapshot(self) -> Tuple[int, int, int, bool]: + """Return ``(prompt, completion, total, estimated)`` for the current context.""" + if self._exact_completion is not None: + completion = self._exact_completion + estimated = self._estimated + else: + completion, est = count_text_tokens(self._inflight_text, self.model) + estimated = self._estimated or est + return ( + self._prompt_tokens, + completion, + self._prompt_tokens + completion, + estimated, + ) + + +__all__ = [ + "ContextTokenTracker", + "count_message_tokens", + "count_text_tokens", + "estimate_tokens", +] diff --git a/oli_bot/tools/files.py b/oli_bot/tools/files.py index cd470bb..569a7c3 100644 --- a/oli_bot/tools/files.py +++ b/oli_bot/tools/files.py @@ -58,13 +58,14 @@ def register_tools(manager: BuiltinToolManager) -> None: handler=_read_file_handler, ) + write_limit = manager._config.max_write_chars manager.register_tool( name="write_file", description="Write content to a file. " "Creates parent directories if they don't exist. " "Use this to generate code, create configuration files, " "save notes, or modify project files. " - "Maximum content length is 100,000 characters.", + f"Maximum content length is {write_limit:,} characters.", parameters={ "type": "object", "properties": { @@ -79,7 +80,12 @@ def register_tools(manager: BuiltinToolManager) -> None: }, "required": ["file_path", "content"], }, - handler=_write_file_handler, + # Read the cap live from the manager's config so a `/config` change + # takes effect without rebuilding the tool set (the description above + # reflects the value at registration time). + handler=lambda file_path, content: _write_file_handler( + file_path, content, max_chars=manager._config.max_write_chars + ), ) manager.register_tool( @@ -187,11 +193,11 @@ def _read_file_handler(file_path: str, offset: int = None, length: int = None) - return f"Error reading file: {e}" -def _write_file_handler(file_path: str, content: str) -> str: +def _write_file_handler(file_path: str, content: str, max_chars: int = 100_000) -> str: path = Path(file_path).expanduser().resolve() - if len(content) > 100_000: + if len(content) > max_chars: return ( - f"Error: Content exceeds maximum length of 100,000 characters " + f"Error: Content exceeds maximum length of {max_chars:,} characters " f"({len(content)} given)." ) try: diff --git a/pyproject.toml b/pyproject.toml index 3248bbf..a7d66d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "openai>=2.0,<3", "huggingface_hub>=1.0,<2", "transformers>=4.40", + "tiktoken>=0.7", "fastapi>=0.110", "uvicorn[standard]>=0.29", "mcp>=2.0,<3", diff --git a/tests/unit/test_agent_process.py b/tests/unit/test_agent_process.py index 082c71c..b795626 100644 --- a/tests/unit/test_agent_process.py +++ b/tests/unit/test_agent_process.py @@ -19,6 +19,7 @@ Message, StreamChunk, TextChunk, + TokenCount, ToolCall, ToolCallChunk, ToolCallExecuting, @@ -35,7 +36,7 @@ class _TextStub: model = "stub" - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): yield TextChunk("hello ") yield TextChunk("world") @@ -48,7 +49,7 @@ def __init__(self, text_before_tool: str = ""): self._calls = 0 self._text_before_tool = text_before_tool - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): self._calls += 1 if self._calls == 1: if self._text_before_tool: @@ -67,7 +68,7 @@ def __init__(self): self.model = "stub" self._calls = 0 - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): self._calls += 1 if self._calls == 1: yield ToolCallChunk( @@ -85,7 +86,7 @@ class _HangingStub: model = "stub" - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): await asyncio.sleep(10) yield TextChunk("never") @@ -98,7 +99,7 @@ class _RaisingStub: def __init__(self, exc): self._exc = exc - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): raise self._exc yield # pragma: no cover — makes this an async generator @@ -152,11 +153,9 @@ async def test_text_only_response_yields_stream_then_done_with_full_text(): dones = [e for e in events if isinstance(e, Done)] # Every StreamChunk fires before the terminal Done, no duplicates. - assert [type(e).__name__ for e in events] == [ - "StreamChunk", - "StreamChunk", - "Done", - ] + # (TokenCount snapshots are interleaved by design and filtered here.) + kinds = [type(e).__name__ for e in events if not isinstance(e, TokenCount)] + assert kinds == ["StreamChunk", "StreamChunk", "Done"] assert "".join(e.text for e in stream_events) == "hello world" assert len(dones) == 1 assert dones[0].full_text == "hello world" @@ -227,7 +226,8 @@ async def test_tool_loop_event_ordering_and_synced_messages(): events = await _run(a, msgs) # Exact ordering: tool executes → tool result → final text stream → done. - kinds = [type(e).__name__ for e in events] + # (TokenCount snapshots are interleaved by design and filtered here.) + kinds = [type(e).__name__ for e in events if not isinstance(e, TokenCount)] assert kinds == ["ToolCallExecuting", "ToolCallResult", "StreamChunk", "Done"] executing = next(e for e in events if isinstance(e, ToolCallExecuting)) @@ -368,7 +368,7 @@ class _AlwaysToolThenFinalStub: def __init__(self): self.tools_seen: list = [] - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): self.tools_seen.append(tools) if len(self.tools_seen) <= 2: yield ToolCallChunk( @@ -403,6 +403,56 @@ async def test_max_iterations_exhausted_falls_back_to_final_stream(): assert dones[0].full_text == "final answer after exhausting iterations" +# --------------------------------------------------------------------------- # +# Truncated tool-call arguments # +# --------------------------------------------------------------------------- # + + +class _ParseErrorStub: + """First call yields a tool call whose arguments failed to parse, second + call yields a final answer.""" + + def __init__(self): + self.model = "stub" + self._calls = 0 + + async def stream_generate(self, messages, tools=None, max_tokens=None): + self._calls += 1 + if self._calls == 1: + yield ToolCallChunk( + [ + ToolCall( + id="c1", + name="builtin__write_file", + parameters={}, + parse_error="Error: truncated JSON", + ) + ] + ) + else: + yield TextChunk("retried") + + +@pytest.mark.asyncio +async def test_parse_error_surfaces_as_tool_result_without_executing(): + mcp = _StubMCP() + a = _agent(_ParseErrorStub(), mcp_manager=mcp) + msgs = [Message(role="user", content="hi")] + events = await _run(a, msgs) + + # The malformed call must NOT be dispatched to the tool manager. + assert mcp.calls == [] + + results = [e for e in events if isinstance(e, ToolCallResult)] + assert len(results) == 1 + assert results[0].name == "builtin__write_file" + assert "truncated" in results[0].result + + # The loop continues, so the model gets a chance to retry. + dones = [e for e in events if isinstance(e, Done)] + assert dones and dones[-1].full_text == "retried" + + # --------------------------------------------------------------------------- # # Chat mode # # --------------------------------------------------------------------------- # @@ -416,11 +466,9 @@ async def test_chat_mode_skips_tool_loop_entirely(): events = await _run(a, msgs) # Ordered, no duplicates: two stream chunks then exactly one Done. - assert [type(e).__name__ for e in events] == [ - "StreamChunk", - "StreamChunk", - "Done", - ] + # (TokenCount snapshots are interleaved by design and filtered here.) + kinds = [type(e).__name__ for e in events if not isinstance(e, TokenCount)] + assert kinds == ["StreamChunk", "StreamChunk", "Done"] # Chat mode must NOT invoke MCP at all — the whole point of the mode. assert mcp.calls == [] # Done carries the streamed text. @@ -438,7 +486,7 @@ async def test_chat_mode_ignores_tool_call_chunks_from_backend(): class _RogueBackend: model = "stub" - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): yield TextChunk("part ") yield ToolCallChunk([ToolCall(id="x", name="builtin__nope", parameters={})]) yield TextChunk("done") @@ -506,12 +554,11 @@ async def test_backend_exception_yields_error_then_done(): a = _agent(_RaisingStub(RuntimeError("upstream 500"))) events = await _run(a, [Message(role="user", content="hi")]) - kinds = [type(e).__name__ for e in events] + kinds = [type(e).__name__ for e in events if not isinstance(e, TokenCount)] assert kinds == ["Error", "Done"] - err, done = events - assert isinstance(err, Error) - assert isinstance(done, Done) + err = next(e for e in events if isinstance(e, Error)) + done = next(e for e in events if isinstance(e, Done)) assert "upstream 500" in err.message # Done.full_text must be empty on error so the UI layer's "skip empty # assistant append" branch fires and the error text never poisons history. @@ -558,10 +605,9 @@ async def test_tool_loop_timeout_yields_error_then_done_and_flips_generating_fla ) events = await _run(a, [Message(role="user", content="hi")]) - kinds = [type(e).__name__ for e in events] + kinds = [type(e).__name__ for e in events if not isinstance(e, TokenCount)] assert kinds == ["Error", "Done"] - err = events[0] - assert isinstance(err, Error) + err = next(e for e in events if isinstance(e, Error)) assert "timed out" in err.message # Regression: the `finally` in process() must reset _generating even on diff --git a/tests/unit/test_agent_scaffolding.py b/tests/unit/test_agent_scaffolding.py index e6e8303..6b03d16 100644 --- a/tests/unit/test_agent_scaffolding.py +++ b/tests/unit/test_agent_scaffolding.py @@ -716,7 +716,7 @@ def test_run_agent_dispatch_aggregates_and_streams_events(): class _StreamingBackend: model = "stub" - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): yield TextChunk(f"answer for {messages[-1].content}") yield Done(full_text=(f"final-{messages[-1].content}")) diff --git a/tests/unit/test_api_server.py b/tests/unit/test_api_server.py index f81c7a9..04a5729 100644 --- a/tests/unit/test_api_server.py +++ b/tests/unit/test_api_server.py @@ -21,7 +21,13 @@ from oli_bot.agent import Agent from oli_bot.config import AppConfig from oli_bot.mcp_client import MCPClientManager -from oli_bot.models import TextChunk, ToolCall, ToolCallChunk +from oli_bot.models import ( + TextChunk, + ToolCall, + ToolCallChunk, + Usage, + UsageChunk, +) from oli_bot.sessions import Session from oli_bot.tools.manager import BuiltinToolManager @@ -29,18 +35,26 @@ class _TextStub: model = "stub-text" - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): yield TextChunk("Hello ") yield TextChunk("from stub") +class _UsageStub: + model = "stub-usage" + + async def stream_generate(self, messages, tools=None, max_tokens=None): + yield TextChunk("counted") + yield UsageChunk(Usage(prompt_tokens=7, completion_tokens=3)) + + class _ToolThenTextStub: model = "stub-tool" def __init__(self): self._calls = 0 - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): self._calls += 1 if self._calls == 1: yield ToolCallChunk( @@ -53,7 +67,7 @@ async def stream_generate(self, messages, tools=None): class _RaisingStub: model = "stub-raise" - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): raise RuntimeError("boom") yield # pragma: no cover @@ -72,7 +86,7 @@ def __init__(self): self.max_active = 0 self._mutex = threading.Lock() - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): with self._mutex: self.active += 1 self.max_active = max(self.max_active, self.active) @@ -90,7 +104,7 @@ class _TodoThenTextStub: def __init__(self): self._calls = 0 - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): self._calls += 1 if self._calls == 1: yield ToolCallChunk( @@ -120,7 +134,7 @@ class _HistoryProbeStub: def __init__(self): self.seen = [] - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): self.seen.append(len(messages)) yield TextChunk(f"n={len(messages)}") @@ -133,7 +147,7 @@ class _MessageProbeStub: def __init__(self): self.messages = [] - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): self.messages = list(messages) yield TextChunk("ok") @@ -267,6 +281,20 @@ def test_chat_completion_non_stream(api): assert body["usage"]["completion_tokens"] >= 1 +def test_chat_completion_reports_exact_usage(api): + api.reset(_UsageStub()) + resp = api.client.post( + "/v1/chat/completions", + json={"messages": [{"role": "user", "content": "hi"}]}, + ) + assert resp.status_code == 200 + assert resp.json()["usage"] == { + "prompt_tokens": 7, + "completion_tokens": 3, + "total_tokens": 10, + } + + def test_chat_completion_tool_loop(api): api.reset(_ToolThenTextStub()) # The first stream_generate yields a tool call, so the harness executes @@ -398,6 +426,10 @@ def test_websocket_chat(api): frames = _recv_until_done(ws) types = [f["type"] for f in frames] assert "text_chunk" in types + assert "token_count" in types + token_frame = next(f for f in frames if f["type"] == "token_count") + assert token_frame["data"]["total_tokens"] > 0 + assert token_frame["data"]["phase"] == "prompt" assert ( "".join(f["data"]["text"] for f in frames if f["type"] == "text_chunk") == "Hello from stub" @@ -433,11 +465,13 @@ def test_websocket_stateful_history(api): ws.send_json({"content": "first"}) frames = _recv_until_done(ws) # stream_generate sees [system header, user1] on the first turn - assert frames[0]["data"]["text"] == "n=2" + text_frames = [f for f in frames if f["type"] == "text_chunk"] + assert text_frames[0]["data"]["text"] == "n=2" ws.send_json({"content": "second"}) frames = _recv_until_done(ws) # system + user1 + assistant1 + user2 = 4 messages on the second turn - assert frames[0]["data"]["text"] == "n=4" + text_frames = [f for f in frames if f["type"] == "text_chunk"] + assert text_frames[0]["data"]["text"] == "n=4" assert probe.seen == [2, 4] @@ -453,7 +487,8 @@ def test_websocket_clear(api): ws.send_json({"content": "after clear"}) frames = _recv_until_done(ws) # history was reset: back down to system + user = 2 messages - assert frames[0]["data"]["text"] == "n=2" + text_frames = [f for f in frames if f["type"] == "text_chunk"] + assert text_frames[0]["data"]["text"] == "n=2" assert probe.seen == [2, 2] diff --git a/tests/unit/test_api_sessions.py b/tests/unit/test_api_sessions.py index ab39cf2..512e33a 100644 --- a/tests/unit/test_api_sessions.py +++ b/tests/unit/test_api_sessions.py @@ -24,7 +24,7 @@ class _TextStub: model = "stub-text" - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): yield TextChunk("Hello ") yield TextChunk("from stub") @@ -37,7 +37,7 @@ class _ToolStub: def __init__(self): self._calls = 0 - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): self._calls += 1 if self._calls == 1: yield ToolCallChunk( @@ -55,7 +55,7 @@ class _HistoryProbeStub: def __init__(self): self.seen = [] - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): self.seen.append([m.role for m in messages]) yield TextChunk(f"n={len(messages)}") diff --git a/tests/unit/test_api_workspace.py b/tests/unit/test_api_workspace.py index 9659b01..3ad2068 100644 --- a/tests/unit/test_api_workspace.py +++ b/tests/unit/test_api_workspace.py @@ -23,7 +23,7 @@ class _TextStub: model = "stub-text" - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): yield TextChunk("Hello from stub") diff --git a/tests/unit/test_backend_tool_call_flush.py b/tests/unit/test_backend_tool_call_flush.py index a7bc990..0b972f9 100644 --- a/tests/unit/test_backend_tool_call_flush.py +++ b/tests/unit/test_backend_tool_call_flush.py @@ -292,3 +292,109 @@ async def create(self, **_k): with pytest.raises(RuntimeError, match="upstream 500"): [ev async for ev in b.stream_generate([], tools=[])] + + +# --------------------------------------------------------------------------- # +# Truncated / malformed arguments # +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_truncated_arguments_yields_parse_error_not_exception(): + """Regression: when generation is cut off mid-JSON (output-token limit), + the stream must surface a ``ToolCall.parse_error`` instead of raising an + unhandled ``JSONDecodeError``.""" + chunks = [ + _make_chunk( + tool_deltas=[ + _tool_delta( + tc_id="c1", + name="builtin__write_file", + args='{"file_path": "a.py", "content": "def f(', + ) + ], + ), + _make_chunk(finish="tool_calls"), + ] + events = [ev async for ev in _make_backend(chunks).stream_generate([], tools=[])] + + call = next(e for e in events if isinstance(e, ToolCallChunk)).tool_calls[0] + assert call.id == "c1" + assert call.name == "builtin__write_file" + assert call.parameters == {} + assert call.parse_error is not None + assert "truncated" in call.parse_error.lower() + + +@pytest.mark.asyncio +async def test_valid_arguments_have_no_parse_error(): + chunks = [ + _make_chunk(tool_deltas=[_tool_delta(tc_id="c1", name="do", args='{"a": 1}')]), + _make_chunk(finish="tool_calls"), + ] + call = next( + e + for e in [ + ev async for ev in _make_backend(chunks).stream_generate([], tools=[]) + ] + if isinstance(e, ToolCallChunk) + ).tool_calls[0] + assert call.parse_error is None + + +# --------------------------------------------------------------------------- # +# max_tokens threading # +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_stream_generate_forwards_max_tokens(): + """The caller-supplied max_tokens must reach the provider request so a + large write isn't truncated by a stale import-time default.""" + + captured: dict = {} + + class _CaptureCompletions: + async def create(self, **kwargs): + captured.update(kwargs) + + class _Iter: + def __aiter__(self_inner): + return self_inner._agen() + + async def _agen(self_inner): + return + yield # pragma: no cover - empty stream + + return _Iter() + + b = OpenAIBackend(api_key="x", base_url="https://ex/", model="gpt-fake") + b.client = SimpleNamespace(chat=SimpleNamespace(completions=_CaptureCompletions())) + + [ev async for ev in b.stream_generate([], tools=[], max_tokens=8192)] + assert captured.get("max_tokens") == 8192 + + +@pytest.mark.asyncio +async def test_stream_generate_omits_max_tokens_when_none(): + captured: dict = {} + + class _CaptureCompletions: + async def create(self, **kwargs): + captured.update(kwargs) + + class _Iter: + def __aiter__(self_inner): + return self_inner._agen() + + async def _agen(self_inner): + return + yield # pragma: no cover - empty stream + + return _Iter() + + b = OpenAIBackend(api_key="x", base_url="https://ex/", model="gpt-fake") + b.client = SimpleNamespace(chat=SimpleNamespace(completions=_CaptureCompletions())) + + [ev async for ev in b.stream_generate([], tools=[])] + assert "max_tokens" not in captured diff --git a/tests/unit/test_config_settings.py b/tests/unit/test_config_settings.py index 99fa49a..df1d4d8 100644 --- a/tests/unit/test_config_settings.py +++ b/tests/unit/test_config_settings.py @@ -177,6 +177,16 @@ def test_transformers_small_model_roundtrip(tmp_path): assert back["transformers"]["is_multi_model"] is True +def test_max_write_chars_roundtrip(tmp_path): + mgr = SettingsManager(config_dir=tmp_path) + settings = mgr.get_defaults() + settings["model_params"]["max_write_chars"] = 12345 + cfg = mgr.to_appconfig(settings) + assert cfg.max_write_chars == 12345 + back = mgr.from_appconfig(cfg) + assert back["model_params"]["max_write_chars"] == 12345 + + def test_to_appconfig_defaults_match_default_settings(tmp_path): mgr = SettingsManager(config_dir=tmp_path) defaults = mgr.get_defaults() diff --git a/tests/unit/test_profiles.py b/tests/unit/test_profiles.py index 10a0c1f..6818eff 100644 --- a/tests/unit/test_profiles.py +++ b/tests/unit/test_profiles.py @@ -25,7 +25,7 @@ class _TextStub: model = "stub-text" - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): yield TextChunk("Hello from stub") diff --git a/tests/unit/test_session_activity.py b/tests/unit/test_session_activity.py index 39d3afc..0b738aa 100644 --- a/tests/unit/test_session_activity.py +++ b/tests/unit/test_session_activity.py @@ -291,7 +291,7 @@ class _ToolStub: def __init__(self): self._calls = 0 - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): self._calls += 1 if self._calls == 1: yield ThinkingChunk("let me check") @@ -347,7 +347,7 @@ async def test_text_only_turn_has_no_activity(): class _TextOnly: model = "stub" - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): yield TextChunk("hello") agent = _agent(_TextOnly()) diff --git a/tests/unit/test_token_usage.py b/tests/unit/test_token_usage.py index b1d41ac..196c137 100644 --- a/tests/unit/test_token_usage.py +++ b/tests/unit/test_token_usage.py @@ -10,6 +10,8 @@ Done, Message, TextChunk, + ThinkingChunk, + TokenCount, ToolCall, ToolCallChunk, Usage, @@ -17,8 +19,82 @@ UsageEvent, ) from oli_bot.sessions import ConversationStore +from oli_bot.tokens import ( + ContextTokenTracker, + count_message_tokens, + count_text_tokens, + estimate_tokens, +) from oli_bot.backends import OllamaBackend, OpenAIBackend +# --------------------------------------------------------------------------- # +# Local token counting # +# --------------------------------------------------------------------------- # + + +class _FakeEncoding: + def encode(self, text, disallowed_special=()): + return list(range(len(text.split()))) + + +def test_count_text_tokens_uses_tokenizer(monkeypatch): + monkeypatch.setattr( + "oli_bot.tokens._resolve_encoding", lambda model: _FakeEncoding() + ) + count, estimated = count_text_tokens("hello world", "gpt-fake") + assert count == 2 + assert estimated is False + + +def test_count_text_tokens_falls_back_to_heuristic(monkeypatch): + monkeypatch.setattr("oli_bot.tokens._resolve_encoding", lambda model: None) + count, estimated = count_text_tokens("hello world", "unknown-model") + assert count == estimate_tokens("hello world") + assert estimated is True + + +def test_count_message_tokens_includes_framing(monkeypatch): + monkeypatch.setattr("oli_bot.tokens._resolve_encoding", lambda model: None) + messages = [Message(role="user", content="hello")] + count, estimated = count_message_tokens(messages, "unknown-model") + assert count == estimate_tokens("user\nhello") + 4 + assert estimated is True + + +def test_context_tracker_tracks_history_and_inflight(monkeypatch): + monkeypatch.setattr("oli_bot.tokens._resolve_encoding", lambda model: None) + tracker = ContextTokenTracker([Message(role="user", content="hi")], "unknown-model") + prompt, completion, total, estimated = tracker.snapshot() + assert prompt > 0 + assert completion == 0 + assert total == prompt + assert estimated is True + + tracker.add_message(Message(role="assistant", content="hello")) + assert tracker.snapshot()[0] > prompt + + tracker.add_inflight("thinking about it") + prompt2, completion2, total2, _ = tracker.snapshot() + assert completion2 > 0 + assert total2 == prompt2 + completion2 + + tracker.set_exact(100, 20, False) + assert tracker.snapshot() == (100, 20, 120, False) + + tracker.clear_inflight() + assert tracker.snapshot() == (100, 0, 100, False) + + +def test_context_tracker_keeps_local_prompt_when_provider_reports_zero(monkeypatch): + monkeypatch.setattr("oli_bot.tokens._resolve_encoding", lambda model: None) + tracker = ContextTokenTracker([Message(role="user", content="hi")], "unknown-model") + prompt, _, _, _ = tracker.snapshot() + + tracker.set_exact(0, 5, False) + + assert tracker.snapshot() == (prompt, 5, prompt + 5, True) + + # --------------------------------------------------------------------------- # # OpenAI streaming # # --------------------------------------------------------------------------- # @@ -192,7 +268,7 @@ class _UsageToolStub: def __init__(self): self._calls = 0 - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): self._calls += 1 if self._calls == 1: yield ToolCallChunk( @@ -212,10 +288,19 @@ async def call_tool(self, name, params, confirm_callback=None, **kwargs): class _NoUsageStub: model = "stub" - async def stream_generate(self, messages, tools=None): + async def stream_generate(self, messages, tools=None, max_tokens=None): yield TextChunk("plain text") +class _ThinkingStub: + model = "stub" + + async def stream_generate(self, messages, tools=None, max_tokens=None): + yield ThinkingChunk("hmm") + yield TextChunk("done") + yield UsageChunk(Usage(prompt_tokens=4, completion_tokens=2)) + + def _agent(backend) -> Agent: return Agent( role="default", @@ -244,6 +329,13 @@ async def test_agent_emits_single_usage_event_summed_across_tool_iterations(): assert not any(isinstance(e, UsageChunk) for e in events) assert any(isinstance(e, Done) for e in events) + token_events = [e for e in events if isinstance(e, TokenCount)] + phases = [e.phase for e in token_events] + assert phases[0] == "prompt" + assert "round" in phases + assert "tool_result" in phases + assert phases[-1] == "final" + @pytest.mark.asyncio async def test_agent_emits_no_usage_event_when_backend_reports_none(): @@ -254,6 +346,24 @@ async def test_agent_emits_no_usage_event_when_backend_reports_none(): assert not any(isinstance(e, UsageEvent) for e in events) assert any(isinstance(e, Done) for e in events) + token_events = [e for e in events if isinstance(e, TokenCount)] + assert token_events[0].phase == "prompt" + assert token_events[-1].phase == "final" + + +@pytest.mark.asyncio +async def test_agent_emits_token_count_when_thinking_finishes(): + a = _agent(_ThinkingStub()) + msgs = [Message(role="system", content="sys"), Message(role="user", content="hi")] + events = [ev async for ev in a.process(msgs)] + + token_events = [e for e in events if isinstance(e, TokenCount)] + phases = [e.phase for e in token_events] + assert phases[0] == "prompt" + assert "thinking" in phases + assert "round" in phases + assert phases[-1] == "final" + # --------------------------------------------------------------------------- # # Session persistence # diff --git a/tests/unit/test_tool_manager.py b/tests/unit/test_tool_manager.py index 7423516..9cd9435 100644 --- a/tests/unit/test_tool_manager.py +++ b/tests/unit/test_tool_manager.py @@ -344,6 +344,39 @@ def test_notebook_set_auto_increments_plan_pages(tmp_path, monkeypatch): assert (notes_dir / "plan-foo-3.md").read_text() == "v3" +# ---------- configurable write cap ------------------------------------------- + + +@pytest.mark.asyncio +async def test_write_file_cap_is_configurable(tmp_path): + session = Session(workspace=tmp_path) + session.grant(SCOPE_WRITE, session=True) + cfg = AppConfig(_env_file=None, offline_mode=False, max_write_chars=5) + m = BuiltinToolManager(session=session, config=cfg) + + too_big = await m.call_tool( + "write_file", + {"file_path": str(tmp_path / "a.txt"), "content": "123456"}, + ) + assert "exceeds maximum length" in too_big + assert "5" in too_big + + ok = await m.call_tool( + "write_file", + {"file_path": str(tmp_path / "b.txt"), "content": "12345"}, + ) + assert ok.startswith("Successfully wrote") + + +def test_write_file_description_reflects_configured_cap(): + cfg = AppConfig(_env_file=None, offline_mode=False, max_write_chars=42) + m = BuiltinToolManager(config=cfg) + defn = next( + d for d in m.get_tool_definitions() if d["name"] == "builtin__write_file" + ) + assert "42" in defn["description"] + + def test_notebook_set_overwrites_non_plan_pages(tmp_path, monkeypatch): from pathlib import Path