diff --git a/.gitignore b/.gitignore index 6381bf2..c4b9a01 100644 --- a/.gitignore +++ b/.gitignore @@ -89,9 +89,9 @@ _embedded_resources.py _build_stamp.py /raw_reasoning *.mp4 -!frontend/background.mp4 !demo.mp4 /Auto_Use/api_key +/Auto_Use/agent_conversation/settings.json /Auto_Use/windows_use/mcp_tracking *.exe *.ico @@ -117,3 +117,8 @@ screen_record.py /Auto_Use/macOS_use/scratchpad /cli_minion_result /Auto_Use/windows_use/scratchpad/cli_minion + +# Resumable chat memory (UI) — runtime session data lives inside the package; +# the code (service.py / __init__.py) is tracked, the saved chats are not. +/Auto_Use/agent_conversation/index.json +/Auto_Use/agent_conversation/chat_*/ diff --git a/Auto_Use/agent_conversation/__init__.py b/Auto_Use/agent_conversation/__init__.py new file mode 100644 index 0000000..0d75aee --- /dev/null +++ b/Auto_Use/agent_conversation/__init__.py @@ -0,0 +1,31 @@ +# Copyright 2026 Autouse AI — https://github.com/auto-use/Auto-Use +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# If you build on this project, please keep this header and credit +# Autouse AI (https://github.com/auto-use/Auto-Use) in forks and derivative works. +# A small attribution goes a long way toward a healthy open-source +# community — thank you for contributing. + +"""Permanent (resumable) chat memory for the UI path. + +All conversation-memory management lives in this package — saving, loading and +optimizing a session's conversation, and the on-disk session folders + index. +The agent stays a thin one-shot loop; app.py talks ONLY to ConversationService +(load a session -> trigger the main agent -> save the run). main.py / cli.py +never import this package, so they remain direct one-shot entry points. +""" + +from .service import ConversationService, conversation + +__all__ = ["ConversationService", "conversation"] diff --git a/Auto_Use/agent_conversation/service.py b/Auto_Use/agent_conversation/service.py new file mode 100644 index 0000000..22f4abc --- /dev/null +++ b/Auto_Use/agent_conversation/service.py @@ -0,0 +1,474 @@ +# Copyright 2026 Autouse AI — https://github.com/auto-use/Auto-Use +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# If you build on this project, please keep this header and credit +# Autouse AI (https://github.com/auto-use/Auto-Use) in forks and derivative works. +# A small attribution goes a long way toward a healthy open-source +# community — thank you for contributing. + +"""ConversationService — permanent, resumable chat memory for the UI path. + +This is the ONE place that knows how a chat is persisted, optimized and +replayed. app.py is thin glue: + + sid, prior = conversation.start_or_resume(session_id, task) # load + agent = AgentService(..., prior_history=prior) # trigger + agent.process_request(task) + conversation.save_run(sid, agent.assistant_messages, # save + agent.tool_responses, status, message, task) + +On-disk layout (data lives INSIDE this package's folder, in sub-folders): + + Auto_Use/agent_conversation/ + index.json <- { id: {title, created_at, + updated_at, last_done_message} } + /conversation.json <- one folder per session: the + optimized, resumable history snapshot + +The saved history is the agent's FULL per-step memory: each assistant step with +its thinking / eval / decision / memory / next_goal / action intact, paired 1:1 +with the agent's already-compacted tool results, and capped with a single +"terminal note" step carrying a clean done-message so a resumed run always knows +how the previous run ended — on EVERY ending (done, user stop, error, provider +kill, step limit, parse fails). Storing the FULL steps (not a trimmed copy) is +deliberate: the debug download (render_readable) is then the TRUE agent memory, +readable like main.py's conversation_N.txt. Resume token-efficiency is +unaffected — the agent's OWN message builder trims older steps to +decision/memory/next_goal when it sends them to the LLM. + +It contains NO system prompt and NO screenshot bytes, so a resumed run never +double-loads the system prompt: the agent's message builder always prepends +exactly one system message and seeds these two lists underneath it. (The debug +log adds the system prompt for readability only, supplied by app.py at export.) +""" + +import os +import re +import sys +import json +import time +import shutil +import logging +from pathlib import Path + +logger = logging.getLogger("agent_conversation") + +# Compiled (Nuitka) binary vs dev run — mirrors the detection in app.py so the +# data root lands next to the executable in a packaged build and inside this +# package in development. +_IS_COMPILED = bool(getattr(sys, "frozen", False)) or ("__compiled__" in globals()) + + +class ConversationService: + """Owns all conversation-memory persistence + optimization for the UI.""" + + # ── storage paths ────────────────────────────────────────────────────── + def root(self) -> Path: + """Root folder holding index.json + one sub-folder per session.""" + if _IS_COMPILED: + base = Path(sys.executable).parent / "Auto_Use" / "agent_conversation" + else: + base = Path(__file__).resolve().parent + base.mkdir(parents=True, exist_ok=True) + return base + + def _index_file(self) -> Path: + return self.root() / "index.json" + + def _session_dir(self, session_id) -> Path: + return self.root() / str(session_id) + + def _session_file(self, session_id) -> Path: + return self._session_dir(session_id) / "conversation.json" + + def _memory_log_file(self, session_id) -> Path: + # The TRUE, human-readable memory log (exact payload sent to the model). + return self._session_dir(session_id) / "memory_log.txt" + + # ── index.json (session -> display meta) ─────────────────────────────── + def _read_index(self) -> dict: + f = self._index_file() + if f.exists(): + try: + with open(f, "r", encoding="utf-8") as fh: + data = json.load(fh) + if isinstance(data, dict): + return data + except Exception: + logger.exception("read index.json") + return {} + + def _write_index(self, index: dict) -> None: + f = self._index_file() + try: + f.parent.mkdir(parents=True, exist_ok=True) + with open(f, "w", encoding="utf-8") as fh: + json.dump(index, fh, indent=2) + except Exception: + logger.exception("write index.json") + + def _touch_index(self, session_id, *, title=None, last_done_message=None, + context_tokens=None, context_cap=None) -> dict: + """Create-or-update one index entry. created_at + title are set ONCE + (title never overwritten, so the original objective stays the label); + updated_at + last_done_message + context_tokens/context_cap refresh + whenever provided. + + context_tokens is the memory bar's LATEST context size (not a sum) and + context_cap is the fixed memory budget the bar fills toward (MemoryTracker + MEMORY_CAP = 300k — NOT the model context window) — together they let a + reopened chat restore the bar to where memory was.""" + index = self._read_index() + now = int(time.time()) + entry = index.get(str(session_id)) or { + "title": None, "created_at": now, + "updated_at": now, "last_done_message": "", + "context_tokens": 0, "context_cap": 0, + } + if title and not entry.get("title"): + entry["title"] = title + if last_done_message is not None: + entry["last_done_message"] = last_done_message + if context_tokens is not None: + entry["context_tokens"] = int(context_tokens or 0) + if context_cap is not None: + entry["context_cap"] = int(context_cap or 0) + entry["updated_at"] = now + index[str(session_id)] = entry + self._write_index(index) + return entry + + # ── per-session conversation.json ────────────────────────────────────── + def _read_session(self, session_id): + """Return the stored optimized history dict, or None on missing/corrupt + (so the caller can resume the SAME id with a fresh agent context).""" + f = self._session_file(session_id) + if f.exists(): + try: + with open(f, "r", encoding="utf-8") as fh: + data = json.load(fh) + if isinstance(data, dict) and isinstance(data.get("history"), dict): + return data["history"] + except Exception: + logger.exception("read session %s", session_id) + return None + + def _write_session(self, session_id, history: dict) -> None: + f = self._session_file(session_id) + try: + f.parent.mkdir(parents=True, exist_ok=True) + with open(f, "w", encoding="utf-8") as fh: + json.dump({"session_id": str(session_id), "history": history}, fh, indent=2) + except Exception: + logger.exception("write session %s", session_id) + + # ── helpers ──────────────────────────────────────────────────────────── + def _new_id(self) -> str: + """Filesystem-safe persistent chat id (distinct from app.py's per-run + guard token).""" + return f"chat_{int(time.time() * 1000):x}" + + def _title_from_task(self, task) -> str: + """First non-empty line of the task, trimmed to a short sidebar label.""" + text = (task or "").strip() + if not text: + return "New chat" + first = text.splitlines()[0].strip() + return first if len(first) <= 60 else first[:57].rstrip() + "..." + + def _terminal_message(self, status: str, message: str) -> str: + """Clean 'done message' for ANY ending — so an interrupted / failed / + provider-killed run still records a sensible conclusion in memory.""" + message = message or "" + if status == "success": + return f"Task completed: {message}".strip().rstrip(":") + if status == "error": + return f"Agent terminated (error / provider): {message}".strip().rstrip(":") + return f"Agent stopped before completing: {message}".strip().rstrip(":") + + def _build_history(self, task, assistant_messages, tool_responses, + status, message, prior_task=None) -> dict: + """Build the saved conversation = the agent's FULL per-step memory + (thinking / eval / decision / memory / next_goal / action all preserved, + tool results as-is), capped with a terminal note recording how the run + ended. + + We store the FULL steps — NOT a trimmed copy — so the debug download is + the true, human-readable agent memory. Resume token-efficiency is + unaffected: the agent's OWN message builder trims older steps to + decision/memory/next_goal when it sends them to the LLM, so storing full + here costs only a little disk and loses nothing for debugging. + prior_task preserves the ORIGINAL objective across resumes.""" + full_assistant = list(assistant_messages or []) + tools = list(tool_responses or []) + # Force exact 1:1 alignment defensively. + if len(tools) < len(full_assistant): + tools += [None] * (len(full_assistant) - len(tools)) + else: + tools = tools[:len(full_assistant)] + done_message = self._terminal_message(status, message) + # Cap with a terminal note so a resumed run (and the reader) always sees + # how the previous run ended — on EVERY ending (done / stop / error). On + # resume it becomes the most-recent step, so the agent's builder replays + # every real step's tool result beneath it. + full_assistant.append(json.dumps({ + "decision": "Previous run concluded.", + "memory": done_message, + "next_goal": "Awaiting the user's next request; resume from this point.", + }, ensure_ascii=False, indent=2)) + tools.append(None) + return { + "version": 1, + "task": prior_task or task, + "assistant_messages": full_assistant, + "tool_responses": tools, + "final_status": status, + "final_message": message, + "done_message": done_message, + } + + # ── public API used by app.py ────────────────────────────────────────── + def start_or_resume(self, req_session_id, task): + """Resolve the chat session for an incoming run. + + Returns (session_id, prior_history): + - existing session (folder present) -> CONTINUATION: load its saved + optimized history (None if missing/corrupt -> fresh agent context on + the SAME id). + - otherwise -> fresh start: mint a new id and seed its index entry so + the sidebar shows it immediately. prior_history is None. + """ + if req_session_id and req_session_id != "new" and self._session_dir(req_session_id).exists(): + sid = str(req_session_id) + return sid, self._read_session(sid) + sid = self._new_id() + self._touch_index(sid, title=self._title_from_task(task)) + return sid, None + + def save_run(self, session_id, assistant_messages, tool_responses, status, + message, task, last_messages=None, context_tokens=None, + context_cap=None): + """Persist a finished run + refresh index meta. Writes TWO things: + - conversation.json — the lean resume seed (assistant/tool turns). + - memory_log.txt — the TRUE debug memory: the exact final payload + sent to the model (system + interleaved history + the live user + message that re-injects user_request/todo/scratchpad/element_tree). + Called for EVERY ending (completion, user stop, error, provider kill).""" + try: + existing = self._read_session(session_id) + prior_task = existing.get("task") if isinstance(existing, dict) else None + payload = self._build_history( + task, assistant_messages, tool_responses, status, message, prior_task + ) + self._write_session(session_id, payload) + done_message = payload["done_message"] + + # TRUE memory log (debug): render the exact payload the model received. + if last_messages: + header = [ + "=== AGENT MEMORY — exact payload sent to the model (true snapshot) ===", + f"Session: {session_id}", + f"Task: {payload.get('task', '')}", + f"Final status: {status}", + f"Done: {done_message}", + ] + # The latest step's response is generated AFTER last_messages was + # built, so append it as the final assistant turn (like main.py). + final_response = (assistant_messages or [None])[-1] + text = self._render_messages(last_messages, header, final_response) + self._write_text(self._memory_log_file(session_id), text) + + self._touch_index( + session_id, + title=self._title_from_task(prior_task or task), # no-op if already set + last_done_message=done_message, + context_tokens=context_tokens, # latest context size for the memory bar + context_cap=context_cap, # fixed 300k memory budget (MEMORY_CAP) + ) + return done_message + except Exception: + logger.exception("save_run %s", session_id) + return None + + def _write_text(self, path: Path, text: str) -> None: + try: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + fh.write(text) + except Exception: + logger.exception("write text %s", path) + + @staticmethod + def _content_text(content): + """Flatten a message's content to text. Prompt-cached turns carry content + as a list of {type, text, ...} blocks rather than a plain string.""" + if isinstance(content, list): + return "\n".join(b.get("text", "") for b in content if isinstance(b, dict)) + return content if isinstance(content, str) else str(content) + + def _render_messages(self, messages, header_lines=None, final_response=None): + """Render an exact messages payload to the human-readable conversation log + (same shape as main.py's conversation_N.txt): SYSTEM PROMPT block, then + each ASSISTANT / USER turn in the order sent — the USER turns include the + live ///.""" + bar = "=" * 60 + out = list(header_lines or []) + if header_lines: + out += [bar, ""] + for m in messages or []: + role = str(m.get("role", "?")).upper() + content = self._content_text(m.get("content", "")) + if role == "SYSTEM": + out += ["=== SYSTEM PROMPT ===", content, "", bar, ""] + else: + out += [f"--- {role} ---", content, ""] + if final_response: + out += ["--- ASSISTANT (final response) ---", str(final_response), ""] + return "\n".join(out) + "\n" + + def list_sessions(self): + """All sessions, newest-first, as {id, name, updated_at, last_done_message}.""" + index = self._read_index() + items = [{ + "id": sid, + "name": meta.get("title") or "New chat", + "updated_at": meta.get("updated_at", 0), + "last_done_message": meta.get("last_done_message", ""), + } for sid, meta in index.items()] + items.sort(key=lambda x: x.get("updated_at", 0), reverse=True) + return items + + def get_session(self, session_id): + """{id, name, last_done_message, context_tokens, context_cap} for the + reopen view, or None if unknown. The full transcript is intentionally NOT + returned — reopen shows only the last done message (+ the memory-bar + latest context size and its cap). Legacy rows that only carried the old + cumulative `tokens_used` are ignored (context_tokens defaults to 0, so the + bar starts empty and the next run corrects it).""" + meta = self._read_index().get(str(session_id)) + if not meta: + return None + return { + "id": str(session_id), + "name": meta.get("title") or "New chat", + "context_tokens": int(meta.get("context_tokens", 0) or 0), + "context_cap": int(meta.get("context_cap", 0) or 0), + "last_done_message": meta.get("last_done_message", ""), + } + + def delete_session(self, session_id) -> bool: + """Remove a session's folder and its index entry (idempotent).""" + try: + d = self._session_dir(session_id) + if d.exists(): + shutil.rmtree(d, ignore_errors=True) + index = self._read_index() + if str(session_id) in index: + del index[str(session_id)] + self._write_index(index) + return True + except Exception: + logger.exception("delete_session %s", session_id) + return False + + def read_raw(self, session_id): + """Full saved conversation.json payload ({session_id, history}) exactly as + on disk, or None if missing/corrupt. This is the agent's real optimized + memory — used by the debug export.""" + f = self._session_file(session_id) + if f.exists(): + try: + with open(f, "r", encoding="utf-8") as fh: + data = json.load(fh) + if isinstance(data, dict): + return data + except Exception: + logger.exception("read_raw %s", session_id) + return None + + def render_readable(self, session_id, system_prompt=None): + """Render a session's saved memory as a HUMAN-READABLE conversation log — + the same shape main.py's save_conversation writes (conversation_N.txt): + a SYSTEM PROMPT block, then the real interleaved ASSISTANT (per-step JSON, + full thinking/eval/decision/memory/next_goal/action) / USER + () turns. This is the true agent memory, formatted to read + and critique — not raw JSON. Returns the text, or None if nothing saved.""" + data = self.read_raw(session_id) + if data is None: + return None + hist = data.get("history") or {} + meta = self.get_session(session_id) or {} + asst = hist.get("assistant_messages") or [] + tools = hist.get("tool_responses") or [] + task = hist.get("task", "") or "" + bar = "=" * 60 + out = [ + "=== AGENT MEMORY (conversation log) ===", + f"Chat: {meta.get('name', '')}", + f"Session: {session_id}", + f"Steps: {len(asst)}", + f"Final status: {hist.get('final_status', '')}", + f"Done: {hist.get('done_message', '')}", + f"Task: {task}", + bar, + ] + if system_prompt: + out += ["", "=== SYSTEM PROMPT ===", system_prompt.strip(), bar] + for i, step in enumerate(asst): + out += ["", "--- ASSISTANT ---"] + if i == 0 and task: + out += ["", task, "", ""] + out.append(str(step)) + tr = tools[i] if i < len(tools) else None + if tr: + out += ["", "--- USER ---", "", str(tr), ""] + return "\n".join(out) + "\n" + + def export_to_downloads(self, session_id, system_prompt=None): + """Debug aid: write a session's TRUE memory log (.txt) to the user's + Downloads folder. Prefers memory_log.txt (the exact payload sent to the + model); falls back to a reconstruction for chats saved before memory_log + existed. Returns the destination path, or None if nothing's saved.""" + text = None + log_file = self._memory_log_file(session_id) + if log_file.exists(): + try: + text = log_file.read_text(encoding="utf-8") + except Exception: + logger.exception("read memory_log %s", session_id) + text = None + if text is None: + text = self.render_readable(session_id, system_prompt) # back-compat fallback + if text is None: + return None + meta = self.get_session(session_id) or {} + slug = re.sub(r"[^A-Za-z0-9._-]+", "-", meta.get("name") or str(session_id)).strip("-")[:40] + # System date-time in a clear, readable form so each download is easy to tell apart. + fname = f"autouse-{slug or session_id}-{time.strftime('%Y-%m-%d_%H-%M-%S')}.txt" + home = Path.home() + for folder in (home / "Downloads", home / "Desktop", home): + try: + if folder.exists(): + dest = folder / fname + with open(dest, "w", encoding="utf-8") as fh: + fh.write(text) + return str(dest) + except Exception: + logger.exception("export_to_downloads -> %s", folder) + continue + return None + + +# Module-level singleton — app.py imports this directly. +conversation = ConversationService() diff --git a/Auto_Use/macOS_use/agent/coder/service.py b/Auto_Use/macOS_use/agent/coder/service.py index 7d06de4..f89a019 100644 --- a/Auto_Use/macOS_use/agent/coder/service.py +++ b/Auto_Use/macOS_use/agent/coder/service.py @@ -282,7 +282,9 @@ def _run_agent_loop(self, task: str) -> str: return "Max iterations reached" - safe_print(f"cli agent running step {step_number}") + # Step heartbeat → debug log only (NOT the UI). The card shows ONLY the agent's + # own validated output; this "running step N" marker is ours, not the model's. + debug_log(f"cli agent running step {step_number}") # Get agent sitting info agent_sitting = self._get_agent_sitting() @@ -353,12 +355,6 @@ def _run_agent_loop(self, task: str) -> str: # Call LLM raw_response = self.llm.send_request(messages) - # Stream the raw LLM response to stdout. The parent main agent's - # subprocess pipe reader picks it up and renders it line-by-line - # in the CLI streaming pill UI (no decoration, just raw text). - if raw_response: - safe_print(raw_response) - # Check stop after LLM if self.stop_event and self.stop_event.is_set(): break @@ -369,10 +365,11 @@ def _run_agent_loop(self, task: str) -> str: # Validate and normalize response success, normalized, failed_raw = CLIAgentResponseFormatter.normalize_response(raw_response) - # If JSON parse failed, discard response and retry with fresh context + # If JSON parse failed, discard response and retry with fresh context. + # Emit NOTHING to the UI on failure — raw/partial/malformed JSON would glitch + # the streaming card; the raw response is already saved above for debugging. if not success: json_fail_count += 1 - safe_print(f"⚠️ JSON parse failed ({json_fail_count}/3). Discarding and retrying...") if json_fail_count >= 3: break @@ -383,6 +380,11 @@ def _run_agent_loop(self, task: str) -> str: # Reset consecutive JSON fail counter on success json_fail_count = 0 + # Stream the validated, complete response (action included) to the card's + # `>` line. The frontend parses the `action` array out of THIS JSON to drive + # the action-icon chain + scratchpad — no extra backend events needed. + safe_print(CLIAgentResponseFormatter.format_stream_json(normalized)) + # Save TRUE agent memory snapshot self._save_conversation_snapshot(messages, normalized, step_number) diff --git a/Auto_Use/macOS_use/agent/coder/view.py b/Auto_Use/macOS_use/agent/coder/view.py index d28fb0b..3f34466 100644 --- a/Auto_Use/macOS_use/agent/coder/view.py +++ b/Auto_Use/macOS_use/agent/coder/view.py @@ -141,4 +141,17 @@ def get_action_block(normalized_response: str) -> list: return json_data.get("action", []) except: pass - return [] \ No newline at end of file + return [] + + @staticmethod + def format_stream_json(normalized_response: str) -> str: + """Per-step payload for the streaming UI terminal — ONLY the action block (a real + terminal shows what's being executed, not thinking/goal/memory). The frontend reads + `action` from this for both the `>` stream and the tool-icon chain. On parse error, + fall back to the input unchanged. + """ + try: + json_data = json.loads(normalized_response) + return json.dumps({"action": json_data.get("action", [])}, indent=2, ensure_ascii=False) + except Exception: + return normalized_response \ No newline at end of file diff --git a/Auto_Use/macOS_use/agent/main_driver/service.py b/Auto_Use/macOS_use/agent/main_driver/service.py index a575737..41df294 100644 --- a/Auto_Use/macOS_use/agent/main_driver/service.py +++ b/Auto_Use/macOS_use/agent/main_driver/service.py @@ -76,7 +76,7 @@ def _cleanup_scratchpad(): class AgentService: """Service for Windows automation agent""" - def __init__(self, provider: str, model: str, save_conversation: bool = False, thinking: bool = True, frontend_callback=None, text_callback=None, web_callback=None, shell_callback=None, cli_callback=None, api_key: str = None, stop_event=None, external_terminal: bool = False): + def __init__(self, provider: str, model: str, save_conversation: bool = False, thinking: bool = True, frontend_callback=None, text_callback=None, web_callback=None, shell_callback=None, cli_callback=None, tool_callback=None, token_callback=None, api_key: str = None, stop_event=None, external_terminal: bool = False, prior_history: Optional[dict] = None): """Initialize the Agent Service""" # Clean up scratchpad for a fresh start _cleanup_scratchpad() @@ -107,6 +107,18 @@ def __init__(self, provider: str, model: str, save_conversation: bool = False, t # Store CLI callback for streaming CLI agent subprocess output to the frontend self.cli_callback = cli_callback + # Store tool-flow callback for the bottom "Tool response" chain animation. + # Signature: tool_callback(event: str, payload: dict|None) + # events: run_start | turn{hasImage} | received{tools:[{name,clicks?}]} | run_end + # The per-turn tools come straight from the parsed action block (this driver), + # so the controller needs no per-action plumbing. + self.tool_callback = tool_callback + + # Memory bar: called after each LLM call with the provider's exact token + # usage (llm_manager.last_usage). The agent only forwards — accumulation + + # persistence live in app.py / memory_compression. + self.token_callback = token_callback + # Initialize Controller with provider and actual API model name (pass api_key for CLI agent subprocess) self.controller = ControllerView(provider=provider, model=self.llm_manager.get_model_name(), web_callback=web_callback, shell_callback=shell_callback, cli_callback=cli_callback, api_key=api_key, stop_event=stop_event, external_terminal=external_terminal) @@ -149,7 +161,24 @@ def __init__(self, provider: str, model: str, save_conversation: bool = False, t # Start fresh each session self.interaction_count = 0 - + + # Resumable chat memory (UI path only). prior_history is an optimized + # snapshot from a PRIOR run of this session, built + loaded by + # Auto_Use.agent_conversation.service (ALL memory management lives there, + # not here). When present, process_request seeds the conversation lists + # from it so the agent "remembers" earlier turns. Scratchpad/todo are + # still wiped above — continuity comes ONLY from this memory, not files. + # The final lists are exposed on self so the conversation service can + # persist them after the run. + self.prior_history = prior_history if isinstance(prior_history, dict) else None + self.assistant_messages = [] + self.tool_responses = [] + # The exact final messages payload sent to the model (system + interleaved + # history + the live user message that re-injects user_request/todo/ + # scratchpad/element_tree each step). Captured for the debug memory log so + # the download is the TRUE conversation, not a reconstruction. + self.last_messages = None + def _load_system_prompt(self) -> str: """Load the system prompt from system_prompt.md file""" try: @@ -337,6 +366,29 @@ def compact(entry): return res return compact(res) + def _backfill_web_findings(self, tool_response_str: str, findings: str) -> str: + """Replace a web step's 'refer to scratchpad' placeholder with the actual + numbered findings the agent distilled into the scratchpad on the digest + step. The scratchpad is wiped on the next user request, so this keeps the + distilled web info in durable conversation memory. Preserves the + surrounding shape (a single web result or a 'multiple' results list).""" + try: + data = json.loads(tool_response_str) + except Exception: + return tool_response_str + + def fill(entry): + if isinstance(entry, dict) and entry.get("tool") == "web": + entry.pop("message", None) + entry["web_result"] = findings + return entry + + if isinstance(data, dict) and data.get("action") == "multiple" and "results" in data: + data["results"] = [fill(r) for r in data["results"]] + else: + data = fill(data) + return json.dumps(data, indent=2, ensure_ascii=False) + def _trim_history_entry(self, entry: str) -> str: """Trim an OLDER history step for context: drop 'thinking', 'eval', and 'action' (keep decision/memory/next_goal). Only the most recent step is kept @@ -353,6 +405,15 @@ def _trim_history_entry(self, entry: str) -> str: except Exception: return entry + def _emit_flow(self, event, payload=None): + """Push a tool-flow event to the frontend bottom chain (best-effort).""" + if not self.tool_callback: + return + try: + self.tool_callback(event, payload) + except Exception: + pass + def process_request(self, task: str) -> dict: """Process a user request in an iterative loop until completion. @@ -371,16 +432,39 @@ def process_request(self, task: str) -> dict: tool_responses = [] # Per-step tool result, aligned 1:1 with assistant_messages; replayed as user turns cli_await_result = None # Stores cli_await result for next iteration's light message pending_web_response = None # Stores web tool response for light digest iteration + web_memory_index = None # tool_responses slot of the web step to backfill with the digest's scratchpad + is_web_digest = False # True only on the iteration that digests a web result into the scratchpad json_fail_count = 0 # Track consecutive JSON parse failures (max 3 before exit) # Final outcome reported to the caller. Defaults to "incomplete" so any # loop exit that doesn't explicitly set success/error is reported as # not-finished rather than silently looking like a completion. final_status = "incomplete" final_message = "Agent stopped before completing the task" - + + # ---- Resumable memory seed (UI continuation) ------------------------- + # If a prior optimized snapshot was supplied (continuing a saved chat), + # replay it into the two memory lists so the existing message-build loop + # re-emits the earlier conversation. Entries are ALREADY trimmed and + # tool_responses already compacted, so no re-processing. The saved memory + # ends with a terminal-note step, so on resume that note is the most-recent + # entry — which makes the loop replay EVERY real step's tool_response. + is_resumed = False + if self.prior_history: + seeded = self.prior_history.get("assistant_messages") or [] + if seeded: + assistant_messages = list(seeded) + tool_responses = list(self.prior_history.get("tool_responses") or [])[:len(assistant_messages)] + tool_responses += [None] * (len(assistant_messages) - len(tool_responses)) + is_resumed = True + is_first_iteration = False # continuation, not a fresh dialogue + last_response = self.prior_history.get("done_message") or "Resuming previous session." + print(f"🧠 Resumed memory: {len(assistant_messages)} prior step(s) loaded") + # ---------------------------------------------------------------------- + # Print model info once at the start print(f"\n🔄 Processing with {self.llm_manager.get_model_name()}") - + self._emit_flow("run_start") # tool-flow chain: clear + take over the demo + # Main agent loop while True: # Check for stop signal @@ -392,13 +476,18 @@ def process_request(self, task: str) -> dict: break step_number += 1 - + is_web_digest = False # set True below only when this iteration digests a web result + # Max step limit - prevent infinite loops if step_number > 100: print("\n🛑 Max step limit reached (100). Exiting agent loop.") final_status, final_message = "incomplete", "Reached the 100-step limit without finishing" break + # Tool-flow chain: announce the turn. Digest iterations send no image, + # so the chain skips screenshot+mapping (true to the agent's behaviour). + self._emit_flow("turn", {"hasImage": not (cli_await_result or pending_web_response)}) + # Skip scan for light digest iterations (CLI await or web tool response) if cli_await_result or pending_web_response: print(f"\n{'='*60}") @@ -514,10 +603,14 @@ def process_request(self, task: str) -> dict: except: pass - # Subsequent iterations - include last_response, todo_list - user_message = f""" + # Subsequent iterations - include last_response, todo_list. + # On a RESUMED session, mark the request as + # so the agent knows this is a continuation of the same session + # (its prior steps are already in the replayed history above). + request_tag = "updated_user_request" if is_resumed else "user_request" + user_message = f"""<{request_tag}> {task} - + {last_response} @@ -593,9 +686,11 @@ def process_request(self, task: str) -> dict: image_sent = False annotated_image_base64 = None - - # Clear flag after consumption + + # Clear flag after consumption; mark this as the web-digest step so its + # scratchpad response can be folded into the web memory below. pending_web_response = None + is_web_digest = True else: # Normal iteration - include full context cli_status = self.controller.get_cli_agent_status() @@ -674,9 +769,15 @@ def process_request(self, task: str) -> dict: if tr: messages.append({"role": "user", "content": f"\n{tr}\n"}) - # Current user message (live screen + most recent ) + # Current user message (live screen + most recent ). + # The system prompt is added exactly once (above) — the resumable seed + # never carries a system prompt, so a continued session can't double it. messages.append({"role": "user", "content": user_message}) + # Capture the exact payload for the debug memory log (the last + # iteration's value is retained — i.e. the agent's final true memory). + self.last_messages = messages + # Apply prompt caching for OpenRouter/Anthropic (explicit cache_control). # messages[-1] = current user message (dynamic, never cache) # messages[-2] = most recent assistant step (full now -> trimmed next step) @@ -706,7 +807,14 @@ def process_request(self, task: str) -> dict: # Get raw response from LLM - pass annotated image raw_response = self.llm_manager.send_request(messages, annotated_image_base64) - + + # Memory bar: forward this call's exact token usage (input+output). + if self.token_callback: + try: + self.token_callback(self.llm_manager.last_usage) + except Exception: + pass + # CRITICAL Check Stop AFTER LLM (discards result if stopped while waiting) if self.stop_event and self.stop_event.is_set(): print("\n🛑 Agent stopped by user (response discarded).") @@ -715,7 +823,7 @@ def process_request(self, task: str) -> dict: break print("✓ LLM response received") - + # Save raw response before any parsing (for debugging) self._save_raw_response(raw_response, step_number) @@ -759,7 +867,11 @@ def process_request(self, task: str) -> dict: # Send to frontend if callback exists (omit action from stream) if self.text_callback: self.text_callback(AgentResponseFormatter.format_response(normalized_json, include_action=False)) - + + # Tool-flow chain: the packet arrived — tick "packet received" and hand + # the frontend this turn's tools, read straight from the action block. + self._emit_flow("received", {"tools": AgentResponseFormatter.extract_tools(normalized_json)}) + # Parse the normalized JSON to extract fields and check for done try: agent_response = json.loads(normalized_json) @@ -808,6 +920,9 @@ def process_request(self, task: str) -> dict: if web_results_list: pending_web_response = "\n" + "\n".join(web_results_list) + "\n" + # Remember this web step's memory slot so the NEXT (digest) + # iteration can backfill it with the scratchpad findings. + web_memory_index = len(tool_responses) - 1 print(f"🌐 Web results captured for digest iteration") else: pending_web_response = None @@ -834,6 +949,26 @@ def process_request(self, task: str) -> dict: indent=2, ensure_ascii=False ) + # Web-result memory: on the digest step, fold the numbered + # scratchpad findings the agent JUST wrote into the original web + # step's tool_response — replacing the 'refer to scratchpad' + # pointer. The scratchpad is wiped on the next user request, so + # this keeps the distilled web info in durable conversation memory. + if (is_web_digest and web_memory_index is not None + and 0 <= web_memory_index < len(tool_responses) + and tool_responses[web_memory_index]): + entries = [ + str(a.get("value", "")).strip() + for a in (agent_response.get("action") or []) + if isinstance(a, dict) and a.get("type") == "scratchpad" and a.get("value") + ] + if entries: + numbered = "\n".join(f"{i + 1}. {e}" for i, e in enumerate(entries)) + tool_responses[web_memory_index] = self._backfill_web_findings( + tool_responses[web_memory_index], numbered + ) + web_memory_index = None + # Wait before next scan (default 3 seconds, unless wait action was used) wait_time = 3.0 # Default wait if action_result.get("tool") == "wait": @@ -888,4 +1023,17 @@ def process_request(self, task: str) -> dict: # Cleanup: Stop CLI agent subprocess if running self.controller.stop_cli_agent() + # tool-flow chain: if the run didn't finish cleanly, cap it with a "!" drop. + if final_status == "error": + self._emit_flow("error", {"text": "llm service not responding"}) + elif final_status == "incomplete" and final_message and "stop" in final_message.lower(): + self._emit_flow("error", {"text": "agent interrupted"}) + self._emit_flow("run_end") # tool-flow chain: run finished + + # Expose the final conversation lists so the conversation service (NOT the + # agent) can optimize + persist them after the run. Memory management is + # kept entirely out of the agent. Return shape is unchanged. + self.assistant_messages = assistant_messages + self.tool_responses = tool_responses + return {"status": final_status, "message": final_message} \ No newline at end of file diff --git a/Auto_Use/macOS_use/agent/main_driver/system_prompt.md b/Auto_Use/macOS_use/agent/main_driver/system_prompt.md index 853d3d9..813c571 100644 --- a/Auto_Use/macOS_use/agent/main_driver/system_prompt.md +++ b/Auto_Use/macOS_use/agent/main_driver/system_prompt.md @@ -19,6 +19,7 @@ Core strengths: 2. Ignore grammar or spelling mistakes and focus on what the user wants to do. 3. This is the ultimate objective that must be completed. 4. Use to turn the user_request into a clear objective and tasks. +5. On a resumed session you instead receive : the same session continued — your prior steps are already in ; treat it as the current request and pick up from where you left off. 1. Using your vision capability, understand the images provided to you at each iteration and perform actions to complete the Objective using . diff --git a/Auto_Use/macOS_use/agent/main_driver/view.py b/Auto_Use/macOS_use/agent/main_driver/view.py index 75f1138..6871fb8 100644 --- a/Auto_Use/macOS_use/agent/main_driver/view.py +++ b/Auto_Use/macOS_use/agent/main_driver/view.py @@ -204,7 +204,35 @@ def normalize_response(raw_response: str) -> tuple: except Exception as e: # If any error occurs, return failure with raw response return (False, None, raw_response) - + + @staticmethod + def extract_tools(normalized_response: str) -> list: + """Pull this turn's tools from the parsed action block, in execution order, + for the frontend tool-flow chain. e.g. + [{"name": "left_click", "clicks": 2}, {"name": "input"}, {"name": "web"}] + """ + tools = [] + try: + data = json.loads(normalized_response) + actions = data.get("action", []) + if isinstance(actions, dict): + actions = [actions] + for item in actions: + if not isinstance(item, dict): + continue + name = item.get("type") + if not name: + continue + tool = {"name": name} + if "clicks" in item: + tool["clicks"] = item.get("clicks") + if "direction" in item: + tool["direction"] = item.get("direction") + tools.append(tool) + except Exception: + pass + return tools + @staticmethod def format_response(normalized_response: str, include_action: bool = False) -> str: """Format normalized JSON response into readable terminal output with emojis. diff --git a/Auto_Use/macOS_use/agent/minions/service.py b/Auto_Use/macOS_use/agent/minions/service.py index 50ea67e..0720a48 100644 --- a/Auto_Use/macOS_use/agent/minions/service.py +++ b/Auto_Use/macOS_use/agent/minions/service.py @@ -245,7 +245,9 @@ def _run_agent_loop(self, task: str) -> str: return "Max iterations reached" - safe_print(f"minion running step {step_number}") + # Step heartbeat → debug log only (NOT the UI). The row shows ONLY the minion's + # own validated output; this "running step N" marker is ours, not the model's. + debug_log(f"minion running step {step_number}") agent_sitting = self._get_agent_sitting() @@ -303,9 +305,6 @@ def _run_agent_loop(self, task: str) -> str: try: raw_response = self.llm.send_request(messages) - if raw_response: - safe_print(raw_response) - if self.stop_event and self.stop_event.is_set(): break @@ -314,8 +313,9 @@ def _run_agent_loop(self, task: str) -> str: success, normalized, failed_raw = self.formatter.normalize_response(raw_response) if not success: + # Emit nothing to the UI on failure — raw/partial/malformed JSON would + # glitch the streaming card; raw response already saved above for debugging. json_fail_count += 1 - safe_print(f"⚠️ JSON parse failed ({json_fail_count}/3). Discarding and retrying...") if json_fail_count >= 3: break @@ -325,6 +325,11 @@ def _run_agent_loop(self, task: str) -> str: json_fail_count = 0 + # Stream the validated, complete response (action included) to the minion row. + # The frontend parses the `action` array out of THIS JSON to drive the row's + # own action sub-chain — no extra backend events needed. + safe_print(self.formatter.format_stream_json(normalized)) + self._save_conversation_snapshot(messages, normalized, step_number) normalized_without_thinking = self._remove_thinking_from_response(normalized) diff --git a/Auto_Use/macOS_use/agent/minions/view.py b/Auto_Use/macOS_use/agent/minions/view.py index 4e2296f..6785fff 100644 --- a/Auto_Use/macOS_use/agent/minions/view.py +++ b/Auto_Use/macOS_use/agent/minions/view.py @@ -126,3 +126,17 @@ def get_action_block(normalized_response: str) -> list: except Exception: pass return [] + + @staticmethod + def format_stream_json(normalized_response: str) -> str: + """Per-step payload for the streaming UI card. For now it returns the COMPLETE + validated JSON (action INCLUDED). All display shaping is centralized here, so + changing what the card shows later — e.g. splitting `action` into its own section + (see get_action_block) — is a one-spot edit. On any parse error, fall back to the + input unchanged. + """ + try: + json_data = json.loads(normalized_response) + return json.dumps(json_data, indent=2, ensure_ascii=False) + except Exception: + return normalized_response diff --git a/Auto_Use/macOS_use/llm_provider/anthropic/service.py b/Auto_Use/macOS_use/llm_provider/anthropic/service.py index fd149e9..f3f99e8 100644 --- a/Auto_Use/macOS_use/llm_provider/anthropic/service.py +++ b/Auto_Use/macOS_use/llm_provider/anthropic/service.py @@ -143,13 +143,14 @@ def send_request(self, messages: list, model: str, annotated_screenshot_base64: if block.get("type") == "text": text_content = block.get("text", "") break - + return { "choices": [{ "message": { "content": text_content } - }] + }], + "usage": result.get("usage", {}), } except requests.exceptions.RequestException as e: diff --git a/Auto_Use/macOS_use/llm_provider/google/service.py b/Auto_Use/macOS_use/llm_provider/google/service.py index b26b3d2..f50c1dc 100644 --- a/Auto_Use/macOS_use/llm_provider/google/service.py +++ b/Auto_Use/macOS_use/llm_provider/google/service.py @@ -128,12 +128,17 @@ def send_request(self, messages: list, model: str, annotated_screenshot_base64: ) # Normalize to OpenAI-style format + um = getattr(response, "usage_metadata", None) return { "choices": [{ "message": { "content": response.text } - }] + }], + "usage": { + "input_tokens": getattr(um, "prompt_token_count", 0) or 0, + "output_tokens": getattr(um, "candidates_token_count", 0) or 0, + } if um else {}, } except Exception as e: raise Exception(f"Google Gemini API request failed: {str(e)}") \ No newline at end of file diff --git a/Auto_Use/macOS_use/llm_provider/llm_manager.py b/Auto_Use/macOS_use/llm_provider/llm_manager.py index 7d7b292..e7e8a08 100644 --- a/Auto_Use/macOS_use/llm_provider/llm_manager.py +++ b/Auto_Use/macOS_use/llm_provider/llm_manager.py @@ -365,6 +365,10 @@ def __init__(self, provider: str, model: str, thinking: bool = True, api_key: st else: self.schema = AGENT_OUTPUT_SCHEMA + # Most recent send_request's normalized token usage (input/output/total). + # Captured as a side effect so callers (e.g. the memory bar) can read it + # without changing send_request's return shape. + self.last_usage = {} self.provider_instance = self._initialize_provider() def _initialize_provider(self): @@ -439,6 +443,34 @@ def _initialize_provider(self): else: raise ValueError(f"Unsupported provider: {self.provider}") + def _normalize_usage(self, u): + """Normalize a provider usage dict to {input_tokens, output_tokens, + total_tokens, context_tokens}, tolerating both key styles (Anthropic-style + input/output and OpenAI-style prompt/completion). Empty/missing -> zeros. + + context_tokens is the TRUE size of the prompt actually sent this turn — the + memory-bar number. A cached token still occupies the context window, so we + add the cache classes back: input_tokens + cache_read + cache_creation. + This is exact for every provider: + - Anthropic: input_tokens EXCLUDES cache, so the cache fields are added. + - OpenAI/Google/Perplexity/OpenRouter/Groq: prompt_tokens already INCLUDES + cached tokens and the Anthropic cache keys are absent (0), so this + collapses to the full prompt count — no double-count. + """ + u = u or {} + inp = int(u.get("input_tokens", u.get("prompt_tokens", 0)) or 0) + out = int(u.get("output_tokens", u.get("completion_tokens", 0)) or 0) + tot = int(u.get("total_tokens", 0) or 0) or (inp + out) + cache_read = int(u.get("cache_read_input_tokens", 0) or 0) + cache_create = int(u.get("cache_creation_input_tokens", 0) or 0) + context_tokens = inp + cache_read + cache_create + return { + "input_tokens": inp, + "output_tokens": out, + "total_tokens": tot, + "context_tokens": context_tokens, + } + def send_request(self, messages: list, annotated_screenshot_base64: Optional[str] = None): """Send request to the selected provider with idempotent retries.""" last_error = None @@ -451,6 +483,7 @@ def send_request(self, messages: list, annotated_screenshot_base64: Optional[str response = self.provider_instance.send_request( attempt_messages, self.model, annotated_screenshot_base64 ) + self.last_usage = self._normalize_usage(response.get("usage")) return response['choices'][0]['message']['content'] except Exception as e: last_error = e @@ -493,6 +526,7 @@ def send_request(self, messages: list, annotated_screenshot_base64: Optional[str response = self.provider_instance.send_request( copy.deepcopy(messages), self.model, annotated_screenshot_base64 ) + self.last_usage = self._normalize_usage(response.get("usage")) return response['choices'][0]['message']['content'] except Exception as fallback_e: print(f"❌ CLI Agent: Fallback {self.display_name} also failed: {fallback_e}") diff --git a/Auto_Use/macOS_use/llm_provider/openai/service.py b/Auto_Use/macOS_use/llm_provider/openai/service.py index 1931d83..e372309 100644 --- a/Auto_Use/macOS_use/llm_provider/openai/service.py +++ b/Auto_Use/macOS_use/llm_provider/openai/service.py @@ -92,14 +92,20 @@ def send_request(self, messages: list, model: str, annotated_screenshot_base64: try: response = self.client.chat.completions.create(**params) - + # Return in the same format as other providers + usage = getattr(response, "usage", None) return { "choices": [{ "message": { "content": response.choices[0].message.content } - }] + }], + "usage": { + "input_tokens": getattr(usage, "prompt_tokens", 0) or 0, + "output_tokens": getattr(usage, "completion_tokens", 0) or 0, + "total_tokens": getattr(usage, "total_tokens", 0) or 0, + } if usage else {}, } except Exception as e: error_msg = f"OpenAI API request failed: {str(e)}" diff --git a/Auto_Use/macOS_use/llm_provider/perplexity/service.py b/Auto_Use/macOS_use/llm_provider/perplexity/service.py index e49db4c..692d7ad 100644 --- a/Auto_Use/macOS_use/llm_provider/perplexity/service.py +++ b/Auto_Use/macOS_use/llm_provider/perplexity/service.py @@ -111,13 +111,14 @@ def send_request(self, messages: list, model: str, annotated_screenshot_base64: break if text_content: break - + return { "choices": [{ "message": { "content": text_content } - }] + }], + "usage": result.get("usage", {}), } except requests.exceptions.RequestException as e: diff --git a/Auto_Use/macOS_use/remote_connection/banner.py b/Auto_Use/macOS_use/remote_connection/banner.py index b25921b..96851fb 100644 --- a/Auto_Use/macOS_use/remote_connection/banner.py +++ b/Auto_Use/macOS_use/remote_connection/banner.py @@ -175,7 +175,7 @@
-
Starting… diff --git a/Auto_Use/macOS_use/tree/element.py b/Auto_Use/macOS_use/tree/element.py index 8af9a88..b17fee1 100644 --- a/Auto_Use/macOS_use/tree/element.py +++ b/Auto_Use/macOS_use/tree/element.py @@ -100,7 +100,7 @@ "AXGroup": { "track": True, "is_enabled_flag": False, - "fallback": ["AXTitle", "AXDescription", "AXRoleDescription"], + "fallback": ["AXTitle", "AXDescription", "_title_ui_element", "AXRoleDescription"], }, "AXRadioButton": { "track": True, @@ -452,6 +452,18 @@ def _wait_for_ax_web_content(app_ax, window, timeout=AX_TREE_READY_TIMEOUT): def build_label(element, cfg): """Try each fallback attribute, return first non-empty, non-generic string.""" for attr in cfg.get("fallback", []): + if attr == "_title_ui_element": + linked = ax_attr(element, "AXTitleUIElement") + if linked: + for sub in ("AXValue", "AXTitle", "AXDescription"): + v = ax_attr(linked, sub) + if v: + label = str(v).replace("\n", " ").strip() + if label.lower() in GENERIC_LABELS: + continue + return label[:50] if len(label) > 50 else label + continue + if attr == "_children_text": children = ax_attr(element, "AXChildren") if children: @@ -670,6 +682,12 @@ def walk(element, results, depth, screen, clip=None, parent_frame=None, my_entry = (my_frame, role_str) if my_frame else None child_ancestors = ancestors + [my_entry] if my_entry else ancestors children = ax_attr(element, "AXChildren") + if not children: + # NSBrowser column views (e.g. the Open/Save file dialog) expose their + # column scroll areas with an EMPTY AXChildren — the actual content + # (an AXList of file rows) is reachable only via AXContents. Fall back + # so these lazy containers get traversed instead of dead-ending here. + children = ax_attr(element, "AXContents") if children: try: for child in children: diff --git a/Auto_Use/memory_compression/__init__.py b/Auto_Use/memory_compression/__init__.py new file mode 100644 index 0000000..f391105 --- /dev/null +++ b/Auto_Use/memory_compression/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2026 Autouse AI — https://github.com/auto-use/Auto-Use +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# If you build on this project, please keep this header and credit +# Autouse AI (https://github.com/auto-use/Auto-Use) in forks and derivative works. +# A small attribution goes a long way toward a healthy open-source +# community — thank you for contributing. + +"""Memory compression / accounting for the agent. + +For now this package hosts MemoryTracker, which measures the MAIN agent's current +context-window fullness (the live "memory bar" gauge) — the exact size of the +latest prompt sent, including cached tokens. Memory-compression logic that keeps +that fullness down will live here too. +""" + +from .memory_tracker import MemoryTracker + +__all__ = ["MemoryTracker"] diff --git a/Auto_Use/memory_compression/memory_tracker.py b/Auto_Use/memory_compression/memory_tracker.py new file mode 100644 index 0000000..e8cbda7 --- /dev/null +++ b/Auto_Use/memory_compression/memory_tracker.py @@ -0,0 +1,85 @@ +# Copyright 2026 Autouse AI — https://github.com/auto-use/Auto-Use +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# If you build on this project, please keep this header and credit +# Autouse AI (https://github.com/auto-use/Auto-Use) in forks and derivative works. +# A small attribution goes a long way toward a healthy open-source +# community — thank you for contributing. + +"""MemoryTracker — current memory fullness for the MAIN agent's bar. + +The bar answers "how full is the MAIN agent's working memory right now?", NOT +"how many tokens have we burned in total". Every turn the agent rebuilds and +re-sends its whole (trimmed) context, and the provider reports the true size of +that exact prompt. We show that latest size against a FIXED budget — MEMORY_CAP +(300k) — so the bar rises as history grows and DROPS when the runtime memory +optimization (stripping old images / element trees) kicks in. + +The cap is fixed on purpose, NOT the model's context window: a future +memory-compression system will keep the agent's real context under this budget, +so 300k is the headroom gauge for it. + +`context_tokens` is the full prompt size INCLUDING cached tokens — a cache-read +token still occupies memory, so it counts whether it was cached or not (see +LLMManager._normalize_usage). Only the main agent is tracked; CLI / coder / +minion sub-agents run on their own LLMManager and never feed this bar. On reopen +the caller seeds `initial_context` from the chat's last saved value. +""" + +# Fixed memory budget the bar fills toward (headroom for future compression). +MEMORY_CAP = 300_000 + + +class MemoryTracker: + """Holds the latest context size for a single chat (overwrite, never sum). + + PURELY COSMETIC. This is a display gauge ONLY — it never gates the agent. The + agent does not stop at the cap (or at 1M); `self.current` keeps the REAL, + unclamped token count, and only the bar's `pct` saturates at 100% so a full + bar reads "memory full" while the agent keeps running. (A future, user-driven + compaction system is what will actually manage memory.) Do NOT wire any agent + stop/limit to this class.""" + + def __init__(self, cap: int = MEMORY_CAP, initial_context: int = 0): + self.cap = int(cap or MEMORY_CAP) + self.current = int(initial_context or 0) + self.last_input = 0 + self.last_output = 0 + + def record(self, usage: dict) -> dict: + """Set the bar to this LLM call's full prompt size. `usage` is the + normalized {input_tokens, output_tokens, context_tokens, ...} from + llm_manager. A turn that reports 0 context (e.g. a failed call left + last_usage stale) is ignored so the bar holds its last good value + instead of collapsing to empty. Returns the current payload.""" + usage = usage or {} + ctx = int(usage.get("context_tokens", 0) or 0) + self.last_input = int(usage.get("input_tokens", 0) or 0) + self.last_output = int(usage.get("output_tokens", 0) or 0) + if ctx > 0: + self.current = ctx + return self.payload() + + def payload(self) -> dict: + """The frontend-facing snapshot: current size, cap, and % (clamped). + `used` is the real count (may exceed `cap`); `pct` clamps to 100 for the + bar height only — it is not a limit the agent obeys.""" + pct = min(100.0, (self.current / self.cap) * 100) if self.cap else 0.0 + return { + "used": self.current, + "cap": self.cap, + "pct": pct, + "last_input": self.last_input, + "last_output": self.last_output, + } diff --git a/Auto_Use/windows_use/agent/coder/service.py b/Auto_Use/windows_use/agent/coder/service.py index e2854b5..c65fdad 100644 --- a/Auto_Use/windows_use/agent/coder/service.py +++ b/Auto_Use/windows_use/agent/coder/service.py @@ -282,7 +282,9 @@ def _run_agent_loop(self, task: str) -> str: return "Max iterations reached" - safe_print(f"cli agent running step {step_number}") + # Step heartbeat → debug log only (NOT the UI). The card shows ONLY the agent's + # own validated output; this "running step N" marker is ours, not the model's. + debug_log(f"cli agent running step {step_number}") # Get agent sitting info agent_sitting = self._get_agent_sitting() @@ -353,12 +355,6 @@ def _run_agent_loop(self, task: str) -> str: # Call LLM raw_response = self.llm.send_request(messages) - # Stream the raw LLM response to stdout. The parent main agent's - # subprocess pipe reader picks it up and renders it line-by-line - # in the CLI streaming pill UI (no decoration, just raw text). - if raw_response: - safe_print(raw_response) - # Check stop after LLM if self.stop_event and self.stop_event.is_set(): break @@ -369,10 +365,11 @@ def _run_agent_loop(self, task: str) -> str: # Validate and normalize response success, normalized, failed_raw = CLIAgentResponseFormatter.normalize_response(raw_response) - # If JSON parse failed, discard response and retry with fresh context + # If JSON parse failed, discard response and retry with fresh context. + # Emit NOTHING to the UI on failure — raw/partial/malformed JSON would glitch + # the streaming card; the raw response is already saved above for debugging. if not success: json_fail_count += 1 - safe_print(f"⚠️ JSON parse failed ({json_fail_count}/3). Discarding and retrying...") if json_fail_count >= 3: break @@ -383,6 +380,11 @@ def _run_agent_loop(self, task: str) -> str: # Reset consecutive JSON fail counter on success json_fail_count = 0 + # Stream the validated, complete response (action included) to the card's + # `>` line. The frontend parses the `action` array out of THIS JSON to drive + # the action-icon chain + scratchpad — no extra backend events needed. + safe_print(CLIAgentResponseFormatter.format_stream_json(normalized)) + # Save TRUE agent memory snapshot self._save_conversation_snapshot(messages, normalized, step_number) diff --git a/Auto_Use/windows_use/agent/coder/view.py b/Auto_Use/windows_use/agent/coder/view.py index d28fb0b..3f34466 100644 --- a/Auto_Use/windows_use/agent/coder/view.py +++ b/Auto_Use/windows_use/agent/coder/view.py @@ -141,4 +141,17 @@ def get_action_block(normalized_response: str) -> list: return json_data.get("action", []) except: pass - return [] \ No newline at end of file + return [] + + @staticmethod + def format_stream_json(normalized_response: str) -> str: + """Per-step payload for the streaming UI terminal — ONLY the action block (a real + terminal shows what's being executed, not thinking/goal/memory). The frontend reads + `action` from this for both the `>` stream and the tool-icon chain. On parse error, + fall back to the input unchanged. + """ + try: + json_data = json.loads(normalized_response) + return json.dumps({"action": json_data.get("action", [])}, indent=2, ensure_ascii=False) + except Exception: + return normalized_response \ No newline at end of file diff --git a/Auto_Use/windows_use/agent/main_driver/service.py b/Auto_Use/windows_use/agent/main_driver/service.py index 9323400..072be7c 100644 --- a/Auto_Use/windows_use/agent/main_driver/service.py +++ b/Auto_Use/windows_use/agent/main_driver/service.py @@ -76,7 +76,7 @@ def _cleanup_scratchpad(): class AgentService: """Service for Windows automation agent""" - def __init__(self, provider: str, model: str, save_conversation: bool = False, thinking: bool = True, frontend_callback=None, text_callback=None, web_callback=None, shell_callback=None, cli_callback=None, api_key: str = None, stop_event=None, external_terminal: bool = False): + def __init__(self, provider: str, model: str, save_conversation: bool = False, thinking: bool = True, frontend_callback=None, text_callback=None, web_callback=None, shell_callback=None, cli_callback=None, tool_callback=None, token_callback=None, api_key: str = None, stop_event=None, external_terminal: bool = False, prior_history: Optional[dict] = None): """Initialize the Agent Service""" # Clean up scratchpad for a fresh start _cleanup_scratchpad() @@ -102,6 +102,18 @@ def __init__(self, provider: str, model: str, save_conversation: bool = False, t # Store CLI callback for streaming CLI agent subprocess output to the frontend self.cli_callback = cli_callback + # Store tool-flow callback for the bottom "Tool response" chain animation. + # Signature: tool_callback(event: str, payload: dict|None) + # events: run_start | turn{hasImage} | received{tools:[{name,clicks?}]} | run_end + # The per-turn tools come straight from the parsed action block (this driver), + # so the controller needs no per-action plumbing. + self.tool_callback = tool_callback + + # Memory bar: called after each LLM call with the provider's exact token + # usage (llm_manager.last_usage). The agent only forwards — accumulation + + # persistence live in app.py / memory_compression. + self.token_callback = token_callback + # Initialize Controller with provider and actual API model name (pass api_key for CLI agent subprocess) self.controller = ControllerView(provider=provider, model=self.llm_manager.get_model_name(), web_callback=web_callback, shell_callback=shell_callback, cli_callback=cli_callback, api_key=api_key, stop_event=stop_event, external_terminal=external_terminal) @@ -144,7 +156,24 @@ def __init__(self, provider: str, model: str, save_conversation: bool = False, t # Start fresh each session self.interaction_count = 0 - + + # Resumable chat memory (UI path only). prior_history is an optimized + # snapshot from a PRIOR run of this session, built + loaded by + # Auto_Use.agent_conversation.service (ALL memory management lives there, + # not here). When present, process_request seeds the conversation lists + # from it so the agent "remembers" earlier turns. Scratchpad/todo are + # still wiped above — continuity comes ONLY from this memory, not files. + # The final lists are exposed on self so the conversation service can + # persist them after the run. + self.prior_history = prior_history if isinstance(prior_history, dict) else None + self.assistant_messages = [] + self.tool_responses = [] + # The exact final messages payload sent to the model (system + interleaved + # history + the live user message that re-injects user_request/todo/ + # scratchpad/element_tree each step). Captured for the debug memory log so + # the download is the TRUE conversation, not a reconstruction. + self.last_messages = None + def _load_system_prompt(self) -> str: """Load the system prompt from system_prompt.md file""" try: @@ -332,6 +361,29 @@ def compact(entry): return res return compact(res) + def _backfill_web_findings(self, tool_response_str: str, findings: str) -> str: + """Replace a web step's 'refer to scratchpad' placeholder with the actual + numbered findings the agent distilled into the scratchpad on the digest + step. The scratchpad is wiped on the next user request, so this keeps the + distilled web info in durable conversation memory. Preserves the + surrounding shape (a single web result or a 'multiple' results list).""" + try: + data = json.loads(tool_response_str) + except Exception: + return tool_response_str + + def fill(entry): + if isinstance(entry, dict) and entry.get("tool") == "web": + entry.pop("message", None) + entry["web_result"] = findings + return entry + + if isinstance(data, dict) and data.get("action") == "multiple" and "results" in data: + data["results"] = [fill(r) for r in data["results"]] + else: + data = fill(data) + return json.dumps(data, indent=2, ensure_ascii=False) + def _trim_history_entry(self, entry: str) -> str: """Trim an OLDER history step for context: drop 'thinking', 'eval', and 'action' (keep decision/memory/next_goal). Only the most recent step is kept @@ -348,6 +400,15 @@ def _trim_history_entry(self, entry: str) -> str: except Exception: return entry + def _emit_flow(self, event, payload=None): + """Push a tool-flow event to the frontend bottom chain (best-effort).""" + if not self.tool_callback: + return + try: + self.tool_callback(event, payload) + except Exception: + pass + def process_request(self, task: str) -> dict: """Process a user request in an iterative loop until completion. @@ -366,16 +427,39 @@ def process_request(self, task: str) -> dict: tool_responses = [] # Per-step tool result, aligned 1:1 with assistant_messages; replayed as user turns cli_await_result = None # Stores cli_await result for next iteration's light message pending_web_response = None # Stores web tool response for light digest iteration + web_memory_index = None # tool_responses slot of the web step to backfill with the digest's scratchpad + is_web_digest = False # True only on the iteration that digests a web result into the scratchpad json_fail_count = 0 # Track consecutive JSON parse failures (max 3 before exit) # Final outcome reported to the caller. Defaults to "incomplete" so any # loop exit that doesn't explicitly set success/error is reported as # not-finished rather than silently looking like a completion. final_status = "incomplete" final_message = "Agent stopped before completing the task" - + + # ---- Resumable memory seed (UI continuation) ------------------------- + # If a prior optimized snapshot was supplied (continuing a saved chat), + # replay it into the two memory lists so the existing message-build loop + # re-emits the earlier conversation. Entries are ALREADY trimmed and + # tool_responses already compacted, so no re-processing. The saved memory + # ends with a terminal-note step, so on resume that note is the most-recent + # entry — which makes the loop replay EVERY real step's tool_response. + is_resumed = False + if self.prior_history: + seeded = self.prior_history.get("assistant_messages") or [] + if seeded: + assistant_messages = list(seeded) + tool_responses = list(self.prior_history.get("tool_responses") or [])[:len(assistant_messages)] + tool_responses += [None] * (len(assistant_messages) - len(tool_responses)) + is_resumed = True + is_first_iteration = False # continuation, not a fresh dialogue + last_response = self.prior_history.get("done_message") or "Resuming previous session." + print(f"🧠 Resumed memory: {len(assistant_messages)} prior step(s) loaded") + # ---------------------------------------------------------------------- + # Print model info once at the start print(f"\n🔄 Processing with {self.llm_manager.get_model_name()}") - + self._emit_flow("run_start") # tool-flow chain: clear + take over the demo + # Main agent loop while True: # Check for stop signal @@ -387,13 +471,18 @@ def process_request(self, task: str) -> dict: break step_number += 1 - + is_web_digest = False # set True below only when this iteration digests a web result + # Max step limit - prevent infinite loops if step_number > 100: print("\n🛑 Max step limit reached (100). Exiting agent loop.") final_status, final_message = "incomplete", "Reached the 100-step limit without finishing" break + # Tool-flow chain: announce the turn. Digest iterations send no image, + # so the chain skips screenshot+mapping (true to the agent's behaviour). + self._emit_flow("turn", {"hasImage": not (cli_await_result or pending_web_response)}) + # Skip scan for light digest iterations (CLI await or web tool response) if cli_await_result or pending_web_response: print(f"\n{'='*60}") @@ -509,10 +598,14 @@ def process_request(self, task: str) -> dict: except: pass - # Subsequent iterations - include last_response, todo_list - user_message = f""" + # Subsequent iterations - include last_response, todo_list. + # On a RESUMED session, mark the request as + # so the agent knows this is a continuation of the same session + # (its prior steps are already in the replayed history above). + request_tag = "updated_user_request" if is_resumed else "user_request" + user_message = f"""<{request_tag}> {task} - + {last_response} @@ -588,9 +681,11 @@ def process_request(self, task: str) -> dict: image_sent = False annotated_image_base64 = None - - # Clear flag after consumption + + # Clear flag after consumption; mark this as the web-digest step so its + # scratchpad response can be folded into the web memory below. pending_web_response = None + is_web_digest = True else: # Normal iteration - include full context cli_status = self.controller.get_cli_agent_status() @@ -669,9 +764,15 @@ def process_request(self, task: str) -> dict: if tr: messages.append({"role": "user", "content": f"\n{tr}\n"}) - # Current user message (live screen + most recent ) + # Current user message (live screen + most recent ). + # The system prompt is added exactly once (above) — the resumable seed + # never carries a system prompt, so a continued session can't double it. messages.append({"role": "user", "content": user_message}) + # Capture the exact payload for the debug memory log (the last + # iteration's value is retained — i.e. the agent's final true memory). + self.last_messages = messages + # Apply prompt caching for OpenRouter/Anthropic (explicit cache_control). # messages[-1] = current user message (dynamic, never cache) # messages[-2] = most recent assistant step (full now -> trimmed next step) @@ -701,7 +802,14 @@ def process_request(self, task: str) -> dict: # Get raw response from LLM - pass annotated image raw_response = self.llm_manager.send_request(messages, annotated_image_base64) - + + # Memory bar: forward this call's exact token usage (input+output). + if self.token_callback: + try: + self.token_callback(self.llm_manager.last_usage) + except Exception: + pass + # CRITICAL Check Stop AFTER LLM (discards result if stopped while waiting) if self.stop_event and self.stop_event.is_set(): print("\n🛑 Agent stopped by user (response discarded).") @@ -710,7 +818,7 @@ def process_request(self, task: str) -> dict: break print("✓ LLM response received") - + # Save raw response before any parsing (for debugging) self._save_raw_response(raw_response, step_number) @@ -754,7 +862,11 @@ def process_request(self, task: str) -> dict: # Send to frontend if callback exists (omit action from stream) if self.text_callback: self.text_callback(AgentResponseFormatter.format_response(normalized_json, include_action=False)) - + + # Tool-flow chain: the packet arrived — tick "packet received" and hand + # the frontend this turn's tools, read straight from the action block. + self._emit_flow("received", {"tools": AgentResponseFormatter.extract_tools(normalized_json)}) + # Parse the normalized JSON to extract fields and check for done try: agent_response = json.loads(normalized_json) @@ -803,6 +915,9 @@ def process_request(self, task: str) -> dict: if web_results_list: pending_web_response = "\n" + "\n".join(web_results_list) + "\n" + # Remember this web step's memory slot so the NEXT (digest) + # iteration can backfill it with the scratchpad findings. + web_memory_index = len(tool_responses) - 1 print(f"🌐 Web results captured for digest iteration") else: pending_web_response = None @@ -829,6 +944,26 @@ def process_request(self, task: str) -> dict: indent=2, ensure_ascii=False ) + # Web-result memory: on the digest step, fold the numbered + # scratchpad findings the agent JUST wrote into the original web + # step's tool_response — replacing the 'refer to scratchpad' + # pointer. The scratchpad is wiped on the next user request, so + # this keeps the distilled web info in durable conversation memory. + if (is_web_digest and web_memory_index is not None + and 0 <= web_memory_index < len(tool_responses) + and tool_responses[web_memory_index]): + entries = [ + str(a.get("value", "")).strip() + for a in (agent_response.get("action") or []) + if isinstance(a, dict) and a.get("type") == "scratchpad" and a.get("value") + ] + if entries: + numbered = "\n".join(f"{i + 1}. {e}" for i, e in enumerate(entries)) + tool_responses[web_memory_index] = self._backfill_web_findings( + tool_responses[web_memory_index], numbered + ) + web_memory_index = None + # Wait before next scan (default 3 seconds, unless wait action was used) wait_time = 3.0 # Default wait if action_result.get("tool") == "wait": @@ -883,4 +1018,17 @@ def process_request(self, task: str) -> dict: # Cleanup: Stop CLI agent subprocess if running self.controller.stop_cli_agent() + # tool-flow chain: if the run didn't finish cleanly, cap it with a "!" drop. + if final_status == "error": + self._emit_flow("error", {"text": "llm service not responding"}) + elif final_status == "incomplete" and final_message and "stop" in final_message.lower(): + self._emit_flow("error", {"text": "agent interrupted"}) + self._emit_flow("run_end") # tool-flow chain: run finished + + # Expose the final conversation lists so the conversation service (NOT the + # agent) can optimize + persist them after the run. Memory management is + # kept entirely out of the agent. Return shape is unchanged. + self.assistant_messages = assistant_messages + self.tool_responses = tool_responses + return {"status": final_status, "message": final_message} \ No newline at end of file diff --git a/Auto_Use/windows_use/agent/main_driver/system_prompt.md b/Auto_Use/windows_use/agent/main_driver/system_prompt.md index 3dcdb41..f7ddde9 100644 --- a/Auto_Use/windows_use/agent/main_driver/system_prompt.md +++ b/Auto_Use/windows_use/agent/main_driver/system_prompt.md @@ -19,6 +19,7 @@ Core strengths: 2. Ignore grammar or spelling mistakes and focus on what the user wants to do. 3. This is the ultimate objective that must be completed. 4. Use to turn the user_request into a clear objective and tasks. +5. On a resumed session you instead receive : the same session continued — your prior steps are already in ; treat it as the current request and pick up from where you left off. 1. Using your vision capability, understand the images provided to you at each iteration and perform actions to complete the Objective using . diff --git a/Auto_Use/windows_use/agent/main_driver/view.py b/Auto_Use/windows_use/agent/main_driver/view.py index 75f1138..6871fb8 100644 --- a/Auto_Use/windows_use/agent/main_driver/view.py +++ b/Auto_Use/windows_use/agent/main_driver/view.py @@ -204,7 +204,35 @@ def normalize_response(raw_response: str) -> tuple: except Exception as e: # If any error occurs, return failure with raw response return (False, None, raw_response) - + + @staticmethod + def extract_tools(normalized_response: str) -> list: + """Pull this turn's tools from the parsed action block, in execution order, + for the frontend tool-flow chain. e.g. + [{"name": "left_click", "clicks": 2}, {"name": "input"}, {"name": "web"}] + """ + tools = [] + try: + data = json.loads(normalized_response) + actions = data.get("action", []) + if isinstance(actions, dict): + actions = [actions] + for item in actions: + if not isinstance(item, dict): + continue + name = item.get("type") + if not name: + continue + tool = {"name": name} + if "clicks" in item: + tool["clicks"] = item.get("clicks") + if "direction" in item: + tool["direction"] = item.get("direction") + tools.append(tool) + except Exception: + pass + return tools + @staticmethod def format_response(normalized_response: str, include_action: bool = False) -> str: """Format normalized JSON response into readable terminal output with emojis. diff --git a/Auto_Use/windows_use/agent/minions/service.py b/Auto_Use/windows_use/agent/minions/service.py index 7e9d52d..3932575 100644 --- a/Auto_Use/windows_use/agent/minions/service.py +++ b/Auto_Use/windows_use/agent/minions/service.py @@ -255,7 +255,9 @@ def _run_agent_loop(self, task: str) -> str: return "Max iterations reached" - safe_print(f"minion running step {step_number}") + # Step heartbeat → debug log only (NOT the UI). The row shows ONLY the minion's + # own validated output; this "running step N" marker is ours, not the model's. + debug_log(f"minion running step {step_number}") agent_sitting = self._get_agent_sitting() @@ -315,9 +317,6 @@ def _run_agent_loop(self, task: str) -> str: try: raw_response = self.llm.send_request(messages) - if raw_response: - safe_print(raw_response) - if self.stop_event and self.stop_event.is_set(): break @@ -326,8 +325,9 @@ def _run_agent_loop(self, task: str) -> str: success, normalized, failed_raw = self.formatter.normalize_response(raw_response) if not success: + # Emit nothing to the UI on failure — raw/partial/malformed JSON would + # glitch the streaming card; raw response already saved above for debugging. json_fail_count += 1 - safe_print(f"⚠️ JSON parse failed ({json_fail_count}/3). Discarding and retrying...") if json_fail_count >= 3: break @@ -337,6 +337,11 @@ def _run_agent_loop(self, task: str) -> str: json_fail_count = 0 + # Stream the validated, complete response (action included) to the minion row. + # The frontend parses the `action` array out of THIS JSON to drive the row's + # own action sub-chain — no extra backend events needed. + safe_print(self.formatter.format_stream_json(normalized)) + self._save_conversation_snapshot(messages, normalized, step_number) normalized_without_thinking = self._remove_thinking_from_response(normalized) diff --git a/Auto_Use/windows_use/agent/minions/view.py b/Auto_Use/windows_use/agent/minions/view.py index 4e2296f..6785fff 100644 --- a/Auto_Use/windows_use/agent/minions/view.py +++ b/Auto_Use/windows_use/agent/minions/view.py @@ -126,3 +126,17 @@ def get_action_block(normalized_response: str) -> list: except Exception: pass return [] + + @staticmethod + def format_stream_json(normalized_response: str) -> str: + """Per-step payload for the streaming UI card. For now it returns the COMPLETE + validated JSON (action INCLUDED). All display shaping is centralized here, so + changing what the card shows later — e.g. splitting `action` into its own section + (see get_action_block) — is a one-spot edit. On any parse error, fall back to the + input unchanged. + """ + try: + json_data = json.loads(normalized_response) + return json.dumps(json_data, indent=2, ensure_ascii=False) + except Exception: + return normalized_response diff --git a/Auto_Use/windows_use/llm_provider/anthropic/service.py b/Auto_Use/windows_use/llm_provider/anthropic/service.py index fd149e9..f3f99e8 100644 --- a/Auto_Use/windows_use/llm_provider/anthropic/service.py +++ b/Auto_Use/windows_use/llm_provider/anthropic/service.py @@ -143,13 +143,14 @@ def send_request(self, messages: list, model: str, annotated_screenshot_base64: if block.get("type") == "text": text_content = block.get("text", "") break - + return { "choices": [{ "message": { "content": text_content } - }] + }], + "usage": result.get("usage", {}), } except requests.exceptions.RequestException as e: diff --git a/Auto_Use/windows_use/llm_provider/google/service.py b/Auto_Use/windows_use/llm_provider/google/service.py index b26b3d2..f50c1dc 100644 --- a/Auto_Use/windows_use/llm_provider/google/service.py +++ b/Auto_Use/windows_use/llm_provider/google/service.py @@ -128,12 +128,17 @@ def send_request(self, messages: list, model: str, annotated_screenshot_base64: ) # Normalize to OpenAI-style format + um = getattr(response, "usage_metadata", None) return { "choices": [{ "message": { "content": response.text } - }] + }], + "usage": { + "input_tokens": getattr(um, "prompt_token_count", 0) or 0, + "output_tokens": getattr(um, "candidates_token_count", 0) or 0, + } if um else {}, } except Exception as e: raise Exception(f"Google Gemini API request failed: {str(e)}") \ No newline at end of file diff --git a/Auto_Use/windows_use/llm_provider/llm_manager.py b/Auto_Use/windows_use/llm_provider/llm_manager.py index 852a1ef..147bb11 100644 --- a/Auto_Use/windows_use/llm_provider/llm_manager.py +++ b/Auto_Use/windows_use/llm_provider/llm_manager.py @@ -450,6 +450,10 @@ def __init__(self, provider: str, model: str, thinking: bool = True, api_key: st else: self.schema = AGENT_OUTPUT_SCHEMA + # Most recent send_request's normalized token usage (input/output/total). + # Captured as a side effect so callers (e.g. the memory bar) can read it + # without changing send_request's return shape. + self.last_usage = {} self.provider_instance = self._initialize_provider() def _initialize_provider(self): @@ -525,6 +529,34 @@ def _initialize_provider(self): else: raise ValueError(f"Unsupported provider: {self.provider}") + def _normalize_usage(self, u): + """Normalize a provider usage dict to {input_tokens, output_tokens, + total_tokens, context_tokens}, tolerating both key styles (Anthropic-style + input/output and OpenAI-style prompt/completion). Empty/missing -> zeros. + + context_tokens is the TRUE size of the prompt actually sent this turn — the + memory-bar number. A cached token still occupies the context window, so we + add the cache classes back: input_tokens + cache_read + cache_creation. + This is exact for every provider: + - Anthropic: input_tokens EXCLUDES cache, so the cache fields are added. + - OpenAI/Google/Perplexity/OpenRouter/Groq: prompt_tokens already INCLUDES + cached tokens and the Anthropic cache keys are absent (0), so this + collapses to the full prompt count — no double-count. + """ + u = u or {} + inp = int(u.get("input_tokens", u.get("prompt_tokens", 0)) or 0) + out = int(u.get("output_tokens", u.get("completion_tokens", 0)) or 0) + tot = int(u.get("total_tokens", 0) or 0) or (inp + out) + cache_read = int(u.get("cache_read_input_tokens", 0) or 0) + cache_create = int(u.get("cache_creation_input_tokens", 0) or 0) + context_tokens = inp + cache_read + cache_create + return { + "input_tokens": inp, + "output_tokens": out, + "total_tokens": tot, + "context_tokens": context_tokens, + } + def send_request(self, messages: list, annotated_screenshot_base64: Optional[str] = None): """Send request to the selected provider with idempotent retries.""" last_error = None @@ -537,6 +569,7 @@ def send_request(self, messages: list, annotated_screenshot_base64: Optional[str response = self.provider_instance.send_request( attempt_messages, self.model, annotated_screenshot_base64 ) + self.last_usage = self._normalize_usage(response.get("usage")) return response['choices'][0]['message']['content'] except Exception as e: last_error = e @@ -579,6 +612,7 @@ def send_request(self, messages: list, annotated_screenshot_base64: Optional[str response = self.provider_instance.send_request( copy.deepcopy(messages), self.model, annotated_screenshot_base64 ) + self.last_usage = self._normalize_usage(response.get("usage")) return response['choices'][0]['message']['content'] except Exception as fallback_e: print(f"❌ CLI Agent: Fallback {self.display_name} also failed: {fallback_e}") diff --git a/Auto_Use/windows_use/llm_provider/openai/service.py b/Auto_Use/windows_use/llm_provider/openai/service.py index 1931d83..e372309 100644 --- a/Auto_Use/windows_use/llm_provider/openai/service.py +++ b/Auto_Use/windows_use/llm_provider/openai/service.py @@ -92,14 +92,20 @@ def send_request(self, messages: list, model: str, annotated_screenshot_base64: try: response = self.client.chat.completions.create(**params) - + # Return in the same format as other providers + usage = getattr(response, "usage", None) return { "choices": [{ "message": { "content": response.choices[0].message.content } - }] + }], + "usage": { + "input_tokens": getattr(usage, "prompt_tokens", 0) or 0, + "output_tokens": getattr(usage, "completion_tokens", 0) or 0, + "total_tokens": getattr(usage, "total_tokens", 0) or 0, + } if usage else {}, } except Exception as e: error_msg = f"OpenAI API request failed: {str(e)}" diff --git a/Auto_Use/windows_use/llm_provider/perplexity/service.py b/Auto_Use/windows_use/llm_provider/perplexity/service.py index a7bd26b..6906751 100644 --- a/Auto_Use/windows_use/llm_provider/perplexity/service.py +++ b/Auto_Use/windows_use/llm_provider/perplexity/service.py @@ -112,13 +112,14 @@ def send_request(self, messages: list, model: str, annotated_screenshot_base64: break if text_content: break - + return { "choices": [{ "message": { "content": text_content } - }] + }], + "usage": result.get("usage", {}), } except requests.exceptions.RequestException as e: diff --git a/Auto_Use/windows_use/remote_connection/banner.py b/Auto_Use/windows_use/remote_connection/banner.py index 233035e..ea2833e 100644 --- a/Auto_Use/windows_use/remote_connection/banner.py +++ b/Auto_Use/windows_use/remote_connection/banner.py @@ -382,7 +382,7 @@ def _redact_secrets(text) -> str: