diff --git a/python/frameworks/mcp-use/CHANGELOG.md b/python/frameworks/mcp-use/CHANGELOG.md new file mode 100644 index 000000000..e82116599 --- /dev/null +++ b/python/frameworks/mcp-use/CHANGELOG.md @@ -0,0 +1,34 @@ +# Changelog + +## 0.1.0 + +- First release. `FutureAGICallback`, a LangChain callback handler for + `mcp_use.MCPAgent(callbacks=[...])`, records one `AGENT` span per agent + run (`run()`, `stream()` or `stream_events()`), one `LLM` span per model + call and one `TOOL` span per tool call. LLM and tool spans are children + of the agent span; LangGraph nodes and middleware are not spans. A + streamed completion is one LLM span with chunk events. Nothing is + patched: only agents given the callback are traced. +- Spans come from `FITracer`: `TraceConfig` and `using_session`, + `using_user`, `using_metadata` and `using_tags` apply. +- Content (prompts, messages, model output, tool arguments and results, + error messages) follows `TraceConfig` like every traceAI instrumentor: + recorded by default, hidden with `FI_HIDE_INPUTS` / `FI_HIDE_OUTPUTS` + (and the message/text variants) or `config=TraceConfig(...)`. There is no + package-specific switch. With either hide flag, error messages are + dropped (exception type only). Under `hide_outputs`, model replies and + tool results that `MCPAgent` memory or `external_history` sends back on a + later turn are outputs too. +- Never recorded: stack traces (no `exception.stacktrace`; the `stack` of + mcp-use's formatted tool error is cut from the tool message the model + gets), absolute local paths, and this machine's user and host name. +- Secrets (values passed in `redact=`, secret-named environment variables, + bearer tokens and key-shaped strings) are removed from everything written, + names and ids included. Recorded text is size-capped. +- Failed tool calls are ERROR tool spans, including an MCP failure mcp-use + returns as the tool's result (recognised by its `tool` key matching the + tool, so a tool's own error-shaped result stays a result); a failed run + is an ERROR agent span and the exception reaches the caller unchanged; + cancelled runs end their spans as `cancelled`. Callback failures never + reach the agent. +- Tested with mcp-use 1.7.1 on Python 3.11, 3.12 and 3.13. diff --git a/python/frameworks/mcp-use/README.md b/python/frameworks/mcp-use/README.md new file mode 100644 index 000000000..96330c1bc --- /dev/null +++ b/python/frameworks/mcp-use/README.md @@ -0,0 +1,366 @@ +# traceAI-mcp-use + +OpenTelemetry tracing for [mcp-use](https://pypi.org/project/mcp-use/) +agents (`mcp_use.MCPAgent`), sent to Future AGI. + +`FutureAGICallback` is a LangChain callback handler. You pass it to the +agent with `MCPAgent(..., callbacks=[...])`. It records one span for each +agent run, each LLM call and each tool call. It patches nothing, imports +neither `langfuse` nor `traceai_mcp`, and sets no environment variables. + +This package traces the agent loop. [`traceAI-mcp`](../mcp) (`traceai_mcp`) +works at the MCP transport level and is a different layer: a green +transport test does not show the agent run, its model calls or its tool +calls. See [Other tracing in the same process](#other-tracing-in-the-same-process). + +## Installation + +```bash +pip install traceAI-mcp-use +``` + +It accepts `mcp-use>=1.7.1,<2`, `langchain-core>=1.0.0,<2` and +`fi-instrumentation-otel>=1.1.0`, on Python `>=3.11,<3.14` (mcp-use itself +requires 3.11). The suite runs with mcp-use 1.7.1 (the latest release on +2026-10-16) on Python 3.11, 3.12 and 3.13, and was run end to end against a +real model and a real stdio MCP server with mcp-use 1.7.1, langchain-core +1.6.12, langchain-openai 1.7.2 and langgraph 1.2.16. + +## Usage + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from mcp_use import MCPAgent, MCPClient +from traceai_mcp_use import FutureAGICallback + +trace_provider = register(project_type=ProjectType.OBSERVE, project_name="mcp-use-agent") + +client = MCPClient.from_dict({"mcpServers": {"calculator": {"command": "python", "args": ["server.py"]}}}) +agent = MCPAgent( + llm=llm, # any LangChain chat model + client=client, + callbacks=[FutureAGICallback(tracer_provider=trace_provider)], +) +result = await agent.run("What is 2 + 3?") +``` + +Pass the provider `register()` returns: `register()` does not set the +global provider unless you ask it to, and without `tracer_provider` the +callback uses the global one. One callback can serve many agents and +concurrent runs. See [`examples/agent_with_tools.py`](examples/agent_with_tools.py). + +`MCPAgent.run()`, `stream()` and `stream_events()` are all traced. The +Python `MCPAgent` is async only. + +## Spans + +| Span | Name | `gen_ai.span.kind` | One per | +|---|---|---|---| +| Agent | `mcp_use.agent` | `AGENT` | `run()`, `stream()` or `stream_events()` call | +| LLM | `chat ` (`text_completion ` for completion models; `chat` when no model name is reported) | `LLM` | model call | +| Tool | `execute_tool ` | `TOOL` | tool call | + +The LLM and tool spans are children of the agent span. In the LangChain +run tree, LangGraph's `model` and `tools` nodes sit between them; graph +nodes and middleware are not spans, so a tool span is a sibling of the LLM +spans, not a child of the LLM call that requested it. The agent span is a +child of the span that is current when the run starts, or a root span. + +A streamed completion is one LLM span, ended when the stream ends. Each +token LangChain reports is a `mcp_use.llm.chunk` event carrying only its +index (`mcp_use.llm.chunk.index`, never the text), for the first 128 +tokens; `mcp_use.llm.chunk_count` holds the exact number. + +The spans are not made current while their run executes, as in +`traceAI-langchain`: a span another instrumentation starts inside a tool +call (for example an HTTP client span) is not a child of the tool span. + +## Span attributes + +| Attribute | Span | Value | +|---|---|---| +| `gen_ai.operation.name` | all | `invoke_agent`, `chat`, `text_completion` or `execute_tool` | +| `mcp_use.agent.llm_call_count`, `mcp_use.agent.tool_call_count`, `mcp_use.agent.tool_error_count` | agent | Model calls, tool calls and failed tool calls in the run | +| `gen_ai.request.model` | LLM | From LangChain's `ls_model_name`, else the invocation `model` / `model_name` / `model_id` | +| `gen_ai.provider.name`, `gen_ai.request.temperature`, `gen_ai.request.max_tokens` | LLM | From LangChain's `ls_provider`, `ls_temperature`, `ls_max_tokens` | +| `gen_ai.response.model`, `gen_ai.response.finish_reasons` | LLM | From the response metadata | +| `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.usage.total_tokens`, `gen_ai.usage.cache_read_tokens` | LLM | From the message's `usage_metadata` (or the provider's `token_usage`) | +| `mcp_use.llm.input_message_count` | LLM | Messages sent to the model | +| `mcp_use.llm.tool_call_count` | LLM | Tool calls the model asked for | +| `mcp_use.llm.chunk_count` | LLM | Streamed tokens, when streamed | +| `gen_ai.tool.name`, `gen_ai.tool.call.id` | tool | The tool and the model's tool-call id | +| `mcp_use.tool.error_type` | tool | Error type, when the tool's failure came back as its result | +| `mcp_use.cancelled`, `mcp_use.incomplete` | any | See [Errors and cancellation](#errors-and-cancellation) | + +A value the callback is not given is omitted, never written as 0. Content +is recorded too, unless a `TraceConfig` hide flag turns it off (see +[Privacy](#privacy)): + +| Attribute | Span | Value | +|---|---|---| +| `input.value` | agent | The last user message of the run's input | +| `output.value` | agent | The final answer (the last assistant message without tool calls) | +| `gen_ai.input.messages..message.role` / `.content` / `.tool_calls..tool_call.*` | LLM | The 32 most recent input messages | +| `gen_ai.output.messages..message.*` | LLM | The model's reply, with up to 16 tool calls (name, id, arguments) | +| `input.value`, `output.value` | LLM | The last input message's text and the reply's text | +| `gen_ai.tool.call.arguments`, `input.value` | tool | The arguments, as JSON | +| `gen_ai.tool.call.result`, `output.value` | tool | The tool's result text | + +Spans come from an `fi_instrumentation.FITracer`, so the attributes of +`using_session`, `using_user`, `using_metadata`, `using_tags` and +`using_attributes` (`session.id`, `user.id`, `metadata`, `tag.tags`) are +set on every span of a run started inside them: + +```python +from fi_instrumentation import using_metadata, using_session, using_tags, using_user + +with using_session("session-1"), using_user("user-1"), using_metadata({"tenant": "t1"}), using_tags(["beta"]): + await agent.run("What is 2 + 3?") +``` + +The Python `MCPAgent` has no `set_metadata` or `set_tags` (the TypeScript +agent does); use these context managers instead. + +## Privacy + +Content follows `TraceConfig`, like every traceAI instrumentor: prompts, +messages, model output, tool arguments, tool results and error messages are +recorded by default, and the standard hide flags turn them off. There is no +separate switch in this package. + +```bash +export FI_HIDE_INPUTS=true # prompts, input messages, tool arguments +export FI_HIDE_OUTPUTS=true # model output, output messages, tool results +``` + +or `FutureAGICallback(tracer_provider=..., config=TraceConfig(hide_inputs=True, hide_outputs=True))`. +With both flags on, spans carry names, kinds, the model and provider, +request parameters, usage, finish reasons, counts, the tool-call id, +statuses and exception types; `input.value` and `output.value` are +`__REDACTED__`. + +Never recorded, whatever the flags: + +- stack traces. The `exception` event has `exception.type` and, when + content is on, `exception.message`; never `exception.stacktrace`. mcp-use + sends a failed tool call back to the model as `format_error()`'s dict, + whose `stack` is a full traceback; in that tool message the trace is + replaced with `[stack trace removed]` and the rest of the JSON is kept. + Any other text loses everything from `Traceback (most recent call last):` + on; +- absolute local paths (`/Users/...`, `/home/...`, `/tmp/...`, `C:\...` + and similar), replaced with `[path]`; +- this machine's user name and host name, replaced with `[user]` and + `[host]` (names of 4 characters or more that are not generic, such as + `root` or `app`). + +```python +FutureAGICallback( + tracer_provider=trace_provider, + config=TraceConfig(hide_inputs=True), # optional; default reads FI_HIDE_* from the environment + redact=["my-mcp-server-token"], # extra secret strings to remove +) +``` + +`redact` must be an iterable of strings; anything else raises `TypeError`. + +Secrets are removed from every text the callback writes, including names +and ids, with content on or hidden. They are replaced by `[redacted]`: + +- the strings passed as `redact=`; +- the value (8 characters or longer) of every environment variable whose + name contains `KEY`, `SECRET`, `TOKEN`, `PASSWORD`, `PASSWD`, + `CREDENTIAL` or `AUTH`, read each time a text is written. That covers + provider keys such as `OPENAI_API_KEY` and `FI_API_KEY`/`FI_SECRET_KEY`; +- `Bearer`/`Basic` credentials, `sk-`/`pk-`/`rk-` style keys and + `api_key=`, `token:`, `password=` or `Authorization:` assignments. + +The callback never reads LangChain's serialized model payload (which can +include the model's repr, and so its key) or the invocation parameters +other than the model name. + +`TraceConfig` settings apply. Pass `config=TraceConfig(...)` or set the +environment variables before creating the callback: + +| Setting | Effect | +|---|---| +| `hide_inputs` / `FI_HIDE_INPUTS=true` | `input.value` is `__REDACTED__`; input messages and tool-call arguments (on tool spans and in model replies) are dropped. Every other text of the run has each input it has seen replaced with `__REDACTED__`: the user messages, the messages sent to the model, and each string value of each tool call's arguments. Error messages are dropped: the status and the `exception` event carry the exception type only. | +| `hide_outputs` / `FI_HIDE_OUTPUTS=true` | `output.value` is `__REDACTED__`; output messages and tool results are dropped. Model replies and tool results sent back to the model as input messages (later in the same run, or from earlier turns; see below) keep their role but lose their content and tool-call arguments, and an LLM span whose last input message is one of them has `input.value` `__REDACTED__`. Every other text of the run has each output it has seen (model replies, their tool-call arguments, tool results, earlier turns' included) replaced with `__REDACTED__`. Error messages are dropped: the status and the `exception` event carry the exception type only. | +| `hide_input_messages`, `hide_output_messages`, `hide_input_text`, `hide_output_text` (`FI_HIDE_INPUT_MESSAGES`, ...) | Applied by `FITracer` to the `gen_ai.input.messages.*` / `gen_ai.output.messages.*` attributes, as for every traceAI instrumentor. | +| `pii_redaction` / `FI_PII_REDACTION=true` | Emails, phone numbers, SSNs, card numbers, IPv4 addresses and `sk-`/`pk-` keys become tokens such as `` in every recorded text: attributes, the error status and the `exception` event. It runs after secrets (and hidden inputs/outputs) are removed and before the size caps. | + +`config` must be a `fi_instrumentation.TraceConfig`; anything else raises +`TypeError`. Other `TraceConfig` fields act only through `FITracer`'s +attribute masking. + +mcp-use's own console logging is not part of the trace and is not changed +by these flags: at its default level it prints query previews and tool +results (INFO), and a failed tool call's formatted error with its stack +trace (ERROR, `mcp_use/errors/error_formatting.py`). `Logger.set_debug(0)` +(`from mcp_use import Logger`) hides the INFO lines; +`logging.getLogger("mcp_use").setLevel(logging.CRITICAL)` also hides the +ERROR line. + +Multi-turn conversations: `MCPAgent` keeps memory by default +(`memory_enabled=True`) and sends the earlier turns, model replies and tool +results included, back to the model as input on the next `run()`, +`stream()` or `stream_events()` call (`mcp_use/agents/mcpagent.py:76`, +`:724-732`, `:879-880`, `:981-983`); `external_history` does the same. +Each call is its own agent span. Under `hide_outputs`, every assistant- or +tool-role message (any role other than user and system) among a run's +input messages counts as an output: its content and tool-call arguments +are dropped from the input messages and removed from every other text of +that run, error text included. The callback reads them when the run +starts, before it writes any of the run's spans. This is +tested over three turns on one agent through `run()`, `stream()`, +`stream_events()` and `external_history`, and on the export. User messages +of earlier turns are inputs: they are recorded unless `hide_inputs` is on. +The texts a run remembers include its replayed history, so in a long +conversation the per-run limit below is reached sooner. + +The hide flags match each text verbatim, and also in its Python-repr and +JSON-quoted forms; a copy that was escaped differently, cut or reworded +stays. Texts shorter than 3 characters are only matched in quoted form, +and non-string argument values (numbers, booleans) are not matched. With a +hide flag on, the callback remembers at most 2048 text forms or about 4 +million characters per run; past that it fails closed and records no +content for the rest of that run (no messages, arguments, results or +error messages). + +## Errors and cancellation + +A failed tool call is a tool span with status ERROR and one `exception` +event. mcp-use 1.7.1 does not raise for a failed MCP call: it returns a +formatted error to the model as the tool's result +(`mcp_use/agents/adapters/langchain_adapter.py:181-206`). The callback +recognises that result (an object with mcp-use's `error`, `details`, +`stack` and `code` keys and a `tool` key equal to the tool's name; a tool's +own result with those keys but no matching `tool` is a result), takes the +error type from it +(`mcp_use.tool.error_type`) and counts it in +`mcp_use.agent.tool_error_count`. A tool call LangChain reports as failed +(for example arguments that fail validation) is recorded the same way, +with the exception's type. Either way the model sees the error and the run +usually goes on, so the agent span ends OK when the run returns an answer. + +When the run itself fails (for example the model call raises), the LLM +span and the agent span end with status ERROR and an `exception` event, +and the exception reaches your code unchanged. + +By default the status description is `: ` and the event +carries `exception.type` and `exception.message` (cut to 1 KB, cleaned as +described in [Privacy](#privacy): secrets, local paths, user and host name +removed). With `hide_inputs` or `hide_outputs` both carry the exception type +only. No stack trace is recorded. + +Cancelling the task that runs the agent (`asyncio.CancelledError`) or +leaving `stream()` before it ends (`GeneratorExit`) ends the open spans +with status ERROR, description `cancelled` and `mcp_use.cancelled` = +`true`, without an exception event. A span whose own end never came +because its parent ended first is ended with `mcp_use.incomplete` = +`true` and status ERROR, `ended without a result`. + +If the callback itself fails (starting a span, reading a payload, setting +an attribute), it logs at debug level and the agent goes on unchanged. +Inside `fi_instrumentation.suppress_tracing()` it records nothing. + +## Turning it off + +Leave the callback out. `callbacks=[]` is the Python switch for no +callbacks at all: mcp-use's `ObservabilityManager` returns that empty list +as is and does not add its default Langfuse handler +(`mcp_use/agents/observability/callbacks_manager.py:72-74`). With +`callbacks=None`, mcp-use uses its Langfuse handler when Langfuse is +configured. There is no `observe=False` on the Python `MCPAgent` (that is +the TypeScript agent). + +## Other tracing in the same process + +- **traceAI-mcp (`traceai_mcp`)** works at the MCP transport. Version + 0.1.2 records no spans of its own; it carries trace context in the + `_meta` of MCP requests. With both packages installed you get the same + spans as with this one alone: one agent span and one tool span per tool + call, none duplicated (tested). Because this package's spans are not made + current, the context `traceai_mcp` sends is whichever span is current in + your code. mcp-use binds `stdio_client` when it is imported + (`mcp_use/client/task_managers/stdio.py:12`), so run + `MCPInstrumentor().instrument()` before importing `mcp_use` for + `traceai_mcp`'s stdio wrapper to apply. +- **traceAI-langchain** adds its own tracer to every LangChain callback + manager, so an `MCPAgent` run would also produce a full LangChain tree + (graph, nodes, model and tool runs). Use one of the two for an agent. +- **mcp-use's own observability**: the Langfuse handler is used only when + `callbacks` is `None` (see above). Laminar is set up when `mcp_use` is + imported if `LAMINAR_PROJECT_API_KEY` is set and `lmnr` is installed, + whatever `callbacks` holds (`mcp_use/agents/observability/__init__.py:6`, + `laminar.py:19-37`); `MCP_USE_LAMINAR=false` turns it off. +- **mcp-use's usage telemetry** sends anonymized product events to PostHog + and Scarf, not spans; this package does not change it. + `MCP_USE_ANONYMIZED_TELEMETRY=false` turns it off + (`mcp_use/telemetry/telemetry.py:139`). + +## Limits + +- Recorded text is cut on a UTF-8 character boundary after it is cleaned + (secrets, then hidden inputs/outputs, then PII): 4 KB for each content + attribute, 256 bytes for names and ids (model, provider, tool name, + tool-call id, exception type) and 1 KB for the error message in the + status and the `exception` event. With `pii_redaction`, `FITracer` runs its own PII pass on every + attribute after the cut; a token such as `` can make an + attribute a few bytes longer than its cap. The status and events are not + passed through it. At most 32 messages and 16 tool calls per message are + recorded per LLM span, and 128 chunk events per streamed LLM span. +- One callback tracks at most 10,000 open runs. Past that, the oldest + agent run is ended with `mcp_use.incomplete` and forgotten. Ending a run + looks only at the open runs of its own agent run, not at other agents'. +- With `use_server_manager=True`, mcp-use restarts the graph run when the + tool set changes during a run, up to 3 times + (`mcp_use/agents/mcpagent.py:741-876`); each restart is a new agent span. + The server manager's own tools return their failures as mcp-use errors + without a `tool` key (`mcp_use/agents/managers/tools/connect_server.py:73`, + `disconnect_server.py:39`, `list_servers_tool.py:47`), so those tool spans + end OK with the error as their result. This path is not covered by the + tests. +- With `output_schema`, mcp-use formats the answer after the graph run with + a model call that gets no callbacks (`mcp_use/agents/mcpagent.py:595`), + so that call has no span. +- A remote agent (`MCPAgent(agent_id=...)`) never reads `callbacks` + (`mcp_use/agents/mcpagent.py:117-120`) and is not traced. +- The Python `MCPAgent` has no `flush()`. The provider `register()` + returns exports what is left when the process exits; in a long-running + process call `trace_provider.force_flush()` when you need the spans now. +- The callback runs on the agent's event loop, not in a thread. With + `register(batch=False)`, each span is exported synchronously on that loop + as it ends. +- Passed to a LangChain runnable other than `MCPAgent`, the callback still + names the outermost run `mcp_use.agent`. +- Tested with mcp-use 1.7.1 only (the latest release on 2026-10-16). + +## Tests + +From the repository root: + +```bash +env -u PYTHONPATH PYTHONPATH="python/frameworks/mcp-use:python:python/tests" \ + uv run --no-project --python 3.11 \ + --with pytest --with pytest-asyncio --with opentelemetry-api --with opentelemetry-sdk \ + --with opentelemetry-instrumentation --with opentelemetry-exporter-otlp-proto-http \ + --with wrapt --with requests --with jsonschema --with protobuf --with opentelemetry-proto \ + --with 'mcp-use==1.7.1' \ + pytest python/frameworks/mcp-use/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +Nothing calls a real model, MCP server or Future AGI. The LLM is a +scripted LangChain chat model. The MCP server is the FastMCP server in +`tests/_mcp_server.py`, on a 127.0.0.1 port for most tests and over stdio +for the contract test; the example test uses the example's own stdio +calculator server. Those send spans through +`fi_instrumentation.register()` to the shared `harness.Receiver` on +127.0.0.1 and check the `/tracer/v1/traces` path, both auth headers, +`project_type=observe`, that content is exported by default and not with +`FI_HIDE_INPUTS`/`FI_HIDE_OUTPUTS`, and that no key, stack trace or local +path is exported either way. +The example test runs `examples/agent_with_tools.py` as its own process, +once alone and once with `traceai_mcp` instrumented. The suite turns off +mcp-use's usage telemetry. diff --git a/python/frameworks/mcp-use/examples/agent_with_tools.py b/python/frameworks/mcp-use/examples/agent_with_tools.py new file mode 100644 index 000000000..822e77372 --- /dev/null +++ b/python/frameworks/mcp-use/examples/agent_with_tools.py @@ -0,0 +1,47 @@ +"""Trace one mcp-use agent run with traceAI-mcp-use. + +Needs FI_API_KEY / FI_SECRET_KEY, langchain-openai and OPENAI_API_KEY (or +pass another LangChain chat model to main()). The MCP server is the local +stdio server next to this file. +""" + +import asyncio +import os +import sys + +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from mcp_use import MCPAgent, MCPClient + +from traceai_mcp_use import FutureAGICallback + +SERVER = os.path.join(os.path.dirname(os.path.abspath(__file__)), "calculator_server.py") + + +def default_llm(): + from langchain_openai import ChatOpenAI + + return ChatOpenAI(model="gpt-4o-mini") + + +async def main(llm=None) -> str: + trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="mcp-use-agent", + ) + client = MCPClient.from_dict( + {"mcpServers": {"calculator": {"command": sys.executable, "args": [SERVER]}}} + ) + agent = MCPAgent( + llm=llm or default_llm(), + client=client, + callbacks=[FutureAGICallback(tracer_provider=trace_provider)], + ) + try: + return await agent.run("What is 2 + 3? Use the add tool.") + finally: + await client.close_all_sessions() + + +if __name__ == "__main__": + print(asyncio.run(main())) diff --git a/python/frameworks/mcp-use/examples/calculator_server.py b/python/frameworks/mcp-use/examples/calculator_server.py new file mode 100644 index 000000000..016e8fb50 --- /dev/null +++ b/python/frameworks/mcp-use/examples/calculator_server.py @@ -0,0 +1,15 @@ +"""A one-tool MCP server over stdio, for examples/agent_with_tools.py.""" + +from mcp.server.fastmcp import FastMCP + +mcp = FastMCP("calculator") + + +@mcp.tool() +def add(a: int, b: int) -> str: + """Add two integers.""" + return str(a + b) + + +if __name__ == "__main__": + mcp.run("stdio") diff --git a/python/frameworks/mcp-use/pyproject.toml b/python/frameworks/mcp-use/pyproject.toml new file mode 100644 index 000000000..9de7eb516 --- /dev/null +++ b/python/frameworks/mcp-use/pyproject.toml @@ -0,0 +1,32 @@ +[tool.poetry] +name = "traceAI-mcp-use" +version = "0.1.0" +description = "OpenTelemetry tracing for mcp-use agents: one span per agent run, LLM call and tool call" +authors = ["Future AGI "] +readme = "README.md" +packages = [ + { include = "traceai_mcp_use" } +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Libraries :: Python Modules", +] + +[tool.poetry.dependencies] +python = ">=3.11,<3.14" +fi-instrumentation-otel = ">=1.1.0" +mcp-use = ">=1.7.1,<2" +langchain-core = ">=1.0.0,<2" + +[tool.poetry.group.dev.dependencies] +pytest = ">=7.0.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/python/frameworks/mcp-use/tests/_example_app.py b/python/frameworks/mcp-use/tests/_example_app.py new file mode 100644 index 000000000..74ad417ad --- /dev/null +++ b/python/frameworks/mcp-use/tests/_example_app.py @@ -0,0 +1,36 @@ +"""Run examples/agent_with_tools.py with the fake LLM, as its own process. + +test_mcp_use_example.py starts this through harness.run. With +``--with-traceai-mcp`` it first instruments the existing transport package +(python/frameworks/mcp, imported, not modified). traceai_mcp wraps +``mcp.client.stdio.stdio_client``; mcp-use binds that name when it is +imported (mcp_use/client/task_managers/stdio.py:12), so the instrumentor +runs before anything imports mcp_use. +""" + +import asyncio +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +if "--with-traceai-mcp" in sys.argv: + sys.path.insert(0, str(HERE.parents[1] / "mcp")) + from traceai_mcp import MCPInstrumentor + + MCPInstrumentor().instrument() + +from _mcp_use_support import ChatFake, answer, tool_call # noqa: E402 + +sys.path.insert(0, str(HERE.parent / "examples")) +import agent_with_tools # noqa: E402 + +if "--with-traceai-mcp" in sys.argv: + from mcp_use.client.task_managers import stdio as stdio_task_manager + + # Proof for the test that the transport instrumentation is in place. + print("traceai_mcp wraps stdio_client:", type(stdio_task_manager.stdio_client).__name__) + +script = [tool_call("add", {"a": 2, "b": 3}), answer("EXAMPLE-ANSWER 2 + 3 = 5")] +print("answer:", asyncio.run(agent_with_tools.main(llm=ChatFake(script=script)))) diff --git a/python/frameworks/mcp-use/tests/_mcp_server.py b/python/frameworks/mcp-use/tests/_mcp_server.py new file mode 100644 index 000000000..f8170eded --- /dev/null +++ b/python/frameworks/mcp-use/tests/_mcp_server.py @@ -0,0 +1,41 @@ +"""A fake MCP server over stdio for the traceAI-mcp-use tests. + +mcp-use starts it as a subprocess (MCPClient "command"/"args"). It talks to +nothing else. Each tool returns a marker string so the tests can tell +whether tool content reached a span. +""" + +import asyncio + +from mcp.server.fastmcp import FastMCP + +mcp = FastMCP("fake") + + +@mcp.tool() +def add(a: int, b: int) -> str: + """Add two integers.""" + return "SUM-RESULT-{0}".format(a + b) + + +@mcp.tool() +def echo(text: str) -> str: + """Return the text.""" + return "ECHO-RESULT:" + text + + +@mcp.tool() +def fail(reason: str) -> str: + """Always fails; the MCP result has isError set.""" + raise ValueError("tool failed because " + reason) + + +@mcp.tool() +async def slow(seconds: float) -> str: + """Sleep, then return.""" + await asyncio.sleep(seconds) + return "SLEPT" + + +if __name__ == "__main__": + mcp.run("stdio") diff --git a/python/frameworks/mcp-use/tests/_mcp_use_support.py b/python/frameworks/mcp-use/tests/_mcp_use_support.py new file mode 100644 index 000000000..d88b17716 --- /dev/null +++ b/python/frameworks/mcp-use/tests/_mcp_use_support.py @@ -0,0 +1,340 @@ +"""Fakes and helpers shared by the traceAI-mcp-use tests. + +The LLM is ``ChatFake``, a LangChain chat model that replays a script. The +MCP server is ``_mcp_server.py``, started by mcp-use over stdio. Nothing +here opens a network connection; every key is a placeholder. + +Import this module before ``mcp_use``: it turns off mcp-use's own usage +telemetry, which otherwise posts to PostHog and Scarf. +""" + +from __future__ import annotations + +import asyncio +import os +import sys +import threading +import time +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple + +# mcp-use 1.7.1 sends anonymized usage events unless this is "false" +# (mcp_use/telemetry/telemetry.py:139). Telemetry is a singleton built on +# first use (telemetry.py:122), so this runs before any MCPAgent/MCPClient. +os.environ["MCP_USE_ANONYMIZED_TELEMETRY"] = "false" +# Importing mcp_use sets up Langfuse and Laminar when their keys are present +# (mcp_use/agents/observability/__init__.py:6). The tests set no keys and +# turn both off as well. +os.environ["MCP_USE_LANGFUSE"] = "false" +os.environ["MCP_USE_LAMINAR"] = "false" + +from langchain_core.callbacks import BaseCallbackHandler # noqa: E402 +from langchain_core.language_models.chat_models import BaseChatModel # noqa: E402 +from langchain_core.messages import AIMessage, AIMessageChunk, BaseMessage # noqa: E402 +from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult # noqa: E402 +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402 + InMemorySpanExporter, +) + +HERE = Path(__file__).resolve().parent +PACKAGE = HERE.parent +REPO_PYTHON = PACKAGE.parents[1] +SERVER = HERE / "_mcp_server.py" + +MODEL = "fake-model-1" +PROMPT = "PROMPT-MARKER please add two and three" +ANSWER = "ANSWER-MARKER the sum is five" +ARG = "ARG-MARKER" +# Placeholders only. LLM_KEY has the sk- shape; ENV_SECRET is only known +# to the callback through the FAKE_PROVIDER_API_KEY environment variable. +LLM_KEY = "sk-placeholder-llm-key-0000000000000000" +ENV_SECRET_NAME = "FAKE_PROVIDER_API_KEY" +ENV_SECRET = "placeholder-env-secret-value-1234" +EXPLICIT_SECRET = "placeholder-mcp-header-token-42" +CONTENT_MARKERS = ( + "PROMPT-MARKER", + "ANSWER-MARKER", + "ARG-MARKER", + "SUM-RESULT", + "ECHO-RESULT", + "tool failed because", +) + +AGENT = "mcp_use.agent" +LLM_SPAN = "chat " + MODEL + + +class ChatFake(BaseChatModel): + """A chat model that replays ``script``. + + Each step is an ``AIMessage`` to return, an exception to raise, or a + callable taking the input messages and returning an ``AIMessage``. + Returned messages carry usage and response metadata like a provider's. + ``streaming=True`` streams word by word through ``on_llm_new_token``. + """ + + script: List[Any] + model_name: str = MODEL + temperature: float = 0.2 + max_tokens: int = 64 + # Held like a provider key. LangChain puts repr(model), and so this + # value, into the serialized payload of on_chat_model_start. + api_key: str = LLM_KEY + streaming: bool = False + index: int = 0 + delay: float = 0.0 + + @property + def _llm_type(self) -> str: + return "fake-chat" + + def bind_tools(self, tools: Any, **kwargs: Any) -> "ChatFake": + return self + + def _next(self, messages: Sequence[BaseMessage]) -> AIMessage: + if self.index >= len(self.script): + step: Any = AIMessage(content="script exhausted") + else: + step = self.script[self.index] + self.index += 1 + if isinstance(step, BaseException): + raise step + if callable(step): + step = step(messages) + return step.model_copy( + update={ + "usage_metadata": {"input_tokens": 11, "output_tokens": 7, "total_tokens": 18}, + "response_metadata": { + "model_name": MODEL, + "finish_reason": "tool_calls" if step.tool_calls else "stop", + }, + } + ) + + def _generate(self, messages, stop=None, run_manager=None, **kwargs): # type: ignore[override] + return ChatResult(generations=[ChatGeneration(message=self._next(messages))]) + + async def _agenerate(self, messages, stop=None, run_manager=None, **kwargs): # type: ignore[override] + if self.delay: + await asyncio.sleep(self.delay) + return self._generate(messages, stop=stop, **kwargs) + + async def _astream(self, messages, stop=None, run_manager=None, **kwargs): # type: ignore[override] + message = self._next(messages) + if message.tool_calls: + yield ChatGenerationChunk( + message=AIMessageChunk( + content="", + tool_call_chunks=[ + { + "name": call["name"], + "args": _json(call["args"]), + "id": call["id"], + "index": index, + } + for index, call in enumerate(message.tool_calls) + ], + usage_metadata=message.usage_metadata, + response_metadata=message.response_metadata, + ) + ) + return + words = str(message.content).split(" ") + for position, word in enumerate(words): + text = word if position == len(words) - 1 else word + " " + last = position == len(words) - 1 + chunk = ChatGenerationChunk( + message=AIMessageChunk( + content=text, + usage_metadata=message.usage_metadata if last else None, + response_metadata=message.response_metadata if last else {}, + ) + ) + if run_manager: + await run_manager.on_llm_new_token(text, chunk=chunk) + yield chunk + + +def _json(value: Any) -> str: + import json + + return json.dumps(value) + + +def tool_call(name: str, args: Dict[str, Any], call_id: str = "call-1") -> AIMessage: + return AIMessage( + content="", tool_calls=[{"name": name, "args": args, "id": call_id, "type": "tool_call"}] + ) + + +def answer(text: str = ANSWER) -> AIMessage: + return AIMessage(content=text) + + +def new_provider() -> Tuple[InMemorySpanExporter, TracerProvider]: + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return exporter, provider + + +class _LoopbackServer: + """``_mcp_server.py`` over streamable HTTP on 127.0.0.1, in a thread. + + Started once per test process: spawning the stdio server costs several + seconds of imports per test. The contract and example tests still use + stdio. + """ + + _lock = threading.Lock() + _url: Optional[str] = None + + @classmethod + def url(cls) -> str: + with cls._lock: + if cls._url is None: + cls._url = cls._start() + return cls._url + + @staticmethod + def _start() -> str: + import socket + + import uvicorn + + sys.path.insert(0, str(HERE)) + from _mcp_server import mcp as fake_server + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + server = uvicorn.Server( + uvicorn.Config(fake_server.streamable_http_app(), log_level="warning") + ) + thread = threading.Thread( + target=lambda: asyncio.run(server.serve(sockets=[sock])), daemon=True + ) + thread.start() + deadline = time.monotonic() + 120 + while not server.started: + if time.monotonic() > deadline or not thread.is_alive(): + raise RuntimeError("the loopback MCP server did not start") + time.sleep(0.02) + return "http://127.0.0.1:{0}/mcp".format(port) + + +def mcp_client(transport: str = "http") -> Any: + """An MCPClient for the fake server: loopback HTTP, or a stdio subprocess.""" + from mcp_use import MCPClient + + if transport == "stdio": + server: Dict[str, Any] = {"command": sys.executable, "args": [str(SERVER)]} + else: + server = {"url": _LoopbackServer.url()} + return MCPClient.from_dict({"mcpServers": {"fake": server}}) + + +def add_script() -> List[Any]: + """The J1 journey: one tool call to add, then the final answer.""" + return [tool_call("add", {"a": 2, "b": 3}), answer()] + + +async def drive( + agent: Any, method: str, query: str = PROMPT, **kwargs: Any +) -> Any: + """Run the agent through one of its three public entry points.""" + if method == "run": + return await agent.run(query, **kwargs) + if method == "stream": + result = None + async for item in agent.stream(query, **kwargs): + result = item + return result + if method == "stream_events": + events = [] + async for event in agent.stream_events(query, **kwargs): + events.append(event) + return events + raise ValueError(method) + + +def run_agent( + script: List[Any], + callbacks: Optional[List[Any]], + method: str = "run", + query: str = PROMPT, + streaming: bool = False, + agent_kwargs: Optional[Dict[str, Any]] = None, + around: Optional[Callable[[], Any]] = None, + transport: str = "http", +) -> Any: + """Build an MCPAgent on the fake LLM and fake server and run it once. + + ``around`` returns a context manager entered around the run (for + using_session, a parent span, suppress_tracing...). + """ + from mcp_use import MCPAgent + + async def go() -> Any: + client = mcp_client(transport) + try: + agent = MCPAgent( + llm=ChatFake(script=script, streaming=streaming), + client=client, + callbacks=callbacks, + **(agent_kwargs or {}), + ) + if around is None: + return await drive(agent, method, query) + with around(): + return await drive(agent, method, query) + finally: + await client.close_all_sessions() + + return asyncio.run(go()) + + +class ToolStarted(BaseCallbackHandler): + """A second LangChain handler that signals when a tool call starts.""" + + run_inline = True + + def __init__(self) -> None: + super().__init__() + self.started = threading.Event() + + def on_tool_start(self, *args: Any, **kwargs: Any) -> None: + self.started.set() + + +def spans_named(spans: Iterable[ReadableSpan], name: str) -> List[ReadableSpan]: + return [span for span in spans if span.name == name] + + +def only(spans: Iterable[ReadableSpan], name: str) -> ReadableSpan: + found = spans_named(spans, name) + assert len(found) == 1, (name, [span.name for span in spans]) + return found[0] + + +def parent_id(span: ReadableSpan) -> Optional[int]: + return span.parent.span_id if span.parent is not None else None + + +def attrs(span: ReadableSpan) -> Dict[str, Any]: + return dict(span.attributes or {}) + + +def wire(spans: Iterable[ReadableSpan]) -> str: + """Every name, attribute, event and status text of the spans, as one string.""" + parts: List[str] = [] + for span in spans: + parts.append(span.name) + parts.append(repr(dict(span.attributes or {}))) + parts.append(str(span.status.description)) + for event in span.events: + parts.append(event.name) + parts.append(repr(dict(event.attributes or {}))) + return "\n".join(parts) diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_agent.py b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py new file mode 100644 index 000000000..28e7aa4e7 --- /dev/null +++ b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py @@ -0,0 +1,607 @@ +"""MCPAgent end to end: the real mcp-use 1.7.1 agent loop on a fake LLM. + +``ChatFake`` replays a script; the MCP server is ``_mcp_server.py`` on a +loopback streamable-HTTP port (see ``_mcp_use_support``). Spans go to an +InMemorySpanExporter through a provider passed to the callback. +""" + +from __future__ import annotations + +import asyncio +import json +from contextlib import contextmanager +from typing import Any, Dict, List + +import pytest +from _mcp_use_support import ( + AGENT, + ANSWER, + ARG, + CONTENT_MARKERS, + ENV_SECRET, + ENV_SECRET_NAME, + EXPLICIT_SECRET, + LLM_KEY, + LLM_SPAN, + MODEL, + PROMPT, + ChatFake, + ToolStarted, + add_script, + answer, + attrs, + drive, + mcp_client, + new_provider, + only, + parent_id, + run_agent, + spans_named, + tool_call, + wire, +) +from fi_instrumentation import ( + TraceConfig, + suppress_tracing, + using_metadata, + using_session, + using_tags, + using_user, +) +from langchain_core.callbacks import BaseCallbackHandler +from mcp_use import MCPAgent +from opentelemetry.trace import StatusCode + +from traceai_mcp_use import FutureAGICallback + +pytest.importorskip("uvicorn", reason="the loopback MCP server runs on uvicorn") + + +def _traced(**kwargs: Any): + exporter, provider = new_provider() + return exporter, FutureAGICallback(tracer_provider=provider, **kwargs) + + +# Content off: the standard traceAI hide flags (FI_HIDE_INPUTS / FI_HIDE_OUTPUTS). +HIDDEN = TraceConfig(hide_inputs=True, hide_outputs=True) + + +def _assert_j1_tree(spans) -> None: + """One agent root; LLM, tool, LLM as its children, in one trace.""" + assert sorted(span.name for span in spans) == sorted([AGENT, LLM_SPAN, LLM_SPAN, "execute_tool add"]) + agent = only(spans, AGENT) + assert parent_id(agent) is None + children = sorted( + (span for span in spans if span is not agent), key=lambda span: span.start_time + ) + assert [span.name for span in children] == [LLM_SPAN, "execute_tool add", LLM_SPAN] + for child in children: + # The callback tree puts the tool run under the "tools" graph node + # and the model run under the "model" node; both nodes are siblings + # under the root, so the tool span is a sibling of the LLM spans. + assert parent_id(child) == agent.context.span_id + assert child.context.trace_id == agent.context.trace_id + assert child.start_time >= agent.start_time and child.end_time <= agent.end_time + for span in spans: + assert span.status.status_code is StatusCode.OK, span.name + assert attrs(agent)["gen_ai.span.kind"] == "AGENT" + assert attrs(agent)["mcp_use.agent.llm_call_count"] == 2 + assert attrs(agent)["mcp_use.agent.tool_call_count"] == 1 + assert attrs(agent)["mcp_use.agent.tool_error_count"] == 0 + llm = attrs(children[0]) + assert llm["gen_ai.span.kind"] == "LLM" + assert llm["gen_ai.request.model"] == MODEL + assert llm["gen_ai.response.model"] == MODEL + assert llm["gen_ai.usage.input_tokens"] == 11 + assert llm["mcp_use.llm.tool_call_count"] == 1 + tool = attrs(children[1]) + assert tool["gen_ai.span.kind"] == "TOOL" + assert tool["gen_ai.tool.name"] == "add" + assert tool["gen_ai.tool.call.id"] == "call-1" + + +@pytest.mark.parametrize("method", ["run", "stream", "stream_events"]) +def test_every_entry_point_gives_one_agent_span_with_llm_and_tool_children(method): + exporter, handler = _traced() + result = run_agent(add_script(), [handler], method=method) + if method == "stream_events": + assert any(event.get("event") == "on_tool_end" for event in result) + else: + assert result == ANSWER + _assert_j1_tree(exporter.get_finished_spans()) + assert handler._runs == {} + + +class _TokenCounter(BaseCallbackHandler): + run_inline = True + + def __init__(self) -> None: + super().__init__() + self.tokens: List[str] = [] + + def on_llm_new_token(self, token: str, **kwargs: Any) -> None: + self.tokens.append(token) + + +@pytest.mark.parametrize("method", ["run", "stream_events"]) +def test_a_streamed_completion_is_one_llm_span(method): + exporter, handler = _traced() + counter = _TokenCounter() + result = run_agent(add_script(), [handler, counter], method=method, streaming=True) + spans = exporter.get_finished_spans() + _assert_j1_tree(spans) + tool_turn, final_turn = sorted(spans_named(spans, LLM_SPAN), key=lambda span: span.start_time) + # "ANSWER-MARKER the sum is five" streams as five word chunks; every + # token LangChain reports is counted and none becomes a span. + assert {"ANSWER-MARKER ", "the ", "sum ", "is ", "five"} <= set(counter.tokens) + chunks = tool_turn.attributes["mcp_use.llm.chunk_count"] + final_turn.attributes["mcp_use.llm.chunk_count"] + assert chunks == len(counter.tokens) + assert final_turn.attributes["mcp_use.llm.chunk_count"] >= 5 + assert {event.name for event in final_turn.events} == {"mcp_use.llm.chunk"} + assert len(final_turn.events) == final_turn.attributes["mcp_use.llm.chunk_count"] + # The answer is the LLM span's output, never a chunk event's. + for event in final_turn.events: + assert set(event.attributes) == {"mcp_use.llm.chunk.index"} + if method == "run": + assert result == ANSWER + + +def test_content_and_keys_are_not_recorded_with_the_hide_flags(monkeypatch): + monkeypatch.setenv(ENV_SECRET_NAME, ENV_SECRET) + exporter, handler = _traced(config=HIDDEN) + script = [tool_call("echo", {"text": ARG + " " + ENV_SECRET}), answer()] + run_agent(script, [handler], query=PROMPT + " " + ENV_SECRET) + spans = exporter.get_finished_spans() + assert len(spans) == 4 + text = wire(spans) + for marker in CONTENT_MARKERS + (LLM_KEY, ENV_SECRET): + assert marker not in text, marker + + +def test_content_is_recorded_by_default_without_secrets(monkeypatch): + monkeypatch.setenv(ENV_SECRET_NAME, ENV_SECRET) + exporter, handler = _traced(redact=[EXPLICIT_SECRET]) + leak = "{0} {1} {2}".format(ENV_SECRET, EXPLICIT_SECRET, LLM_KEY) + script = [tool_call("echo", {"text": ARG + " " + leak}), answer()] + run_agent(script, [handler], query=PROMPT + " " + leak) + spans = exporter.get_finished_spans() + text = wire(spans) + for secret in (ENV_SECRET, EXPLICIT_SECRET, LLM_KEY): + assert secret not in text, secret + agent = attrs(only(spans, AGENT)) + assert agent["input.value"] == PROMPT + " [redacted] [redacted] [redacted]" + assert agent["output.value"] == ANSWER + tool = attrs(only(spans, "execute_tool echo")) + assert json.loads(tool["gen_ai.tool.call.arguments"]) == {"text": ARG + " [redacted] [redacted] [redacted]"} + assert tool["gen_ai.tool.call.result"] == "ECHO-RESULT:" + ARG + " [redacted] [redacted] [redacted]" + + +def test_mcp_tool_error_fails_the_tool_span_and_the_agent_recovers(): + # mcp-use 1.7.1 turns a failed MCP call into a formatted error the LLM + # reads (agents/adapters/langchain_adapter.py:181-206), so the run + # succeeds; the tool span is ERROR and the agent counts the error. + exporter, handler = _traced(config=HIDDEN) + script = [tool_call("fail", {"reason": ARG}), answer()] + assert run_agent(script, [handler]) == ANSWER + spans = exporter.get_finished_spans() + tool = only(spans, "execute_tool fail") + assert tool.status.status_code is StatusCode.ERROR + assert tool.status.description == "RuntimeError" + assert tool.attributes["mcp_use.tool.error_type"] == "RuntimeError" + assert [event.attributes["exception.type"] for event in tool.events] == ["RuntimeError"] + agent = only(spans, AGENT) + assert agent.status.status_code is StatusCode.OK + assert agent.attributes["mcp_use.agent.tool_error_count"] == 1 + assert "tool failed because" not in wire(spans) and ARG not in wire(spans) + + +def test_tool_error_text_is_recorded_by_default_and_dropped_with_hide_inputs(): + exporter, handler = _traced() + run_agent([tool_call("fail", {"reason": ARG}), answer()], [handler]) + spans = exporter.get_finished_spans() + tool = only(spans, "execute_tool fail") + assert tool.status.description == "RuntimeError: Error executing tool fail: tool failed because " + ARG + (event,) = tool.events + assert "exception.stacktrace" not in event.attributes + # mcp-use's format_error "stack" reaches the next model call as the + # tool message; the trace is cut there, the rest of the JSON kept. + final = sorted(spans_named(spans, LLM_SPAN), key=lambda span: span.start_time)[-1] + replayed = json.loads(attrs(final)["gen_ai.input.messages.3.message.content"]) + assert replayed["stack"] == "[stack trace removed]" and replayed["tool"] == "fail" + for needle in ("Traceback", 'File "', "site-packages"): + assert needle not in wire(spans), needle + + exporter, handler = _traced(config=TraceConfig(hide_inputs=True)) + run_agent([tool_call("fail", {"reason": ARG}), answer()], [handler]) + spans = exporter.get_finished_spans() + tool = only(spans, "execute_tool fail") + assert tool.status.description == "RuntimeError" + (event,) = tool.events + assert dict(event.attributes) == {"exception.type": "RuntimeError"} + assert ARG not in wire(spans) and "PROMPT-MARKER" not in wire(spans) + + +SECOND_ANSWER = "SECOND-ANSWER done" +EARLIER_TURN = ("SUM-RESULT", "ANSWER-MARKER") + + +def _quote_the_history(messages: List[Any]) -> Any: + """A model step that fails with an error quoting the earlier turns.""" + earlier = [str(m.content) for m in messages if m.type in ("ai", "tool") and m.content] + raise ValueError("model rejected the history: " + " | ".join(earlier)) + + +def _three_turns(method: str, external: bool, **options: Any) -> List[Any]: + """Three queries on one MCPAgent; returns each turn's spans. + + Turn 1 calls add (SUM-RESULT-5) and answers ANSWER-MARKER. With memory + on (mcp-use's default) or the history passed back as external_history, + turns 2 and 3 send turn 1's messages to the model again. Turn 2 echoes + and answers; in turn 3 the model raises an error quoting the history. + """ + exporter, handler = _traced(**options) + script = add_script() + [ + tool_call("echo", {"text": "SECOND-ARG"}, "call-2"), + answer(SECOND_ANSWER), + _quote_the_history, + ] + turns: List[Any] = [] + + async def go() -> None: + client = mcp_client() + try: + agent = MCPAgent(llm=ChatFake(script=script), client=client, callbacks=[handler]) + assert agent.memory_enabled + kwargs: Dict[str, Any] = {} + await drive(agent, method, PROMPT) + if external: + kwargs["external_history"] = list(agent.get_conversation_history()) + agent.clear_conversation_history() + assert kwargs.get("external_history") or agent.get_conversation_history() + turns.append(exporter.get_finished_spans()) + exporter.clear() + await drive(agent, method, "SECOND-PROMPT echo it", **kwargs) + turns.append(exporter.get_finished_spans()) + exporter.clear() + with pytest.raises(ValueError): + await drive(agent, method, "THIRD-PROMPT", **kwargs) + turns.append(exporter.get_finished_spans()) + finally: + await client.close_all_sessions() + + asyncio.run(go()) + assert handler._runs == {} + return turns + + +_TURN_MODES = [("run", False), ("stream", False), ("stream_events", False), ("run", True)] + + +@pytest.mark.parametrize("method,external", _TURN_MODES) +def test_hide_outputs_keeps_earlier_turns_out_of_later_turns(method, external): + _, second, third = _three_turns(method, external, config=TraceConfig(hide_outputs=True)) + for spans in (second, third): + text = wire(spans) + for marker in EARLIER_TURN + ("ECHO-RESULT", "SECOND-ANSWER"): + assert marker not in text, marker + assert "SECOND-ARG" not in wire(third) # turn 2's tool-call arguments are model output + # The replayed messages keep their roles and the count; only the + # assistant and tool content is dropped. + replay = sorted(spans_named(second, LLM_SPAN), key=lambda span: span.start_time)[0] + values = attrs(replay) + roles = [values["gen_ai.input.messages.{0}.message.role".format(i)] for i in range(5)] + assert roles == ["system", "user", "assistant", "tool", "assistant"] + assert values["mcp_use.llm.input_message_count"] == 6 + assert values["gen_ai.input.messages.5.message.content"] == "SECOND-PROMPT echo it" + for index in (2, 3, 4): + assert "gen_ai.input.messages.{0}.message.content".format(index) not in values + # Inputs are still recorded; the error text (it quotes the earlier + # outputs) is dropped, the exception type kept. + assert attrs(only(third, AGENT))["input.value"] == "THIRD-PROMPT" + for span in (only(third, AGENT), only(third, LLM_SPAN)): + assert span.status.description == "ValueError" + (event,) = span.events + assert dict(event.attributes) == {"exception.type": "ValueError"} + + +@pytest.mark.parametrize("method,external", _TURN_MODES) +def test_without_hide_outputs_later_turns_record_the_earlier_turns(method, external): + # Control for the test above: the same turns carry turn 1's tool result + # and answer as input messages, and the error quotes them. + _, second, third = _three_turns(method, external) + replay = attrs(sorted(spans_named(second, LLM_SPAN), key=lambda span: span.start_time)[0]) + assert replay["gen_ai.input.messages.3.message.content"] == "SUM-RESULT-5" + assert replay["gen_ai.input.messages.4.message.content"] == ANSWER + assert "SUM-RESULT-5" in only(third, LLM_SPAN).status.description + assert ANSWER in only(third, AGENT).status.description + + +@pytest.mark.parametrize("retry_on_error", [True, False]) +def test_invalid_tool_arguments_fail_the_tool_span(retry_on_error): + exporter, handler = _traced() + script = [tool_call("add", {"a": "not-a-number", "b": 3}), answer()] + result = run_agent(script, [handler], agent_kwargs={"retry_on_error": retry_on_error}) + assert result == ANSWER + spans = exporter.get_finished_spans() + tool = only(spans, "execute_tool add") + assert tool.status.status_code is StatusCode.ERROR + assert tool.events[0].attributes["exception.type"] == "pydantic_core._pydantic_core.ValidationError" + assert only(spans, AGENT).attributes["mcp_use.agent.tool_error_count"] == 1 + + +def test_an_llm_error_fails_the_agent_span_and_reaches_the_caller_unchanged(): + exporter, handler = _traced() + error = ConnectionError("provider unreachable") + with pytest.raises(ConnectionError) as raised: + run_agent([error], [handler]) + assert raised.value is error + spans = exporter.get_finished_spans() + assert sorted(span.name for span in spans) == sorted([AGENT, LLM_SPAN]) + for span in spans: + assert span.status.status_code is StatusCode.ERROR + assert span.events[0].attributes["exception.type"] == "ConnectionError" + assert parent_id(only(spans, LLM_SPAN)) == only(spans, AGENT).context.span_id + + +def test_cancelling_the_run_ends_the_open_spans_as_cancelled(): + exporter, handler = _traced() + started = ToolStarted() + + async def go() -> None: + client = mcp_client() + try: + agent = MCPAgent( + llm=ChatFake(script=[tool_call("slow", {"seconds": 30}), answer()]), + client=client, + callbacks=[handler, started], + ) + task = asyncio.ensure_future(agent.run(PROMPT)) + for _ in range(6000): # up to a minute on a slow machine + if started.started.is_set(): + break + await asyncio.sleep(0.01) + assert started.started.is_set() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + finally: + await client.close_all_sessions() + + asyncio.run(go()) + spans = exporter.get_finished_spans() + assert sorted(span.name for span in spans) == sorted([AGENT, LLM_SPAN, "execute_tool slow"]) + for name in (AGENT, "execute_tool slow"): + span = only(spans, name) + assert span.status.status_code is StatusCode.ERROR + assert span.status.description == "cancelled" + assert span.attributes["mcp_use.cancelled"] is True + assert span.events == () + assert only(spans, LLM_SPAN).status.status_code is StatusCode.OK + assert handler._runs == {} + + +def test_leaving_stream_early_ends_the_agent_span_as_cancelled(): + exporter, handler = _traced() + + async def go() -> None: + client = mcp_client() + try: + agent = MCPAgent(llm=ChatFake(script=add_script()), client=client, callbacks=[handler]) + stream = agent.stream(PROMPT) + async for _ in stream: + break # the first item is the (action, observation) of the tool step + await stream.aclose() + for _ in range(6000): + if spans_named(exporter.get_finished_spans(), AGENT): + break + await asyncio.sleep(0.01) + finally: + await client.close_all_sessions() + + asyncio.run(go()) + spans = exporter.get_finished_spans() + agent = only(spans, AGENT) + assert agent.status.status_code is StatusCode.ERROR + assert agent.status.description == "cancelled" + assert agent.attributes["mcp_use.cancelled"] is True + assert only(spans, "execute_tool add").status.status_code is StatusCode.OK + assert handler._runs == {} + + +def test_empty_callbacks_list_yields_no_spans(): + # callbacks=[] is the Python switch: ObservabilityManager.get_callbacks + # returns it as is and adds no default handler + # (agents/observability/callbacks_manager.py:72-74). + exporter, handler = _traced() + seen: List[Any] = [] + + async def go() -> None: + client = mcp_client() + try: + agent = MCPAgent(llm=ChatFake(script=add_script()), client=client, callbacks=[]) + seen.append(agent.callbacks) + assert await agent.run(PROMPT) == ANSWER + finally: + await client.close_all_sessions() + + asyncio.run(go()) + assert seen == [[]] + assert exporter.get_finished_spans() == () + + +def test_an_agent_without_the_callback_is_not_traced(): + # The handler exists in the process but only traces where it is passed. + exporter, handler = _traced() + run_agent(add_script(), [handler]) + exporter.clear() + run_agent(add_script(), None) + run_agent(add_script(), [ToolStarted()]) + assert exporter.get_finished_spans() == () + + +def test_the_callers_current_span_parents_the_agent_span(): + exporter, handler = _traced() + tracer = new_provider()[1].get_tracer("app") + outer: List[Any] = [] + + @contextmanager + def around(): + with tracer.start_as_current_span("request") as span: + outer.append(span) + yield + + run_agent(add_script(), [handler], around=around) + agent = only(exporter.get_finished_spans(), AGENT) + assert parent_id(agent) == outer[0].get_span_context().span_id + assert agent.context.trace_id == outer[0].get_span_context().trace_id + + +def test_session_user_metadata_and_tags_reach_every_span(): + exporter, handler = _traced() + + @contextmanager + def around(): + with using_session("session-1"), using_user("user-1"), using_metadata( + {"tenant": "t1"} + ), using_tags(["beta"]): + yield + + run_agent(add_script(), [handler], around=around) + spans = exporter.get_finished_spans() + assert len(spans) == 4 + for span in spans: + assert span.attributes["session.id"] == "session-1", span.name + assert span.attributes["user.id"] == "user-1" + assert json.loads(span.attributes["metadata"]) == {"tenant": "t1"} + assert span.attributes["tag.tags"] == ("beta",) + + +def test_suppress_tracing_records_nothing_and_the_run_still_answers(): + exporter, handler = _traced() + assert run_agent(add_script(), [handler], around=suppress_tracing) == ANSWER + assert exporter.get_finished_spans() == () + + +def test_concurrent_runs_on_one_handler_get_separate_traces(): + exporter, handler = _traced() + + async def one() -> Any: + client = mcp_client() + try: + agent = MCPAgent(llm=ChatFake(script=add_script(), delay=0.05), client=client, callbacks=[handler]) + return await agent.run(PROMPT) + finally: + await client.close_all_sessions() + + async def both() -> List[Any]: + return await asyncio.gather(one(), one(), one()) + + assert asyncio.run(both()) == [ANSWER] * 3 + spans = exporter.get_finished_spans() + agents = spans_named(spans, AGENT) + assert len(agents) == 3 + for agent in agents: + tree = [span for span in spans if span.context.trace_id == agent.context.trace_id] + _assert_j1_tree(tree) + assert handler._runs == {} + + +def test_a_broken_tracer_does_not_change_the_agent_result(): + class Broken: + def get_tracer(self, *args: Any, **kwargs: Any) -> Any: + return self + + def start_span(self, *args: Any, **kwargs: Any) -> Any: + raise RuntimeError("tracer is broken") + + handler = FutureAGICallback(tracer_provider=Broken()) + assert run_agent(add_script(), [handler]) == ANSWER + for method in ("stream", "stream_events"): + run_agent(add_script(), [handler], method=method) + assert handler._runs == {} + + +def test_sync_and_async_langchain_dispatch_give_the_same_spans(): + # MCPAgent is async only. The handler also runs under LangChain's sync + # callback manager: drive a LangChain agent with a plain tool through + # invoke() and ainvoke() and compare. + from langchain.agents import create_agent + from langchain_core.tools import tool + + @tool + def add(a: int, b: int) -> str: + """Add two integers.""" + return "SUM-RESULT-{0}".format(a + b) + + shapes = [] + for mode in ("sync", "async"): + exporter, handler = _traced() + graph = create_agent(ChatFake(script=add_script()), [add]) + inputs = {"messages": [("user", PROMPT)]} + config: Dict[str, Any] = {"callbacks": [handler]} + if mode == "sync": + graph.invoke(inputs, config=config) + else: + asyncio.run(graph.ainvoke(inputs, config=config)) + spans = exporter.get_finished_spans() + _assert_j1_tree(spans) + assert attrs(only(spans, "execute_tool add"))["gen_ai.tool.call.result"] == "SUM-RESULT-5" + shapes.append(sorted((span.name, tuple(sorted(attrs(span)))) for span in spans)) + assert shapes[0] == shapes[1] + + +def test_the_structured_output_formatting_call_has_no_span(): + # With output_schema, mcp-use 1.7.1 formats the answer after the graph + # run with structured_llm.ainvoke(prompt), passing no callbacks + # (agents/mcpagent.py:595, 886). That call gets no span and does not + # start a second agent span. + from langchain_core.runnables import RunnableLambda + from pydantic import BaseModel + + class Sum(BaseModel): + total: int + + class ChatStructured(ChatFake): + def with_structured_output(self, schema: Any, **kwargs: Any) -> Any: + return RunnableLambda(lambda prompt: Sum(total=5)) + + exporter, handler = _traced() + + async def go() -> Any: + client = mcp_client() + try: + agent = MCPAgent(llm=ChatStructured(script=add_script()), client=client, callbacks=[handler]) + return await agent.run(PROMPT, output_schema=Sum) + finally: + await client.close_all_sessions() + + assert asyncio.run(go()) == Sum(total=5) + _assert_j1_tree(exporter.get_finished_spans()) + + +def test_callbacks_run_on_the_event_loop_thread(): + # run_inline: LangChain would otherwise run each sync callback in an + # executor thread, where a cancelled run can end a parent before its + # child's start callback has finished. + import threading + + exporter, provider = new_provider() + threads: List[int] = [] + + class Recording(FutureAGICallback): + def on_chain_start(self, *args: Any, **kwargs: Any) -> None: + threads.append(threading.get_ident()) + super().on_chain_start(*args, **kwargs) + + def on_chat_model_start(self, *args: Any, **kwargs: Any) -> None: + threads.append(threading.get_ident()) + super().on_chat_model_start(*args, **kwargs) + + def on_tool_start(self, *args: Any, **kwargs: Any) -> None: + threads.append(threading.get_ident()) + super().on_tool_start(*args, **kwargs) + + assert run_agent(add_script(), [Recording(tracer_provider=provider)]) == ANSWER + assert len(threads) >= 4 + assert set(threads) == {threading.get_ident()} + _assert_j1_tree(exporter.get_finished_spans()) diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py new file mode 100644 index 000000000..32f6634d1 --- /dev/null +++ b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py @@ -0,0 +1,1016 @@ +"""Unit tests: a fake LangChain callback sequence, no MCP server and no LLM. + +``Run`` replays the callback tree mcp-use 1.7.1 produces for one agent run +(recorded from MCPAgent.run on the fake LLM and server): a LangGraph root +chain, a ``model`` node chain holding each chat-model run, and a ``tools`` +node chain holding each tool run. Spans go to an InMemorySpanExporter. +""" + +from __future__ import annotations + +import asyncio +import getpass +import json +import os +import socket +from typing import Any, Dict, List, Optional, Sequence +from uuid import UUID, uuid4 + +import pytest +from _mcp_use_support import ( + AGENT, + ANSWER, + ARG, + CONTENT_MARKERS, + ENV_SECRET, + ENV_SECRET_NAME, + EXPLICIT_SECRET, + LLM_KEY, + LLM_SPAN, + MODEL, + PROMPT, + attrs, + new_provider, + only, + parent_id, + spans_named, + wire, +) +from fi_instrumentation import ( + REDACTED_VALUE, + TraceConfig, + suppress_tracing, + using_metadata, + using_session, + using_tags, + using_user, +) +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage +from langchain_core.outputs import ChatGeneration, LLMResult +from opentelemetry import trace as trace_api +from opentelemetry.trace import StatusCode + +import traceai_mcp_use._callback as callback_module +from traceai_mcp_use import AGENT_SPAN_NAME, FutureAGICallback +from traceai_mcp_use._text import ( + MAX_ERROR_BYTES, + MAX_NAME_BYTES, + MAX_VALUE_BYTES, + scrub_local, +) + +TOOL_SPAN = "execute_tool add" +METADATA = { + "ls_provider": "fake", + "ls_model_name": MODEL, + "ls_model_type": "chat", + "ls_temperature": 0.2, + "ls_max_tokens": 64, +} + + +def _result(message: AIMessage) -> LLMResult: + message = message.model_copy( + update={ + "usage_metadata": { + "input_tokens": 11, + "output_tokens": 7, + "total_tokens": 18, + "input_token_details": {"cache_read": 3}, + }, + "response_metadata": { + "model_name": MODEL, + "finish_reason": "tool_calls" if message.tool_calls else "stop", + }, + } + ) + return LLMResult(generations=[[ChatGeneration(message=message)]]) + + +class Run: + """One agent run, driven callback by callback like LangGraph does.""" + + def __init__(self, handler: FutureAGICallback, query: str = PROMPT, earlier: Sequence[Any] = ()) -> None: + self.handler = handler + self.root = uuid4() + # ``earlier``: messages of previous turns (MCPAgent memory). + self.history: List[Any] = list(earlier) + [HumanMessage(content=query)] + handler.on_chain_start( + {}, {"messages": list(self.history)}, run_id=self.root, name="LangGraph" + ) + + def node(self, name: str) -> UUID: + run_id = uuid4() + self.handler.on_chain_start( + None, {"messages": list(self.history)}, run_id=run_id, parent_run_id=self.root, name=name + ) + return run_id + + def end_node(self, run_id: UUID) -> None: + self.handler.on_chain_end({"messages": []}, run_id=run_id, parent_run_id=self.root) + + def llm(self, reply: AIMessage, system: str = "You are an MCP agent.") -> UUID: + node = self.node("model") + run_id = uuid4() + self.handler.on_chat_model_start( + {"name": "ChatFake", "repr": "ChatFake(api_key='{0}')".format(LLM_KEY)}, + [[SystemMessage(content=system)] + list(self.history)], + run_id=run_id, + parent_run_id=node, + invocation_params={"model_name": MODEL, "api_key": LLM_KEY}, + metadata=dict(METADATA), + ) + self.handler.on_llm_end(_result(reply), run_id=run_id, parent_run_id=node) + self.end_node(node) + self.history.append(reply) + return run_id + + def start_tool(self, name: str, args: Dict[str, Any], call_id: str = "call-1"): + node = self.node("tools") + run_id = uuid4() + self.handler.on_tool_start( + {"name": name, "description": "A tool."}, + str(args), + run_id=run_id, + parent_run_id=node, + inputs=dict(args), + tool_call_id=call_id, + name=name, + ) + return node, run_id + + def tool(self, name: str, args: Dict[str, Any], result: str, call_id: str = "call-1") -> None: + node, run_id = self.start_tool(name, args, call_id) + output = ToolMessage(content=result, name=name, tool_call_id=call_id) + self.handler.on_tool_end(output, run_id=run_id, parent_run_id=node) + self.end_node(node) + self.history.append(output) + + def finish(self, text: str = ANSWER) -> None: + self.handler.on_chain_end( + {"messages": self.history + [AIMessage(content=text)]}, run_id=self.root + ) + + +def _journey(handler: FutureAGICallback, query: str = PROMPT, args=None, result="SUM-RESULT-5"): + run = Run(handler, query) + args = args if args is not None else {"a": 2, "b": 3, "note": ARG} + run.llm( + AIMessage( + content="", tool_calls=[{"name": "add", "args": args, "id": "call-1", "type": "tool_call"}] + ) + ) + run.tool("add", args, result) + run.llm(AIMessage(content=ANSWER)) + run.finish() + return run + + +def _handler(**kwargs: Any): + exporter, provider = new_provider() + return exporter, FutureAGICallback(tracer_provider=provider, **kwargs) + + +# Content off: the standard traceAI hide flags (FI_HIDE_INPUTS / FI_HIDE_OUTPUTS). +HIDDEN = TraceConfig(hide_inputs=True, hide_outputs=True) +_CONTENT_PREFIXES = ("input.", "output.", "gen_ai.input.", "gen_ai.output.") +_CONTENT_KEYS = ("gen_ai.tool.call.arguments", "gen_ai.tool.call.result") + + +def _meta(span: Any) -> Dict[str, Any]: + """A span's attributes without the content ones (messages, arguments, results).""" + return { + key: value + for key, value in attrs(span).items() + if not key.startswith(_CONTENT_PREFIXES) and key not in _CONTENT_KEYS + } + + +def test_three_span_kinds_from_a_fake_callback_sequence(): + exporter, handler = _handler() + _journey(handler) + spans = exporter.get_finished_spans() + + # The model/tools/middleware node chains are not spans. + assert sorted(span.name for span in spans) == sorted([AGENT, LLM_SPAN, LLM_SPAN, TOOL_SPAN]) + assert AGENT_SPAN_NAME == AGENT + agent = only(spans, AGENT) + tool = only(spans, TOOL_SPAN) + first_llm, second_llm = spans_named(spans, LLM_SPAN) + assert parent_id(agent) is None + # Parent = nearest traced ancestor in the callback tree: the agent. + for child in (first_llm, tool, second_llm): + assert parent_id(child) == agent.context.span_id + assert child.context.trace_id == agent.context.trace_id + + assert _meta(agent) == { + "gen_ai.span.kind": "AGENT", + "gen_ai.operation.name": "invoke_agent", + "mcp_use.agent.llm_call_count": 2, + "mcp_use.agent.tool_call_count": 1, + "mcp_use.agent.tool_error_count": 0, + } + assert _meta(first_llm) == { + "gen_ai.span.kind": "LLM", + "gen_ai.operation.name": "chat", + "gen_ai.request.model": MODEL, + "gen_ai.provider.name": "fake", + "gen_ai.request.temperature": 0.2, + "gen_ai.request.max_tokens": 64, + "gen_ai.response.model": MODEL, + "gen_ai.response.finish_reasons": ("tool_calls",), + "gen_ai.usage.input_tokens": 11, + "gen_ai.usage.output_tokens": 7, + "gen_ai.usage.total_tokens": 18, + "gen_ai.usage.cache_read_tokens": 3, + "mcp_use.llm.input_message_count": 2, + "mcp_use.llm.tool_call_count": 1, + } + assert attrs(second_llm)["mcp_use.llm.input_message_count"] == 4 + assert attrs(second_llm)["gen_ai.response.finish_reasons"] == ("stop",) + assert _meta(tool) == { + "gen_ai.span.kind": "TOOL", + "gen_ai.operation.name": "execute_tool", + "gen_ai.tool.name": "add", + "gen_ai.tool.call.id": "call-1", + } + for span in spans: + assert span.status.status_code is StatusCode.OK + assert span.events == () + + +def test_hide_inputs_and_outputs_turn_content_off(): + exporter, handler = _handler(config=HIDDEN) + _journey(handler) + text = wire(exporter.get_finished_spans()) + for marker in CONTENT_MARKERS + (LLM_KEY, "You are an MCP agent"): + assert marker not in text, marker + for span in exporter.get_finished_spans(): + for key, value in attrs(span).items(): + if key in ("input.value", "output.value"): + assert value == REDACTED_VALUE, key + continue + assert not key.startswith(_CONTENT_PREFIXES), key + assert key not in _CONTENT_KEYS, key + + +def test_content_is_recorded_by_default(): + # The traceAI TraceConfig convention: content on unless a hide flag is set. + exporter, handler = _handler() + _journey(handler) + spans = exporter.get_finished_spans() + agent = attrs(only(spans, AGENT)) + first_llm, second_llm = (attrs(span) for span in spans_named(spans, LLM_SPAN)) + tool = attrs(only(spans, TOOL_SPAN)) + arguments = json.dumps({"a": 2, "b": 3, "note": ARG}) + + assert agent["input.value"] == PROMPT + assert agent["output.value"] == ANSWER + assert first_llm["gen_ai.input.messages.0.message.role"] == "system" + assert first_llm["gen_ai.input.messages.0.message.content"] == "You are an MCP agent." + assert first_llm["gen_ai.input.messages.1.message.role"] == "user" + assert first_llm["gen_ai.input.messages.1.message.content"] == PROMPT + assert first_llm["input.value"] == PROMPT + call = "gen_ai.output.messages.0.message.tool_calls.0.tool_call." + assert first_llm["gen_ai.output.messages.0.message.role"] == "assistant" + assert first_llm[call + "function.name"] == "add" + assert first_llm[call + "id"] == "call-1" + assert first_llm[call + "function.arguments"] == arguments + assert "output.value" not in first_llm # a tool-call-only reply has no text + assert second_llm["gen_ai.input.messages.3.message.role"] == "tool" + assert second_llm["gen_ai.input.messages.3.message.content"] == "SUM-RESULT-5" + assert second_llm["output.value"] == ANSWER + assert tool["gen_ai.tool.call.arguments"] == arguments + assert tool["input.value"] == arguments + assert tool["gen_ai.tool.call.result"] == "SUM-RESULT-5" + assert tool["output.value"] == "SUM-RESULT-5" + # The serialized model and invocation params are never read for content. + assert LLM_KEY not in wire(spans) + + +def test_hide_inputs_drops_every_input_including_tool_arguments(): + exporter, handler = _handler(config=TraceConfig(hide_inputs=True)) + _journey(handler) + spans = exporter.get_finished_spans() + text = wire(spans) + assert PROMPT not in text and ARG not in text + for span in spans: + values = attrs(span) + assert values.get("input.value", REDACTED_VALUE) == REDACTED_VALUE + assert not any(key.startswith("gen_ai.input.messages") for key in values) + assert not any(key.endswith("function.arguments") for key in values) + assert "gen_ai.tool.call.arguments" not in values + # Outputs stay. + assert attrs(only(spans, AGENT))["output.value"] == ANSWER + assert attrs(only(spans, TOOL_SPAN))["gen_ai.tool.call.result"] == "SUM-RESULT-5" + + +def test_hide_outputs_drops_every_output(): + exporter, handler = _handler(config=TraceConfig(hide_outputs=True)) + _journey(handler) + spans = exporter.get_finished_spans() + text = wire(spans) + assert ANSWER not in text and "SUM-RESULT" not in text + for span in spans: + values = attrs(span) + assert values.get("output.value", REDACTED_VALUE) == REDACTED_VALUE + assert not any(key.startswith("gen_ai.output.messages") for key in values) + assert "gen_ai.tool.call.result" not in values + assert attrs(only(spans, AGENT))["input.value"] == PROMPT + + +def test_hide_outputs_drops_earlier_turns_replayed_as_input(): + # mcp-use's memory (or external_history) sends earlier turns' replies + # and tool results back as input: under hide_outputs they are outputs. + exporter, handler = _handler(config=TraceConfig(hide_outputs=True)) + earlier = [ + HumanMessage(content="OLD-PROMPT"), + AIMessage( + content="OLD-THINKING", + tool_calls=[{"name": "add", "args": {"note": "OLD-ARG"}, "id": "c0", "type": "tool_call"}], + ), + ToolMessage(content="OLD-RESULT", tool_call_id="c0"), + {"role": "assistant", "content": "OLD-ANSWER"}, + ] + run = Run(handler, earlier=earlier) + node = run.node("model") + llm = uuid4() + handler.on_chat_model_start( + {}, [[SystemMessage(content="You are an MCP agent.")] + run.history], run_id=llm, parent_run_id=node + ) + error = ValueError("cannot use OLD-RESULT, OLD-ANSWER, OLD-ARG or OLD-THINKING") + handler.on_llm_error(error, run_id=llm, parent_run_id=node) + handler.on_chain_error(error, run_id=node, parent_run_id=run.root) + handler.on_chain_error(error, run_id=run.root) + spans = exporter.get_finished_spans() + text = wire(spans) + for marker in ("OLD-RESULT", "OLD-ANSWER", "OLD-ARG", "OLD-THINKING"): + assert marker not in text, marker + llm_span = attrs(only(spans, "chat")) + roles = [llm_span["gen_ai.input.messages.{0}.message.role".format(i)] for i in range(6)] + assert roles == ["system", "user", "assistant", "tool", "assistant", "user"] + assert llm_span["gen_ai.input.messages.1.message.content"] == "OLD-PROMPT" + assert llm_span["gen_ai.input.messages.5.message.content"] == PROMPT + for index in (2, 3, 4): + assert "gen_ai.input.messages.{0}.message.content".format(index) not in llm_span + call = "gen_ai.input.messages.2.message.tool_calls.0.tool_call." + assert llm_span[call + "function.name"] == "add" + assert call + "function.arguments" not in llm_span + assert llm_span["input.value"] == PROMPT + for span in spans: + # Error text is content: dropped with either hide flag. + assert span.status.description == "ValueError" + + +def test_hide_outputs_with_a_replayed_reply_last_redacts_the_llm_input_value(): + exporter, handler = _handler(config=TraceConfig(hide_outputs=True)) + run = Run(handler) + run.history.append(ToolMessage(content="OLD-RESULT", tool_call_id="c0")) + run.llm(AIMessage(content=ANSWER)) + run.finish() + llm = attrs(only(exporter.get_finished_spans(), LLM_SPAN)) + assert llm["input.value"] == REDACTED_VALUE + assert "OLD-RESULT" not in wire(exporter.get_finished_spans()) + + +def test_hide_flags_are_read_from_the_environment(monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + monkeypatch.setenv("FI_HIDE_OUTPUTS", "true") + exporter, handler = _handler() + _journey(handler) + text = wire(exporter.get_finished_spans()) + for marker in CONTENT_MARKERS: + assert marker not in text, marker + + +def _failing_tool(handler: FutureAGICallback, error: BaseException, query: str = PROMPT): + run = Run(handler, query) + args = {"reason": ARG + " jane.doe@example.com"} + run.llm( + AIMessage(content="", tool_calls=[{"name": "fail", "args": args, "id": "c", "type": "tool_call"}]) + ) + node, tool = run.start_tool("fail", args, "c") + try: + raise error + except BaseException as raised: # give it a traceback + handler.on_tool_error(raised, run_id=tool, parent_run_id=node) + run.end_node(node) + run.llm(AIMessage(content=ANSWER)) + run.finish() + + +def test_tool_error_records_its_message_by_default_and_never_a_stack_trace(): + exporter, handler = _handler() + _failing_tool(handler, ValueError("tool failed because " + ARG)) + spans = exporter.get_finished_spans() + tool = only(spans, "execute_tool fail") + assert tool.status.status_code is StatusCode.ERROR + assert tool.status.description == "ValueError: tool failed because " + ARG + (event,) = tool.events + assert event.name == "exception" + assert dict(event.attributes) == { + "exception.type": "ValueError", + "exception.message": "tool failed because " + ARG, + } + agent = only(spans, AGENT) + # The agent recovered (mcp-use returns tool errors to the LLM). + assert agent.status.status_code is StatusCode.OK + assert attrs(agent)["mcp_use.agent.tool_error_count"] == 1 + assert "Traceback" not in wire(spans) and 'File "' not in wire(spans) + + +@pytest.mark.parametrize( + "config", + [TraceConfig(hide_inputs=True), TraceConfig(hide_outputs=True), HIDDEN], + ids=["hide_inputs", "hide_outputs", "both"], +) +def test_either_hide_flag_drops_the_error_text(config): + # Error text quotes tool arguments, tool results and a server's own data. + exporter, handler = _handler(config=config) + _failing_tool(handler, ValueError("tool failed because " + ARG + " SERVER-SIDE-TEXT")) + spans = exporter.get_finished_spans() + tool = only(spans, "execute_tool fail") + assert tool.status.status_code is StatusCode.ERROR + assert tool.status.description == "ValueError" + assert dict(tool.events[0].attributes) == {"exception.type": "ValueError"} + assert "SERVER-SIDE-TEXT" not in wire(spans) and "tool failed because" not in wire(spans) + + +def test_stack_traces_paths_user_and_host_are_never_recorded(): + # mcp-use sends a failed MCP call back to the model as format_error()'s + # dict, whose "stack" is a full traceback; exception text quotes paths. + user, host = getpass.getuser(), socket.gethostname() + home = os.path.expanduser("~") + exporter, handler = _handler() + run = Run(handler) + run.llm(AIMessage(content="", tool_calls=[{"name": "fail", "args": {"topic": "t"}, "id": "c", "type": "tool_call"}])) + node, tool = run.start_tool("fail", {"topic": "t"}, "c") + try: + raise OSError("cannot open {0}/notes/archive.db on {1} as {2}".format(home, host, user)) + except OSError as raised: + handler.on_tool_error(raised, run_id=tool, parent_run_id=node) + run.end_node(node) + formatted = json.dumps( + { + "error": "RuntimeError", + "details": "Error executing tool fail: down at /home/someone/srv/db.sqlite", + "stack": 'Traceback (most recent call last):\n File "{0}/x.py", line 3, in f\nRuntimeError: down'.format(home), + "code": "UNKNOWN", + "tool": "fail", + } + ) + run.history.append(ToolMessage(content=formatted, tool_call_id="c", name="fail")) + run.llm(AIMessage(content="It failed at C:\\Users\\someone\\x.txt")) + run.finish(text="done") + spans = exporter.get_finished_spans() + text = wire(spans) + for needle in ("Traceback", 'File "', "/notes/archive.db", "/home/someone", "C:\\Users", home): + assert needle not in text, needle + if len(user) >= 4 and user.lower() not in ("root", "runner"): + assert user not in text + if len(host.split(".")[0]) >= 4 and host != "localhost": + assert host.split(".")[0] not in text + span = only(spans, "execute_tool fail") + assert "exception.stacktrace" not in span.events[0].attributes + assert span.status.description.startswith("OSError: cannot open [path] on ") + replayed = attrs(spans_named(spans, LLM_SPAN)[1])["gen_ai.input.messages.3.message.content"] + assert json.loads(replayed) == { + "error": "RuntimeError", + "details": "Error executing tool fail: down at [path]", + "stack": "[stack trace removed]", + "code": "UNKNOWN", + "tool": "fail", + } + + +def test_scrub_local_keeps_ordinary_text(): + for text in ( + "see https://example.com/home/docs and /api/v1/items", + "2 + 3 = 5", + "the root cause was a timeout", + ): + assert scrub_local(text) == text + assert scrub_local("x Traceback (most recent call last):\n File \"a.py\"") == "x [stack trace removed]" + assert scrub_local("open /tmp/abc/d.txt, then /var/log/x.") == "open [path], then [path]" + + +def test_pii_redaction_covers_attributes_status_and_exception_event(): + exporter, handler = _handler(config=TraceConfig(pii_redaction=True)) + _failing_tool(handler, ValueError("mail jane.doe@example.com failed"), query="ask jane.doe@example.com") + spans = exporter.get_finished_spans() + assert "jane.doe@example.com" not in wire(spans) + assert attrs(only(spans, AGENT))["input.value"] == "ask " + tool = only(spans, "execute_tool fail") + assert tool.status.description == "ValueError: mail failed" + assert tool.events[0].attributes["exception.message"] == "mail failed" + + +def test_secrets_are_removed_from_attributes_status_and_events(monkeypatch): + monkeypatch.setenv(ENV_SECRET_NAME, ENV_SECRET) + exporter, handler = _handler(redact=[EXPLICIT_SECRET]) + leaky = "keys {0} {1} {2} Authorization: Bearer abcdefghijklmnop".format( + ENV_SECRET, EXPLICIT_SECRET, LLM_KEY + ) + run = Run(handler, query=leaky) + run.llm(AIMessage(content=leaky)) + node, tool = run.start_tool("echo", {"text": leaky}) + try: + raise RuntimeError(leaky) + except RuntimeError as error: + handler.on_tool_error(error, run_id=tool, parent_run_id=node) + run.end_node(node) + run.finish(text=leaky) + spans = exporter.get_finished_spans() + text = wire(spans) + for secret in (ENV_SECRET, EXPLICIT_SECRET, LLM_KEY, "abcdefghijklmnop"): + assert secret not in text, secret + assert attrs(only(spans, AGENT))["input.value"].startswith("keys [redacted] [redacted] [redacted]") + + +def test_secrets_are_removed_from_names_with_content_off(monkeypatch): + monkeypatch.setenv(ENV_SECRET_NAME, ENV_SECRET) + exporter, handler = _handler(config=HIDDEN) + run = Run(handler) + node, tool = run.start_tool("tool-" + ENV_SECRET, {}, call_id="id-" + ENV_SECRET) + handler.on_tool_end("done", run_id=tool, parent_run_id=node) + run.end_node(node) + run.finish() + assert ENV_SECRET not in wire(exporter.get_finished_spans()) + + +def test_size_caps(): + exporter, handler = _handler() + long_text = "é" * 10_000 # 2 bytes each in UTF-8 + run = Run(handler, query=long_text) + many = [HumanMessage(content="m{0}".format(index)) for index in range(40)] + run.history = many + calls = [ + {"name": "t{0}".format(index), "args": {}, "id": "i{0}".format(index), "type": "tool_call"} + for index in range(20) + ] + run.llm(AIMessage(content=long_text, tool_calls=calls)) + long_name = "n" * 1000 + node, tool = run.start_tool(long_name, {"text": long_text}) + # A chained exception with a long message: no stack trace is recorded, + # only the message, cut at its cap. + try: + try: + raise RuntimeError("c" * 20_000) + except RuntimeError as cause: + raise ValueError("x" * 5000 + " LAST-LINE-END") from cause + except ValueError as error: + handler.on_tool_error(error, run_id=tool, parent_run_id=node) + run.end_node(node) + run.finish(text=long_text) + spans = exporter.get_finished_spans() + + for span in spans: + for key, value in attrs(span).items(): + if isinstance(value, str): + assert len(value.encode("utf-8")) <= MAX_VALUE_BYTES, key + agent = attrs(only(spans, AGENT)) + assert agent["input.value"] == "é" * (MAX_VALUE_BYTES // 2) + llm = attrs(only(spans, LLM_SPAN)) + assert llm["mcp_use.llm.input_message_count"] == 41 + recorded = {key.split(".")[3] for key in llm if key.startswith("gen_ai.input.messages.")} + assert len(recorded) == callback_module.MAX_MESSAGES == 32 + # The most recent 32 messages: m9..m39 and the system prompt is dropped. + assert llm["gen_ai.input.messages.31.message.content"] == "m39" + assert llm["mcp_use.llm.tool_call_count"] == 20 + names = [key for key in llm if key.endswith("function.name")] + assert len(names) == callback_module.MAX_TOOL_CALLS == 16 + + # The tool name is capped before it is put into the span name. + tool_span = only(spans, "execute_tool " + "n" * MAX_NAME_BYTES) + assert tool_span.attributes["gen_ai.tool.name"] == "n" * MAX_NAME_BYTES + (event,) = tool_span.events + message = event.attributes["exception.message"] + assert message == "x" * MAX_ERROR_BYTES + assert tool_span.status.description == "ValueError: " + "x" * MAX_ERROR_BYTES + assert "exception.stacktrace" not in event.attributes + assert "c" * 100 not in wire(spans) # the cause is not recorded + + +def test_streamed_tokens_are_events_on_one_llm_span_with_capped_count(): + exporter, handler = _handler() + run = Run(handler) + node = run.node("model") + llm = uuid4() + handler.on_chat_model_start({}, [[HumanMessage(content=PROMPT)]], run_id=llm, parent_run_id=node, metadata=METADATA) + for index in range(200): + handler.on_llm_new_token("TOKEN-TEXT-{0} ".format(index), run_id=llm, parent_run_id=node) + handler.on_llm_end(_result(AIMessage(content="streamed")), run_id=llm, parent_run_id=node) + run.end_node(node) + run.finish() + spans = exporter.get_finished_spans() + assert [span.name for span in spans].count(LLM_SPAN) == 1 + span = only(spans, LLM_SPAN) + assert span.attributes["mcp_use.llm.chunk_count"] == 200 + assert len(span.events) == callback_module.MAX_CHUNK_EVENTS == 128 + assert [event.attributes["mcp_use.llm.chunk.index"] for event in span.events] == list(range(128)) + assert all(event.name == "mcp_use.llm.chunk" for event in span.events) + assert all(set(event.attributes) == {"mcp_use.llm.chunk.index"} for event in span.events) + assert "TOKEN-TEXT" not in wire(spans) + + +@pytest.mark.parametrize("hidden", [False, True]) +def test_llm_error_fails_the_llm_and_agent_spans(hidden): + exporter, handler = _handler(**({"config": HIDDEN} if hidden else {})) + run = Run(handler) + node = run.node("model") + llm = uuid4() + handler.on_chat_model_start({}, [[HumanMessage(content=PROMPT)]], run_id=llm, parent_run_id=node, metadata=METADATA) + error = ConnectionError("provider down " + PROMPT) + handler.on_llm_error(error, run_id=llm, parent_run_id=node) + handler.on_chain_error(error, run_id=node, parent_run_id=run.root) + handler.on_chain_error(error, run_id=run.root) + spans = exporter.get_finished_spans() + assert sorted(span.name for span in spans) == sorted([AGENT, LLM_SPAN]) + for span in spans: + assert span.status.status_code is StatusCode.ERROR + if hidden: + assert span.status.description == "ConnectionError" + else: + assert span.status.description == "ConnectionError: provider down " + PROMPT + assert [event.name for event in span.events] == ["exception"] + assert span.events[0].attributes["exception.type"] == "ConnectionError" + assert "exception.stacktrace" not in span.events[0].attributes + assert attrs(only(spans, AGENT))["mcp_use.agent.llm_call_count"] == 1 + assert (PROMPT in wire(spans)) is not hidden + + +def test_mcp_use_formatted_tool_error_is_an_error_span(): + # mcp-use 1.7.1 returns a failed MCP call as format_error()'s dict, as + # JSON in the ToolMessage (agents/adapters/langchain_adapter.py:181-206). + for capture in (False, True): + exporter, handler = _handler(**({} if capture else {"config": HIDDEN})) + run = Run(handler) + node, tool = run.start_tool("fail", {"reason": ARG}) + content = json.dumps( + { + "error": "RuntimeError", + "details": "Error executing tool fail: tool failed because " + ARG, + "stack": "Traceback (most recent call last):\n ...\nRuntimeError: boom", + "code": "UNKNOWN", + "tool": "fail", + } + ) + handler.on_tool_end(ToolMessage(content=content, tool_call_id="call-1"), run_id=tool, parent_run_id=node) + run.end_node(node) + run.finish() + spans = exporter.get_finished_spans() + span = only(spans, "execute_tool fail") + assert span.status.status_code is StatusCode.ERROR + assert span.attributes["mcp_use.tool.error_type"] == "RuntimeError" + (event,) = span.events + assert event.attributes["exception.type"] == "RuntimeError" + assert "gen_ai.tool.call.result" not in span.attributes + assert attrs(only(spans, AGENT))["mcp_use.agent.tool_error_count"] == 1 + assert "exception.stacktrace" not in event.attributes + assert "Traceback" not in wire(spans) # format_error's "stack" is never read + if capture: + assert span.status.description == "RuntimeError: Error executing tool fail: tool failed because " + ARG + else: + assert span.status.description == "RuntimeError" + assert ARG not in wire(spans) + + +@pytest.mark.parametrize("as_message", [True, False]) +@pytest.mark.parametrize("tool_key", [None, "fail", 42]) +def test_a_tool_result_shaped_like_an_mcp_use_error_is_still_a_result(tool_key, as_message): + # format_error output always carries tool= + # (langchain_adapter.py:197-205, tool_error_middleware.py:61). A tool's + # own result with the error/details/stack/code keys (an incident + # lookup, say) and no matching tool key is a successful call. + exporter, handler = _handler() + run = Run(handler) + node, tool = run.start_tool("incidents", {"id": "INC-1"}) + body: Dict[str, Any] = { + "error": "DiskFull", + "details": "INCIDENT-MARKER disk full on db-1", + "stack": "n/a", + "code": "E42", + } + if tool_key is not None: + body["tool"] = tool_key + content = json.dumps(body) + output = ToolMessage(content=content, tool_call_id="call-1") if as_message else body + handler.on_tool_end(output, run_id=tool, parent_run_id=node) + run.end_node(node) + run.finish() + spans = exporter.get_finished_spans() + span = only(spans, "execute_tool incidents") + assert span.status.status_code is StatusCode.OK + assert span.events == () + assert "mcp_use.tool.error_type" not in span.attributes + assert span.attributes["gen_ai.tool.call.result"] == content + assert attrs(only(spans, AGENT))["mcp_use.agent.tool_error_count"] == 0 + + +def test_tool_message_with_error_status_is_an_error_span(): + exporter, handler = _handler() + run = Run(handler) + node, tool = run.start_tool("add", {"a": 1}) + output = ToolMessage(content="Error: bad input", tool_call_id="call-1", status="error") + handler.on_tool_end(output, run_id=tool, parent_run_id=node) + run.end_node(node) + run.finish() + span = only(exporter.get_finished_spans(), TOOL_SPAN) + assert span.status.status_code is StatusCode.ERROR + assert span.events[0].attributes["exception.type"] == "langchain_core.tools.ToolException" + + +@pytest.mark.parametrize("error", [asyncio.CancelledError(), GeneratorExit()]) +def test_cancellation_ends_open_spans_as_cancelled_without_an_exception_event(error): + exporter, handler = _handler() + run = Run(handler) + node, tool = run.start_tool("slow", {"seconds": 30}) + # What LangGraph reports when the task is cancelled during the tool + # call: chain errors for the tools node and the root, no tool callback. + handler.on_chain_error(error, run_id=node, parent_run_id=run.root) + handler.on_chain_error(error, run_id=run.root) + spans = exporter.get_finished_spans() + assert sorted(span.name for span in spans) == sorted([AGENT, "execute_tool slow"]) + for span in spans: + assert span.status.status_code is StatusCode.ERROR + assert span.status.description == "cancelled" + assert span.attributes["mcp_use.cancelled"] is True + assert span.events == () + + +def test_a_run_whose_parent_ended_first_is_ended_as_incomplete(): + exporter, handler = _handler() + run = Run(handler) + node = run.node("model") + handler.on_chat_model_start({}, [[HumanMessage(content="x")]], run_id=uuid4(), parent_run_id=node) + run.finish() + spans = exporter.get_finished_spans() + llm = only(spans, "chat") + assert llm.attributes["mcp_use.incomplete"] is True + assert llm.status.status_code is StatusCode.ERROR + assert llm.status.description == "ended without a result" + assert only(spans, AGENT).status.status_code is StatusCode.OK + assert handler._runs == {} and len(handler._roots) == 0 + + +def test_text_completion_models_get_an_llm_span(): + exporter, handler = _handler() + run = Run(handler) + node = run.node("model") + llm = uuid4() + handler.on_llm_start({}, ["complete " + PROMPT], run_id=llm, parent_run_id=node, invocation_params={"model": "completion-1"}) + handler.on_llm_end(LLMResult(generations=[[]], llm_output={"model_name": "completion-1"}), run_id=llm, parent_run_id=node) + run.end_node(node) + run.finish() + span = only(exporter.get_finished_spans(), "text_completion completion-1") + assert span.attributes["gen_ai.operation.name"] == "text_completion" + assert span.attributes["input.value"] == "complete " + PROMPT + + +def test_interleaved_agent_runs_on_one_handler_stay_separate(): + exporter, handler = _handler() + first, second = Run(handler), Run(handler) + first.llm(AIMessage(content="a")) + second.llm(AIMessage(content="b")) + second.tool("add", {"a": 1}, "r") + first.finish() + second.finish() + spans = exporter.get_finished_spans() + agents = spans_named(spans, AGENT) + assert len(agents) == 2 + for agent in agents: + children = [span for span in spans if parent_id(span) == agent.context.span_id] + assert all(child.context.trace_id == agent.context.trace_id for child in children) + counts = sorted((a.attributes["mcp_use.agent.llm_call_count"], a.attributes["mcp_use.agent.tool_call_count"]) for a in agents) + assert counts == [(1, 0), (1, 1)] + assert handler._runs == {} + + +def test_open_runs_are_bounded(monkeypatch): + monkeypatch.setattr(callback_module, "MAX_OPEN_RUNS", 3) + exporter, handler = _handler() + abandoned = Run(handler) + abandoned.start_tool("slow", {}) + current = Run(handler) + current.node("model") # 5 open runs: the abandoned agent run is evicted + spans = exporter.get_finished_spans() + assert sorted(span.name for span in spans) == sorted([AGENT, "execute_tool slow"]) + assert all(span.attributes["mcp_use.incomplete"] is True for span in spans) + current.finish() + assert handler._runs == {} + + +def test_ending_a_run_only_looks_at_its_own_agent_run(monkeypatch): + # The open-descendant check on each run end covers the runs under the + # same agent run, not every open run of every agent on the handler. + exporter, handler = _handler() + busy = Run(handler) + for _ in range(50): + busy.start_tool("slow", {}) + looked_at: List[UUID] = [] + descends = handler._descends + + def spy(run: Any, ancestor: UUID) -> bool: + looked_at.append(run.root) + return descends(run, ancestor) + + monkeypatch.setattr(handler, "_descends", spy) + journey = _journey(handler) + assert looked_at and set(looked_at) == {journey.root} + assert len(handler._runs) == 101 # busy: the root, 50 tools nodes, 50 tools + exporter.clear() + busy.finish() + spans = exporter.get_finished_spans() + assert [span.name for span in spans].count("execute_tool slow") == 50 + assert all(span.attributes.get("mcp_use.incomplete") for span in spans if span.name != AGENT) + assert handler._runs == {} and len(handler._roots) == 0 + + +def test_a_repeated_start_of_an_open_agent_run_still_cleans_up(): + exporter, handler = _handler() + run = Run(handler) + run.start_tool("slow", {}) + handler.on_chain_start({}, {"messages": []}, run_id=run.root) # same id again + run.finish() + assert handler._runs == {} and len(handler._roots) == 0 + assert only(exporter.get_finished_spans(), "execute_tool slow").attributes["mcp_use.incomplete"] is True + + +def test_the_current_span_parents_the_agent_span(): + exporter, handler = _handler() + tracer = new_provider()[1].get_tracer("test") + with tracer.start_as_current_span("request") as outer: + _journey(handler) + agent = only(exporter.get_finished_spans(), AGENT) + assert parent_id(agent) == outer.get_span_context().span_id + assert agent.context.trace_id == outer.get_span_context().trace_id + + +def test_context_attributes_are_stamped_on_every_span(): + exporter, handler = _handler() + with using_session("session-1"), using_user("user-1"), using_metadata({"tenant": "t1"}), using_tags(["beta"]): + _journey(handler) + spans = exporter.get_finished_spans() + assert len(spans) == 4 + for span in spans: + assert span.attributes["session.id"] == "session-1" + assert span.attributes["user.id"] == "user-1" + assert json.loads(span.attributes["metadata"]) == {"tenant": "t1"} + assert span.attributes["tag.tags"] == ("beta",) + + +def test_suppress_tracing_records_nothing(): + exporter, handler = _handler() + with suppress_tracing(): + _journey(handler) + assert exporter.get_finished_spans() == () + assert handler._runs == {} + + +class _Broken: + """A tracer provider whose tracers and spans raise on every call.""" + + def get_tracer(self, *args: Any, **kwargs: Any) -> Any: + return self + + def start_span(self, *args: Any, **kwargs: Any) -> Any: + raise RuntimeError("tracer is broken") + + +class _BrokenSpans(trace_api.TracerProvider): + """Spans start, then raise from set_status, add_event and end. + + With ``on_attribute`` they also raise from set_attribute, so starting + one fails too (FITracer sets the start attributes). + """ + + def __init__(self, on_attribute: bool) -> None: + self.on_attribute = on_attribute + + def get_tracer(self, *args: Any, **kwargs: Any) -> Any: + return self + + def start_span(self, *args: Any, **kwargs: Any) -> Any: + return _BrokenSpan(self.on_attribute) + + +class _BrokenSpan(trace_api.NonRecordingSpan): + def __init__(self, on_attribute: bool) -> None: + super().__init__(trace_api.INVALID_SPAN_CONTEXT) + self.on_attribute = on_attribute + + def is_recording(self) -> bool: + return True + + def set_attribute(self, *args: Any, **kwargs: Any) -> None: + if self.on_attribute: + raise RuntimeError("span is broken") + + def set_status(self, *args: Any, **kwargs: Any) -> None: + raise RuntimeError("span is broken") + + def add_event(self, *args: Any, **kwargs: Any) -> None: + raise RuntimeError("span is broken") + + def end(self, *args: Any, **kwargs: Any) -> None: + raise RuntimeError("span is broken") + + +@pytest.mark.parametrize( + "provider", [_Broken(), _BrokenSpans(on_attribute=False), _BrokenSpans(on_attribute=True)] +) +def test_tracer_failures_never_reach_the_caller(provider): + handler = FutureAGICallback(tracer_provider=provider) + _journey(handler) + _failing_tool(handler, ValueError("boom")) + run = Run(handler) + node = run.node("model") + llm = uuid4() + handler.on_chat_model_start({}, [[HumanMessage(content="x")]], run_id=llm, parent_run_id=node) + handler.on_llm_new_token("t", run_id=llm, parent_run_id=node) + handler.on_chain_error(asyncio.CancelledError(), run_id=run.root) + assert handler._runs == {} + + +@pytest.mark.parametrize("capture", [False, True]) +def test_malformed_callback_payloads_never_raise(capture): + exporter, handler = _handler(**({} if capture else {"config": HIDDEN})) + root, child = uuid4(), uuid4() + weird: List[Any] = [None, object(), 42, "text", [object()], {"messages": object()}] + for value in weird: + handler.on_chain_start(value, value, run_id=root) + handler.on_chat_model_start(value, [[value]] if value is not None else value, run_id=child, parent_run_id=root, metadata=value, invocation_params=value) + handler.on_llm_new_token(value, run_id=child) + handler.on_llm_end(value, run_id=child) + handler.on_llm_start(value, value, run_id=child, parent_run_id=root) + handler.on_llm_error(RuntimeError(), run_id=child) + handler.on_tool_start(value, value, run_id=child, parent_run_id=root, inputs=value) + handler.on_tool_end(value, run_id=child) + handler.on_tool_error(RuntimeError(), run_id=child) + handler.on_retriever_start(value, value, run_id=child, parent_run_id=root) + handler.on_retriever_end(value, run_id=child) + handler.on_chain_end(value, run_id=root) + handler.on_chain_error(RuntimeError(), run_id=uuid4()) + assert handler._runs == {} + + +def test_constructor_validates_its_arguments(): + with pytest.raises(TypeError, match="TraceConfig"): + FutureAGICallback(config={"hide_inputs": True}) # type: ignore[arg-type] + # No package-specific privacy switch: TraceConfig / FI_HIDE_* only. + with pytest.raises(TypeError, match="capture_content"): + FutureAGICallback(capture_content=True) # type: ignore[call-arg] + with pytest.raises(TypeError, match="not a string"): + FutureAGICallback(redact="secret-value") # type: ignore[arg-type] + with pytest.raises(TypeError, match="strings"): + FutureAGICallback(redact=[b"secret-value"]) # type: ignore[list-item] + + +def test_hide_flags_fail_closed_when_a_run_remembers_too_much(monkeypatch): + import traceai_mcp_use._text as text_module + + monkeypatch.setattr(text_module, "MAX_KNOWN_TEXTS", 4) + exporter, handler = _handler(config=TraceConfig(hide_inputs=True)) + _journey(handler) + _failing_tool(handler, ValueError("tool failed because " + ARG)) + spans = exporter.get_finished_spans() + text = wire(spans) + for marker in CONTENT_MARKERS + (ANSWER,): + assert marker not in text, marker + tool = only(spans, "execute_tool fail") + assert tool.status.description == "ValueError" + assert dict(tool.events[0].attributes) == {"exception.type": "ValueError"} + + +def test_an_internal_failure_in_any_callback_never_reaches_langchain(monkeypatch): + # Every public callback catches its own failure, past the inner guards + # the other tests exercise: make the bookkeeping itself raise. + exporter, handler = _handler() + + def boom(*args: Any, **kwargs: Any) -> Any: + raise RuntimeError("internal failure") + + class Exploding(dict): + def get(self, *args: Any, **kwargs: Any) -> Any: + raise RuntimeError("internal failure") + + for name in ("_begin", "_pop", "_finish_error"): + monkeypatch.setattr(handler, name, boom) + monkeypatch.setattr(handler, "_runs", Exploding()) + run_id, parent = uuid4(), uuid4() + reply = _result(AIMessage(content="x")) + handler.on_chain_start({}, {"messages": []}, run_id=run_id) + handler.on_chain_end({}, run_id=run_id) + handler.on_chain_error(RuntimeError(), run_id=run_id) + handler.on_retriever_start({}, "q", run_id=run_id, parent_run_id=parent) + handler.on_retriever_end([], run_id=run_id) + handler.on_retriever_error(RuntimeError(), run_id=run_id) + handler.on_chat_model_start({}, [[HumanMessage(content="x")]], run_id=run_id) + handler.on_llm_start({}, ["x"], run_id=run_id) + handler.on_llm_new_token("t", run_id=run_id) + handler.on_llm_end(reply, run_id=run_id) + handler.on_llm_error(RuntimeError(), run_id=run_id) + handler.on_tool_start({"name": "add"}, "{}", run_id=run_id, inputs={}) + handler.on_tool_end("result", run_id=run_id) + handler.on_tool_error(RuntimeError(), run_id=run_id) + assert exporter.get_finished_spans() == () diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_contract.py b/python/frameworks/mcp-use/tests/test_mcp_use_contract.py new file mode 100644 index 000000000..a92ce4da2 --- /dev/null +++ b/python/frameworks/mcp-use/tests/test_mcp_use_contract.py @@ -0,0 +1,235 @@ +"""Shared-harness contract test for traceAI-mcp-use (TH-8331, harness Receiver). + +The real MCPAgent runs on the fake LLM against the fake MCP server over +stdio. Spans leave through the real ``fi_instrumentation.register()`` OTLP +exporter into ``harness.Receiver`` on 127.0.0.1. Every key is a +placeholder. +""" + +from __future__ import annotations + +import asyncio +import json +import sys +from collections import Counter +from pathlib import Path +from typing import Any, Dict, List + +import pytest +from _mcp_use_support import ( + AGENT, + ANSWER, + ARG, + CONTENT_MARKERS, + ENV_SECRET, + ENV_SECRET_NAME, + LLM_KEY, + LLM_SPAN, + MODEL, + PROMPT, + ChatFake, + add_script, + answer, + mcp_client, + tool_call, +) + +pytest.importorskip("opentelemetry.proto", reason="the harness decodes OTLP protobuf") + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) + +from harness import Receiver, _flatten_attributes # noqa: E402 + +from traceai_mcp_use import FutureAGICallback # noqa: E402 + +FI_API_KEY = "placeholder-fi-api-key" +FI_SECRET_KEY = "placeholder-fi-secret-key" +PROJECT = "mcp-use-contract" + + +def _register(receiver: Receiver, monkeypatch) -> Any: + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv("FI_API_KEY", FI_API_KEY) + monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET_KEY) + monkeypatch.setenv(ENV_SECRET_NAME, ENV_SECRET) + return register(project_type=ProjectType.OBSERVE, project_name=PROJECT, verbose=False) + + +def _journey(receiver: Receiver, monkeypatch, **options: Any) -> List[Dict[str, Any]]: + """J1 (add, then answer) and a failing tool call, through register().""" + from mcp_use import MCPAgent + + provider = _register(receiver, monkeypatch) + handler = FutureAGICallback(tracer_provider=provider, **options) + + async def go() -> None: + client = mcp_client("stdio") + try: + agent = MCPAgent(llm=ChatFake(script=add_script()), client=client, callbacks=[handler]) + assert await agent.run(PROMPT + " " + ENV_SECRET) == ANSWER + failing = MCPAgent( + llm=ChatFake(script=[tool_call("fail", {"reason": ARG}), answer()]), + client=client, + callbacks=[handler], + ) + assert await failing.run(PROMPT) == ANSWER + finally: + await client.close_all_sessions() + + try: + asyncio.run(go()) + assert provider.force_flush(timeout_millis=10_000) + finally: + provider.shutdown() + return receiver.spans() + + +def _trees(spans: List[Dict[str, Any]]) -> List[List[Dict[str, Any]]]: + """Spans grouped by trace, agent span first, in start order.""" + traces: Dict[str, List[Dict[str, Any]]] = {} + for span in sorted(spans, key=lambda span: int(span["startTimeUnixNano"])): + traces.setdefault(span["traceId"], []).append(span) + return sorted( + (sorted(group, key=lambda span: span["name"] != AGENT) for group in traces.values()), + key=lambda group: int(group[0]["startTimeUnixNano"]), + ) + + +def _attrs(span: Dict[str, Any]) -> Dict[str, Any]: + return _flatten_attributes(span.get("attributes", [])) + + +def test_agent_runs_reach_the_collector_contract(monkeypatch): + with Receiver() as receiver: + spans = _journey(receiver, monkeypatch) + exports = receiver.requests() + + assert exports + for export in exports: + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_API_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET_KEY + assert "authorization" not in export["headers"] + for resource in export["resource_attributes"]: + assert resource["project_name"] == PROJECT + assert resource["project_type"] == "observe" + + j1, failed = _trees(spans) + agent, *children = j1 + assert agent["name"] == AGENT + assert not agent.get("parentSpanId") + assert [child["name"] for child in children] == [LLM_SPAN, "execute_tool add", LLM_SPAN] + for child in children: + assert child["parentSpanId"] == agent["spanId"] + kinds = [_attrs(span)["gen_ai.span.kind"] for span in j1] + assert kinds == ["AGENT", "LLM", "TOOL", "LLM"] + assert _attrs(children[0])["gen_ai.request.model"] == MODEL + assert _attrs(children[1])["gen_ai.tool.name"] == "add" + assert int(_attrs(agent)["mcp_use.agent.tool_call_count"]) == 1 + # register()'s processor also promotes UNSET to OK on export, so the + # handler's own OK is pinned by the in-memory tests. + assert all(span["status"]["code"] == "STATUS_CODE_OK" for span in j1) + + failed_agent, *failed_children = failed + tool = next(span for span in failed_children if span["name"] == "execute_tool fail") + assert tool["parentSpanId"] == failed_agent["spanId"] + assert tool["status"]["code"] == "STATUS_CODE_ERROR" + # Default TraceConfig: the error text is content and is recorded. + assert tool["status"]["message"] == "RuntimeError: Error executing tool fail: tool failed because " + ARG + assert [event["name"] for event in tool["events"]] == ["exception"] + assert failed_agent["status"]["code"] == "STATUS_CODE_OK" + assert int(_attrs(failed_agent)["mcp_use.agent.tool_error_count"]) == 1 + + # One span per tool call: nothing else shares a tool span's name and parent. + for tree in (j1, failed): + tools = Counter((span["name"], span.get("parentSpanId")) for span in tree if span["name"].startswith("execute_tool")) + assert set(tools.values()) == {1} + + +def test_no_key_or_content_is_exported_with_the_hide_flags(monkeypatch): + # Content off the standard traceAI way: FI_HIDE_INPUTS / FI_HIDE_OUTPUTS. + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + monkeypatch.setenv("FI_HIDE_OUTPUTS", "true") + with Receiver() as receiver: + spans = _journey(receiver, monkeypatch) + + assert len(spans) == 8 # two runs: agent, LLM, tool, LLM each + text = json.dumps(spans) + for secret in (FI_API_KEY, FI_SECRET_KEY, LLM_KEY, ENV_SECRET) + CONTENT_MARKERS: + assert secret not in text, secret + for span in spans: + for key, value in _attrs(span).items(): + if key in ("input.value", "output.value"): + assert value == "__REDACTED__", key + continue + assert not key.startswith(("input.", "output.", "gen_ai.input.", "gen_ai.output.")), key + assert key not in ("gen_ai.tool.call.arguments", "gen_ai.tool.call.result"), key + for needle in ("Traceback", 'File "', "site-packages", "exception.stacktrace"): + assert needle not in text, needle + + +def test_content_reaches_the_collector_by_default_without_keys(monkeypatch): + # Control for the test above: with the default TraceConfig the same + # journey carries the prompt, arguments, results and answer, still + # without keys, stack traces or local paths. + monkeypatch.delenv("FI_HIDE_INPUTS", raising=False) + monkeypatch.delenv("FI_HIDE_OUTPUTS", raising=False) + with Receiver() as receiver: + spans = _journey(receiver, monkeypatch) + + text = json.dumps(spans) + for marker in ("PROMPT-MARKER", "ANSWER-MARKER", "SUM-RESULT-5", ARG, "tool failed because"): + assert marker in text, marker + for secret in (FI_API_KEY, FI_SECRET_KEY, LLM_KEY, ENV_SECRET): + assert secret not in text, secret + for needle in ("Traceback", 'File "', "site-packages", "exception.stacktrace"): + assert needle not in text, needle + agent = _attrs(_trees(spans)[0][0]) + assert agent["input.value"] == PROMPT + " [redacted]" + + +def _two_turns(receiver: Receiver, monkeypatch, **options: Any) -> List[Dict[str, Any]]: + """J1, then a second query on the same MCPAgent (memory on, the default).""" + from mcp_use import MCPAgent + + provider = _register(receiver, monkeypatch) + handler = FutureAGICallback(tracer_provider=provider, **options) + + async def go() -> None: + client = mcp_client("stdio") + try: + agent = MCPAgent( + llm=ChatFake(script=add_script() + [answer("SECOND-ANSWER done")]), + client=client, + callbacks=[handler], + ) + assert await agent.run(PROMPT) == ANSWER + assert await agent.run("SECOND-PROMPT") == "SECOND-ANSWER done" + finally: + await client.close_all_sessions() + + try: + asyncio.run(go()) + assert provider.force_flush(timeout_millis=10_000) + finally: + provider.shutdown() + return receiver.spans() + + +@pytest.mark.parametrize("hide_outputs", [False, True]) +def test_hide_outputs_keeps_the_first_turn_out_of_the_second_turn_export(monkeypatch, hide_outputs): + from fi_instrumentation import TraceConfig + + with Receiver() as receiver: + spans = _two_turns(receiver, monkeypatch, config=TraceConfig(hide_outputs=hide_outputs)) + + _, second = _trees(spans) + assert [span["name"] for span in second] == [AGENT, LLM_SPAN] + assert _attrs(second[0])["input.value"] == "SECOND-PROMPT" + text = json.dumps(second) + for marker in ("SUM-RESULT-5", "ANSWER-MARKER"): + # The control (hide_outputs off) shows the second turn replays them. + assert (marker in text) is not hide_outputs, marker diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_example.py b/python/frameworks/mcp-use/tests/test_mcp_use_example.py new file mode 100644 index 000000000..b7e08bd70 --- /dev/null +++ b/python/frameworks/mcp-use/tests/test_mcp_use_example.py @@ -0,0 +1,121 @@ +"""examples/agent_with_tools.py runs end to end through the shared harness. + +harness.run starts ``_example_app.py`` as a subprocess with FI_BASE_URL +pointed at harness.Receiver. The example's own code runs: register(), an +MCPClient on the stdio calculator server next to it, and an MCPAgent with +FutureAGICallback. Only the LLM is replaced by the fake. The process exits +without flushing explicitly; the spans it exported prove the flush at exit +(PRD J7). + +The same run with the existing transport package traceai_mcp instrumented +first gives the same spans: traceai_mcp 0.1.2 records no spans of its own +(it only carries trace context in MCP request metadata), so the agent span +is there and no tool span is duplicated under the same name and parent. +""" + +from __future__ import annotations + +import json +import os +import sys +from collections import Counter +from pathlib import Path +from typing import Any, Dict, List, Tuple + +import pytest +from _mcp_use_support import AGENT, LLM_KEY, LLM_SPAN, MODEL + +pytest.importorskip("opentelemetry.proto", reason="the harness decodes OTLP protobuf") + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) + +from harness import Receiver, _flatten_attributes, run # noqa: E402 + +APP = Path(__file__).resolve().parent / "_example_app.py" +FI_API_KEY = "placeholder-fi-api-key" +FI_SECRET_KEY = "placeholder-fi-secret-key" + + +def _run_example(*flags: str) -> Tuple[Any, List[Dict[str, Any]], List[Dict[str, Any]]]: + with Receiver() as receiver: + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("FI_", "OTEL_", "LANGFUSE_", "LAMINAR_", "MCP_USE_")) + } + env.update( + { + "FI_BASE_URL": receiver.origin, + "FI_API_KEY": FI_API_KEY, + "FI_SECRET_KEY": FI_SECRET_KEY, + # mcp-use's own usage telemetry posts to PostHog and Scarf. + "MCP_USE_ANONYMIZED_TELEMETRY": "false", + # The subprocess imports exactly what this test process imports. + "PYTHONPATH": os.pathsep.join(sys.path), + } + ) + result = run([sys.executable, str(APP), *flags], env, None, timeout=600) + spans = receiver.spans() + exports = receiver.requests() + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode(errors="replace")[-4000:] + return result, spans, exports + + +def _shape(spans: List[Dict[str, Any]]) -> List[Tuple[str, str, bool]]: + """(name, kind, parented to the agent span) in start order.""" + (agent,) = [span for span in spans if span["name"] == AGENT] + ordered = sorted(spans, key=lambda span: int(span["startTimeUnixNano"])) + return [ + ( + span["name"], + _flatten_attributes(span["attributes"])["gen_ai.span.kind"], + span.get("parentSpanId") == agent["spanId"], + ) + for span in ordered + ] + + +def _check(result: Any, spans: List[Dict[str, Any]], exports: List[Dict[str, Any]]) -> None: + assert b"answer: EXAMPLE-ANSWER 2 + 3 = 5" in result.stdout + assert _shape(spans) == [ + (AGENT, "AGENT", False), + (LLM_SPAN, "LLM", True), + ("execute_tool add", "TOOL", True), + (LLM_SPAN, "LLM", True), + ] + assert len({span["traceId"] for span in spans}) == 1 + assert all(span["status"]["code"] == "STATUS_CODE_OK" for span in spans) + tool = next(span for span in spans if span["name"] == "execute_tool add") + assert _flatten_attributes(tool["attributes"])["gen_ai.tool.name"] == "add" + llm = next(span for span in spans if span["name"] == LLM_SPAN) + assert _flatten_attributes(llm["attributes"])["gen_ai.request.model"] == MODEL + + assert exports + for export in exports: + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_API_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET_KEY + for resource in export["resource_attributes"]: + assert resource["project_name"] == "mcp-use-agent" + assert resource["project_type"] == "observe" + + wire = json.dumps(spans) + for text in (FI_API_KEY, FI_SECRET_KEY, LLM_KEY): + assert text not in wire, text + # Content is captured by default (TraceConfig); FI_HIDE_* turns it off. + for text in ("What is 2 + 3", "EXAMPLE-ANSWER"): + assert text in wire, text + + +def test_example_exports_one_agent_tree_at_exit(): + _check(*_run_example()) + + +def test_with_traceai_mcp_the_agent_span_is_there_and_no_tool_span_is_duplicated(): + result, spans, exports = _run_example("--with-traceai-mcp") + assert b"traceai_mcp wraps stdio_client: FunctionWrapper" in result.stdout + _check(result, spans, exports) + by_name_and_parent = Counter((span["name"], span.get("parentSpanId")) for span in spans) + assert by_name_and_parent[("execute_tool add", next(s["spanId"] for s in spans if s["name"] == AGENT))] == 1 + assert [name for (name, _), count in by_name_and_parent.items() if count > 1] == [LLM_SPAN] diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_isolation.py b/python/frameworks/mcp-use/tests/test_mcp_use_isolation.py new file mode 100644 index 000000000..cc66ebdff --- /dev/null +++ b/python/frameworks/mcp-use/tests/test_mcp_use_isolation.py @@ -0,0 +1,96 @@ +"""The package only acts where the callback is passed. + +Importing traceai_mcp_use, building FutureAGICallback and running an agent +with it change no attribute of mcp-use, LangChain or mcp classes and +modules and no environment variable. The test suite itself never sends +mcp-use usage telemetry. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap + +import pytest +from _mcp_use_support import ANSWER, add_script, new_provider, run_agent + +from traceai_mcp_use import FutureAGICallback + +pytest.importorskip("uvicorn", reason="the loopback MCP server runs on uvicorn") + +SNAPSHOT = textwrap.dedent( + """ + import importlib, json, os, sys + import _mcp_use_support # telemetry off, as in every test + TARGETS = { + "mcp_use.agents.mcpagent": ["MCPAgent"], + "mcp_use.agents.observability.callbacks_manager": ["ObservabilityManager"], + "mcp_use.agents.adapters.langchain_adapter": ["LangChainAdapter"], + "mcp_use.client.task_managers.stdio": [], + "mcp.client.stdio": [], + "langchain_core.callbacks.manager": ["CallbackManager", "AsyncCallbackManager"], + "langchain_core.callbacks.base": ["BaseCallbackHandler", "BaseCallbackManager"], + "langchain_core.language_models.chat_models": ["BaseChatModel"], + "langchain_core.tools.base": ["BaseTool"], + } + + def snapshot(): + found = {} + for module_name, classes in TARGETS.items(): + module = importlib.import_module(module_name) + for key, value in vars(module).items(): + found[module_name + ":" + key] = id(value) + for class_name in classes: + for key, value in vars(getattr(module, class_name)).items(): + found[module_name + "." + class_name + ":" + key] = id(value) + return found + + before, env_before = snapshot(), dict(os.environ) + import traceai_mcp_use + from uuid import uuid4 + from _mcp_use_support import new_provider + handler = traceai_mcp_use.FutureAGICallback(tracer_provider=new_provider()[1]) + root = uuid4() + handler.on_chain_start({}, {"messages": []}, run_id=root) + handler.on_chain_end({"messages": []}, run_id=root) + after, env_after = snapshot(), dict(os.environ) + changed = sorted(key for key in set(before) | set(after) if before.get(key) != after.get(key)) + print(json.dumps({ + "changed": changed, + "env": sorted(set(env_before.items()) ^ set(env_after.items())), + "modules": sorted(name for name in sys.modules if name.split(".")[0] in ("langfuse", "lmnr", "traceai_mcp")), + })) + """ +) + + +def test_import_and_use_patch_nothing_and_set_no_environment_variable(): + env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path)) + result = subprocess.run( + [sys.executable, "-c", SNAPSHOT], env=env, capture_output=True, timeout=600 + ) + assert result.returncode == 0, result.stderr.decode(errors="replace")[-4000:] + report = json.loads(result.stdout.decode().strip().splitlines()[-1]) + assert report == {"changed": [], "env": [], "modules": []} + + +def test_a_traced_run_leaves_the_agent_callbacks_as_passed(): + from mcp_use import MCPAgent + + exporter, provider = new_provider() + handler = FutureAGICallback(tracer_provider=provider) + before = {key: id(value) for key, value in vars(MCPAgent).items()} + assert run_agent(add_script(), [handler]) == ANSWER + assert {key: id(value) for key, value in vars(MCPAgent).items()} == before + assert len(exporter.get_finished_spans()) == 4 + + +def test_mcp_use_usage_telemetry_is_off_in_this_suite(): + from mcp_use.telemetry.telemetry import Telemetry + + telemetry = Telemetry() + assert telemetry._posthog_client is None + assert telemetry._scarf_client is None diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_packaging.py b/python/frameworks/mcp-use/tests/test_mcp_use_packaging.py new file mode 100644 index 000000000..be85f515f --- /dev/null +++ b/python/frameworks/mcp-use/tests/test_mcp_use_packaging.py @@ -0,0 +1,105 @@ +"""Packaging contract: dependency ranges, the tested versions, Pythons, version.""" + +from __future__ import annotations + +import importlib.metadata +import re +import sys +from pathlib import Path + +from packaging.specifiers import SpecifierSet +from packaging.version import Version + +import traceai_mcp_use + +PACKAGE = Path(__file__).resolve().parents[1] +PYPROJECT = PACKAGE / "pyproject.toml" +README = PACKAGE / "README.md" +CHANGELOG = PACKAGE / "CHANGELOG.md" + +# The architecture's CI pin (mcp-use 1.7.1, commit 1e14ac9d). The README and +# CHANGELOG claim exactly this version; the suite runs on nothing else. +TESTED_MCP_USE = "1.7.1" +TESTED_PYTHONS = ["3.11", "3.12", "3.13"] + + +def _dependency(name: str) -> SpecifierSet: + match = re.search( + r'^{0}\s*=\s*"([^"]+)"'.format(re.escape(name)), PYPROJECT.read_text(), re.M + ) + assert match, name + return SpecifierSet(match.group(1)) + + +def test_the_suite_runs_on_the_tested_mcp_use_version(): + assert importlib.metadata.version("mcp-use") == TESTED_MCP_USE + + +def test_mcp_use_is_a_1x_range_from_the_tested_version(): + spec = _dependency("mcp-use") + assert spec.contains(TESTED_MCP_USE) + for version in ("1.7.2", "1.99.0"): + assert spec.contains(version), version + # 1.7.0 and earlier are not tested; 2.x drops the deprecated paths. + for version in ("1.7.0", "1.6.0", "2.0.0"): + assert not spec.contains(version), version + + +def test_langchain_core_range_admits_the_installed_version(): + spec = _dependency("langchain-core") + assert spec.contains(importlib.metadata.version("langchain-core")) + assert not spec.contains("0.3.80") + assert not spec.contains("2.0.0") + + +def test_fi_instrumentation_floor_is_1_1_0(): + # 0.1.11/0.1.12 export to the old create_otel_span path and the published + # 1.0.0 fails while building its headers; 1.1.0 is the first good release. + spec = _dependency("fi-instrumentation-otel") + assert spec.contains("1.1.0") + for version in ("0.1.11", "0.1.12", "0.1.16", "1.0.0"): + assert not spec.contains(version), version + + +def test_classifiers_and_python_range_match_the_tested_versions(): + # mcp-use requires Python >=3.11 (its Requires-Python), so the floor is + # 3.11, not the 3.9/3.10 floor of the rest of the family. + text = PYPROJECT.read_text() + assert re.findall(r'"Programming Language :: Python :: (3\.\d+)"', text) == TESTED_PYTHONS + python = _dependency("python") + for version in ("3.11.0", "3.12.10", "3.13.1"): + assert python.contains(version), version + for version in ("3.10.14", "3.14.0"): + assert not python.contains(version), version + assert "{0}.{1}".format(*sys.version_info[:2]) in TESTED_PYTHONS + + +def test_version_matches_pyproject_and_changelog(): + version = re.search(r'^version\s*=\s*"([^"]+)"', PYPROJECT.read_text(), re.M).group(1) + assert traceai_mcp_use.__version__ == version == "0.1.0" + assert re.search(r"^## (\S+)", CHANGELOG.read_text(), re.M).group(1) == version + assert re.search(r'^name\s*=\s*"traceAI-mcp-use"', PYPROJECT.read_text(), re.M) + + +def test_readme_and_changelog_claim_only_the_tested_versions(): + for path in (README, CHANGELOG): + text = " ".join(path.read_text().split()) + assert "mcp-use {0}".format(TESTED_MCP_USE) in text, path.name + claimed = set(re.findall(r"mcp-use (\d+\.\d+\.\d+)", text)) + assert claimed == {TESTED_MCP_USE}, (path.name, claimed) + assert "3.11, 3.12 and 3.13" in text, path.name + + +def test_package_source_imports_neither_langfuse_nor_traceai_mcp_nor_patches(): + source = "\n".join(path.read_text() for path in (PACKAGE / "traceai_mcp_use").glob("*.py")) + for forbidden in ( + r"^\s*(from|import)\s+langfuse", + r"^\s*(from|import)\s+lmnr", + r"^\s*(from|import)\s+traceai_mcp\b", + r"^\s*(from|import)\s+wrapt", + r"os\.environ\[[^\]]+\]\s*=", + r"os\.(putenv|environ\.setdefault|environ\.update)", + r"setattr\(", + ): + assert not re.search(forbidden, source, re.M), forbidden + assert Version(traceai_mcp_use.__version__) == Version("0.1.0") diff --git a/python/frameworks/mcp-use/traceai_mcp_use/__init__.py b/python/frameworks/mcp-use/traceai_mcp_use/__init__.py new file mode 100644 index 000000000..f922a60c7 --- /dev/null +++ b/python/frameworks/mcp-use/traceai_mcp_use/__init__.py @@ -0,0 +1,13 @@ +"""Trace mcp-use agents with Future AGI. + +``FutureAGICallback`` is a LangChain callback handler. Pass it to the agent: +``MCPAgent(llm=..., client=..., callbacks=[FutureAGICallback(tracer_provider=...)])``. +It records one span per agent run, LLM call and tool call. It patches +nothing, imports neither ``langfuse`` nor ``traceai_mcp``, and sets no +environment variables. +""" + +from traceai_mcp_use._callback import AGENT_SPAN_NAME, FutureAGICallback +from traceai_mcp_use.version import __version__ + +__all__ = ["AGENT_SPAN_NAME", "FutureAGICallback", "__version__"] diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py new file mode 100644 index 000000000..b79dabad6 --- /dev/null +++ b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py @@ -0,0 +1,1113 @@ +"""A LangChain callback handler that traces an mcp-use ``MCPAgent`` run. + +mcp-use 1.7.1 hands the ``callbacks`` list it was constructed with to the +LangGraph agent on every run (``MCPAgent.stream`` and +``MCPAgent._generate_response_chunks_async`` pass +``config={"callbacks": self.callbacks, ...}``). This handler turns the run +tree LangChain reports into three kinds of spans: + +- one ``AGENT`` span for the outermost run the handler sees (the LangGraph + graph that ``MCPAgent.run`` / ``stream`` / ``stream_events`` drives), +- one ``LLM`` span per chat-model (or completion-model) call, +- one ``TOOL`` span per tool call. + +The graph's internal node and middleware runs are not spans; an LLM or +tool span's parent is its nearest traced ancestor, which for mcp-use is the +agent span. Nothing is patched: the handler only exists where it is passed. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import threading +from collections import OrderedDict +from dataclasses import dataclass, field +from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple +from uuid import UUID + +from fi_instrumentation import FITracer, REDACTED_VALUE, TraceConfig +from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes +from langchain_core.callbacks import BaseCallbackHandler +from opentelemetry import context as context_api +from opentelemetry import trace as trace_api +from opentelemetry.trace import Span, Status, StatusCode + +from traceai_mcp_use._text import ( + MAX_ERROR_BYTES, + MAX_NAME_BYTES, + MAX_VALUE_BYTES, + KnownTexts, + Secrets, + cap, + clean, +) +from traceai_mcp_use.version import __version__ + +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + +AGENT_SPAN_NAME = "mcp_use.agent" + +SPAN_KIND = SpanAttributes.GEN_AI_SPAN_KIND +OPERATION = SpanAttributes.GEN_AI_OPERATION_NAME +INPUT_VALUE = SpanAttributes.INPUT_VALUE +OUTPUT_VALUE = SpanAttributes.OUTPUT_VALUE +INPUT_MESSAGES = SpanAttributes.GEN_AI_INPUT_MESSAGES +OUTPUT_MESSAGES = SpanAttributes.GEN_AI_OUTPUT_MESSAGES +REQUEST_MODEL = SpanAttributes.GEN_AI_REQUEST_MODEL +RESPONSE_MODEL = SpanAttributes.GEN_AI_RESPONSE_MODEL +PROVIDER = SpanAttributes.GEN_AI_PROVIDER_NAME +TEMPERATURE = SpanAttributes.GEN_AI_REQUEST_TEMPERATURE +MAX_TOKENS = SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS +FINISH_REASONS = SpanAttributes.GEN_AI_RESPONSE_FINISH_REASONS +INPUT_TOKENS = SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS +OUTPUT_TOKENS = SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS +TOTAL_TOKENS = SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS +CACHE_READ_TOKENS = SpanAttributes.GEN_AI_USAGE_CACHE_READ_TOKENS +TOOL_NAME = SpanAttributes.GEN_AI_TOOL_NAME +TOOL_CALL_ID = SpanAttributes.GEN_AI_TOOL_CALL_ID +TOOL_ARGUMENTS = SpanAttributes.GEN_AI_TOOL_CALL_ARGUMENTS +TOOL_RESULT = SpanAttributes.GEN_AI_TOOL_CALL_RESULT + +LLM_CALL_COUNT = "mcp_use.agent.llm_call_count" +TOOL_CALL_COUNT = "mcp_use.agent.tool_call_count" +TOOL_ERROR_COUNT = "mcp_use.agent.tool_error_count" +INPUT_MESSAGE_COUNT = "mcp_use.llm.input_message_count" +REQUESTED_TOOL_CALLS = "mcp_use.llm.tool_call_count" +CHUNK_COUNT = "mcp_use.llm.chunk_count" +CHUNK_EVENT = "mcp_use.llm.chunk" +CHUNK_INDEX = "mcp_use.llm.chunk.index" +TOOL_ERROR_TYPE = "mcp_use.tool.error_type" +CANCELLED = "mcp_use.cancelled" +INCOMPLETE = "mcp_use.incomplete" + +# At most this many messages are written per LLM span (the most recent ones); +# mcp_use.llm.input_message_count keeps the exact number. +MAX_MESSAGES = 32 +# At most this many tool calls are written per output message. +MAX_TOOL_CALLS = 16 +# At most this many chunk events per streamed LLM span; the count stays exact. +MAX_CHUNK_EVENTS = 128 +# Open runs tracked at once across all agent runs of one handler. Past it, +# the oldest agent run is ended (mcp_use.incomplete) and forgotten. +MAX_OPEN_RUNS = 10_000 + +# mcp-use 1.7.1 returns a failed MCP tool call as this dict +# (mcp_use/errors/error_formatting.py format_error) instead of raising, plus +# a "tool" key holding the tool's name (agents/adapters/langchain_adapter.py +# :197-205, agents/middleware/tool_error_middleware.py:61). +_MCP_USE_ERROR_KEYS = frozenset(("error", "details", "stack", "code")) + +_ROLES = {"human": "user", "ai": "assistant", "system": "system", "tool": "tool"} + +_AGENT = "agent" +_LLM = "llm" +_TOOL = "tool" +_CHAIN = "chain" + + +@dataclass +class _Failure: + type_name: str + qualified: str + message: Optional[str] + + +@dataclass +class _Root: + inputs: KnownTexts = field(default_factory=KnownTexts) + outputs: KnownTexts = field(default_factory=KnownTexts) + llm_calls: int = 0 + tool_calls: int = 0 + tool_errors: int = 0 + # Ids of the open runs under this agent run, its own included (a dict + # as an insertion-ordered set), so ending a run looks at its own tree only. + runs: Dict[UUID, None] = field(default_factory=dict) + + +@dataclass +class _Run: + run_id: UUID + kind: str + root: UUID + parent: Optional[UUID] + span: Optional[Span] = None + chunks: int = 0 + # A tool run's name as LangChain reported it (not capped or redacted). + tool_name: Optional[str] = None + + +def _describe(error: BaseException) -> str: + try: + return str(error) + except Exception: + return "".format(type(error).__name__) + + +def _qualified(error: BaseException) -> str: + error_type = type(error) + module = getattr(error_type, "__module__", "") + if module and module != "builtins": + return "{0}.{1}".format(module, error_type.__qualname__) + return error_type.__qualname__ + + +def _failure_from_exception(error: BaseException) -> _Failure: + # No stack trace: it carries local file paths and the user name. + return _Failure(type(error).__name__, _qualified(error), _describe(error)) + + +def _is_cancellation(error: BaseException) -> bool: + # asyncio cancellation, or a consumer that stopped reading MCPAgent.stream() + # (LangGraph reports the closed generator as GeneratorExit). + return isinstance(error, (asyncio.CancelledError, GeneratorExit)) + + +def _text_of(content: Any) -> Optional[str]: + """Plain text of a LangChain message content (str or list of blocks).""" + if content is None: + return None + if isinstance(content, str): + return content + if isinstance(content, list): + parts: List[str] = [] + for block in content: + if isinstance(block, str): + parts.append(block) + elif isinstance(block, Mapping) and isinstance(block.get("text"), str): + parts.append(block["text"]) + return "".join(parts) + return str(content) + + +def _role(message: Any) -> str: + if isinstance(message, Mapping): + role = message.get("role") or message.get("type") + return _ROLES.get(role, role) if isinstance(role, str) else "unknown" + kind = getattr(message, "type", None) + if isinstance(kind, str): + return _ROLES.get(kind, kind) + return "unknown" + + +def _is_output_role(role: str) -> bool: + """A message the model or a tool wrote: assistant, tool, or any role + other than user and system (fail closed for hide_outputs).""" + return role not in ("user", "system") + + +def _content(message: Any) -> Optional[str]: + """Text of a LangChain message object or a {"role", "content"} dict.""" + if isinstance(message, Mapping): + return _text_of(message.get("content")) + return _text_of(getattr(message, "content", None)) + + +def _tool_calls(message: Any) -> List[Mapping[str, Any]]: + if isinstance(message, Mapping): + calls = message.get("tool_calls") + else: + calls = getattr(message, "tool_calls", None) + if isinstance(calls, list): + return [call for call in calls if isinstance(call, Mapping)] + return [] + + +def _json(value: Any) -> str: + try: + return json.dumps(value, ensure_ascii=False, default=str) + except Exception: + return str(value) + + +def _strings_in(value: Any, depth: int = 0) -> Iterable[str]: + """Every string inside a tool-argument structure (bounded depth).""" + if depth > 6: + return + if isinstance(value, str): + yield value + elif isinstance(value, Mapping): + for item in value.values(): + yield from _strings_in(item, depth + 1) + elif isinstance(value, (list, tuple)): + for item in value: + yield from _strings_in(item, depth + 1) + + +def _replayed_outputs(messages: Iterable[Any]) -> List[Optional[str]]: + """Content and tool-call argument strings of the output-role messages + among a run's input messages. + + MCPAgent keeps memory by default and sends earlier turns (their replies, + tool calls and tool results) back as input on the next run, as does + ``external_history``: for hide_outputs these are outputs. + """ + texts: List[Optional[str]] = [] + for message in messages: + if not _is_output_role(_role(message)): + continue + texts.append(_content(message)) + for call in _tool_calls(message): + texts.extend(_strings_in(call.get("args"))) + return texts + + +def _mcp_use_error(output: Any, tool: Optional[str]) -> Optional[Mapping[str, Any]]: + """The mcp-use formatted error the tool named ``tool`` returned, if this + output is one. A tool's own result with the same keys but no matching + "tool" key is a result, not an error.""" + if not isinstance(tool, str): + return None + candidate: Any = output + content = getattr(output, "content", None) + if content is not None and not isinstance(output, Mapping): + candidate = content + if isinstance(candidate, str): + stripped = candidate.strip() + if not (stripped.startswith("{") and stripped.endswith("}")): + return None + try: + candidate = json.loads(stripped) + except ValueError: + return None + if ( + isinstance(candidate, Mapping) + and _MCP_USE_ERROR_KEYS.issubset(candidate.keys()) + and isinstance(candidate.get("error"), str) + and candidate.get("tool") == tool + ): + return candidate + return None + + +def _tool_output_text(output: Any) -> Optional[str]: + content = getattr(output, "content", None) + if content is not None: + return _text_of(content) + if isinstance(output, str): + return output + if output is None: + return None + return _json(output) + + +def _int(value: Any) -> Optional[int]: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + return None + + +def _number(value: Any) -> Optional[float]: + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return value + return None + + +class FutureAGICallback(BaseCallbackHandler): + """Trace an mcp-use agent: pass it as ``MCPAgent(..., callbacks=[...])``. + + Args: + tracer_provider: the provider spans go to; defaults to the global + provider. Pass the one ``fi_instrumentation.register()`` returned. + config: a ``fi_instrumentation.TraceConfig``; by default read from + the environment like every traceAI instrumentor. Content + (prompts, messages, model output, tool arguments and results, + error messages) is recorded unless the config hides it: ``hide_inputs``, + ``hide_outputs`` (``FI_HIDE_INPUTS``, ``FI_HIDE_OUTPUTS``) and + the message/text variants, plus ``pii_redaction`` + (``FI_PII_REDACTION``). Stack traces, local paths, the user + name and the host name are never recorded. + redact: secret strings to remove from everything written (for + example an LLM key or an MCP server header the environment does + not hold). + """ + + raise_error = False + # Run on the event loop, not in an executor thread. LangChain otherwise + # hands each sync callback to run_in_executor; a cancelled run can then + # deliver on_chain_error for a parent while on_tool_start for its child + # is still running in a thread, and the child span would be orphaned. + # Inline, every callback runs to completion in order. The handler does + # no I/O of its own; span export is the span processor's. + run_inline = True + + def __init__( + self, + tracer_provider: Optional[trace_api.TracerProvider] = None, + config: Optional[TraceConfig] = None, + redact: Iterable[str] = (), + ) -> None: + super().__init__() + if config is None: + config = TraceConfig() + elif not isinstance(config, TraceConfig): + raise TypeError( + "config must be a fi_instrumentation.TraceConfig, got {0}".format( + type(config).__name__ + ) + ) + if isinstance(redact, str): + raise TypeError("redact must be an iterable of strings, not a string") + secrets = list(redact) + for value in secrets: + if not isinstance(value, str): + raise TypeError( + "redact must contain strings, got {0}".format(type(value).__name__) + ) + provider = tracer_provider or trace_api.get_tracer_provider() + # FITracer applies the config's masking and PII pass to attributes and + # stamps using_session / using_user / using_metadata / using_tags. + self._tracer = FITracer( + trace_api.get_tracer(__name__, __version__, provider), config=config + ) + self._hide_inputs = bool(config.hide_inputs) + self._hide_outputs = bool(config.hide_outputs) + self._pii = bool(config.pii_redaction) + self._secrets = Secrets(secrets) + self._lock = threading.RLock() + self._runs: Dict[UUID, _Run] = {} + self._roots: "OrderedDict[UUID, _Root]" = OrderedDict() + + # ------------------------------------------------------------------ text + + def _scrubs(self, root: Optional[_Root]) -> List[KnownTexts]: + if root is None: + return [] + scrubs: List[KnownTexts] = [] + if self._hide_inputs: + scrubs.append(root.inputs) + if self._hide_outputs: + scrubs.append(root.outputs) + return scrubs + + def _clean(self, text: Optional[str], root: Optional[_Root], limit: int) -> Optional[str]: + if not isinstance(text, str): + return None + return clean(text, self._secrets, limit, self._pii, self._scrubs(root)) + + def _name(self, text: Any) -> Optional[str]: + """An identifier (model, tool name, id): secrets removed, 256-byte cap.""" + if not isinstance(text, str) or not text: + return None + return cap(self._secrets.redact(text), MAX_NAME_BYTES) + + def _remember(self, root: Optional[_Root], texts: Iterable[Optional[str]], output: bool) -> None: + if root is None: + return + if output and not self._hide_outputs: + return + if not output and not self._hide_inputs: + return + known = root.outputs if output else root.inputs + with self._lock: + for text in texts: + if isinstance(text, str): + known.add(text, self._secrets) + + # ------------------------------------------------------------ bookkeeping + + def _begin( + self, + run_id: UUID, + parent_run_id: Optional[UUID], + kind: str, + name: str, + attributes: Dict[str, Any], + ) -> Tuple[Optional[_Run], Optional[_Root]]: + """Track a run and start its span (none for collapsed chain runs).""" + if context_api.get_value(context_api._SUPPRESS_INSTRUMENTATION_KEY): + return None, None + evicted: List[_Run] = [] + with self._lock: + parent = self._runs.get(parent_run_id) if parent_run_id is not None else None + if parent is None: + root_id = run_id + if kind == _CHAIN: + kind = _AGENT + # setdefault: a repeated start of an open root keeps its index. + self._roots.setdefault(root_id, _Root()) + else: + root_id = parent.root + root = self._roots.get(root_id) + if root is None: # the root was evicted while this run started + return None, None + if kind == _LLM: + root.llm_calls += 1 + elif kind == _TOOL: + root.tool_calls += 1 + run = _Run(run_id=run_id, kind=kind, root=root_id, parent=parent_run_id) + self._runs[run_id] = run + root.runs[run_id] = None + traced_parent = self._traced_ancestor(parent) + while len(self._runs) > MAX_OPEN_RUNS and len(self._roots) > 1: + oldest = next(iter(self._roots)) + if oldest == root_id: + break + evicted.extend(self._forget_root(oldest)) + for stale in evicted: + self._end_incomplete(stale, None) + if kind == _CHAIN: + return run, root + if kind == _AGENT: + name = AGENT_SPAN_NAME + try: + parent_context = ( + trace_api.set_span_in_context(traced_parent.span) + if traced_parent is not None and traced_parent.span is not None + else None + ) + run.span = self._tracer.start_span(name, context=parent_context, attributes=attributes) + except Exception: + logger.debug("Could not start the mcp-use span", exc_info=True) + run.span = None + return run, root + + def _traced_ancestor(self, run: Optional[_Run]) -> Optional[_Run]: + while run is not None: + if run.kind != _CHAIN: + return run + run = self._runs.get(run.parent) if run.parent is not None else None + return None + + def _forget_root(self, root_id: UUID) -> List[_Run]: + """Drop an agent run and every run under it; caller holds the lock.""" + root = self._roots.pop(root_id, None) + if root is None: + return [] + dropped = [self._runs.pop(run_id, None) for run_id in root.runs] + root.runs.clear() + return [run for run in dropped if run is not None and run.span is not None] + + def _descends(self, run: _Run, ancestor: UUID) -> bool: + parent = run.parent + while parent is not None: + if parent == ancestor: + return True + above = self._runs.get(parent) + if above is None: + return False + parent = above.parent + return False + + def _pop(self, run_id: UUID) -> Tuple[Optional[_Run], Optional[_Root], List[_Run]]: + """Remove a finished run and any still-open run below it.""" + with self._lock: + run = self._runs.pop(run_id, None) + if run is None: + return None, None, [] + root = self._roots.get(run.root) + if root is None: # not expected: a root's runs leave with it + return run, None, [] + root.runs.pop(run_id, None) + if run.run_id == run.root: + # The agent run ended: whatever is still open under it. + self._roots.pop(run.root, None) + orphans = [self._runs.pop(other) for other in root.runs if other in self._runs] + root.runs.clear() + else: + orphans = [ + self._runs[other] + for other in root.runs + if other in self._runs and self._descends(self._runs[other], run_id) + ] + for orphan in orphans: + self._runs.pop(orphan.run_id, None) + root.runs.pop(orphan.run_id, None) + return run, root, [orphan for orphan in orphans if orphan.span is not None] + + # ----------------------------------------------------------------- ending + + def _set(self, span: Span, key: str, value: Any) -> None: + if value is None: + return + try: + span.set_attribute(key, value) + except Exception: + logger.debug("Could not set %s", key, exc_info=True) + + def _end(self, span: Optional[Span]) -> None: + if span is None: + return + try: + span.end() + except Exception: + logger.debug("Could not end the mcp-use span", exc_info=True) + + def _ok(self, span: Span) -> None: + try: + span.set_status(Status(StatusCode.OK)) + except Exception: + logger.debug("Could not set the span status", exc_info=True) + + def _cancelled(self, span: Span) -> None: + # Cancellation is not an exception: no event, no message. + self._set(span, CANCELLED, True) + try: + span.set_status(Status(StatusCode.ERROR, "cancelled")) + except Exception: + logger.debug("Could not mark the span cancelled", exc_info=True) + + def _end_incomplete(self, run: _Run, cause: Optional[BaseException]) -> None: + """End a span whose own end callback never came (its parent ended first).""" + span = run.span + if span is None: + return + if cause is not None and _is_cancellation(cause): + self._cancelled(span) + else: + self._set(span, INCOMPLETE, True) + try: + span.set_status(Status(StatusCode.ERROR, "ended without a result")) + except Exception: + logger.debug("Could not set the span status", exc_info=True) + self._end(span) + + def _record_failure(self, span: Span, failure: _Failure, root: Optional[_Root]) -> None: + type_name = self._name(failure.type_name) or "Error" + qualified = self._name(failure.qualified) or type_name + message: Optional[str] = None + if not (self._hide_inputs or self._hide_outputs): + # Error text quotes prompts, tool arguments and tool results, and + # a server's own data: it is content, dropped when either hide + # flag is on. Stack traces are never recorded. + message = self._clean(failure.message, root, MAX_ERROR_BYTES) + description = "{0}: {1}".format(type_name, message) if message else type_name + try: + span.set_status(Status(StatusCode.ERROR, description)) + except Exception: + logger.debug("Could not set the error status", exc_info=True) + event: Dict[str, Any] = {"exception.type": qualified} + if message: + event["exception.message"] = message + try: + span.add_event("exception", event) + except Exception: + logger.debug("Could not record the exception", exc_info=True) + + def _finish_error(self, run_id: UUID, error: BaseException) -> None: + run, root, orphans = self._pop(run_id) + if run is None: + return + for orphan in orphans: + self._end_incomplete(orphan, error) + if run.kind == _TOOL and root is not None and not _is_cancellation(error): + with self._lock: + root.tool_errors += 1 + span = run.span + if span is None: + return + if run.kind == _AGENT: + self._agent_counts(span, root) + if run.kind == _LLM and run.chunks: + self._set(span, CHUNK_COUNT, run.chunks) + if _is_cancellation(error): + self._cancelled(span) + else: + self._record_failure(span, _failure_from_exception(error), root) + self._end(span) + + def _agent_counts(self, span: Span, root: Optional[_Root]) -> None: + if root is None: + return + self._set(span, LLM_CALL_COUNT, root.llm_calls) + self._set(span, TOOL_CALL_COUNT, root.tool_calls) + self._set(span, TOOL_ERROR_COUNT, root.tool_errors) + + # ------------------------------------------------------------- callbacks + + def on_chain_start( + self, + serialized: Dict[str, Any], + inputs: Dict[str, Any], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + attributes = { + SPAN_KIND: FiSpanKindValues.AGENT.value, + OPERATION: "invoke_agent", + } + run, root = self._begin(run_id, parent_run_id, _CHAIN, AGENT_SPAN_NAME, attributes) + if run is None or run.kind != _AGENT: + return + messages = inputs.get("messages") if isinstance(inputs, Mapping) else None + history = messages if isinstance(messages, list) else [] + self._remember(root, [_content(m) for m in history], output=False) + # Earlier turns' outputs, known before any content of this run is written. + self._remember(root, _replayed_outputs(history), output=True) + if run.span is None: + return + if self._hide_inputs: + self._set(run.span, INPUT_VALUE, REDACTED_VALUE) + return + query = None + for message in reversed(history): + if _role(message) == "user": + query = _content(message) + break + self._set(run.span, INPUT_VALUE, self._clean(query, root, MAX_VALUE_BYTES)) + except Exception: + logger.debug("mcp-use callback on_chain_start failed", exc_info=True) + + def on_chain_end( + self, + outputs: Dict[str, Any], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + run, root, orphans = self._pop(run_id) + if run is None: + return + for orphan in orphans: + self._end_incomplete(orphan, None) + if run.kind != _AGENT or run.span is None: + return + span = run.span + self._agent_counts(span, root) + answer = None + messages = outputs.get("messages") if isinstance(outputs, Mapping) else None + for message in reversed(messages) if isinstance(messages, list) else []: + if _role(message) == "assistant" and not _tool_calls(message): + answer = _content(message) + break + if answer is not None: + if self._hide_outputs: + self._set(span, OUTPUT_VALUE, REDACTED_VALUE) + else: + self._set(span, OUTPUT_VALUE, self._clean(answer, root, MAX_VALUE_BYTES)) + self._ok(span) + self._end(span) + except Exception: + logger.debug("mcp-use callback on_chain_end failed", exc_info=True) + + def on_chain_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + self._finish_error(run_id, error) + except Exception: + logger.debug("mcp-use callback on_chain_error failed", exc_info=True) + + # Retriever runs are not spans, but their children need a known parent. + def on_retriever_start( + self, + serialized: Dict[str, Any], + query: str, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + if parent_run_id is not None: + with self._lock: + known = parent_run_id in self._runs + if known: + self._begin(run_id, parent_run_id, _CHAIN, "", {}) + except Exception: + logger.debug("mcp-use callback on_retriever_start failed", exc_info=True) + + def on_retriever_end(self, documents: Any, *, run_id: UUID, **kwargs: Any) -> None: + try: + self._pop(run_id) + except Exception: + logger.debug("mcp-use callback on_retriever_end failed", exc_info=True) + + def on_retriever_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None: + try: + self._pop(run_id) + except Exception: + logger.debug("mcp-use callback on_retriever_error failed", exc_info=True) + + def _llm_attributes( + self, operation: str, kwargs: Mapping[str, Any] + ) -> Tuple[str, Dict[str, Any]]: + metadata = kwargs.get("metadata") or {} + params = kwargs.get("invocation_params") or {} + model = None + for source, key in ( + (metadata, "ls_model_name"), + (params, "model"), + (params, "model_name"), + (params, "model_id"), + ): + if isinstance(source, Mapping) and isinstance(source.get(key), str) and source.get(key): + model = source.get(key) + break + attributes: Dict[str, Any] = { + SPAN_KIND: FiSpanKindValues.LLM.value, + OPERATION: operation, + } + model_name = self._name(model) + if model_name: + attributes[REQUEST_MODEL] = model_name + if isinstance(metadata, Mapping): + provider = self._name(metadata.get("ls_provider")) + if provider: + attributes[PROVIDER] = provider + temperature = _number(metadata.get("ls_temperature")) + if temperature is not None: + attributes[TEMPERATURE] = temperature + max_tokens = _int(metadata.get("ls_max_tokens")) + if max_tokens is not None: + attributes[MAX_TOKENS] = max_tokens + name = "{0} {1}".format(operation, model_name) if model_name else operation + return name, attributes + + def _message_attributes( + self, prefix: str, messages: Sequence[Any], root: Optional[_Root] + ) -> Dict[str, Any]: + attributes: Dict[str, Any] = {} + for index, message in enumerate(list(messages)[-MAX_MESSAGES:]): + base = "{0}.{1}.message".format(prefix, index) + role = _role(message) + attributes[base + ".role"] = role + # Replies and tool results sent back as input (the run's own, or + # earlier turns' from MCPAgent memory) are outputs: hide_outputs + # drops their content and tool-call arguments. + hidden = self._hide_outputs and _is_output_role(role) + text = None if hidden else self._clean(_content(message), root, MAX_VALUE_BYTES) + if text: + attributes[base + ".content"] = text + for call_index, call in enumerate(_tool_calls(message)[:MAX_TOOL_CALLS]): + call_base = "{0}.tool_calls.{1}.tool_call".format(base, call_index) + call_name = self._name(call.get("name")) + if call_name: + attributes[call_base + ".function.name"] = call_name + call_id = self._name(call.get("id")) + if call_id: + attributes[call_base + ".id"] = call_id + # Tool-call arguments are the tool's input, also when they + # appear in the model's output: hide_inputs drops them. + if self._hide_inputs or hidden: + continue + arguments = self._clean(_json(call.get("args")), root, MAX_VALUE_BYTES) + if arguments: + attributes[call_base + ".function.arguments"] = arguments + return attributes + + def _start_llm( + self, + operation: str, + messages: Sequence[Any], + run_id: UUID, + parent_run_id: Optional[UUID], + kwargs: Mapping[str, Any], + ) -> None: + name, attributes = self._llm_attributes(operation, kwargs) + attributes[INPUT_MESSAGE_COUNT] = len(messages) + run, root = self._begin(run_id, parent_run_id, _LLM, name, attributes) + if run is None or run.span is None: + return + texts: List[Optional[str]] = [] + for message in messages: + texts.append(_content(message)) + for call in _tool_calls(message): + texts.extend(_strings_in(call.get("args"))) + self._remember(root, texts, output=False) + self._remember(root, _replayed_outputs(messages), output=True) + if self._hide_inputs: + self._set(run.span, INPUT_VALUE, REDACTED_VALUE) + return + for key, value in self._message_attributes(INPUT_MESSAGES, messages, root).items(): + self._set(run.span, key, value) + if messages: + if self._hide_outputs and _is_output_role(_role(messages[-1])): + self._set(run.span, INPUT_VALUE, REDACTED_VALUE) + return + last = self._clean(_content(messages[-1]), root, MAX_VALUE_BYTES) + if last: + self._set(run.span, INPUT_VALUE, last) + + def on_chat_model_start( + self, + serialized: Dict[str, Any], + messages: List[List[Any]], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + first = messages[0] if messages else [] + self._start_llm("chat", list(first), run_id, parent_run_id, kwargs) + except Exception: + logger.debug("mcp-use callback on_chat_model_start failed", exc_info=True) + + def on_llm_start( + self, + serialized: Dict[str, Any], + prompts: List[str], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + from langchain_core.messages import HumanMessage + + messages = [HumanMessage(content=prompt) for prompt in prompts[:1]] + self._start_llm("text_completion", messages, run_id, parent_run_id, kwargs) + except Exception: + logger.debug("mcp-use callback on_llm_start failed", exc_info=True) + + def on_llm_new_token( + self, + token: Any, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + # One event per chunk (index only, never the text), up to + # MAX_CHUNK_EVENTS; mcp_use.llm.chunk_count keeps the exact number. + try: + with self._lock: + run = self._runs.get(run_id) + if run is None or run.span is None: + return + run.chunks += 1 + index = run.chunks - 1 + if index < MAX_CHUNK_EVENTS: + run.span.add_event(CHUNK_EVENT, {CHUNK_INDEX: index}) + except Exception: + logger.debug("mcp-use callback on_llm_new_token failed", exc_info=True) + + def on_llm_end( + self, + response: Any, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + run, root, orphans = self._pop(run_id) + if run is None: + return + for orphan in orphans: + self._end_incomplete(orphan, None) + span = run.span + if span is None: + return + if run.chunks: + self._set(span, CHUNK_COUNT, run.chunks) + try: + self._llm_response(span, response, root) + except Exception: + logger.debug("Could not read the LLM response", exc_info=True) + self._ok(span) + self._end(span) + except Exception: + logger.debug("mcp-use callback on_llm_end failed", exc_info=True) + + def _llm_response(self, span: Span, response: Any, root: Optional[_Root]) -> None: + generations = getattr(response, "generations", None) or [] + first = list(generations[0]) if generations else [] + llm_output = getattr(response, "llm_output", None) or {} + messages: List[Any] = [] + reasons: List[str] = [] + usage: Optional[Mapping[str, Any]] = None + response_model: Any = None + for generation in first: + message = getattr(generation, "message", None) + if message is None: + from langchain_core.messages import AIMessage + + message = AIMessage(content=getattr(generation, "text", "") or "") + messages.append(message) + metadata = getattr(message, "response_metadata", None) or {} + info = getattr(generation, "generation_info", None) or {} + for source in (metadata, info): + reason = source.get("finish_reason") or source.get("stop_reason") + if isinstance(reason, str) and reason: + reasons.append(reason) + break + response_model = response_model or metadata.get("model_name") or metadata.get("model") + usage = usage or getattr(message, "usage_metadata", None) + if isinstance(llm_output, Mapping): + response_model = response_model or llm_output.get("model_name") + name = self._name(response_model) + if name: + self._set(span, RESPONSE_MODEL, name) + if reasons: + self._set(span, FINISH_REASONS, [self._name(reason) for reason in reasons]) + if isinstance(usage, Mapping): + self._set(span, INPUT_TOKENS, _int(usage.get("input_tokens"))) + self._set(span, OUTPUT_TOKENS, _int(usage.get("output_tokens"))) + self._set(span, TOTAL_TOKENS, _int(usage.get("total_tokens"))) + details = usage.get("input_token_details") + if isinstance(details, Mapping): + self._set(span, CACHE_READ_TOKENS, _int(details.get("cache_read"))) + elif isinstance(llm_output, Mapping) and isinstance(llm_output.get("token_usage"), Mapping): + token_usage = llm_output["token_usage"] + self._set(span, INPUT_TOKENS, _int(token_usage.get("prompt_tokens"))) + self._set(span, OUTPUT_TOKENS, _int(token_usage.get("completion_tokens"))) + self._set(span, TOTAL_TOKENS, _int(token_usage.get("total_tokens"))) + if messages: + self._set(span, REQUESTED_TOOL_CALLS, sum(len(_tool_calls(m)) for m in messages)) + texts: List[Optional[str]] = [] + arguments: List[Optional[str]] = [] + for message in messages: + texts.append(_content(message)) + for call in _tool_calls(message): + arguments.extend(_strings_in(call.get("args"))) + self._remember(root, texts + arguments, output=True) + # The arguments the model chose are the next tool call's input. + self._remember(root, arguments, output=False) + if not messages: + return + if self._hide_outputs: + self._set(span, OUTPUT_VALUE, REDACTED_VALUE) + return + for key, value in self._message_attributes(OUTPUT_MESSAGES, messages, root).items(): + self._set(span, key, value) + answer = self._clean(_content(messages[0]), root, MAX_VALUE_BYTES) + if answer: + self._set(span, OUTPUT_VALUE, answer) + + def on_llm_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + self._finish_error(run_id, error) + except Exception: + logger.debug("mcp-use callback on_llm_error failed", exc_info=True) + + def on_tool_start( + self, + serialized: Dict[str, Any], + input_str: str, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + inputs: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> None: + try: + raw_name = None + if isinstance(serialized, Mapping): + raw_name = serialized.get("name") + raw_name = raw_name or kwargs.get("name") + tool = self._name(raw_name) or "unknown" + attributes: Dict[str, Any] = { + SPAN_KIND: FiSpanKindValues.TOOL.value, + OPERATION: "execute_tool", + TOOL_NAME: tool, + } + call_id = self._name(kwargs.get("tool_call_id")) + if call_id: + attributes[TOOL_CALL_ID] = call_id + run, root = self._begin( + run_id, parent_run_id, _TOOL, "execute_tool {0}".format(tool), attributes + ) + if run is None: + return + if isinstance(raw_name, str): + run.tool_name = raw_name + if run.span is None: + return + arguments = _json(inputs) if isinstance(inputs, Mapping) else input_str + texts: List[Optional[str]] = [arguments, input_str] + if isinstance(inputs, Mapping): + texts.extend(_strings_in(inputs)) + self._remember(root, texts, output=False) + if self._hide_inputs: + self._set(run.span, INPUT_VALUE, REDACTED_VALUE) + return + text = self._clean(arguments, root, MAX_VALUE_BYTES) + self._set(run.span, TOOL_ARGUMENTS, text) + self._set(run.span, INPUT_VALUE, text) + except Exception: + logger.debug("mcp-use callback on_tool_start failed", exc_info=True) + + def on_tool_end( + self, + output: Any, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + run, root, orphans = self._pop(run_id) + if run is None: + return + for orphan in orphans: + self._end_incomplete(orphan, None) + failure: Optional[_Failure] = None + try: + formatted = _mcp_use_error(output, run.tool_name) + if formatted is not None: + error_type = str(formatted.get("error")) + details = formatted.get("details") + # format_error's "stack" is never read. + failure = _Failure( + error_type, + error_type, + details if isinstance(details, str) else None, + ) + elif getattr(output, "status", None) == "error": + failure = _Failure( + "ToolException", + "langchain_core.tools.ToolException", + _tool_output_text(output), + ) + except Exception: + logger.debug("Could not classify the tool result", exc_info=True) + text = _tool_output_text(output) + self._remember(root, [text], output=True) + if failure is not None and root is not None: + with self._lock: + root.tool_errors += 1 + span = run.span + if span is None: + return + if failure is not None: + self._set(span, TOOL_ERROR_TYPE, self._name(failure.type_name)) + self._record_failure(span, failure, root) + else: + if self._hide_outputs: + self._set(span, OUTPUT_VALUE, REDACTED_VALUE) + else: + cleaned = self._clean(text, root, MAX_VALUE_BYTES) + self._set(span, TOOL_RESULT, cleaned) + self._set(span, OUTPUT_VALUE, cleaned) + self._ok(span) + self._end(span) + except Exception: + logger.debug("mcp-use callback on_tool_end failed", exc_info=True) + + def on_tool_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + self._finish_error(run_id, error) + except Exception: + logger.debug("mcp-use callback on_tool_error failed", exc_info=True) diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_text.py b/python/frameworks/mcp-use/traceai_mcp_use/_text.py new file mode 100644 index 000000000..078e4327e --- /dev/null +++ b/python/frameworks/mcp-use/traceai_mcp_use/_text.py @@ -0,0 +1,245 @@ +"""Text cleaning applied to every string the callback writes to a span. + +Order for every value: remove secrets, then (when a TraceConfig hide flag is +on) remove text the run already saw as an input or output, then cut Python +stack traces and replace local paths, the user name and the host name, then +PII when ``pii_redaction`` is on, then cut to a UTF-8 byte cap. Secrets go +first so a cap can never leave the start of a key behind; hidden texts are +matched before the local pass rewrites them; PII goes before the cap so a +cut cannot leave part of an email address. +""" + +from __future__ import annotations + +import getpass +import json +import os +import re +import socket +from typing import Iterable, List, Optional, Sequence, Tuple + +from fi_instrumentation import REDACTED_VALUE +from fi_instrumentation.instrumentation.pii_redaction import redact_pii_in_string + +REDACTED = "[redacted]" + +MAX_VALUE_BYTES = 4096 +MAX_NAME_BYTES = 256 +MAX_ERROR_BYTES = 1024 + +# A secret shorter than this is not searched for: replacing a 3-character +# value everywhere would shred unrelated text. +MIN_SECRET_CHARS = 8 +# Known inputs/outputs shorter than this are only removed in their quoted +# forms ('ab', "ab"); bare short strings would match inside unrelated words. +MIN_SCRUB_CHARS = 3 +# Bound on the text one agent run remembers for hide_inputs/hide_outputs. +MAX_KNOWN_TEXTS = 2048 +MAX_KNOWN_CHARS = 4 * 1024 * 1024 + +_SECRET_ENV_NAME = re.compile(r"KEY|SECRET|TOKEN|PASSWORD|PASSWD|CREDENTIAL|AUTH", re.I) +_BEARER = re.compile(r"(?i)\b(bearer|basic)(\s+)[A-Za-z0-9._~+/=-]{8,}") +_ASSIGNED = re.compile( + r"(?i)\b((?:x[-_])?api[-_]?key|authorization|secret(?:[-_]?key)?|access[-_]?token" + r"|refresh[-_]?token|token|password|passwd)" + r"([\"']?\s*[:=]\s*[\"']?)" + r"([^\s\"',;}\]]{4,})" +) +_KEY_SHAPE = re.compile(r"\b(?:sk|pk|rk)-[A-Za-z0-9_-]{16,}") + + +def cap(value: str, limit: int) -> str: + """Return the longest whole-character prefix of at most ``limit`` UTF-8 bytes.""" + return value.encode("utf-8", "replace")[:limit].decode("utf-8", "ignore") + + +# --- stack traces, local paths, user and host names are never exported ----- +# +# mcp-use returns a failed MCP tool call to the model as format_error()'s dict +# (mcp_use/errors/error_formatting.py), whose "stack" is traceback.format_exc(): +# absolute file paths, the home directory and so the user name. That text is +# the next LLM call's tool message, and exception messages quote paths too. +# None of it is exported, with content on or hidden. +STACK_TRACE_REMOVED = "[stack trace removed]" +PATH_REMOVED = "[path]" +USER_REMOVED = "[user]" +HOST_REMOVED = "[host]" +_TRACEBACK_HEAD = "Traceback (most recent call last)" +_TRACEBACK = re.compile(r"Traceback \(most recent call last\):.*", re.DOTALL) +_LOCAL_PATH = re.compile( + r"(?|:*?]+" + r"|/(?:Users|home|root|private|var|tmp|opt|usr|srv|mnt|app|workspace|etc|Volumes)/[^\s\"'<>|:*?,;)\]\\]+)" +) +# Account names too generic to replace inside ordinary text. +_GENERIC_NAMES = frozenset( + "root admin administrator user users app ubuntu runner node nobody guest test dev " + "default jovyan vscode daemon localhost".split() +) + + +def _identity_patterns() -> List[Tuple["re.Pattern[str]", str]]: + """Whole-word patterns for this machine's user name and host name.""" + names: List[Tuple[str, str]] = [] + try: + names.append((getpass.getuser(), USER_REMOVED)) + except Exception: # no user database entry (some containers) + pass + try: + host = socket.gethostname() + names.extend(((host, HOST_REMOVED), (host.split(".")[0], HOST_REMOVED))) + except Exception: + pass + patterns: List[Tuple["re.Pattern[str]", str]] = [] + for name, token in names: + if len(name) >= 4 and name.lower() not in _GENERIC_NAMES: + word = re.compile(r"(? object: + if isinstance(value, str): + return _TRACEBACK.sub(STACK_TRACE_REMOVED, value) if _TRACEBACK_HEAD in value else value + if isinstance(value, list): + return [_cut_tracebacks(item) for item in value] + if isinstance(value, dict): + return {key: _cut_tracebacks(item) for key, item in value.items()} + return value + + +def scrub_local(text: str) -> str: + """Cut Python stack traces; replace local paths, the user and the host name. + + A JSON object or array (an mcp-use formatted tool error is one) has the + trace cut inside each string value, so the rest of the JSON survives; + any other text loses everything from ``Traceback (most recent call + last):`` on. + """ + if _TRACEBACK_HEAD in text: + stripped = text.strip() + if stripped[:1] in ("{", "["): + try: + text = json.dumps(_cut_tracebacks(json.loads(stripped)), ensure_ascii=False) + except ValueError: + pass + if _TRACEBACK_HEAD in text: + text = _TRACEBACK.sub(STACK_TRACE_REMOVED, text) + text = _LOCAL_PATH.sub(PATH_REMOVED, text) + for pattern, token in _IDENTITY: + text = pattern.sub(token, text) + return text + + +class Secrets: + """Secret values the callback removes from every text it writes. + + Sources: the values passed as ``redact=`` and, read on every call, the + values of environment variables whose name contains KEY, SECRET, TOKEN, + PASSWORD, PASSWD, CREDENTIAL or AUTH (LLM provider keys such as + OPENAI_API_KEY, and FI_API_KEY / FI_SECRET_KEY). Values shorter than + ``MIN_SECRET_CHARS`` are ignored. Bearer/Basic credentials, ``sk-`` + style keys and ``api_key=...`` / ``Authorization: ...`` assignments are + also replaced by pattern, for secrets the callback was never told about. + """ + + def __init__(self, explicit: Iterable[str] = ()) -> None: + self._explicit: Tuple[str, ...] = tuple( + value for value in dict.fromkeys(explicit) if len(value) >= MIN_SECRET_CHARS + ) + + def values(self) -> List[str]: + found = list(self._explicit) + try: + for name, value in list(os.environ.items()): + if ( + value + and len(value) >= MIN_SECRET_CHARS + and _SECRET_ENV_NAME.search(name) + and value not in found + ): + found.append(value) + except Exception: # an unreadable environment must not break redaction + pass + return sorted(found, key=len, reverse=True) + + def redact(self, text: str) -> str: + for value in self.values(): + if value in text: + text = text.replace(value, REDACTED) + text = _BEARER.sub(lambda match: match.group(1) + match.group(2) + REDACTED, text) + text = _ASSIGNED.sub(lambda match: match.group(1) + match.group(2) + REDACTED, text) + return _KEY_SHAPE.sub(REDACTED, text) + + +class KnownTexts: + """Inputs (or outputs) one agent run has seen, for ``hide_inputs``/``hide_outputs``. + + Every text is stored raw, with secrets already removed (so it still + matches after the secret pass), and in its Python-repr and JSON-quoted + forms (so an echo inside a repr'd dict or a JSON body matches too). + Past ``MAX_KNOWN_TEXTS`` entries or ``MAX_KNOWN_CHARS`` characters the + set is marked overflowed and ``scrub`` fails closed by returning None. + """ + + def __init__(self) -> None: + self._items: set = set() + self._chars = 0 + self._sorted: Optional[Sequence[str]] = None + self.overflowed = False + + def add(self, text: str, secrets: Secrets) -> None: + if not isinstance(text, str) or not text or self.overflowed: + return + forms = {repr(text), json.dumps(text, ensure_ascii=False), json.dumps(text)} + if len(text) >= MIN_SCRUB_CHARS: + forms.add(text) + redacted = secrets.redact(text) + if len(redacted) >= MIN_SCRUB_CHARS: + forms.add(redacted) + for form in forms: + if form in self._items: + continue + self._items.add(form) + self._chars += len(form) + self._sorted = None + if len(self._items) > MAX_KNOWN_TEXTS or self._chars > MAX_KNOWN_CHARS: + self.overflowed = True + self._items.clear() + + def scrub(self, text: str) -> Optional[str]: + if self.overflowed: + return None + if self._sorted is None: + self._sorted = sorted(self._items, key=len, reverse=True) + for item in self._sorted: + if item in text: + text = text.replace(item, REDACTED_VALUE) + return text + + +def clean( + text: str, + secrets: Secrets, + limit: int, + pii: bool = False, + scrub: Sequence[KnownTexts] = (), +) -> Optional[str]: + """Secrets, then known inputs/outputs, then stack traces and local + identity, then PII, then the byte cap. + + Returns None when a hide flag's text set overflowed: the caller must + then record nothing rather than an unscrubbed value. + """ + text = secrets.redact(text) + for known in scrub: + scrubbed = known.scrub(text) + if scrubbed is None: + return None + text = scrubbed + text = scrub_local(text) + if pii: + text = redact_pii_in_string(text) + return cap(text, limit) diff --git a/python/frameworks/mcp-use/traceai_mcp_use/py.typed b/python/frameworks/mcp-use/traceai_mcp_use/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/python/frameworks/mcp-use/traceai_mcp_use/version.py b/python/frameworks/mcp-use/traceai_mcp_use/version.py new file mode 100644 index 000000000..3dc1f76bc --- /dev/null +++ b/python/frameworks/mcp-use/traceai_mcp_use/version.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..cf79047f1 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,238 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + body = self._read_body() + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + + def clear(self) -> None: + """Remove every decoded span and request record received so far.""" + with self._lock: + self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..bd61d5adc --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,189 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out