From 5936e0960f01a23e842d3df90b7c101874ed8149 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 18:58:31 -0700 Subject: [PATCH 01/19] test(harness): add shared OTLP harness for TH-8103 contract tests --- python/tests/harness/__init__.py | 184 +++++++++++++++++++++++++++ python/tests/harness/test_harness.py | 101 +++++++++++++++ 2 files changed, 285 insertions(+) create mode 100644 python/tests/harness/__init__.py create mode 100644 python/tests/harness/test_harness.py diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..dfd9198e4 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,184 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + if urlsplit(self.path).path != "/v1/traces": + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def clear(self) -> None: + """Remove every decoded span received so far.""" + with self._lock: + self._spans.clear() + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..4e76ceabe --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,101 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out From 32f5c9d2b3b80f97a70e00b3f3ca1b408b14dc5c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 23:21:29 -0700 Subject: [PATCH 02/19] test(harness): accept the collector /tracer/v1/traces path --- python/tests/harness/__init__.py | 3 ++- python/tests/harness/test_harness.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index dfd9198e4..cb510ad10 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -28,7 +28,7 @@ def __init__(self) -> None: class Handler(BaseHTTPRequestHandler): def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path != "/v1/traces": + if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return @@ -76,6 +76,7 @@ def log_message(self, _format: str, *args: Any) -> None: self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) self._thread.start() diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 4e76ceabe..6a32c2af8 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -38,6 +38,18 @@ def test_receiver_stores_posted_span_and_can_clear() -> None: assert receiver.spans() == [] +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From 3eaadc8bdbf428a472db6779091bbce3d6d89015 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 15:59:06 -0700 Subject: [PATCH 03/19] fix(harness): decode chunked OTLP bodies and record request metadata The Node OTLP exporter streams with Transfer-Encoding: chunked and sends no Content-Length. The Receiver read Content-Length only, so every Node export came back 400 with zero spans; two TH-8103 children had to put a de-chunking relay in front of it. Receiver now accepts both framings. It also records one entry per accepted export (path, lower-cased headers, flattened resource attributes) via requests(), so a contract test can assert X-Api-Key/X-Secret-Key and project_name through the shared harness instead of a private recorder. Verified: 6 harness tests pass, and the TanStack example's real Node exporter delivered 4 chunked exports (4 spans, collector path, both auth headers, project_name/project_type) with no relay. Refs: TH-8339, TH-8103 --- python/tests/harness/__init__.py | 61 ++++++++++++++++++++-- python/tests/harness/test_harness.py | 76 ++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index cb510ad10..cf79047f1 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -23,18 +23,36 @@ class Receiver: def __init__(self) -> None: self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] self._lock = threading.Lock() owner = self class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return try: - length = int(self.headers.get("Content-Length", "0")) - body = self.rfile.read(length) + body = self._read_body() content_type = self.headers.get("Content-Type", "").lower() if content_type.startswith("application/json"): @@ -53,9 +71,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler return spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] for resource_spans in request.get( "resourceSpans", request.get("resource_spans", []) ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) for scope_spans in resource_spans.get( "scopeSpans", resource_spans.get("scope_spans", []) ): @@ -66,6 +90,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler with owner._lock: owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) self.send_response(HTTPStatus.OK) self.send_header("Content-Length", "0") self.end_headers() @@ -93,10 +126,30 @@ def spans(self) -> list[dict[str, Any]]: with self._lock: return copy.deepcopy(self._spans) + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + def clear(self) -> None: - """Remove every decoded span received so far.""" + """Remove every decoded span and request record received so far.""" with self._lock: self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat def run( diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 6a32c2af8..bd61d5adc 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -50,6 +50,82 @@ def test_receiver_accepts_the_collector_path() -> None: assert receiver.spans() == [{"name": "collector.path"}] +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From d777b1f2d9f18de485994788f7660984519aa67b Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 07:36:07 -0700 Subject: [PATCH 04/19] feat(mcp-use): add traceAI-mcp-use, a LangChain callback for MCPAgent New package python/frameworks/mcp-use (traceAI-mcp-use, import traceai_mcp_use). FutureAGICallback is passed as MCPAgent(..., callbacks=[...]); mcp-use 1.7.1 hands that list to the LangGraph agent on every run. It records one AGENT span per agent run, one LLM span per model call and one TOOL span per tool call, parented to the nearest traced ancestor in the LangChain run tree. Graph node and middleware runs are not spans. Nothing is patched. Spans come from an FITracer, so TraceConfig masking, pii_redaction and using_session/user/metadata/tags apply. Content is off by default (capture_content=True turns it on); secrets are removed from every text written; hide_inputs/hide_outputs also scrub error text; every value is capped. Callbacks never raise into the agent. approved: Nikhil 2026-10-03 blanket Refs: TH-8331 --- python/frameworks/mcp-use/CHANGELOG.md | 15 + .../mcp-use/examples/agent_with_tools.py | 47 + .../mcp-use/examples/calculator_server.py | 15 + python/frameworks/mcp-use/pyproject.toml | 32 + .../mcp-use/traceai_mcp_use/__init__.py | 13 + .../mcp-use/traceai_mcp_use/_callback.py | 1083 +++++++++++++++++ .../mcp-use/traceai_mcp_use/_text.py | 174 +++ .../mcp-use/traceai_mcp_use/py.typed | 0 .../mcp-use/traceai_mcp_use/version.py | 1 + 9 files changed, 1380 insertions(+) create mode 100644 python/frameworks/mcp-use/CHANGELOG.md create mode 100644 python/frameworks/mcp-use/examples/agent_with_tools.py create mode 100644 python/frameworks/mcp-use/examples/calculator_server.py create mode 100644 python/frameworks/mcp-use/pyproject.toml create mode 100644 python/frameworks/mcp-use/traceai_mcp_use/__init__.py create mode 100644 python/frameworks/mcp-use/traceai_mcp_use/_callback.py create mode 100644 python/frameworks/mcp-use/traceai_mcp_use/_text.py create mode 100644 python/frameworks/mcp-use/traceai_mcp_use/py.typed create mode 100644 python/frameworks/mcp-use/traceai_mcp_use/version.py diff --git a/python/frameworks/mcp-use/CHANGELOG.md b/python/frameworks/mcp-use/CHANGELOG.md new file mode 100644 index 000000000..b21a0146b --- /dev/null +++ b/python/frameworks/mcp-use/CHANGELOG.md @@ -0,0 +1,15 @@ +# Changelog + +## 0.1.0 + +- First release. `FutureAGICallback`, a LangChain callback handler for + `mcp_use.MCPAgent(callbacks=[...])`, records one `AGENT` span per agent + run, one `LLM` span per model call and one `TOOL` span per tool call, + parented as LangChain reports the run tree. +- Content (prompts, model output, tool arguments, tool results, error + messages) is off by default; `capture_content=True` turns it on. + `TraceConfig` `hide_inputs`, `hide_outputs` and `pii_redaction` apply to + every text written, including error text. +- Secrets (values passed in `redact=`, secret-named environment variables, + bearer tokens and key-shaped strings) are removed from everything written. +- Tested with mcp-use 1.7.1 on Python 3.11, 3.12 and 3.13. diff --git a/python/frameworks/mcp-use/examples/agent_with_tools.py b/python/frameworks/mcp-use/examples/agent_with_tools.py new file mode 100644 index 000000000..822e77372 --- /dev/null +++ b/python/frameworks/mcp-use/examples/agent_with_tools.py @@ -0,0 +1,47 @@ +"""Trace one mcp-use agent run with traceAI-mcp-use. + +Needs FI_API_KEY / FI_SECRET_KEY, langchain-openai and OPENAI_API_KEY (or +pass another LangChain chat model to main()). The MCP server is the local +stdio server next to this file. +""" + +import asyncio +import os +import sys + +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from mcp_use import MCPAgent, MCPClient + +from traceai_mcp_use import FutureAGICallback + +SERVER = os.path.join(os.path.dirname(os.path.abspath(__file__)), "calculator_server.py") + + +def default_llm(): + from langchain_openai import ChatOpenAI + + return ChatOpenAI(model="gpt-4o-mini") + + +async def main(llm=None) -> str: + trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="mcp-use-agent", + ) + client = MCPClient.from_dict( + {"mcpServers": {"calculator": {"command": sys.executable, "args": [SERVER]}}} + ) + agent = MCPAgent( + llm=llm or default_llm(), + client=client, + callbacks=[FutureAGICallback(tracer_provider=trace_provider)], + ) + try: + return await agent.run("What is 2 + 3? Use the add tool.") + finally: + await client.close_all_sessions() + + +if __name__ == "__main__": + print(asyncio.run(main())) diff --git a/python/frameworks/mcp-use/examples/calculator_server.py b/python/frameworks/mcp-use/examples/calculator_server.py new file mode 100644 index 000000000..016e8fb50 --- /dev/null +++ b/python/frameworks/mcp-use/examples/calculator_server.py @@ -0,0 +1,15 @@ +"""A one-tool MCP server over stdio, for examples/agent_with_tools.py.""" + +from mcp.server.fastmcp import FastMCP + +mcp = FastMCP("calculator") + + +@mcp.tool() +def add(a: int, b: int) -> str: + """Add two integers.""" + return str(a + b) + + +if __name__ == "__main__": + mcp.run("stdio") diff --git a/python/frameworks/mcp-use/pyproject.toml b/python/frameworks/mcp-use/pyproject.toml new file mode 100644 index 000000000..9de7eb516 --- /dev/null +++ b/python/frameworks/mcp-use/pyproject.toml @@ -0,0 +1,32 @@ +[tool.poetry] +name = "traceAI-mcp-use" +version = "0.1.0" +description = "OpenTelemetry tracing for mcp-use agents: one span per agent run, LLM call and tool call" +authors = ["Future AGI "] +readme = "README.md" +packages = [ + { include = "traceai_mcp_use" } +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Libraries :: Python Modules", +] + +[tool.poetry.dependencies] +python = ">=3.11,<3.14" +fi-instrumentation-otel = ">=1.1.0" +mcp-use = ">=1.7.1,<2" +langchain-core = ">=1.0.0,<2" + +[tool.poetry.group.dev.dependencies] +pytest = ">=7.0.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/python/frameworks/mcp-use/traceai_mcp_use/__init__.py b/python/frameworks/mcp-use/traceai_mcp_use/__init__.py new file mode 100644 index 000000000..f922a60c7 --- /dev/null +++ b/python/frameworks/mcp-use/traceai_mcp_use/__init__.py @@ -0,0 +1,13 @@ +"""Trace mcp-use agents with Future AGI. + +``FutureAGICallback`` is a LangChain callback handler. Pass it to the agent: +``MCPAgent(llm=..., client=..., callbacks=[FutureAGICallback(tracer_provider=...)])``. +It records one span per agent run, LLM call and tool call. It patches +nothing, imports neither ``langfuse`` nor ``traceai_mcp``, and sets no +environment variables. +""" + +from traceai_mcp_use._callback import AGENT_SPAN_NAME, FutureAGICallback +from traceai_mcp_use.version import __version__ + +__all__ = ["AGENT_SPAN_NAME", "FutureAGICallback", "__version__"] diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py new file mode 100644 index 000000000..2d965c0be --- /dev/null +++ b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py @@ -0,0 +1,1083 @@ +"""A LangChain callback handler that traces an mcp-use ``MCPAgent`` run. + +mcp-use 1.7.1 hands the ``callbacks`` list it was constructed with to the +LangGraph agent on every run (``MCPAgent.stream`` and +``MCPAgent._generate_response_chunks_async`` pass +``config={"callbacks": self.callbacks, ...}``). This handler turns the run +tree LangChain reports into three kinds of spans: + +- one ``AGENT`` span for the outermost run the handler sees (the LangGraph + graph that ``MCPAgent.run`` / ``stream`` / ``stream_events`` drives), +- one ``LLM`` span per chat-model (or completion-model) call, +- one ``TOOL`` span per tool call. + +The graph's internal node and middleware runs are not spans; an LLM or +tool span's parent is its nearest traced ancestor, which for mcp-use is the +agent span. Nothing is patched: the handler only exists where it is passed. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import threading +import traceback +from collections import OrderedDict +from dataclasses import dataclass, field +from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple +from uuid import UUID + +from fi_instrumentation import FITracer, REDACTED_VALUE, TraceConfig +from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes +from langchain_core.callbacks import BaseCallbackHandler +from opentelemetry import context as context_api +from opentelemetry import trace as trace_api +from opentelemetry.trace import Span, Status, StatusCode + +from traceai_mcp_use._text import ( + MAX_ERROR_BYTES, + MAX_NAME_BYTES, + MAX_STACKTRACE_BYTES, + MAX_VALUE_BYTES, + KnownTexts, + Secrets, + cap, + clean, +) +from traceai_mcp_use.version import __version__ + +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + +AGENT_SPAN_NAME = "mcp_use.agent" + +SPAN_KIND = SpanAttributes.GEN_AI_SPAN_KIND +OPERATION = SpanAttributes.GEN_AI_OPERATION_NAME +INPUT_VALUE = SpanAttributes.INPUT_VALUE +OUTPUT_VALUE = SpanAttributes.OUTPUT_VALUE +INPUT_MESSAGES = SpanAttributes.GEN_AI_INPUT_MESSAGES +OUTPUT_MESSAGES = SpanAttributes.GEN_AI_OUTPUT_MESSAGES +REQUEST_MODEL = SpanAttributes.GEN_AI_REQUEST_MODEL +RESPONSE_MODEL = SpanAttributes.GEN_AI_RESPONSE_MODEL +PROVIDER = SpanAttributes.GEN_AI_PROVIDER_NAME +TEMPERATURE = SpanAttributes.GEN_AI_REQUEST_TEMPERATURE +MAX_TOKENS = SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS +FINISH_REASONS = SpanAttributes.GEN_AI_RESPONSE_FINISH_REASONS +INPUT_TOKENS = SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS +OUTPUT_TOKENS = SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS +TOTAL_TOKENS = SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS +CACHE_READ_TOKENS = SpanAttributes.GEN_AI_USAGE_CACHE_READ_TOKENS +TOOL_NAME = SpanAttributes.GEN_AI_TOOL_NAME +TOOL_CALL_ID = SpanAttributes.GEN_AI_TOOL_CALL_ID +TOOL_ARGUMENTS = SpanAttributes.GEN_AI_TOOL_CALL_ARGUMENTS +TOOL_RESULT = SpanAttributes.GEN_AI_TOOL_CALL_RESULT + +LLM_CALL_COUNT = "mcp_use.agent.llm_call_count" +TOOL_CALL_COUNT = "mcp_use.agent.tool_call_count" +TOOL_ERROR_COUNT = "mcp_use.agent.tool_error_count" +INPUT_MESSAGE_COUNT = "mcp_use.llm.input_message_count" +REQUESTED_TOOL_CALLS = "mcp_use.llm.tool_call_count" +CHUNK_COUNT = "mcp_use.llm.chunk_count" +CHUNK_EVENT = "mcp_use.llm.chunk" +CHUNK_INDEX = "mcp_use.llm.chunk.index" +TOOL_ERROR_TYPE = "mcp_use.tool.error_type" +CANCELLED = "mcp_use.cancelled" +INCOMPLETE = "mcp_use.incomplete" + +# At most this many messages are written per LLM span (the most recent ones); +# mcp_use.llm.input_message_count keeps the exact number. +MAX_MESSAGES = 32 +# At most this many tool calls are written per output message. +MAX_TOOL_CALLS = 16 +# At most this many chunk events per streamed LLM span; the count stays exact. +MAX_CHUNK_EVENTS = 128 +# Open runs tracked at once across all agent runs of one handler. Past it, +# the oldest agent run is ended (mcp_use.incomplete) and forgotten. +MAX_OPEN_RUNS = 10_000 + +# mcp-use 1.7.1 returns a failed MCP tool call as this dict +# (mcp_use/errors/error_formatting.py format_error) instead of raising. +_MCP_USE_ERROR_KEYS = frozenset(("error", "details", "stack", "code")) + +_ROLES = {"human": "user", "ai": "assistant", "system": "system", "tool": "tool"} + +_AGENT = "agent" +_LLM = "llm" +_TOOL = "tool" +_CHAIN = "chain" + + +@dataclass +class _Failure: + type_name: str + qualified: str + message: Optional[str] + stacktrace: Optional[str] + # The message is the tool's own output (an MCP error result). + from_output: bool = False + + +@dataclass +class _Root: + inputs: KnownTexts = field(default_factory=KnownTexts) + outputs: KnownTexts = field(default_factory=KnownTexts) + llm_calls: int = 0 + tool_calls: int = 0 + tool_errors: int = 0 + + +@dataclass +class _Run: + run_id: UUID + kind: str + root: UUID + parent: Optional[UUID] + span: Optional[Span] = None + chunks: int = 0 + + +def _describe(error: BaseException) -> str: + try: + return str(error) + except Exception: + return "".format(type(error).__name__) + + +def _qualified(error: BaseException) -> str: + error_type = type(error) + module = getattr(error_type, "__module__", "") + if module and module != "builtins": + return "{0}.{1}".format(module, error_type.__qualname__) + return error_type.__qualname__ + + +def _failure_from_exception(error: BaseException) -> _Failure: + stacktrace: Optional[str] + try: + stacktrace = "".join( + traceback.format_exception(type(error), error, error.__traceback__) + ) + except Exception: + stacktrace = None + return _Failure(type(error).__name__, _qualified(error), _describe(error), stacktrace) + + +def _is_cancellation(error: BaseException) -> bool: + # asyncio cancellation, or a consumer that stopped reading MCPAgent.stream() + # (LangGraph reports the closed generator as GeneratorExit). + return isinstance(error, (asyncio.CancelledError, GeneratorExit)) + + +def _text_of(content: Any) -> Optional[str]: + """Plain text of a LangChain message content (str or list of blocks).""" + if content is None: + return None + if isinstance(content, str): + return content + if isinstance(content, list): + parts: List[str] = [] + for block in content: + if isinstance(block, str): + parts.append(block) + elif isinstance(block, Mapping) and isinstance(block.get("text"), str): + parts.append(block["text"]) + return "".join(parts) + return str(content) + + +def _role(message: Any) -> str: + if isinstance(message, Mapping): + role = message.get("role") or message.get("type") + return _ROLES.get(role, role) if isinstance(role, str) else "unknown" + kind = getattr(message, "type", None) + if isinstance(kind, str): + return _ROLES.get(kind, kind) + return "unknown" + + +def _content(message: Any) -> Optional[str]: + """Text of a LangChain message object or a {"role", "content"} dict.""" + if isinstance(message, Mapping): + return _text_of(message.get("content")) + return _text_of(getattr(message, "content", None)) + + +def _tool_calls(message: Any) -> List[Mapping[str, Any]]: + if isinstance(message, Mapping): + calls = message.get("tool_calls") + else: + calls = getattr(message, "tool_calls", None) + if isinstance(calls, list): + return [call for call in calls if isinstance(call, Mapping)] + return [] + + +def _json(value: Any) -> str: + try: + return json.dumps(value, ensure_ascii=False, default=str) + except Exception: + return str(value) + + +def _strings_in(value: Any, depth: int = 0) -> Iterable[str]: + """Every string inside a tool-argument structure (bounded depth).""" + if depth > 6: + return + if isinstance(value, str): + yield value + elif isinstance(value, Mapping): + for item in value.values(): + yield from _strings_in(item, depth + 1) + elif isinstance(value, (list, tuple)): + for item in value: + yield from _strings_in(item, depth + 1) + + +def _mcp_use_error(output: Any) -> Optional[Mapping[str, Any]]: + """The mcp-use formatted error a tool returned, if this output is one.""" + candidate: Any = output + content = getattr(output, "content", None) + if content is not None and not isinstance(output, Mapping): + candidate = content + if isinstance(candidate, str): + stripped = candidate.strip() + if not (stripped.startswith("{") and stripped.endswith("}")): + return None + try: + candidate = json.loads(stripped) + except ValueError: + return None + if ( + isinstance(candidate, Mapping) + and _MCP_USE_ERROR_KEYS.issubset(candidate.keys()) + and isinstance(candidate.get("error"), str) + ): + return candidate + return None + + +def _tool_output_text(output: Any) -> Optional[str]: + content = getattr(output, "content", None) + if content is not None: + return _text_of(content) + if isinstance(output, str): + return output + if output is None: + return None + return _json(output) + + +def _int(value: Any) -> Optional[int]: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + return None + + +def _number(value: Any) -> Optional[float]: + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return value + return None + + +class FutureAGICallback(BaseCallbackHandler): + """Trace an mcp-use agent: pass it as ``MCPAgent(..., callbacks=[...])``. + + Args: + tracer_provider: the provider spans go to; defaults to the global + provider. Pass the one ``fi_instrumentation.register()`` returned. + config: a ``fi_instrumentation.TraceConfig``. ``hide_inputs``, + ``hide_outputs`` and ``pii_redaction`` (or ``FI_HIDE_INPUTS``, + ``FI_HIDE_OUTPUTS``, ``FI_PII_REDACTION``) apply to every text + this handler writes. + capture_content: record prompts, model output, tool arguments, tool + results and error messages. Off by default: spans then carry + names, kinds, model, usage, counts, statuses and exception types. + redact: secret strings to remove from everything written (for + example an LLM key or an MCP server header the environment does + not hold). + """ + + raise_error = False + # Run on the event loop, not in an executor thread. LangChain otherwise + # hands each sync callback to run_in_executor; a cancelled run can then + # deliver on_chain_error for a parent while on_tool_start for its child + # is still running in a thread, and the child span would be orphaned. + # Inline, every callback runs to completion in order. The handler does + # no I/O of its own; span export is the span processor's. + run_inline = True + + def __init__( + self, + tracer_provider: Optional[trace_api.TracerProvider] = None, + config: Optional[TraceConfig] = None, + capture_content: bool = False, + redact: Iterable[str] = (), + ) -> None: + super().__init__() + if config is None: + config = TraceConfig() + elif not isinstance(config, TraceConfig): + raise TypeError( + "config must be a fi_instrumentation.TraceConfig, got {0}".format( + type(config).__name__ + ) + ) + if not isinstance(capture_content, bool): + raise TypeError( + "capture_content must be a bool, got {0}".format(type(capture_content).__name__) + ) + if isinstance(redact, str): + raise TypeError("redact must be an iterable of strings, not a string") + secrets = list(redact) + for value in secrets: + if not isinstance(value, str): + raise TypeError( + "redact must contain strings, got {0}".format(type(value).__name__) + ) + provider = tracer_provider or trace_api.get_tracer_provider() + # FITracer applies the config's masking and PII pass to attributes and + # stamps using_session / using_user / using_metadata / using_tags. + self._tracer = FITracer( + trace_api.get_tracer(__name__, __version__, provider), config=config + ) + self._capture = capture_content + self._hide_inputs = bool(config.hide_inputs) + self._hide_outputs = bool(config.hide_outputs) + self._pii = bool(config.pii_redaction) + self._secrets = Secrets(secrets) + self._lock = threading.RLock() + self._runs: Dict[UUID, _Run] = {} + self._roots: "OrderedDict[UUID, _Root]" = OrderedDict() + + # ------------------------------------------------------------------ text + + def _scrubs(self, root: Optional[_Root]) -> List[KnownTexts]: + if root is None: + return [] + scrubs: List[KnownTexts] = [] + if self._hide_inputs: + scrubs.append(root.inputs) + if self._hide_outputs: + scrubs.append(root.outputs) + return scrubs + + def _clean( + self, text: Optional[str], root: Optional[_Root], limit: int, tail: bool = False + ) -> Optional[str]: + if not isinstance(text, str): + return None + return clean(text, self._secrets, limit, self._pii, self._scrubs(root), tail) + + def _name(self, text: Any) -> Optional[str]: + """An identifier (model, tool name, id): secrets removed, 256-byte cap.""" + if not isinstance(text, str) or not text: + return None + return cap(self._secrets.redact(text), MAX_NAME_BYTES) + + def _remember(self, root: Optional[_Root], texts: Iterable[Optional[str]], output: bool) -> None: + if root is None or not self._capture: + return + if output and not self._hide_outputs: + return + if not output and not self._hide_inputs: + return + known = root.outputs if output else root.inputs + with self._lock: + for text in texts: + if isinstance(text, str): + known.add(text, self._secrets) + + # ------------------------------------------------------------ bookkeeping + + def _begin( + self, + run_id: UUID, + parent_run_id: Optional[UUID], + kind: str, + name: str, + attributes: Dict[str, Any], + ) -> Tuple[Optional[_Run], Optional[_Root]]: + """Track a run and start its span (none for collapsed chain runs).""" + if context_api.get_value(context_api._SUPPRESS_INSTRUMENTATION_KEY): + return None, None + evicted: List[_Run] = [] + with self._lock: + parent = self._runs.get(parent_run_id) if parent_run_id is not None else None + if parent is None: + root_id = run_id + if kind == _CHAIN: + kind = _AGENT + self._roots[root_id] = _Root() + else: + root_id = parent.root + root = self._roots.get(root_id) + if root is None: # the root was evicted while this run started + return None, None + if kind == _LLM: + root.llm_calls += 1 + elif kind == _TOOL: + root.tool_calls += 1 + run = _Run(run_id=run_id, kind=kind, root=root_id, parent=parent_run_id) + self._runs[run_id] = run + traced_parent = self._traced_ancestor(parent) + while len(self._runs) > MAX_OPEN_RUNS and len(self._roots) > 1: + oldest = next(iter(self._roots)) + if oldest == root_id: + break + evicted.extend(self._forget_root(oldest)) + for stale in evicted: + self._end_incomplete(stale, None) + if kind == _CHAIN: + return run, root + if kind == _AGENT: + name = AGENT_SPAN_NAME + try: + parent_context = ( + trace_api.set_span_in_context(traced_parent.span) + if traced_parent is not None and traced_parent.span is not None + else None + ) + run.span = self._tracer.start_span(name, context=parent_context, attributes=attributes) + except Exception: + logger.debug("Could not start the mcp-use span", exc_info=True) + run.span = None + return run, root + + def _traced_ancestor(self, run: Optional[_Run]) -> Optional[_Run]: + while run is not None: + if run.kind != _CHAIN: + return run + run = self._runs.get(run.parent) if run.parent is not None else None + return None + + def _forget_root(self, root_id: UUID) -> List[_Run]: + """Drop an agent run and every run under it; caller holds the lock.""" + self._roots.pop(root_id, None) + dropped = [run for run in self._runs.values() if run.root == root_id] + for run in dropped: + self._runs.pop(run.run_id, None) + return [run for run in dropped if run.span is not None] + + def _descends(self, run: _Run, ancestor: UUID) -> bool: + parent = run.parent + while parent is not None: + if parent == ancestor: + return True + above = self._runs.get(parent) + if above is None: + return False + parent = above.parent + return False + + def _pop(self, run_id: UUID) -> Tuple[Optional[_Run], Optional[_Root], List[_Run]]: + """Remove a finished run and any still-open run below it.""" + with self._lock: + run = self._runs.pop(run_id, None) + if run is None: + return None, None, [] + orphans = [other for other in self._runs.values() if self._descends(other, run_id)] + for orphan in orphans: + self._runs.pop(orphan.run_id, None) + root = self._roots.get(run.root) + if run.run_id == run.root: + self._roots.pop(run.root, None) + for leftover in [other for other in self._runs.values() if other.root == run.root]: + self._runs.pop(leftover.run_id, None) + orphans.append(leftover) + return run, root, [orphan for orphan in orphans if orphan.span is not None] + + # ----------------------------------------------------------------- ending + + def _set(self, span: Span, key: str, value: Any) -> None: + if value is None: + return + try: + span.set_attribute(key, value) + except Exception: + logger.debug("Could not set %s", key, exc_info=True) + + def _end(self, span: Optional[Span]) -> None: + if span is None: + return + try: + span.end() + except Exception: + logger.debug("Could not end the mcp-use span", exc_info=True) + + def _ok(self, span: Span) -> None: + try: + span.set_status(Status(StatusCode.OK)) + except Exception: + logger.debug("Could not set the span status", exc_info=True) + + def _cancelled(self, span: Span) -> None: + # Cancellation is not an exception: no event, no message. + self._set(span, CANCELLED, True) + try: + span.set_status(Status(StatusCode.ERROR, "cancelled")) + except Exception: + logger.debug("Could not mark the span cancelled", exc_info=True) + + def _end_incomplete(self, run: _Run, cause: Optional[BaseException]) -> None: + """End a span whose own end callback never came (its parent ended first).""" + span = run.span + if span is None: + return + if cause is not None and _is_cancellation(cause): + self._cancelled(span) + else: + self._set(span, INCOMPLETE, True) + try: + span.set_status(Status(StatusCode.ERROR, "ended without a result")) + except Exception: + logger.debug("Could not set the span status", exc_info=True) + self._end(span) + + def _record_failure(self, span: Span, failure: _Failure, root: Optional[_Root]) -> None: + type_name = self._name(failure.type_name) or "Error" + qualified = self._name(failure.qualified) or type_name + message: Optional[str] = None + stacktrace: Optional[str] = None + if self._capture and not (failure.from_output and self._hide_outputs): + # Error text can quote prompts, tool arguments and tool results, + # so it is content: recorded only with capture_content, and with + # every hidden input/output removed from it. + message = self._clean(failure.message, root, MAX_ERROR_BYTES) + stacktrace = self._clean(failure.stacktrace, root, MAX_STACKTRACE_BYTES, tail=True) + description = "{0}: {1}".format(type_name, message) if message else type_name + try: + span.set_status(Status(StatusCode.ERROR, description)) + except Exception: + logger.debug("Could not set the error status", exc_info=True) + event: Dict[str, Any] = {"exception.type": qualified} + if message: + event["exception.message"] = message + if stacktrace: + event["exception.stacktrace"] = stacktrace + try: + span.add_event("exception", event) + except Exception: + logger.debug("Could not record the exception", exc_info=True) + + def _finish_error(self, run_id: UUID, error: BaseException) -> None: + run, root, orphans = self._pop(run_id) + if run is None: + return + for orphan in orphans: + self._end_incomplete(orphan, error) + if run.kind == _TOOL and root is not None and not _is_cancellation(error): + with self._lock: + root.tool_errors += 1 + span = run.span + if span is None: + return + if run.kind == _AGENT: + self._agent_counts(span, root) + if run.kind == _LLM and run.chunks: + self._set(span, CHUNK_COUNT, run.chunks) + if _is_cancellation(error): + self._cancelled(span) + else: + self._record_failure(span, _failure_from_exception(error), root) + self._end(span) + + def _agent_counts(self, span: Span, root: Optional[_Root]) -> None: + if root is None: + return + self._set(span, LLM_CALL_COUNT, root.llm_calls) + self._set(span, TOOL_CALL_COUNT, root.tool_calls) + self._set(span, TOOL_ERROR_COUNT, root.tool_errors) + + # ------------------------------------------------------------- callbacks + + def on_chain_start( + self, + serialized: Dict[str, Any], + inputs: Dict[str, Any], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + attributes = { + SPAN_KIND: FiSpanKindValues.AGENT.value, + OPERATION: "invoke_agent", + } + run, root = self._begin(run_id, parent_run_id, _CHAIN, AGENT_SPAN_NAME, attributes) + if run is None or run.kind != _AGENT or run.span is None: + return + messages = inputs.get("messages") if isinstance(inputs, Mapping) else None + texts = [_content(m) for m in messages or []] if isinstance( + messages, list + ) else [] + self._remember(root, texts, output=False) + if not self._capture: + return + if self._hide_inputs: + self._set(run.span, INPUT_VALUE, REDACTED_VALUE) + return + query = None + for message in reversed(messages or []) if isinstance(messages, list) else []: + if _role(message) == "user": + query = _content(message) + break + self._set(run.span, INPUT_VALUE, self._clean(query, root, MAX_VALUE_BYTES)) + except Exception: + logger.debug("mcp-use callback on_chain_start failed", exc_info=True) + + def on_chain_end( + self, + outputs: Dict[str, Any], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + run, root, orphans = self._pop(run_id) + if run is None: + return + for orphan in orphans: + self._end_incomplete(orphan, None) + if run.kind != _AGENT or run.span is None: + return + span = run.span + self._agent_counts(span, root) + if self._capture: + answer = None + messages = outputs.get("messages") if isinstance(outputs, Mapping) else None + for message in reversed(messages) if isinstance(messages, list) else []: + if _role(message) == "assistant" and not _tool_calls(message): + answer = _content(message) + break + if answer is not None: + if self._hide_outputs: + self._set(span, OUTPUT_VALUE, REDACTED_VALUE) + else: + self._set(span, OUTPUT_VALUE, self._clean(answer, root, MAX_VALUE_BYTES)) + self._ok(span) + self._end(span) + except Exception: + logger.debug("mcp-use callback on_chain_end failed", exc_info=True) + + def on_chain_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + self._finish_error(run_id, error) + except Exception: + logger.debug("mcp-use callback on_chain_error failed", exc_info=True) + + # Retriever runs are not spans, but their children need a known parent. + def on_retriever_start( + self, + serialized: Dict[str, Any], + query: str, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + if parent_run_id is not None: + with self._lock: + known = parent_run_id in self._runs + if known: + self._begin(run_id, parent_run_id, _CHAIN, "", {}) + except Exception: + logger.debug("mcp-use callback on_retriever_start failed", exc_info=True) + + def on_retriever_end(self, documents: Any, *, run_id: UUID, **kwargs: Any) -> None: + try: + self._pop(run_id) + except Exception: + logger.debug("mcp-use callback on_retriever_end failed", exc_info=True) + + def on_retriever_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None: + try: + self._pop(run_id) + except Exception: + logger.debug("mcp-use callback on_retriever_error failed", exc_info=True) + + def _llm_attributes( + self, operation: str, kwargs: Mapping[str, Any] + ) -> Tuple[str, Dict[str, Any]]: + metadata = kwargs.get("metadata") or {} + params = kwargs.get("invocation_params") or {} + model = None + for source, key in ( + (metadata, "ls_model_name"), + (params, "model"), + (params, "model_name"), + (params, "model_id"), + ): + if isinstance(source, Mapping) and isinstance(source.get(key), str) and source.get(key): + model = source.get(key) + break + attributes: Dict[str, Any] = { + SPAN_KIND: FiSpanKindValues.LLM.value, + OPERATION: operation, + } + model_name = self._name(model) + if model_name: + attributes[REQUEST_MODEL] = model_name + if isinstance(metadata, Mapping): + provider = self._name(metadata.get("ls_provider")) + if provider: + attributes[PROVIDER] = provider + temperature = _number(metadata.get("ls_temperature")) + if temperature is not None: + attributes[TEMPERATURE] = temperature + max_tokens = _int(metadata.get("ls_max_tokens")) + if max_tokens is not None: + attributes[MAX_TOKENS] = max_tokens + name = "{0} {1}".format(operation, model_name) if model_name else operation + return name, attributes + + def _message_attributes( + self, prefix: str, messages: Sequence[Any], root: Optional[_Root] + ) -> Dict[str, Any]: + attributes: Dict[str, Any] = {} + for index, message in enumerate(list(messages)[-MAX_MESSAGES:]): + base = "{0}.{1}.message".format(prefix, index) + attributes[base + ".role"] = _role(message) + text = self._clean(_content(message), root, MAX_VALUE_BYTES) + if text: + attributes[base + ".content"] = text + for call_index, call in enumerate(_tool_calls(message)[:MAX_TOOL_CALLS]): + call_base = "{0}.tool_calls.{1}.tool_call".format(base, call_index) + call_name = self._name(call.get("name")) + if call_name: + attributes[call_base + ".function.name"] = call_name + call_id = self._name(call.get("id")) + if call_id: + attributes[call_base + ".id"] = call_id + # Tool-call arguments are the tool's input, also when they + # appear in the model's output: hide_inputs drops them. + if self._hide_inputs: + continue + arguments = self._clean(_json(call.get("args")), root, MAX_VALUE_BYTES) + if arguments: + attributes[call_base + ".function.arguments"] = arguments + return attributes + + def _start_llm( + self, + operation: str, + messages: Sequence[Any], + run_id: UUID, + parent_run_id: Optional[UUID], + kwargs: Mapping[str, Any], + ) -> None: + name, attributes = self._llm_attributes(operation, kwargs) + attributes[INPUT_MESSAGE_COUNT] = len(messages) + run, root = self._begin(run_id, parent_run_id, _LLM, name, attributes) + if run is None or run.span is None: + return + texts: List[Optional[str]] = [] + for message in messages: + texts.append(_content(message)) + for call in _tool_calls(message): + texts.extend(_strings_in(call.get("args"))) + self._remember(root, texts, output=False) + if not self._capture: + return + if self._hide_inputs: + self._set(run.span, INPUT_VALUE, REDACTED_VALUE) + return + for key, value in self._message_attributes(INPUT_MESSAGES, messages, root).items(): + self._set(run.span, key, value) + if messages: + last = self._clean(_content(messages[-1]), root, MAX_VALUE_BYTES) + if last: + self._set(run.span, INPUT_VALUE, last) + + def on_chat_model_start( + self, + serialized: Dict[str, Any], + messages: List[List[Any]], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + first = messages[0] if messages else [] + self._start_llm("chat", list(first), run_id, parent_run_id, kwargs) + except Exception: + logger.debug("mcp-use callback on_chat_model_start failed", exc_info=True) + + def on_llm_start( + self, + serialized: Dict[str, Any], + prompts: List[str], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + from langchain_core.messages import HumanMessage + + messages = [HumanMessage(content=prompt) for prompt in prompts[:1]] + self._start_llm("text_completion", messages, run_id, parent_run_id, kwargs) + except Exception: + logger.debug("mcp-use callback on_llm_start failed", exc_info=True) + + def on_llm_new_token( + self, + token: Any, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + # One event per chunk (index only, never the text), up to + # MAX_CHUNK_EVENTS; mcp_use.llm.chunk_count keeps the exact number. + try: + with self._lock: + run = self._runs.get(run_id) + if run is None or run.span is None: + return + run.chunks += 1 + index = run.chunks - 1 + if index < MAX_CHUNK_EVENTS: + run.span.add_event(CHUNK_EVENT, {CHUNK_INDEX: index}) + except Exception: + logger.debug("mcp-use callback on_llm_new_token failed", exc_info=True) + + def on_llm_end( + self, + response: Any, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + run, root, orphans = self._pop(run_id) + if run is None: + return + for orphan in orphans: + self._end_incomplete(orphan, None) + span = run.span + if span is None: + return + if run.chunks: + self._set(span, CHUNK_COUNT, run.chunks) + try: + self._llm_response(span, response, root) + except Exception: + logger.debug("Could not read the LLM response", exc_info=True) + self._ok(span) + self._end(span) + except Exception: + logger.debug("mcp-use callback on_llm_end failed", exc_info=True) + + def _llm_response(self, span: Span, response: Any, root: Optional[_Root]) -> None: + generations = getattr(response, "generations", None) or [] + first = list(generations[0]) if generations else [] + llm_output = getattr(response, "llm_output", None) or {} + messages: List[Any] = [] + reasons: List[str] = [] + usage: Optional[Mapping[str, Any]] = None + response_model: Any = None + for generation in first: + message = getattr(generation, "message", None) + if message is None: + from langchain_core.messages import AIMessage + + message = AIMessage(content=getattr(generation, "text", "") or "") + messages.append(message) + metadata = getattr(message, "response_metadata", None) or {} + info = getattr(generation, "generation_info", None) or {} + for source in (metadata, info): + reason = source.get("finish_reason") or source.get("stop_reason") + if isinstance(reason, str) and reason: + reasons.append(reason) + break + response_model = response_model or metadata.get("model_name") or metadata.get("model") + usage = usage or getattr(message, "usage_metadata", None) + if isinstance(llm_output, Mapping): + response_model = response_model or llm_output.get("model_name") + name = self._name(response_model) + if name: + self._set(span, RESPONSE_MODEL, name) + if reasons: + self._set(span, FINISH_REASONS, [cap(reason, MAX_NAME_BYTES) for reason in reasons]) + if isinstance(usage, Mapping): + self._set(span, INPUT_TOKENS, _int(usage.get("input_tokens"))) + self._set(span, OUTPUT_TOKENS, _int(usage.get("output_tokens"))) + self._set(span, TOTAL_TOKENS, _int(usage.get("total_tokens"))) + details = usage.get("input_token_details") + if isinstance(details, Mapping): + self._set(span, CACHE_READ_TOKENS, _int(details.get("cache_read"))) + elif isinstance(llm_output, Mapping) and isinstance(llm_output.get("token_usage"), Mapping): + token_usage = llm_output["token_usage"] + self._set(span, INPUT_TOKENS, _int(token_usage.get("prompt_tokens"))) + self._set(span, OUTPUT_TOKENS, _int(token_usage.get("completion_tokens"))) + self._set(span, TOTAL_TOKENS, _int(token_usage.get("total_tokens"))) + if messages: + self._set(span, REQUESTED_TOOL_CALLS, sum(len(_tool_calls(m)) for m in messages)) + texts: List[Optional[str]] = [] + arguments: List[Optional[str]] = [] + for message in messages: + texts.append(_content(message)) + for call in _tool_calls(message): + arguments.extend(_strings_in(call.get("args"))) + self._remember(root, texts + arguments, output=True) + # The arguments the model chose are the next tool call's input. + self._remember(root, arguments, output=False) + if not self._capture or not messages: + return + if self._hide_outputs: + self._set(span, OUTPUT_VALUE, REDACTED_VALUE) + return + for key, value in self._message_attributes(OUTPUT_MESSAGES, messages, root).items(): + self._set(span, key, value) + answer = self._clean(_content(messages[0]), root, MAX_VALUE_BYTES) + if answer: + self._set(span, OUTPUT_VALUE, answer) + + def on_llm_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + self._finish_error(run_id, error) + except Exception: + logger.debug("mcp-use callback on_llm_error failed", exc_info=True) + + def on_tool_start( + self, + serialized: Dict[str, Any], + input_str: str, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + inputs: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> None: + try: + tool = None + if isinstance(serialized, Mapping): + tool = serialized.get("name") + tool = self._name(tool or kwargs.get("name")) or "unknown" + attributes: Dict[str, Any] = { + SPAN_KIND: FiSpanKindValues.TOOL.value, + OPERATION: "execute_tool", + TOOL_NAME: tool, + } + call_id = self._name(kwargs.get("tool_call_id")) + if call_id: + attributes[TOOL_CALL_ID] = call_id + run, root = self._begin( + run_id, parent_run_id, _TOOL, "execute_tool {0}".format(tool), attributes + ) + if run is None or run.span is None: + return + arguments = _json(inputs) if isinstance(inputs, Mapping) else input_str + texts: List[Optional[str]] = [arguments, input_str] + if isinstance(inputs, Mapping): + texts.extend(_strings_in(inputs)) + self._remember(root, texts, output=False) + if not self._capture: + return + if self._hide_inputs: + self._set(run.span, INPUT_VALUE, REDACTED_VALUE) + return + text = self._clean(arguments, root, MAX_VALUE_BYTES) + self._set(run.span, TOOL_ARGUMENTS, text) + self._set(run.span, INPUT_VALUE, text) + except Exception: + logger.debug("mcp-use callback on_tool_start failed", exc_info=True) + + def on_tool_end( + self, + output: Any, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + run, root, orphans = self._pop(run_id) + if run is None: + return + for orphan in orphans: + self._end_incomplete(orphan, None) + failure: Optional[_Failure] = None + try: + formatted = _mcp_use_error(output) + if formatted is not None: + error_type = str(formatted.get("error")) + details = formatted.get("details") + stack = formatted.get("stack") + failure = _Failure( + error_type, + error_type, + details if isinstance(details, str) else None, + stack if isinstance(stack, str) else None, + from_output=True, + ) + elif getattr(output, "status", None) == "error": + failure = _Failure( + "ToolException", + "langchain_core.tools.ToolException", + _tool_output_text(output), + None, + from_output=True, + ) + except Exception: + logger.debug("Could not classify the tool result", exc_info=True) + text = _tool_output_text(output) + self._remember(root, [text], output=True) + if failure is not None and root is not None: + with self._lock: + root.tool_errors += 1 + span = run.span + if span is None: + return + if failure is not None: + self._set(span, TOOL_ERROR_TYPE, self._name(failure.type_name)) + self._record_failure(span, failure, root) + else: + if self._capture: + if self._hide_outputs: + self._set(span, OUTPUT_VALUE, REDACTED_VALUE) + else: + cleaned = self._clean(text, root, MAX_VALUE_BYTES) + self._set(span, TOOL_RESULT, cleaned) + self._set(span, OUTPUT_VALUE, cleaned) + self._ok(span) + self._end(span) + except Exception: + logger.debug("mcp-use callback on_tool_end failed", exc_info=True) + + def on_tool_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + try: + self._finish_error(run_id, error) + except Exception: + logger.debug("mcp-use callback on_tool_error failed", exc_info=True) diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_text.py b/python/frameworks/mcp-use/traceai_mcp_use/_text.py new file mode 100644 index 000000000..cbf3b8254 --- /dev/null +++ b/python/frameworks/mcp-use/traceai_mcp_use/_text.py @@ -0,0 +1,174 @@ +"""Text cleaning applied to every string the callback writes to a span. + +Order for every value: remove secrets, then (when a TraceConfig hide flag is +on) remove text the run already saw as an input or output, then PII when +``pii_redaction`` is on, then cut to a UTF-8 byte cap. Secrets go first so a +cap can never leave the start of a key behind; PII goes before the cap so a +cut cannot leave part of an email address. +""" + +from __future__ import annotations + +import json +import os +import re +from typing import Iterable, List, Optional, Sequence, Tuple + +from fi_instrumentation import REDACTED_VALUE +from fi_instrumentation.instrumentation.pii_redaction import redact_pii_in_string + +REDACTED = "[redacted]" + +MAX_VALUE_BYTES = 4096 +MAX_NAME_BYTES = 256 +MAX_ERROR_BYTES = 1024 +MAX_STACKTRACE_BYTES = 16 * 1024 + +# A secret shorter than this is not searched for: replacing a 3-character +# value everywhere would shred unrelated text. +MIN_SECRET_CHARS = 8 +# Known inputs/outputs shorter than this are only removed in their quoted +# forms ('ab', "ab"); bare short strings would match inside unrelated words. +MIN_SCRUB_CHARS = 3 +# Bound on the text one agent run remembers for hide_inputs/hide_outputs. +MAX_KNOWN_TEXTS = 2048 +MAX_KNOWN_CHARS = 4 * 1024 * 1024 + +_SECRET_ENV_NAME = re.compile(r"KEY|SECRET|TOKEN|PASSWORD|PASSWD|CREDENTIAL|AUTH", re.I) +_BEARER = re.compile(r"(?i)\b(bearer|basic)(\s+)[A-Za-z0-9._~+/=-]{8,}") +_ASSIGNED = re.compile( + r"(?i)\b((?:x[-_])?api[-_]?key|authorization|secret(?:[-_]?key)?|access[-_]?token" + r"|refresh[-_]?token|token|password|passwd)" + r"([\"']?\s*[:=]\s*[\"']?)" + r"([^\s\"',;}\]]{4,})" +) +_KEY_SHAPE = re.compile(r"\b(?:sk|pk|rk)-[A-Za-z0-9_-]{16,}") + + +def cap(value: str, limit: int) -> str: + """Return the longest whole-character prefix of at most ``limit`` UTF-8 bytes.""" + return value.encode("utf-8", "replace")[:limit].decode("utf-8", "ignore") + + +def cap_tail(value: str, limit: int) -> str: + """Return the longest whole-character suffix of at most ``limit`` UTF-8 bytes. + + Used for stack traces, whose last lines (the raising frame and the + exception line) matter most. + """ + encoded = value.encode("utf-8", "replace") + if len(encoded) <= limit: + return value + return encoded[len(encoded) - limit :].decode("utf-8", "ignore") + + +class Secrets: + """Secret values the callback removes from every text it writes. + + Sources: the values passed as ``redact=`` and, read on every call, the + values of environment variables whose name contains KEY, SECRET, TOKEN, + PASSWORD, PASSWD, CREDENTIAL or AUTH (LLM provider keys such as + OPENAI_API_KEY, and FI_API_KEY / FI_SECRET_KEY). Values shorter than + ``MIN_SECRET_CHARS`` are ignored. Bearer/Basic credentials, ``sk-`` + style keys and ``api_key=...`` / ``Authorization: ...`` assignments are + also replaced by pattern, for secrets the callback was never told about. + """ + + def __init__(self, explicit: Iterable[str] = ()) -> None: + self._explicit: Tuple[str, ...] = tuple( + value for value in dict.fromkeys(explicit) if len(value) >= MIN_SECRET_CHARS + ) + + def values(self) -> List[str]: + found = list(self._explicit) + try: + for name, value in list(os.environ.items()): + if ( + value + and len(value) >= MIN_SECRET_CHARS + and _SECRET_ENV_NAME.search(name) + and value not in found + ): + found.append(value) + except Exception: # an unreadable environment must not break redaction + pass + return sorted(found, key=len, reverse=True) + + def redact(self, text: str) -> str: + for value in self.values(): + if value in text: + text = text.replace(value, REDACTED) + text = _BEARER.sub(lambda match: match.group(1) + match.group(2) + REDACTED, text) + text = _ASSIGNED.sub(lambda match: match.group(1) + match.group(2) + REDACTED, text) + return _KEY_SHAPE.sub(REDACTED, text) + + +class KnownTexts: + """Inputs (or outputs) one agent run has seen, for ``hide_inputs``/``hide_outputs``. + + Every text is stored raw, with secrets already removed (so it still + matches after the secret pass), and in its Python-repr and JSON-quoted + forms (so an echo inside a repr'd dict or a JSON body matches too). + Past ``MAX_KNOWN_TEXTS`` entries or ``MAX_KNOWN_CHARS`` characters the + set is marked overflowed and ``scrub`` fails closed by returning None. + """ + + def __init__(self) -> None: + self._items: set = set() + self._chars = 0 + self._sorted: Optional[Sequence[str]] = None + self.overflowed = False + + def add(self, text: str, secrets: Secrets) -> None: + if not isinstance(text, str) or not text or self.overflowed: + return + forms = {repr(text), json.dumps(text, ensure_ascii=False), json.dumps(text)} + if len(text) >= MIN_SCRUB_CHARS: + forms.add(text) + redacted = secrets.redact(text) + if len(redacted) >= MIN_SCRUB_CHARS: + forms.add(redacted) + for form in forms: + if form in self._items: + continue + self._items.add(form) + self._chars += len(form) + self._sorted = None + if len(self._items) > MAX_KNOWN_TEXTS or self._chars > MAX_KNOWN_CHARS: + self.overflowed = True + self._items.clear() + + def scrub(self, text: str) -> Optional[str]: + if self.overflowed: + return None + if self._sorted is None: + self._sorted = sorted(self._items, key=len, reverse=True) + for item in self._sorted: + if item in text: + text = text.replace(item, REDACTED_VALUE) + return text + + +def clean( + text: str, + secrets: Secrets, + limit: int, + pii: bool = False, + scrub: Sequence[KnownTexts] = (), + tail: bool = False, +) -> Optional[str]: + """Secrets, then known inputs/outputs, then PII, then the byte cap. + + ``tail`` keeps the end of the text instead of the start. Returns None + when a hide flag's text set overflowed: the caller must then record + nothing rather than an unscrubbed value. + """ + text = secrets.redact(text) + for known in scrub: + scrubbed = known.scrub(text) + if scrubbed is None: + return None + text = scrubbed + if pii: + text = redact_pii_in_string(text) + return cap_tail(text, limit) if tail else cap(text, limit) diff --git a/python/frameworks/mcp-use/traceai_mcp_use/py.typed b/python/frameworks/mcp-use/traceai_mcp_use/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/python/frameworks/mcp-use/traceai_mcp_use/version.py b/python/frameworks/mcp-use/traceai_mcp_use/version.py new file mode 100644 index 000000000..3dc1f76bc --- /dev/null +++ b/python/frameworks/mcp-use/traceai_mcp_use/version.py @@ -0,0 +1 @@ +__version__ = "0.1.0" From f7c3274cc647811ed321cf6bb32579ca795e2c91 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 07:36:07 -0700 Subject: [PATCH 05/19] test(mcp-use): unit-test the callback against a fake callback sequence No MCP server and no LLM: the tests replay the LangChain callback tree mcp-use 1.7.1 produces for one agent run into an InMemorySpanExporter. They cover the three span kinds and their parents, content off by default, capture_content, hide_inputs/hide_outputs (including tool-call arguments and error text), pii_redaction, secrets, size caps, streamed tokens as capped events, errors, cancellation, incomplete runs, bounded open runs, context attributes, suppress_tracing, and tracer failures and malformed payloads never raising. Shared fakes (a scripted chat model and a FastMCP server) are added for the agent tests. approved: Nikhil 2026-10-03 blanket Refs: TH-8331 --- .../frameworks/mcp-use/tests/_mcp_server.py | 41 + .../mcp-use/tests/_mcp_use_support.py | 340 ++++++++ .../mcp-use/tests/test_mcp_use_callback.py | 751 ++++++++++++++++++ 3 files changed, 1132 insertions(+) create mode 100644 python/frameworks/mcp-use/tests/_mcp_server.py create mode 100644 python/frameworks/mcp-use/tests/_mcp_use_support.py create mode 100644 python/frameworks/mcp-use/tests/test_mcp_use_callback.py diff --git a/python/frameworks/mcp-use/tests/_mcp_server.py b/python/frameworks/mcp-use/tests/_mcp_server.py new file mode 100644 index 000000000..f8170eded --- /dev/null +++ b/python/frameworks/mcp-use/tests/_mcp_server.py @@ -0,0 +1,41 @@ +"""A fake MCP server over stdio for the traceAI-mcp-use tests. + +mcp-use starts it as a subprocess (MCPClient "command"/"args"). It talks to +nothing else. Each tool returns a marker string so the tests can tell +whether tool content reached a span. +""" + +import asyncio + +from mcp.server.fastmcp import FastMCP + +mcp = FastMCP("fake") + + +@mcp.tool() +def add(a: int, b: int) -> str: + """Add two integers.""" + return "SUM-RESULT-{0}".format(a + b) + + +@mcp.tool() +def echo(text: str) -> str: + """Return the text.""" + return "ECHO-RESULT:" + text + + +@mcp.tool() +def fail(reason: str) -> str: + """Always fails; the MCP result has isError set.""" + raise ValueError("tool failed because " + reason) + + +@mcp.tool() +async def slow(seconds: float) -> str: + """Sleep, then return.""" + await asyncio.sleep(seconds) + return "SLEPT" + + +if __name__ == "__main__": + mcp.run("stdio") diff --git a/python/frameworks/mcp-use/tests/_mcp_use_support.py b/python/frameworks/mcp-use/tests/_mcp_use_support.py new file mode 100644 index 000000000..8a4c1ae1e --- /dev/null +++ b/python/frameworks/mcp-use/tests/_mcp_use_support.py @@ -0,0 +1,340 @@ +"""Fakes and helpers shared by the traceAI-mcp-use tests. + +The LLM is ``ChatFake``, a LangChain chat model that replays a script. The +MCP server is ``_mcp_server.py``, started by mcp-use over stdio. Nothing +here opens a network connection; every key is a placeholder. + +Import this module before ``mcp_use``: it turns off mcp-use's own usage +telemetry, which otherwise posts to PostHog and Scarf. +""" + +from __future__ import annotations + +import asyncio +import os +import sys +import threading +import time +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple + +# mcp-use 1.7.1 sends anonymized usage events unless this is "false" +# (mcp_use/telemetry/telemetry.py:139). Telemetry is a singleton built on +# first use (telemetry.py:122), so this runs before any MCPAgent/MCPClient. +os.environ["MCP_USE_ANONYMIZED_TELEMETRY"] = "false" +# Importing mcp_use sets up Langfuse and Laminar when their keys are present +# (mcp_use/agents/observability/__init__.py:6). The tests set no keys and +# turn both off as well. +os.environ["MCP_USE_LANGFUSE"] = "false" +os.environ["MCP_USE_LAMINAR"] = "false" + +from langchain_core.callbacks import BaseCallbackHandler # noqa: E402 +from langchain_core.language_models.chat_models import BaseChatModel # noqa: E402 +from langchain_core.messages import AIMessage, AIMessageChunk, BaseMessage # noqa: E402 +from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult # noqa: E402 +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402 + InMemorySpanExporter, +) + +HERE = Path(__file__).resolve().parent +PACKAGE = HERE.parent +REPO_PYTHON = PACKAGE.parents[1] +SERVER = HERE / "_mcp_server.py" + +MODEL = "fake-model-1" +PROMPT = "PROMPT-MARKER please add two and three" +ANSWER = "ANSWER-MARKER the sum is five" +ARG = "ARG-MARKER" +# Placeholders only. LLM_KEY has the sk- shape; ENV_SECRET is only known +# to the callback through the FAKE_PROVIDER_API_KEY environment variable. +LLM_KEY = "sk-placeholder-llm-key-0000000000000000" +ENV_SECRET_NAME = "FAKE_PROVIDER_API_KEY" +ENV_SECRET = "placeholder-env-secret-value-1234" +EXPLICIT_SECRET = "placeholder-mcp-header-token-42" +CONTENT_MARKERS = ( + "PROMPT-MARKER", + "ANSWER-MARKER", + "ARG-MARKER", + "SUM-RESULT", + "ECHO-RESULT", + "tool failed because", +) + +AGENT = "mcp_use.agent" +LLM_SPAN = "chat " + MODEL + + +class ChatFake(BaseChatModel): + """A chat model that replays ``script``. + + Each step is an ``AIMessage`` to return, an exception to raise, or a + callable taking the input messages and returning an ``AIMessage``. + Returned messages carry usage and response metadata like a provider's. + ``streaming=True`` streams word by word through ``on_llm_new_token``. + """ + + script: List[Any] + model_name: str = MODEL + temperature: float = 0.2 + max_tokens: int = 64 + # Held like a provider key. LangChain puts repr(model), and so this + # value, into the serialized payload of on_chat_model_start. + api_key: str = LLM_KEY + streaming: bool = False + index: int = 0 + delay: float = 0.0 + + @property + def _llm_type(self) -> str: + return "fake-chat" + + def bind_tools(self, tools: Any, **kwargs: Any) -> "ChatFake": + return self + + def _next(self, messages: Sequence[BaseMessage]) -> AIMessage: + if self.index >= len(self.script): + step: Any = AIMessage(content="script exhausted") + else: + step = self.script[self.index] + self.index += 1 + if isinstance(step, BaseException): + raise step + if callable(step): + step = step(messages) + return step.model_copy( + update={ + "usage_metadata": {"input_tokens": 11, "output_tokens": 7, "total_tokens": 18}, + "response_metadata": { + "model_name": MODEL, + "finish_reason": "tool_calls" if step.tool_calls else "stop", + }, + } + ) + + def _generate(self, messages, stop=None, run_manager=None, **kwargs): # type: ignore[override] + return ChatResult(generations=[ChatGeneration(message=self._next(messages))]) + + async def _agenerate(self, messages, stop=None, run_manager=None, **kwargs): # type: ignore[override] + if self.delay: + await asyncio.sleep(self.delay) + return self._generate(messages, stop=stop, **kwargs) + + async def _astream(self, messages, stop=None, run_manager=None, **kwargs): # type: ignore[override] + message = self._next(messages) + if message.tool_calls: + yield ChatGenerationChunk( + message=AIMessageChunk( + content="", + tool_call_chunks=[ + { + "name": call["name"], + "args": _json(call["args"]), + "id": call["id"], + "index": index, + } + for index, call in enumerate(message.tool_calls) + ], + usage_metadata=message.usage_metadata, + response_metadata=message.response_metadata, + ) + ) + return + words = str(message.content).split(" ") + for position, word in enumerate(words): + text = word if position == len(words) - 1 else word + " " + last = position == len(words) - 1 + chunk = ChatGenerationChunk( + message=AIMessageChunk( + content=text, + usage_metadata=message.usage_metadata if last else None, + response_metadata=message.response_metadata if last else {}, + ) + ) + if run_manager: + await run_manager.on_llm_new_token(text, chunk=chunk) + yield chunk + + +def _json(value: Any) -> str: + import json + + return json.dumps(value) + + +def tool_call(name: str, args: Dict[str, Any], call_id: str = "call-1") -> AIMessage: + return AIMessage( + content="", tool_calls=[{"name": name, "args": args, "id": call_id, "type": "tool_call"}] + ) + + +def answer(text: str = ANSWER) -> AIMessage: + return AIMessage(content=text) + + +def new_provider() -> Tuple[InMemorySpanExporter, TracerProvider]: + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return exporter, provider + + +class _LoopbackServer: + """``_mcp_server.py`` over streamable HTTP on 127.0.0.1, in a thread. + + Started once per test process: spawning the stdio server costs several + seconds of imports per test. The contract and example tests still use + stdio. + """ + + _lock = threading.Lock() + _url: Optional[str] = None + + @classmethod + def url(cls) -> str: + with cls._lock: + if cls._url is None: + cls._url = cls._start() + return cls._url + + @staticmethod + def _start() -> str: + import socket + + import uvicorn + + sys.path.insert(0, str(HERE)) + from _mcp_server import mcp as fake_server + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + server = uvicorn.Server( + uvicorn.Config(fake_server.streamable_http_app(), log_level="warning") + ) + thread = threading.Thread( + target=lambda: asyncio.run(server.serve(sockets=[sock])), daemon=True + ) + thread.start() + deadline = time.monotonic() + 30 + while not server.started: + if time.monotonic() > deadline or not thread.is_alive(): + raise RuntimeError("the loopback MCP server did not start") + time.sleep(0.02) + return "http://127.0.0.1:{0}/mcp".format(port) + + +def mcp_client(transport: str = "http") -> Any: + """An MCPClient for the fake server: loopback HTTP, or a stdio subprocess.""" + from mcp_use import MCPClient + + if transport == "stdio": + server: Dict[str, Any] = {"command": sys.executable, "args": [str(SERVER)]} + else: + server = {"url": _LoopbackServer.url()} + return MCPClient.from_dict({"mcpServers": {"fake": server}}) + + +def add_script() -> List[Any]: + """The J1 journey: one tool call to add, then the final answer.""" + return [tool_call("add", {"a": 2, "b": 3}), answer()] + + +async def drive( + agent: Any, method: str, query: str = PROMPT, **kwargs: Any +) -> Any: + """Run the agent through one of its three public entry points.""" + if method == "run": + return await agent.run(query, **kwargs) + if method == "stream": + result = None + async for item in agent.stream(query, **kwargs): + result = item + return result + if method == "stream_events": + events = [] + async for event in agent.stream_events(query, **kwargs): + events.append(event) + return events + raise ValueError(method) + + +def run_agent( + script: List[Any], + callbacks: Optional[List[Any]], + method: str = "run", + query: str = PROMPT, + streaming: bool = False, + agent_kwargs: Optional[Dict[str, Any]] = None, + around: Optional[Callable[[], Any]] = None, + transport: str = "http", +) -> Any: + """Build an MCPAgent on the fake LLM and fake server and run it once. + + ``around`` returns a context manager entered around the run (for + using_session, a parent span, suppress_tracing...). + """ + from mcp_use import MCPAgent + + async def go() -> Any: + client = mcp_client(transport) + try: + agent = MCPAgent( + llm=ChatFake(script=script, streaming=streaming), + client=client, + callbacks=callbacks, + **(agent_kwargs or {}), + ) + if around is None: + return await drive(agent, method, query) + with around(): + return await drive(agent, method, query) + finally: + await client.close_all_sessions() + + return asyncio.run(go()) + + +class ToolStarted(BaseCallbackHandler): + """A second LangChain handler that signals when a tool call starts.""" + + run_inline = True + + def __init__(self) -> None: + super().__init__() + self.started = threading.Event() + + def on_tool_start(self, *args: Any, **kwargs: Any) -> None: + self.started.set() + + +def spans_named(spans: Iterable[ReadableSpan], name: str) -> List[ReadableSpan]: + return [span for span in spans if span.name == name] + + +def only(spans: Iterable[ReadableSpan], name: str) -> ReadableSpan: + found = spans_named(spans, name) + assert len(found) == 1, (name, [span.name for span in spans]) + return found[0] + + +def parent_id(span: ReadableSpan) -> Optional[int]: + return span.parent.span_id if span.parent is not None else None + + +def attrs(span: ReadableSpan) -> Dict[str, Any]: + return dict(span.attributes or {}) + + +def wire(spans: Iterable[ReadableSpan]) -> str: + """Every name, attribute, event and status text of the spans, as one string.""" + parts: List[str] = [] + for span in spans: + parts.append(span.name) + parts.append(repr(dict(span.attributes or {}))) + parts.append(str(span.status.description)) + for event in span.events: + parts.append(event.name) + parts.append(repr(dict(event.attributes or {}))) + return "\n".join(parts) diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py new file mode 100644 index 000000000..2f495dd48 --- /dev/null +++ b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py @@ -0,0 +1,751 @@ +"""Unit tests: a fake LangChain callback sequence, no MCP server and no LLM. + +``Run`` replays the callback tree mcp-use 1.7.1 produces for one agent run +(recorded from MCPAgent.run on the fake LLM and server): a LangGraph root +chain, a ``model`` node chain holding each chat-model run, and a ``tools`` +node chain holding each tool run. Spans go to an InMemorySpanExporter. +""" + +from __future__ import annotations + +import asyncio +import json +import os +from typing import Any, Dict, List, Optional +from uuid import UUID, uuid4 + +import pytest +from _mcp_use_support import ( + AGENT, + ANSWER, + ARG, + CONTENT_MARKERS, + ENV_SECRET, + ENV_SECRET_NAME, + EXPLICIT_SECRET, + LLM_KEY, + LLM_SPAN, + MODEL, + PROMPT, + attrs, + new_provider, + only, + parent_id, + spans_named, + wire, +) +from fi_instrumentation import ( + REDACTED_VALUE, + TraceConfig, + suppress_tracing, + using_metadata, + using_session, + using_tags, + using_user, +) +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage +from langchain_core.outputs import ChatGeneration, LLMResult +from opentelemetry import trace as trace_api +from opentelemetry.trace import StatusCode + +import traceai_mcp_use._callback as callback_module +from traceai_mcp_use import AGENT_SPAN_NAME, FutureAGICallback +from traceai_mcp_use._text import ( + MAX_ERROR_BYTES, + MAX_NAME_BYTES, + MAX_STACKTRACE_BYTES, + MAX_VALUE_BYTES, +) + +TOOL_SPAN = "execute_tool add" +METADATA = { + "ls_provider": "fake", + "ls_model_name": MODEL, + "ls_model_type": "chat", + "ls_temperature": 0.2, + "ls_max_tokens": 64, +} + + +def _result(message: AIMessage) -> LLMResult: + message = message.model_copy( + update={ + "usage_metadata": { + "input_tokens": 11, + "output_tokens": 7, + "total_tokens": 18, + "input_token_details": {"cache_read": 3}, + }, + "response_metadata": { + "model_name": MODEL, + "finish_reason": "tool_calls" if message.tool_calls else "stop", + }, + } + ) + return LLMResult(generations=[[ChatGeneration(message=message)]]) + + +class Run: + """One agent run, driven callback by callback like LangGraph does.""" + + def __init__(self, handler: FutureAGICallback, query: str = PROMPT) -> None: + self.handler = handler + self.root = uuid4() + self.history: List[Any] = [HumanMessage(content=query)] + handler.on_chain_start( + {}, {"messages": list(self.history)}, run_id=self.root, name="LangGraph" + ) + + def node(self, name: str) -> UUID: + run_id = uuid4() + self.handler.on_chain_start( + None, {"messages": list(self.history)}, run_id=run_id, parent_run_id=self.root, name=name + ) + return run_id + + def end_node(self, run_id: UUID) -> None: + self.handler.on_chain_end({"messages": []}, run_id=run_id, parent_run_id=self.root) + + def llm(self, reply: AIMessage, system: str = "You are an MCP agent.") -> UUID: + node = self.node("model") + run_id = uuid4() + self.handler.on_chat_model_start( + {"name": "ChatFake", "repr": "ChatFake(api_key='{0}')".format(LLM_KEY)}, + [[SystemMessage(content=system)] + list(self.history)], + run_id=run_id, + parent_run_id=node, + invocation_params={"model_name": MODEL, "api_key": LLM_KEY}, + metadata=dict(METADATA), + ) + self.handler.on_llm_end(_result(reply), run_id=run_id, parent_run_id=node) + self.end_node(node) + self.history.append(reply) + return run_id + + def start_tool(self, name: str, args: Dict[str, Any], call_id: str = "call-1"): + node = self.node("tools") + run_id = uuid4() + self.handler.on_tool_start( + {"name": name, "description": "A tool."}, + str(args), + run_id=run_id, + parent_run_id=node, + inputs=dict(args), + tool_call_id=call_id, + name=name, + ) + return node, run_id + + def tool(self, name: str, args: Dict[str, Any], result: str, call_id: str = "call-1") -> None: + node, run_id = self.start_tool(name, args, call_id) + output = ToolMessage(content=result, name=name, tool_call_id=call_id) + self.handler.on_tool_end(output, run_id=run_id, parent_run_id=node) + self.end_node(node) + self.history.append(output) + + def finish(self, text: str = ANSWER) -> None: + self.handler.on_chain_end( + {"messages": self.history + [AIMessage(content=text)]}, run_id=self.root + ) + + +def _journey(handler: FutureAGICallback, query: str = PROMPT, args=None, result="SUM-RESULT-5"): + run = Run(handler, query) + args = args if args is not None else {"a": 2, "b": 3, "note": ARG} + run.llm( + AIMessage( + content="", tool_calls=[{"name": "add", "args": args, "id": "call-1", "type": "tool_call"}] + ) + ) + run.tool("add", args, result) + run.llm(AIMessage(content=ANSWER)) + run.finish() + return run + + +def _handler(**kwargs: Any): + exporter, provider = new_provider() + return exporter, FutureAGICallback(tracer_provider=provider, **kwargs) + + +def test_three_span_kinds_from_a_fake_callback_sequence(): + exporter, handler = _handler() + _journey(handler) + spans = exporter.get_finished_spans() + + # The model/tools/middleware node chains are not spans. + assert sorted(span.name for span in spans) == sorted([AGENT, LLM_SPAN, LLM_SPAN, TOOL_SPAN]) + assert AGENT_SPAN_NAME == AGENT + agent = only(spans, AGENT) + tool = only(spans, TOOL_SPAN) + first_llm, second_llm = spans_named(spans, LLM_SPAN) + assert parent_id(agent) is None + # Parent = nearest traced ancestor in the callback tree: the agent. + for child in (first_llm, tool, second_llm): + assert parent_id(child) == agent.context.span_id + assert child.context.trace_id == agent.context.trace_id + + assert attrs(agent) == { + "gen_ai.span.kind": "AGENT", + "gen_ai.operation.name": "invoke_agent", + "mcp_use.agent.llm_call_count": 2, + "mcp_use.agent.tool_call_count": 1, + "mcp_use.agent.tool_error_count": 0, + } + assert attrs(first_llm) == { + "gen_ai.span.kind": "LLM", + "gen_ai.operation.name": "chat", + "gen_ai.request.model": MODEL, + "gen_ai.provider.name": "fake", + "gen_ai.request.temperature": 0.2, + "gen_ai.request.max_tokens": 64, + "gen_ai.response.model": MODEL, + "gen_ai.response.finish_reasons": ("tool_calls",), + "gen_ai.usage.input_tokens": 11, + "gen_ai.usage.output_tokens": 7, + "gen_ai.usage.total_tokens": 18, + "gen_ai.usage.cache_read_tokens": 3, + "mcp_use.llm.input_message_count": 2, + "mcp_use.llm.tool_call_count": 1, + } + assert attrs(second_llm)["mcp_use.llm.input_message_count"] == 4 + assert attrs(second_llm)["gen_ai.response.finish_reasons"] == ("stop",) + assert attrs(tool) == { + "gen_ai.span.kind": "TOOL", + "gen_ai.operation.name": "execute_tool", + "gen_ai.tool.name": "add", + "gen_ai.tool.call.id": "call-1", + } + for span in spans: + assert span.status.status_code is StatusCode.OK + assert span.events == () + + +def test_content_is_off_by_default(): + exporter, handler = _handler() + _journey(handler) + text = wire(exporter.get_finished_spans()) + for marker in CONTENT_MARKERS + (LLM_KEY, "You are an MCP agent"): + assert marker not in text, marker + for span in exporter.get_finished_spans(): + for key in attrs(span): + assert not key.startswith(("input.", "output.", "gen_ai.input.", "gen_ai.output.")) + assert key not in ("gen_ai.tool.call.arguments", "gen_ai.tool.call.result") + + +def test_capture_content_records_prompt_messages_arguments_and_results(): + exporter, handler = _handler(capture_content=True) + _journey(handler) + spans = exporter.get_finished_spans() + agent = attrs(only(spans, AGENT)) + first_llm, second_llm = (attrs(span) for span in spans_named(spans, LLM_SPAN)) + tool = attrs(only(spans, TOOL_SPAN)) + arguments = json.dumps({"a": 2, "b": 3, "note": ARG}) + + assert agent["input.value"] == PROMPT + assert agent["output.value"] == ANSWER + assert first_llm["gen_ai.input.messages.0.message.role"] == "system" + assert first_llm["gen_ai.input.messages.0.message.content"] == "You are an MCP agent." + assert first_llm["gen_ai.input.messages.1.message.role"] == "user" + assert first_llm["gen_ai.input.messages.1.message.content"] == PROMPT + assert first_llm["input.value"] == PROMPT + call = "gen_ai.output.messages.0.message.tool_calls.0.tool_call." + assert first_llm["gen_ai.output.messages.0.message.role"] == "assistant" + assert first_llm[call + "function.name"] == "add" + assert first_llm[call + "id"] == "call-1" + assert first_llm[call + "function.arguments"] == arguments + assert "output.value" not in first_llm # a tool-call-only reply has no text + assert second_llm["gen_ai.input.messages.3.message.role"] == "tool" + assert second_llm["gen_ai.input.messages.3.message.content"] == "SUM-RESULT-5" + assert second_llm["output.value"] == ANSWER + assert tool["gen_ai.tool.call.arguments"] == arguments + assert tool["input.value"] == arguments + assert tool["gen_ai.tool.call.result"] == "SUM-RESULT-5" + assert tool["output.value"] == "SUM-RESULT-5" + # The serialized model and invocation params are never read for content. + assert LLM_KEY not in wire(spans) + + +def test_hide_inputs_drops_every_input_including_tool_arguments(): + exporter, handler = _handler(capture_content=True, config=TraceConfig(hide_inputs=True)) + _journey(handler) + spans = exporter.get_finished_spans() + text = wire(spans) + assert PROMPT not in text and ARG not in text + for span in spans: + values = attrs(span) + assert values.get("input.value", REDACTED_VALUE) == REDACTED_VALUE + assert not any(key.startswith("gen_ai.input.messages") for key in values) + assert not any(key.endswith("function.arguments") for key in values) + assert "gen_ai.tool.call.arguments" not in values + # Outputs stay. + assert attrs(only(spans, AGENT))["output.value"] == ANSWER + assert attrs(only(spans, TOOL_SPAN))["gen_ai.tool.call.result"] == "SUM-RESULT-5" + + +def test_hide_outputs_drops_every_output(): + exporter, handler = _handler(capture_content=True, config=TraceConfig(hide_outputs=True)) + _journey(handler) + spans = exporter.get_finished_spans() + text = wire(spans) + assert ANSWER not in text and "SUM-RESULT" not in text + for span in spans: + values = attrs(span) + assert values.get("output.value", REDACTED_VALUE) == REDACTED_VALUE + assert not any(key.startswith("gen_ai.output.messages") for key in values) + assert "gen_ai.tool.call.result" not in values + assert attrs(only(spans, AGENT))["input.value"] == PROMPT + + +def test_hide_flags_are_read_from_the_environment(monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + monkeypatch.setenv("FI_HIDE_OUTPUTS", "true") + exporter, handler = _handler(capture_content=True) + _journey(handler) + text = wire(exporter.get_finished_spans()) + for marker in CONTENT_MARKERS: + assert marker not in text, marker + + +def _failing_tool(handler: FutureAGICallback, error: BaseException, query: str = PROMPT): + run = Run(handler, query) + args = {"reason": ARG + " jane.doe@example.com"} + run.llm( + AIMessage(content="", tool_calls=[{"name": "fail", "args": args, "id": "c", "type": "tool_call"}]) + ) + node, tool = run.start_tool("fail", args, "c") + try: + raise error + except BaseException as raised: # give it a traceback + handler.on_tool_error(raised, run_id=tool, parent_run_id=node) + run.end_node(node) + run.llm(AIMessage(content=ANSWER)) + run.finish() + + +def test_tool_error_records_type_only_by_default(): + exporter, handler = _handler() + _failing_tool(handler, ValueError("tool failed because " + ARG)) + spans = exporter.get_finished_spans() + tool = only(spans, "execute_tool fail") + assert tool.status.status_code is StatusCode.ERROR + assert tool.status.description == "ValueError" + (event,) = tool.events + assert event.name == "exception" + assert dict(event.attributes) == {"exception.type": "ValueError"} + agent = only(spans, AGENT) + # The agent recovered (mcp-use returns tool errors to the LLM). + assert agent.status.status_code is StatusCode.OK + assert attrs(agent)["mcp_use.agent.tool_error_count"] == 1 + assert "tool failed because" not in wire(spans) + + +def test_tool_error_text_with_capture_and_hide_inputs_scrubs_the_arguments(): + exporter, handler = _handler(capture_content=True, config=TraceConfig(hide_inputs=True)) + error = ValueError("tool failed because " + ARG + " jane.doe@example.com") + _failing_tool(handler, error) + tool = only(exporter.get_finished_spans(), "execute_tool fail") + (event,) = tool.events + for text in (tool.status.description, event.attributes["exception.message"], event.attributes["exception.stacktrace"]): + assert "tool failed because" in text + assert ARG not in text, text + assert "jane.doe" not in text, text + assert REDACTED_VALUE in text + + +def test_pii_redaction_covers_attributes_status_and_exception_event(): + exporter, handler = _handler(capture_content=True, config=TraceConfig(pii_redaction=True)) + _failing_tool(handler, ValueError("mail jane.doe@example.com failed"), query="ask jane.doe@example.com") + spans = exporter.get_finished_spans() + assert "jane.doe@example.com" not in wire(spans) + assert attrs(only(spans, AGENT))["input.value"] == "ask " + tool = only(spans, "execute_tool fail") + assert tool.status.description == "ValueError: mail failed" + assert tool.events[0].attributes["exception.message"] == "mail failed" + + +def test_secrets_are_removed_from_attributes_status_and_events(monkeypatch): + monkeypatch.setenv(ENV_SECRET_NAME, ENV_SECRET) + exporter, handler = _handler(capture_content=True, redact=[EXPLICIT_SECRET]) + leaky = "keys {0} {1} {2} Authorization: Bearer abcdefghijklmnop".format( + ENV_SECRET, EXPLICIT_SECRET, LLM_KEY + ) + run = Run(handler, query=leaky) + run.llm(AIMessage(content=leaky)) + node, tool = run.start_tool("echo", {"text": leaky}) + try: + raise RuntimeError(leaky) + except RuntimeError as error: + handler.on_tool_error(error, run_id=tool, parent_run_id=node) + run.end_node(node) + run.finish(text=leaky) + spans = exporter.get_finished_spans() + text = wire(spans) + for secret in (ENV_SECRET, EXPLICIT_SECRET, LLM_KEY, "abcdefghijklmnop"): + assert secret not in text, secret + assert attrs(only(spans, AGENT))["input.value"].startswith("keys [redacted] [redacted] [redacted]") + + +def test_secrets_are_removed_from_names_with_content_off(monkeypatch): + monkeypatch.setenv(ENV_SECRET_NAME, ENV_SECRET) + exporter, handler = _handler() + run = Run(handler) + node, tool = run.start_tool("tool-" + ENV_SECRET, {}, call_id="id-" + ENV_SECRET) + handler.on_tool_end("done", run_id=tool, parent_run_id=node) + run.end_node(node) + run.finish() + assert ENV_SECRET not in wire(exporter.get_finished_spans()) + + +def test_size_caps(): + exporter, handler = _handler(capture_content=True) + long_text = "é" * 10_000 # 2 bytes each in UTF-8 + run = Run(handler, query=long_text) + many = [HumanMessage(content="m{0}".format(index)) for index in range(40)] + run.history = many + calls = [ + {"name": "t{0}".format(index), "args": {}, "id": "i{0}".format(index), "type": "tool_call"} + for index in range(20) + ] + run.llm(AIMessage(content=long_text, tool_calls=calls)) + long_name = "n" * 1000 + node, tool = run.start_tool(long_name, {"text": long_text}) + # A traceback longer than the cap: the cause's 20 KB message comes + # first, the raised exception's line last. The cap keeps the end. + try: + try: + raise RuntimeError("c" * 20_000) + except RuntimeError as cause: + raise ValueError("x" * 5000 + " LAST-LINE-END") from cause + except ValueError as error: + handler.on_tool_error(error, run_id=tool, parent_run_id=node) + run.end_node(node) + run.finish(text=long_text) + spans = exporter.get_finished_spans() + + for span in spans: + for key, value in attrs(span).items(): + if isinstance(value, str): + assert len(value.encode("utf-8")) <= MAX_VALUE_BYTES, key + agent = attrs(only(spans, AGENT)) + assert agent["input.value"] == "é" * (MAX_VALUE_BYTES // 2) + llm = attrs(only(spans, LLM_SPAN)) + assert llm["mcp_use.llm.input_message_count"] == 41 + recorded = {key.split(".")[3] for key in llm if key.startswith("gen_ai.input.messages.")} + assert len(recorded) == callback_module.MAX_MESSAGES == 32 + # The most recent 32 messages: m9..m39 and the system prompt is dropped. + assert llm["gen_ai.input.messages.31.message.content"] == "m39" + assert llm["mcp_use.llm.tool_call_count"] == 20 + names = [key for key in llm if key.endswith("function.name")] + assert len(names) == callback_module.MAX_TOOL_CALLS == 16 + + # The tool name is capped before it is put into the span name. + tool_span = only(spans, "execute_tool " + "n" * MAX_NAME_BYTES) + assert tool_span.attributes["gen_ai.tool.name"] == "n" * MAX_NAME_BYTES + (event,) = tool_span.events + message = event.attributes["exception.message"] + assert message == "x" * MAX_ERROR_BYTES + assert tool_span.status.description == "ValueError: " + "x" * MAX_ERROR_BYTES + stacktrace = event.attributes["exception.stacktrace"] + assert MAX_STACKTRACE_BYTES - 4 <= len(stacktrace.encode("utf-8")) <= MAX_STACKTRACE_BYTES + assert stacktrace.rstrip().endswith("LAST-LINE-END") + + +def test_streamed_tokens_are_events_on_one_llm_span_with_capped_count(): + exporter, handler = _handler(capture_content=True) + run = Run(handler) + node = run.node("model") + llm = uuid4() + handler.on_chat_model_start({}, [[HumanMessage(content=PROMPT)]], run_id=llm, parent_run_id=node, metadata=METADATA) + for index in range(200): + handler.on_llm_new_token("TOKEN-TEXT-{0} ".format(index), run_id=llm, parent_run_id=node) + handler.on_llm_end(_result(AIMessage(content="streamed")), run_id=llm, parent_run_id=node) + run.end_node(node) + run.finish() + spans = exporter.get_finished_spans() + assert [span.name for span in spans].count(LLM_SPAN) == 1 + span = only(spans, LLM_SPAN) + assert span.attributes["mcp_use.llm.chunk_count"] == 200 + assert len(span.events) == callback_module.MAX_CHUNK_EVENTS == 128 + assert [event.attributes["mcp_use.llm.chunk.index"] for event in span.events] == list(range(128)) + assert all(event.name == "mcp_use.llm.chunk" for event in span.events) + assert all(set(event.attributes) == {"mcp_use.llm.chunk.index"} for event in span.events) + assert "TOKEN-TEXT" not in wire(spans) + + +def test_llm_error_fails_the_llm_and_agent_spans(): + exporter, handler = _handler() + run = Run(handler) + node = run.node("model") + llm = uuid4() + handler.on_chat_model_start({}, [[HumanMessage(content=PROMPT)]], run_id=llm, parent_run_id=node, metadata=METADATA) + error = ConnectionError("provider down " + PROMPT) + handler.on_llm_error(error, run_id=llm, parent_run_id=node) + handler.on_chain_error(error, run_id=node, parent_run_id=run.root) + handler.on_chain_error(error, run_id=run.root) + spans = exporter.get_finished_spans() + assert sorted(span.name for span in spans) == sorted([AGENT, LLM_SPAN]) + for span in spans: + assert span.status.status_code is StatusCode.ERROR + assert span.status.description == "ConnectionError" + assert [event.name for event in span.events] == ["exception"] + assert span.events[0].attributes["exception.type"] == "ConnectionError" + assert attrs(only(spans, AGENT))["mcp_use.agent.llm_call_count"] == 1 + assert PROMPT not in wire(spans) + + +def test_mcp_use_formatted_tool_error_is_an_error_span(): + # mcp-use 1.7.1 returns a failed MCP call as format_error()'s dict, as + # JSON in the ToolMessage (agents/adapters/langchain_adapter.py:186-205). + for capture in (False, True): + exporter, handler = _handler(capture_content=capture) + run = Run(handler) + node, tool = run.start_tool("fail", {"reason": ARG}) + content = json.dumps( + { + "error": "RuntimeError", + "details": "Error executing tool fail: tool failed because " + ARG, + "stack": "Traceback (most recent call last):\n ...\nRuntimeError: boom", + "code": "UNKNOWN", + "tool": "fail", + } + ) + handler.on_tool_end(ToolMessage(content=content, tool_call_id="call-1"), run_id=tool, parent_run_id=node) + run.end_node(node) + run.finish() + spans = exporter.get_finished_spans() + span = only(spans, "execute_tool fail") + assert span.status.status_code is StatusCode.ERROR + assert span.attributes["mcp_use.tool.error_type"] == "RuntimeError" + (event,) = span.events + assert event.attributes["exception.type"] == "RuntimeError" + assert "gen_ai.tool.call.result" not in span.attributes + assert attrs(only(spans, AGENT))["mcp_use.agent.tool_error_count"] == 1 + if capture: + assert span.status.description == "RuntimeError: Error executing tool fail: tool failed because " + ARG + assert event.attributes["exception.stacktrace"].endswith("RuntimeError: boom") + else: + assert span.status.description == "RuntimeError" + assert ARG not in wire(spans) + + +def test_tool_message_with_error_status_is_an_error_span(): + exporter, handler = _handler() + run = Run(handler) + node, tool = run.start_tool("add", {"a": 1}) + output = ToolMessage(content="Error: bad input", tool_call_id="call-1", status="error") + handler.on_tool_end(output, run_id=tool, parent_run_id=node) + run.end_node(node) + run.finish() + span = only(exporter.get_finished_spans(), TOOL_SPAN) + assert span.status.status_code is StatusCode.ERROR + assert span.events[0].attributes["exception.type"] == "langchain_core.tools.ToolException" + + +@pytest.mark.parametrize("error", [asyncio.CancelledError(), GeneratorExit()]) +def test_cancellation_ends_open_spans_as_cancelled_without_an_exception_event(error): + exporter, handler = _handler() + run = Run(handler) + node, tool = run.start_tool("slow", {"seconds": 30}) + # What LangGraph reports when the task is cancelled during the tool + # call: chain errors for the tools node and the root, no tool callback. + handler.on_chain_error(error, run_id=node, parent_run_id=run.root) + handler.on_chain_error(error, run_id=run.root) + spans = exporter.get_finished_spans() + assert sorted(span.name for span in spans) == sorted([AGENT, "execute_tool slow"]) + for span in spans: + assert span.status.status_code is StatusCode.ERROR + assert span.status.description == "cancelled" + assert span.attributes["mcp_use.cancelled"] is True + assert span.events == () + + +def test_a_run_whose_parent_ended_first_is_ended_as_incomplete(): + exporter, handler = _handler() + run = Run(handler) + node = run.node("model") + handler.on_chat_model_start({}, [[HumanMessage(content="x")]], run_id=uuid4(), parent_run_id=node) + run.finish() + spans = exporter.get_finished_spans() + llm = only(spans, "chat") + assert llm.attributes["mcp_use.incomplete"] is True + assert llm.status.status_code is StatusCode.ERROR + assert llm.status.description == "ended without a result" + assert only(spans, AGENT).status.status_code is StatusCode.OK + assert handler._runs == {} and len(handler._roots) == 0 + + +def test_text_completion_models_get_an_llm_span(): + exporter, handler = _handler(capture_content=True) + run = Run(handler) + node = run.node("model") + llm = uuid4() + handler.on_llm_start({}, ["complete " + PROMPT], run_id=llm, parent_run_id=node, invocation_params={"model": "completion-1"}) + handler.on_llm_end(LLMResult(generations=[[]], llm_output={"model_name": "completion-1"}), run_id=llm, parent_run_id=node) + run.end_node(node) + run.finish() + span = only(exporter.get_finished_spans(), "text_completion completion-1") + assert span.attributes["gen_ai.operation.name"] == "text_completion" + assert span.attributes["input.value"] == "complete " + PROMPT + + +def test_interleaved_agent_runs_on_one_handler_stay_separate(): + exporter, handler = _handler() + first, second = Run(handler), Run(handler) + first.llm(AIMessage(content="a")) + second.llm(AIMessage(content="b")) + second.tool("add", {"a": 1}, "r") + first.finish() + second.finish() + spans = exporter.get_finished_spans() + agents = spans_named(spans, AGENT) + assert len(agents) == 2 + for agent in agents: + children = [span for span in spans if parent_id(span) == agent.context.span_id] + assert all(child.context.trace_id == agent.context.trace_id for child in children) + counts = sorted((a.attributes["mcp_use.agent.llm_call_count"], a.attributes["mcp_use.agent.tool_call_count"]) for a in agents) + assert counts == [(1, 0), (1, 1)] + assert handler._runs == {} + + +def test_open_runs_are_bounded(monkeypatch): + monkeypatch.setattr(callback_module, "MAX_OPEN_RUNS", 3) + exporter, handler = _handler() + abandoned = Run(handler) + abandoned.start_tool("slow", {}) + current = Run(handler) + current.node("model") # 5 open runs: the abandoned agent run is evicted + spans = exporter.get_finished_spans() + assert sorted(span.name for span in spans) == sorted([AGENT, "execute_tool slow"]) + assert all(span.attributes["mcp_use.incomplete"] is True for span in spans) + current.finish() + assert handler._runs == {} + + +def test_the_current_span_parents_the_agent_span(): + exporter, handler = _handler() + tracer = new_provider()[1].get_tracer("test") + with tracer.start_as_current_span("request") as outer: + _journey(handler) + agent = only(exporter.get_finished_spans(), AGENT) + assert parent_id(agent) == outer.get_span_context().span_id + assert agent.context.trace_id == outer.get_span_context().trace_id + + +def test_context_attributes_are_stamped_on_every_span(): + exporter, handler = _handler() + with using_session("session-1"), using_user("user-1"), using_metadata({"tenant": "t1"}), using_tags(["beta"]): + _journey(handler) + spans = exporter.get_finished_spans() + assert len(spans) == 4 + for span in spans: + assert span.attributes["session.id"] == "session-1" + assert span.attributes["user.id"] == "user-1" + assert json.loads(span.attributes["metadata"]) == {"tenant": "t1"} + assert span.attributes["tag.tags"] == ("beta",) + + +def test_suppress_tracing_records_nothing(): + exporter, handler = _handler() + with suppress_tracing(): + _journey(handler) + assert exporter.get_finished_spans() == () + assert handler._runs == {} + + +class _Broken: + """A tracer provider whose tracers and spans raise on every call.""" + + def get_tracer(self, *args: Any, **kwargs: Any) -> Any: + return self + + def start_span(self, *args: Any, **kwargs: Any) -> Any: + raise RuntimeError("tracer is broken") + + +class _BrokenSpans(trace_api.TracerProvider): + """Spans start, then raise from set_status, add_event and end. + + With ``on_attribute`` they also raise from set_attribute, so starting + one fails too (FITracer sets the start attributes). + """ + + def __init__(self, on_attribute: bool) -> None: + self.on_attribute = on_attribute + + def get_tracer(self, *args: Any, **kwargs: Any) -> Any: + return self + + def start_span(self, *args: Any, **kwargs: Any) -> Any: + return _BrokenSpan(self.on_attribute) + + +class _BrokenSpan(trace_api.NonRecordingSpan): + def __init__(self, on_attribute: bool) -> None: + super().__init__(trace_api.INVALID_SPAN_CONTEXT) + self.on_attribute = on_attribute + + def is_recording(self) -> bool: + return True + + def set_attribute(self, *args: Any, **kwargs: Any) -> None: + if self.on_attribute: + raise RuntimeError("span is broken") + + def set_status(self, *args: Any, **kwargs: Any) -> None: + raise RuntimeError("span is broken") + + def add_event(self, *args: Any, **kwargs: Any) -> None: + raise RuntimeError("span is broken") + + def end(self, *args: Any, **kwargs: Any) -> None: + raise RuntimeError("span is broken") + + +@pytest.mark.parametrize( + "provider", [_Broken(), _BrokenSpans(on_attribute=False), _BrokenSpans(on_attribute=True)] +) +def test_tracer_failures_never_reach_the_caller(provider): + handler = FutureAGICallback(tracer_provider=provider, capture_content=True) + _journey(handler) + _failing_tool(handler, ValueError("boom")) + run = Run(handler) + node = run.node("model") + llm = uuid4() + handler.on_chat_model_start({}, [[HumanMessage(content="x")]], run_id=llm, parent_run_id=node) + handler.on_llm_new_token("t", run_id=llm, parent_run_id=node) + handler.on_chain_error(asyncio.CancelledError(), run_id=run.root) + assert handler._runs == {} + + +@pytest.mark.parametrize("capture", [False, True]) +def test_malformed_callback_payloads_never_raise(capture): + exporter, handler = _handler(capture_content=capture) + root, child = uuid4(), uuid4() + weird: List[Any] = [None, object(), 42, "text", [object()], {"messages": object()}] + for value in weird: + handler.on_chain_start(value, value, run_id=root) + handler.on_chat_model_start(value, [[value]] if value is not None else value, run_id=child, parent_run_id=root, metadata=value, invocation_params=value) + handler.on_llm_new_token(value, run_id=child) + handler.on_llm_end(value, run_id=child) + handler.on_llm_start(value, value, run_id=child, parent_run_id=root) + handler.on_llm_error(RuntimeError(), run_id=child) + handler.on_tool_start(value, value, run_id=child, parent_run_id=root, inputs=value) + handler.on_tool_end(value, run_id=child) + handler.on_tool_error(RuntimeError(), run_id=child) + handler.on_retriever_start(value, value, run_id=child, parent_run_id=root) + handler.on_retriever_end(value, run_id=child) + handler.on_chain_end(value, run_id=root) + handler.on_chain_error(RuntimeError(), run_id=uuid4()) + assert handler._runs == {} + + +def test_constructor_validates_its_arguments(): + with pytest.raises(TypeError, match="TraceConfig"): + FutureAGICallback(config={"hide_inputs": True}) # type: ignore[arg-type] + with pytest.raises(TypeError, match="capture_content"): + FutureAGICallback(capture_content="yes") # type: ignore[arg-type] + with pytest.raises(TypeError, match="not a string"): + FutureAGICallback(redact="secret-value") # type: ignore[arg-type] + with pytest.raises(TypeError, match="strings"): + FutureAGICallback(redact=[b"secret-value"]) # type: ignore[list-item] From a24faefe17a01402d15bf691eb7f0cfe34a086d6 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 07:40:03 -0700 Subject: [PATCH 06/19] test(mcp-use): run the real MCPAgent on a fake LLM and loopback MCP server ChatFake replays a script; the FastMCP fake server runs once per test process on a 127.0.0.1 streamable-HTTP port. Covers run, stream and stream_events (one AGENT root with LLM and tool children), streamed completions as one LLM span, content and keys absent by default, capture_content without secrets, MCP tool errors and invalid arguments (tool span ERROR, agent recovers), an LLM error (agent ERROR, the same exception reaches the caller), cancellation and leaving stream() early, callbacks=[] and agents without the handler (no spans), the caller's span as parent, using_* context, suppress_tracing, concurrent runs, a broken tracer, and sync/async LangChain dispatch parity. approved: Nikhil 2026-10-03 blanket Refs: TH-8331 --- .../mcp-use/tests/test_mcp_use_agent.py | 440 ++++++++++++++++++ 1 file changed, 440 insertions(+) create mode 100644 python/frameworks/mcp-use/tests/test_mcp_use_agent.py diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_agent.py b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py new file mode 100644 index 000000000..460ecc98c --- /dev/null +++ b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py @@ -0,0 +1,440 @@ +"""MCPAgent end to end: the real mcp-use 1.7.1 agent loop on a fake LLM. + +``ChatFake`` replays a script; the MCP server is ``_mcp_server.py`` on a +loopback streamable-HTTP port (see ``_mcp_use_support``). Spans go to an +InMemorySpanExporter through a provider passed to the callback. +""" + +from __future__ import annotations + +import asyncio +import json +from contextlib import contextmanager +from typing import Any, Dict, List + +import pytest +from _mcp_use_support import ( + AGENT, + ANSWER, + ARG, + CONTENT_MARKERS, + ENV_SECRET, + ENV_SECRET_NAME, + EXPLICIT_SECRET, + LLM_KEY, + LLM_SPAN, + MODEL, + PROMPT, + ChatFake, + ToolStarted, + add_script, + answer, + attrs, + mcp_client, + new_provider, + only, + parent_id, + run_agent, + spans_named, + tool_call, + wire, +) +from fi_instrumentation import ( + REDACTED_VALUE, + TraceConfig, + suppress_tracing, + using_metadata, + using_session, + using_tags, + using_user, +) +from langchain_core.callbacks import BaseCallbackHandler +from mcp_use import MCPAgent +from opentelemetry.trace import StatusCode + +from traceai_mcp_use import FutureAGICallback + +pytest.importorskip("uvicorn", reason="the loopback MCP server runs on uvicorn") + + +def _traced(**kwargs: Any): + exporter, provider = new_provider() + return exporter, FutureAGICallback(tracer_provider=provider, **kwargs) + + +def _assert_j1_tree(spans) -> None: + """One agent root; LLM, tool, LLM as its children, in one trace.""" + assert sorted(span.name for span in spans) == sorted([AGENT, LLM_SPAN, LLM_SPAN, "execute_tool add"]) + agent = only(spans, AGENT) + assert parent_id(agent) is None + children = sorted( + (span for span in spans if span is not agent), key=lambda span: span.start_time + ) + assert [span.name for span in children] == [LLM_SPAN, "execute_tool add", LLM_SPAN] + for child in children: + # The callback tree puts the tool run under the "tools" graph node + # and the model run under the "model" node; both nodes are siblings + # under the root, so the tool span is a sibling of the LLM spans. + assert parent_id(child) == agent.context.span_id + assert child.context.trace_id == agent.context.trace_id + assert child.start_time >= agent.start_time and child.end_time <= agent.end_time + for span in spans: + assert span.status.status_code is StatusCode.OK, span.name + assert attrs(agent)["gen_ai.span.kind"] == "AGENT" + assert attrs(agent)["mcp_use.agent.llm_call_count"] == 2 + assert attrs(agent)["mcp_use.agent.tool_call_count"] == 1 + assert attrs(agent)["mcp_use.agent.tool_error_count"] == 0 + llm = attrs(children[0]) + assert llm["gen_ai.span.kind"] == "LLM" + assert llm["gen_ai.request.model"] == MODEL + assert llm["gen_ai.response.model"] == MODEL + assert llm["gen_ai.usage.input_tokens"] == 11 + assert llm["mcp_use.llm.tool_call_count"] == 1 + tool = attrs(children[1]) + assert tool["gen_ai.span.kind"] == "TOOL" + assert tool["gen_ai.tool.name"] == "add" + assert tool["gen_ai.tool.call.id"] == "call-1" + + +@pytest.mark.parametrize("method", ["run", "stream", "stream_events"]) +def test_every_entry_point_gives_one_agent_span_with_llm_and_tool_children(method): + exporter, handler = _traced() + result = run_agent(add_script(), [handler], method=method) + if method == "stream_events": + assert any(event.get("event") == "on_tool_end" for event in result) + else: + assert result == ANSWER + _assert_j1_tree(exporter.get_finished_spans()) + assert handler._runs == {} + + +class _TokenCounter(BaseCallbackHandler): + run_inline = True + + def __init__(self) -> None: + super().__init__() + self.tokens: List[str] = [] + + def on_llm_new_token(self, token: str, **kwargs: Any) -> None: + self.tokens.append(token) + + +@pytest.mark.parametrize("method", ["run", "stream_events"]) +def test_a_streamed_completion_is_one_llm_span(method): + exporter, handler = _traced() + counter = _TokenCounter() + result = run_agent(add_script(), [handler, counter], method=method, streaming=True) + spans = exporter.get_finished_spans() + _assert_j1_tree(spans) + tool_turn, final_turn = sorted(spans_named(spans, LLM_SPAN), key=lambda span: span.start_time) + # "ANSWER-MARKER the sum is five" streams as five word chunks; every + # token LangChain reports is counted and none becomes a span. + assert {"ANSWER-MARKER ", "the ", "sum ", "is ", "five"} <= set(counter.tokens) + chunks = tool_turn.attributes["mcp_use.llm.chunk_count"] + final_turn.attributes["mcp_use.llm.chunk_count"] + assert chunks == len(counter.tokens) + assert final_turn.attributes["mcp_use.llm.chunk_count"] >= 5 + assert {event.name for event in final_turn.events} == {"mcp_use.llm.chunk"} + assert len(final_turn.events) == final_turn.attributes["mcp_use.llm.chunk_count"] + assert "ANSWER-MARKER" not in wire(spans) + if method == "run": + assert result == ANSWER + + +def test_content_and_keys_are_not_recorded_by_default(monkeypatch): + monkeypatch.setenv(ENV_SECRET_NAME, ENV_SECRET) + exporter, handler = _traced() + script = [tool_call("echo", {"text": ARG + " " + ENV_SECRET}), answer()] + run_agent(script, [handler], query=PROMPT + " " + ENV_SECRET) + spans = exporter.get_finished_spans() + assert len(spans) == 4 + text = wire(spans) + for marker in CONTENT_MARKERS + (LLM_KEY, ENV_SECRET): + assert marker not in text, marker + + +def test_capture_content_records_content_without_secrets(monkeypatch): + monkeypatch.setenv(ENV_SECRET_NAME, ENV_SECRET) + exporter, handler = _traced(capture_content=True, redact=[EXPLICIT_SECRET]) + leak = "{0} {1} {2}".format(ENV_SECRET, EXPLICIT_SECRET, LLM_KEY) + script = [tool_call("echo", {"text": ARG + " " + leak}), answer()] + run_agent(script, [handler], query=PROMPT + " " + leak) + spans = exporter.get_finished_spans() + text = wire(spans) + for secret in (ENV_SECRET, EXPLICIT_SECRET, LLM_KEY): + assert secret not in text, secret + agent = attrs(only(spans, AGENT)) + assert agent["input.value"] == PROMPT + " [redacted] [redacted] [redacted]" + assert agent["output.value"] == ANSWER + tool = attrs(only(spans, "execute_tool echo")) + assert json.loads(tool["gen_ai.tool.call.arguments"]) == {"text": ARG + " [redacted] [redacted] [redacted]"} + assert tool["gen_ai.tool.call.result"] == "ECHO-RESULT:" + ARG + " [redacted] [redacted] [redacted]" + + +def test_mcp_tool_error_fails_the_tool_span_and_the_agent_recovers(): + # mcp-use 1.7.1 turns a failed MCP call into a formatted error the LLM + # reads (agents/adapters/langchain_adapter.py:186-205), so the run + # succeeds; the tool span is ERROR and the agent counts the error. + exporter, handler = _traced() + script = [tool_call("fail", {"reason": ARG}), answer()] + assert run_agent(script, [handler]) == ANSWER + spans = exporter.get_finished_spans() + tool = only(spans, "execute_tool fail") + assert tool.status.status_code is StatusCode.ERROR + assert tool.status.description == "RuntimeError" + assert tool.attributes["mcp_use.tool.error_type"] == "RuntimeError" + assert [event.attributes["exception.type"] for event in tool.events] == ["RuntimeError"] + agent = only(spans, AGENT) + assert agent.status.status_code is StatusCode.OK + assert agent.attributes["mcp_use.agent.tool_error_count"] == 1 + assert "tool failed because" not in wire(spans) and ARG not in wire(spans) + + +def test_tool_error_text_is_recorded_with_capture_and_scrubbed_with_hide_inputs(): + exporter, handler = _traced(capture_content=True) + run_agent([tool_call("fail", {"reason": ARG}), answer()], [handler]) + tool = only(exporter.get_finished_spans(), "execute_tool fail") + assert tool.status.description == "RuntimeError: Error executing tool fail: tool failed because " + ARG + + exporter, handler = _traced(capture_content=True, config=TraceConfig(hide_inputs=True)) + run_agent([tool_call("fail", {"reason": ARG}), answer()], [handler]) + spans = exporter.get_finished_spans() + tool = only(spans, "execute_tool fail") + assert tool.status.description == "RuntimeError: Error executing tool fail: tool failed because " + REDACTED_VALUE + (event,) = tool.events + assert ARG not in event.attributes["exception.message"] + assert ARG not in event.attributes["exception.stacktrace"] + assert ARG not in wire(spans) and "PROMPT-MARKER" not in wire(spans) + + +@pytest.mark.parametrize("retry_on_error", [True, False]) +def test_invalid_tool_arguments_fail_the_tool_span(retry_on_error): + exporter, handler = _traced() + script = [tool_call("add", {"a": "not-a-number", "b": 3}), answer()] + result = run_agent(script, [handler], agent_kwargs={"retry_on_error": retry_on_error}) + assert result == ANSWER + spans = exporter.get_finished_spans() + tool = only(spans, "execute_tool add") + assert tool.status.status_code is StatusCode.ERROR + assert tool.events[0].attributes["exception.type"] == "pydantic_core._pydantic_core.ValidationError" + assert only(spans, AGENT).attributes["mcp_use.agent.tool_error_count"] == 1 + + +def test_an_llm_error_fails_the_agent_span_and_reaches_the_caller_unchanged(): + exporter, handler = _traced() + error = ConnectionError("provider unreachable") + with pytest.raises(ConnectionError) as raised: + run_agent([error], [handler]) + assert raised.value is error + spans = exporter.get_finished_spans() + assert sorted(span.name for span in spans) == sorted([AGENT, LLM_SPAN]) + for span in spans: + assert span.status.status_code is StatusCode.ERROR + assert span.events[0].attributes["exception.type"] == "ConnectionError" + assert parent_id(only(spans, LLM_SPAN)) == only(spans, AGENT).context.span_id + + +def test_cancelling_the_run_ends_the_open_spans_as_cancelled(): + exporter, handler = _traced() + started = ToolStarted() + + async def go() -> None: + client = mcp_client() + try: + agent = MCPAgent( + llm=ChatFake(script=[tool_call("slow", {"seconds": 30}), answer()]), + client=client, + callbacks=[handler, started], + ) + task = asyncio.ensure_future(agent.run(PROMPT)) + for _ in range(1000): + if started.started.is_set(): + break + await asyncio.sleep(0.01) + assert started.started.is_set() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + finally: + await client.close_all_sessions() + + asyncio.run(go()) + spans = exporter.get_finished_spans() + assert sorted(span.name for span in spans) == sorted([AGENT, LLM_SPAN, "execute_tool slow"]) + for name in (AGENT, "execute_tool slow"): + span = only(spans, name) + assert span.status.status_code is StatusCode.ERROR + assert span.status.description == "cancelled" + assert span.attributes["mcp_use.cancelled"] is True + assert span.events == () + assert only(spans, LLM_SPAN).status.status_code is StatusCode.OK + assert handler._runs == {} + + +def test_leaving_stream_early_ends_the_agent_span_as_cancelled(): + exporter, handler = _traced() + + async def go() -> None: + client = mcp_client() + try: + agent = MCPAgent(llm=ChatFake(script=add_script()), client=client, callbacks=[handler]) + stream = agent.stream(PROMPT) + async for _ in stream: + break # the first item is the (action, observation) of the tool step + await stream.aclose() + for _ in range(500): + if spans_named(exporter.get_finished_spans(), AGENT): + break + await asyncio.sleep(0.01) + finally: + await client.close_all_sessions() + + asyncio.run(go()) + spans = exporter.get_finished_spans() + agent = only(spans, AGENT) + assert agent.status.status_code is StatusCode.ERROR + assert agent.status.description == "cancelled" + assert agent.attributes["mcp_use.cancelled"] is True + assert only(spans, "execute_tool add").status.status_code is StatusCode.OK + assert handler._runs == {} + + +def test_empty_callbacks_list_yields_no_spans(): + # callbacks=[] is the Python switch: ObservabilityManager.get_callbacks + # returns it as is and adds no default handler + # (agents/observability/callbacks_manager.py:72-74). + exporter, handler = _traced() + seen: List[Any] = [] + + async def go() -> None: + client = mcp_client() + try: + agent = MCPAgent(llm=ChatFake(script=add_script()), client=client, callbacks=[]) + seen.append(agent.callbacks) + assert await agent.run(PROMPT) == ANSWER + finally: + await client.close_all_sessions() + + asyncio.run(go()) + assert seen == [[]] + assert exporter.get_finished_spans() == () + + +def test_an_agent_without_the_callback_is_not_traced(): + # The handler exists in the process but only traces where it is passed. + exporter, handler = _traced() + run_agent(add_script(), [handler]) + exporter.clear() + run_agent(add_script(), None) + run_agent(add_script(), [ToolStarted()]) + assert exporter.get_finished_spans() == () + + +def test_the_callers_current_span_parents_the_agent_span(): + exporter, handler = _traced() + tracer = new_provider()[1].get_tracer("app") + outer: List[Any] = [] + + @contextmanager + def around(): + with tracer.start_as_current_span("request") as span: + outer.append(span) + yield + + run_agent(add_script(), [handler], around=around) + agent = only(exporter.get_finished_spans(), AGENT) + assert parent_id(agent) == outer[0].get_span_context().span_id + assert agent.context.trace_id == outer[0].get_span_context().trace_id + + +def test_session_user_metadata_and_tags_reach_every_span(): + exporter, handler = _traced() + + @contextmanager + def around(): + with using_session("session-1"), using_user("user-1"), using_metadata( + {"tenant": "t1"} + ), using_tags(["beta"]): + yield + + run_agent(add_script(), [handler], around=around) + spans = exporter.get_finished_spans() + assert len(spans) == 4 + for span in spans: + assert span.attributes["session.id"] == "session-1", span.name + assert span.attributes["user.id"] == "user-1" + assert json.loads(span.attributes["metadata"]) == {"tenant": "t1"} + assert span.attributes["tag.tags"] == ("beta",) + + +def test_suppress_tracing_records_nothing_and_the_run_still_answers(): + exporter, handler = _traced() + assert run_agent(add_script(), [handler], around=suppress_tracing) == ANSWER + assert exporter.get_finished_spans() == () + + +def test_concurrent_runs_on_one_handler_get_separate_traces(): + exporter, handler = _traced() + + async def one() -> Any: + client = mcp_client() + try: + agent = MCPAgent(llm=ChatFake(script=add_script(), delay=0.05), client=client, callbacks=[handler]) + return await agent.run(PROMPT) + finally: + await client.close_all_sessions() + + async def both() -> List[Any]: + return await asyncio.gather(one(), one(), one()) + + assert asyncio.run(both()) == [ANSWER] * 3 + spans = exporter.get_finished_spans() + agents = spans_named(spans, AGENT) + assert len(agents) == 3 + for agent in agents: + tree = [span for span in spans if span.context.trace_id == agent.context.trace_id] + _assert_j1_tree(tree) + assert handler._runs == {} + + +def test_a_broken_tracer_does_not_change_the_agent_result(): + class Broken: + def get_tracer(self, *args: Any, **kwargs: Any) -> Any: + return self + + def start_span(self, *args: Any, **kwargs: Any) -> Any: + raise RuntimeError("tracer is broken") + + handler = FutureAGICallback(tracer_provider=Broken(), capture_content=True) + assert run_agent(add_script(), [handler]) == ANSWER + for method in ("stream", "stream_events"): + run_agent(add_script(), [handler], method=method) + assert handler._runs == {} + + +def test_sync_and_async_langchain_dispatch_give_the_same_spans(): + # MCPAgent is async only. The handler also runs under LangChain's sync + # callback manager: drive a LangChain agent with a plain tool through + # invoke() and ainvoke() and compare. + from langchain.agents import create_agent + from langchain_core.tools import tool + + @tool + def add(a: int, b: int) -> str: + """Add two integers.""" + return "SUM-RESULT-{0}".format(a + b) + + shapes = [] + for mode in ("sync", "async"): + exporter, handler = _traced(capture_content=True) + graph = create_agent(ChatFake(script=add_script()), [add]) + inputs = {"messages": [("user", PROMPT)]} + config: Dict[str, Any] = {"callbacks": [handler]} + if mode == "sync": + graph.invoke(inputs, config=config) + else: + asyncio.run(graph.ainvoke(inputs, config=config)) + spans = exporter.get_finished_spans() + _assert_j1_tree(spans) + assert attrs(only(spans, "execute_tool add"))["gen_ai.tool.call.result"] == "SUM-RESULT-5" + shapes.append(sorted((span.name, tuple(sorted(attrs(span)))) for span in spans)) + assert shapes[0] == shapes[1] From 5ddc2243c34071252ef662fcd049ff8ebf04ae52 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 07:43:45 -0700 Subject: [PATCH 07/19] test(mcp-use): contract test through register() into the harness Receiver The real MCPAgent runs on the fake LLM against the fake MCP server over stdio; spans leave through fi_instrumentation.register()'s OTLP exporter into harness.Receiver on 127.0.0.1. Asserts /tracer/v1/traces, both x-api-key and x-secret-key, project_type=observe, the agent tree and kinds on the wire, one span per tool call, a failing tool as an ERROR span under an OK agent, and that no key or content is exported by default, with an opt-in capture_content control. approved: Nikhil 2026-10-03 blanket Refs: TH-8331 --- .../mcp-use/tests/test_mcp_use_contract.py | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 python/frameworks/mcp-use/tests/test_mcp_use_contract.py diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_contract.py b/python/frameworks/mcp-use/tests/test_mcp_use_contract.py new file mode 100644 index 000000000..4b3a896f1 --- /dev/null +++ b/python/frameworks/mcp-use/tests/test_mcp_use_contract.py @@ -0,0 +1,171 @@ +"""Shared-harness contract test for traceAI-mcp-use (TH-8331, harness Receiver). + +The real MCPAgent runs on the fake LLM against the fake MCP server over +stdio. Spans leave through the real ``fi_instrumentation.register()`` OTLP +exporter into ``harness.Receiver`` on 127.0.0.1. Every key is a +placeholder. +""" + +from __future__ import annotations + +import asyncio +import json +import sys +from collections import Counter +from pathlib import Path +from typing import Any, Dict, List + +import pytest +from _mcp_use_support import ( + AGENT, + ANSWER, + ARG, + CONTENT_MARKERS, + ENV_SECRET, + ENV_SECRET_NAME, + LLM_KEY, + LLM_SPAN, + MODEL, + PROMPT, + ChatFake, + add_script, + answer, + mcp_client, + tool_call, +) + +pytest.importorskip("opentelemetry.proto", reason="the harness decodes OTLP protobuf") + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) + +from harness import Receiver, _flatten_attributes # noqa: E402 + +from traceai_mcp_use import FutureAGICallback # noqa: E402 + +FI_API_KEY = "placeholder-fi-api-key" +FI_SECRET_KEY = "placeholder-fi-secret-key" +PROJECT = "mcp-use-contract" + + +def _journey(receiver: Receiver, monkeypatch, **options: Any) -> List[Dict[str, Any]]: + """J1 (add, then answer) and a failing tool call, through register().""" + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + from mcp_use import MCPAgent + + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv("FI_API_KEY", FI_API_KEY) + monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET_KEY) + monkeypatch.setenv(ENV_SECRET_NAME, ENV_SECRET) + provider = register(project_type=ProjectType.OBSERVE, project_name=PROJECT, verbose=False) + handler = FutureAGICallback(tracer_provider=provider, **options) + + async def go() -> None: + client = mcp_client("stdio") + try: + agent = MCPAgent(llm=ChatFake(script=add_script()), client=client, callbacks=[handler]) + assert await agent.run(PROMPT + " " + ENV_SECRET) == ANSWER + failing = MCPAgent( + llm=ChatFake(script=[tool_call("fail", {"reason": ARG}), answer()]), + client=client, + callbacks=[handler], + ) + assert await failing.run(PROMPT) == ANSWER + finally: + await client.close_all_sessions() + + try: + asyncio.run(go()) + assert provider.force_flush(timeout_millis=10_000) + finally: + provider.shutdown() + return receiver.spans() + + +def _trees(spans: List[Dict[str, Any]]) -> List[List[Dict[str, Any]]]: + """Spans grouped by trace, agent span first, in start order.""" + traces: Dict[str, List[Dict[str, Any]]] = {} + for span in sorted(spans, key=lambda span: int(span["startTimeUnixNano"])): + traces.setdefault(span["traceId"], []).append(span) + return sorted( + (sorted(group, key=lambda span: span["name"] != AGENT) for group in traces.values()), + key=lambda group: int(group[0]["startTimeUnixNano"]), + ) + + +def _attrs(span: Dict[str, Any]) -> Dict[str, Any]: + return _flatten_attributes(span.get("attributes", [])) + + +def test_agent_runs_reach_the_collector_contract(monkeypatch): + with Receiver() as receiver: + spans = _journey(receiver, monkeypatch) + exports = receiver.requests() + + assert exports + for export in exports: + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_API_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET_KEY + assert "authorization" not in export["headers"] + for resource in export["resource_attributes"]: + assert resource["project_name"] == PROJECT + assert resource["project_type"] == "observe" + + j1, failed = _trees(spans) + agent, *children = j1 + assert agent["name"] == AGENT + assert not agent.get("parentSpanId") + assert [child["name"] for child in children] == [LLM_SPAN, "execute_tool add", LLM_SPAN] + for child in children: + assert child["parentSpanId"] == agent["spanId"] + kinds = [_attrs(span)["gen_ai.span.kind"] for span in j1] + assert kinds == ["AGENT", "LLM", "TOOL", "LLM"] + assert _attrs(children[0])["gen_ai.request.model"] == MODEL + assert _attrs(children[1])["gen_ai.tool.name"] == "add" + assert int(_attrs(agent)["mcp_use.agent.tool_call_count"]) == 1 + # register()'s processor also promotes UNSET to OK on export, so the + # handler's own OK is pinned by the in-memory tests. + assert all(span["status"]["code"] == "STATUS_CODE_OK" for span in j1) + + failed_agent, *failed_children = failed + tool = next(span for span in failed_children if span["name"] == "execute_tool fail") + assert tool["parentSpanId"] == failed_agent["spanId"] + assert tool["status"]["code"] == "STATUS_CODE_ERROR" + assert tool["status"]["message"] == "RuntimeError" + assert [event["name"] for event in tool["events"]] == ["exception"] + assert failed_agent["status"]["code"] == "STATUS_CODE_OK" + assert int(_attrs(failed_agent)["mcp_use.agent.tool_error_count"]) == 1 + + # One span per tool call: nothing else shares a tool span's name and parent. + for tree in (j1, failed): + tools = Counter((span["name"], span.get("parentSpanId")) for span in tree if span["name"].startswith("execute_tool")) + assert set(tools.values()) == {1} + + +def test_no_key_or_content_is_exported_by_default(monkeypatch): + with Receiver() as receiver: + spans = _journey(receiver, monkeypatch) + + assert len(spans) == 8 # two runs: agent, LLM, tool, LLM each + text = json.dumps(spans) + for secret in (FI_API_KEY, FI_SECRET_KEY, LLM_KEY, ENV_SECRET) + CONTENT_MARKERS: + assert secret not in text, secret + for span in spans: + for key in _attrs(span): + assert not key.startswith(("input.", "output.", "gen_ai.input.", "gen_ai.output.")) + + +def test_opt_in_capture_reaches_the_collector_without_keys(monkeypatch): + # Control for the test above: the same journey with capture_content + # carries the prompt, arguments, results and answer, still without keys. + with Receiver() as receiver: + spans = _journey(receiver, monkeypatch, capture_content=True) + + text = json.dumps(spans) + for marker in ("PROMPT-MARKER", "ANSWER-MARKER", "SUM-RESULT-5", ARG, "tool failed because"): + assert marker in text, marker + for secret in (FI_API_KEY, FI_SECRET_KEY, LLM_KEY, ENV_SECRET): + assert secret not in text, secret + agent = _attrs(_trees(spans)[0][0]) + assert agent["input.value"] == PROMPT + " [redacted]" From f51384b6c7e0224260d8cc01ad45f34b4d2928d5 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 07:43:45 -0700 Subject: [PATCH 08/19] test(mcp-use): run the example through harness.run, with and without traceai_mcp _example_app.py runs examples/agent_with_tools.py as its own process with only the LLM replaced by the fake: register(), the stdio calculator server and FutureAGICallback are the example's own code. The process exits without an explicit flush, so the exported spans prove the flush at exit. With traceai_mcp instrumented first (imported from python/frameworks/mcp, not modified) the spans are the same: the agent span is there and no tool span is duplicated under the same name and parent. approved: Nikhil 2026-10-03 blanket Refs: TH-8331 --- .../frameworks/mcp-use/tests/_example_app.py | 36 ++++++ .../mcp-use/tests/test_mcp_use_example.py | 118 ++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 python/frameworks/mcp-use/tests/_example_app.py create mode 100644 python/frameworks/mcp-use/tests/test_mcp_use_example.py diff --git a/python/frameworks/mcp-use/tests/_example_app.py b/python/frameworks/mcp-use/tests/_example_app.py new file mode 100644 index 000000000..74ad417ad --- /dev/null +++ b/python/frameworks/mcp-use/tests/_example_app.py @@ -0,0 +1,36 @@ +"""Run examples/agent_with_tools.py with the fake LLM, as its own process. + +test_mcp_use_example.py starts this through harness.run. With +``--with-traceai-mcp`` it first instruments the existing transport package +(python/frameworks/mcp, imported, not modified). traceai_mcp wraps +``mcp.client.stdio.stdio_client``; mcp-use binds that name when it is +imported (mcp_use/client/task_managers/stdio.py:12), so the instrumentor +runs before anything imports mcp_use. +""" + +import asyncio +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +if "--with-traceai-mcp" in sys.argv: + sys.path.insert(0, str(HERE.parents[1] / "mcp")) + from traceai_mcp import MCPInstrumentor + + MCPInstrumentor().instrument() + +from _mcp_use_support import ChatFake, answer, tool_call # noqa: E402 + +sys.path.insert(0, str(HERE.parent / "examples")) +import agent_with_tools # noqa: E402 + +if "--with-traceai-mcp" in sys.argv: + from mcp_use.client.task_managers import stdio as stdio_task_manager + + # Proof for the test that the transport instrumentation is in place. + print("traceai_mcp wraps stdio_client:", type(stdio_task_manager.stdio_client).__name__) + +script = [tool_call("add", {"a": 2, "b": 3}), answer("EXAMPLE-ANSWER 2 + 3 = 5")] +print("answer:", asyncio.run(agent_with_tools.main(llm=ChatFake(script=script)))) diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_example.py b/python/frameworks/mcp-use/tests/test_mcp_use_example.py new file mode 100644 index 000000000..2ac4891aa --- /dev/null +++ b/python/frameworks/mcp-use/tests/test_mcp_use_example.py @@ -0,0 +1,118 @@ +"""examples/agent_with_tools.py runs end to end through the shared harness. + +harness.run starts ``_example_app.py`` as a subprocess with FI_BASE_URL +pointed at harness.Receiver. The example's own code runs: register(), an +MCPClient on the stdio calculator server next to it, and an MCPAgent with +FutureAGICallback. Only the LLM is replaced by the fake. The process exits +without flushing explicitly; the spans it exported prove the flush at exit +(PRD J7). + +The same run with the existing transport package traceai_mcp instrumented +first gives the same spans: traceai_mcp 0.1.2 records no spans of its own +(it only carries trace context in MCP request metadata), so the agent span +is there and no tool span is duplicated under the same name and parent. +""" + +from __future__ import annotations + +import json +import os +import sys +from collections import Counter +from pathlib import Path +from typing import Any, Dict, List, Tuple + +import pytest +from _mcp_use_support import AGENT, LLM_KEY, LLM_SPAN, MODEL + +pytest.importorskip("opentelemetry.proto", reason="the harness decodes OTLP protobuf") + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) + +from harness import Receiver, _flatten_attributes, run # noqa: E402 + +APP = Path(__file__).resolve().parent / "_example_app.py" +FI_API_KEY = "placeholder-fi-api-key" +FI_SECRET_KEY = "placeholder-fi-secret-key" + + +def _run_example(*flags: str) -> Tuple[Any, List[Dict[str, Any]], List[Dict[str, Any]]]: + with Receiver() as receiver: + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("FI_", "OTEL_", "LANGFUSE_", "LAMINAR_", "MCP_USE_")) + } + env.update( + { + "FI_BASE_URL": receiver.origin, + "FI_API_KEY": FI_API_KEY, + "FI_SECRET_KEY": FI_SECRET_KEY, + # mcp-use's own usage telemetry posts to PostHog and Scarf. + "MCP_USE_ANONYMIZED_TELEMETRY": "false", + # The subprocess imports exactly what this test process imports. + "PYTHONPATH": os.pathsep.join(sys.path), + } + ) + result = run([sys.executable, str(APP), *flags], env, None, timeout=600) + spans = receiver.spans() + exports = receiver.requests() + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode(errors="replace")[-4000:] + return result, spans, exports + + +def _shape(spans: List[Dict[str, Any]]) -> List[Tuple[str, str, bool]]: + """(name, kind, parented to the agent span) in start order.""" + (agent,) = [span for span in spans if span["name"] == AGENT] + ordered = sorted(spans, key=lambda span: int(span["startTimeUnixNano"])) + return [ + ( + span["name"], + _flatten_attributes(span["attributes"])["gen_ai.span.kind"], + span.get("parentSpanId") == agent["spanId"], + ) + for span in ordered + ] + + +def _check(result: Any, spans: List[Dict[str, Any]], exports: List[Dict[str, Any]]) -> None: + assert b"answer: EXAMPLE-ANSWER 2 + 3 = 5" in result.stdout + assert _shape(spans) == [ + (AGENT, "AGENT", False), + (LLM_SPAN, "LLM", True), + ("execute_tool add", "TOOL", True), + (LLM_SPAN, "LLM", True), + ] + assert len({span["traceId"] for span in spans}) == 1 + assert all(span["status"]["code"] == "STATUS_CODE_OK" for span in spans) + tool = next(span for span in spans if span["name"] == "execute_tool add") + assert _flatten_attributes(tool["attributes"])["gen_ai.tool.name"] == "add" + llm = next(span for span in spans if span["name"] == LLM_SPAN) + assert _flatten_attributes(llm["attributes"])["gen_ai.request.model"] == MODEL + + assert exports + for export in exports: + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_API_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET_KEY + for resource in export["resource_attributes"]: + assert resource["project_name"] == "mcp-use-agent" + assert resource["project_type"] == "observe" + + wire = json.dumps(spans) + for text in (FI_API_KEY, FI_SECRET_KEY, LLM_KEY, "What is 2 + 3", "EXAMPLE-ANSWER"): + assert text not in wire, text + + +def test_example_exports_one_agent_tree_at_exit(): + _check(*_run_example()) + + +def test_with_traceai_mcp_the_agent_span_is_there_and_no_tool_span_is_duplicated(): + result, spans, exports = _run_example("--with-traceai-mcp") + assert b"traceai_mcp wraps stdio_client: FunctionWrapper" in result.stdout + _check(result, spans, exports) + by_name_and_parent = Counter((span["name"], span.get("parentSpanId")) for span in spans) + assert by_name_and_parent[("execute_tool add", next(s["spanId"] for s in spans if s["name"] == AGENT))] == 1 + assert [name for (name, _), count in by_name_and_parent.items() if count > 1] == [LLM_SPAN] From b9601949cda45b7dc9b264b64d0f1a1dbe458a8a Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 07:51:22 -0700 Subject: [PATCH 09/19] fix(mcp-use): run finish reasons through the secret pass Finish reasons were only capped. Every other name and id the callback writes goes through _name (secrets removed, 256-byte cap); finish reasons now do too, so the README's "secrets are removed from every text the callback writes" holds for them. approved: Nikhil 2026-10-03 blanket Refs: TH-8331 --- python/frameworks/mcp-use/traceai_mcp_use/_callback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py index 2d965c0be..a8c96463c 100644 --- a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py +++ b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py @@ -915,7 +915,7 @@ def _llm_response(self, span: Span, response: Any, root: Optional[_Root]) -> Non if name: self._set(span, RESPONSE_MODEL, name) if reasons: - self._set(span, FINISH_REASONS, [cap(reason, MAX_NAME_BYTES) for reason in reasons]) + self._set(span, FINISH_REASONS, [self._name(reason) for reason in reasons]) if isinstance(usage, Mapping): self._set(span, INPUT_TOKENS, _int(usage.get("input_tokens"))) self._set(span, OUTPUT_TOKENS, _int(usage.get("output_tokens"))) From 97363d14658cf46f5c2a02bb76a2e00f20a410fa Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 07:51:22 -0700 Subject: [PATCH 10/19] docs(mcp-use): README and CHANGELOG for traceAI-mcp-use 0.1.0 The README covers installation and the tested matrix (mcp-use 1.7.1 on Python 3.11, 3.12 and 3.13), usage with register(), the three span kinds and their parents, every attribute, context attributes, privacy (content off by default, capture_content, secrets, hide_inputs/hide_outputs/ pii_redaction including error text), errors and cancellation, callbacks=[] as the Python off switch, overlap with traceai_mcp, traceAI-langchain and mcp-use's own Langfuse/Laminar/telemetry, limits (size caps, server-manager restarts, structured output, remote agents, no flush()), and how to run the tests. Vendor behaviour is cited to the pinned mcp-use 1.7.1 source. approved: Nikhil 2026-10-03 blanket Refs: TH-8331 --- python/frameworks/mcp-use/CHANGELOG.md | 23 +- python/frameworks/mcp-use/README.md | 303 +++++++++++++++++++++++++ 2 files changed, 319 insertions(+), 7 deletions(-) create mode 100644 python/frameworks/mcp-use/README.md diff --git a/python/frameworks/mcp-use/CHANGELOG.md b/python/frameworks/mcp-use/CHANGELOG.md index b21a0146b..c087657fa 100644 --- a/python/frameworks/mcp-use/CHANGELOG.md +++ b/python/frameworks/mcp-use/CHANGELOG.md @@ -4,12 +4,21 @@ - First release. `FutureAGICallback`, a LangChain callback handler for `mcp_use.MCPAgent(callbacks=[...])`, records one `AGENT` span per agent - run, one `LLM` span per model call and one `TOOL` span per tool call, - parented as LangChain reports the run tree. -- Content (prompts, model output, tool arguments, tool results, error - messages) is off by default; `capture_content=True` turns it on. - `TraceConfig` `hide_inputs`, `hide_outputs` and `pii_redaction` apply to - every text written, including error text. + run (`run()`, `stream()` or `stream_events()`), one `LLM` span per model + call and one `TOOL` span per tool call. LLM and tool spans are children + of the agent span; LangGraph nodes and middleware are not spans. A + streamed completion is one LLM span with chunk events. Nothing is + patched: only agents given the callback are traced. +- Spans come from `FITracer`: `TraceConfig` and `using_session`, + `using_user`, `using_metadata` and `using_tags` apply. +- Content (prompts, messages, model output, tool arguments and results, + error messages and stack traces) is off by default; `capture_content=True` + turns it on. `hide_inputs`, `hide_outputs` and `pii_redaction` also apply + to error text. - Secrets (values passed in `redact=`, secret-named environment variables, - bearer tokens and key-shaped strings) are removed from everything written. + bearer tokens and key-shaped strings) are removed from everything written, + names and ids included. Recorded text is size-capped. +- Failed tool calls are ERROR tool spans; a failed run is an ERROR agent + span and the exception reaches the caller unchanged; cancelled runs end + their spans as `cancelled`. Callback failures never reach the agent. - Tested with mcp-use 1.7.1 on Python 3.11, 3.12 and 3.13. diff --git a/python/frameworks/mcp-use/README.md b/python/frameworks/mcp-use/README.md new file mode 100644 index 000000000..90924c69d --- /dev/null +++ b/python/frameworks/mcp-use/README.md @@ -0,0 +1,303 @@ +# traceAI-mcp-use + +OpenTelemetry tracing for [mcp-use](https://pypi.org/project/mcp-use/) +agents (`mcp_use.MCPAgent`), sent to Future AGI. + +`FutureAGICallback` is a LangChain callback handler. You pass it to the +agent with `MCPAgent(..., callbacks=[...])`. It records one span for each +agent run, each LLM call and each tool call. It patches nothing, imports +neither `langfuse` nor `traceai_mcp`, and sets no environment variables. + +This package traces the agent loop. [`traceAI-mcp`](../mcp) (`traceai_mcp`) +works at the MCP transport level and is a different layer: a green +transport test does not show the agent run, its model calls or its tool +calls. See [Other tracing in the same process](#other-tracing-in-the-same-process). + +## Installation + +```bash +pip install traceAI-mcp-use +``` + +It accepts `mcp-use>=1.7.1,<2`, `langchain-core>=1.0.0,<2` and +`fi-instrumentation-otel>=1.1.0`, on Python `>=3.11,<3.14` (mcp-use itself +requires 3.11). The suite runs with mcp-use 1.7.1 on Python 3.11, 3.12 and +3.13. + +## Usage + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from mcp_use import MCPAgent, MCPClient +from traceai_mcp_use import FutureAGICallback + +trace_provider = register(project_type=ProjectType.OBSERVE, project_name="mcp-use-agent") + +client = MCPClient.from_dict({"mcpServers": {"calculator": {"command": "python", "args": ["server.py"]}}}) +agent = MCPAgent( + llm=llm, # any LangChain chat model + client=client, + callbacks=[FutureAGICallback(tracer_provider=trace_provider)], +) +result = await agent.run("What is 2 + 3?") +``` + +Pass the provider `register()` returns: `register()` does not set the +global provider unless you ask it to, and without `tracer_provider` the +callback uses the global one. One callback can serve many agents and +concurrent runs. See [`examples/agent_with_tools.py`](examples/agent_with_tools.py). + +`MCPAgent.run()`, `stream()` and `stream_events()` are all traced. The +Python `MCPAgent` is async only. + +## Spans + +| Span | Name | `gen_ai.span.kind` | One per | +|---|---|---|---| +| Agent | `mcp_use.agent` | `AGENT` | `run()`, `stream()` or `stream_events()` call | +| LLM | `chat ` (`text_completion ` for completion models; `chat` when no model name is reported) | `LLM` | model call | +| Tool | `execute_tool ` | `TOOL` | tool call | + +The LLM and tool spans are children of the agent span. In the LangChain +run tree, LangGraph's `model` and `tools` nodes sit between them; graph +nodes and middleware are not spans, so a tool span is a sibling of the LLM +spans, not a child of the LLM call that requested it. The agent span is a +child of the span that is current when the run starts, or a root span. + +A streamed completion is one LLM span, ended when the stream ends. Each +token LangChain reports is a `mcp_use.llm.chunk` event carrying only its +index (`mcp_use.llm.chunk.index`, never the text), for the first 128 +tokens; `mcp_use.llm.chunk_count` holds the exact number. + +The spans are not made current while their run executes, as in +`traceAI-langchain`: a span another instrumentation starts inside a tool +call (for example an HTTP client span) is not a child of the tool span. + +## Span attributes + +| Attribute | Span | Value | +|---|---|---| +| `gen_ai.operation.name` | all | `invoke_agent`, `chat`, `text_completion` or `execute_tool` | +| `mcp_use.agent.llm_call_count`, `mcp_use.agent.tool_call_count`, `mcp_use.agent.tool_error_count` | agent | Model calls, tool calls and failed tool calls in the run | +| `gen_ai.request.model` | LLM | From LangChain's `ls_model_name`, else the invocation `model` / `model_name` / `model_id` | +| `gen_ai.provider.name`, `gen_ai.request.temperature`, `gen_ai.request.max_tokens` | LLM | From LangChain's `ls_provider`, `ls_temperature`, `ls_max_tokens` | +| `gen_ai.response.model`, `gen_ai.response.finish_reasons` | LLM | From the response metadata | +| `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.usage.total_tokens`, `gen_ai.usage.cache_read_tokens` | LLM | From the message's `usage_metadata` (or the provider's `token_usage`) | +| `mcp_use.llm.input_message_count` | LLM | Messages sent to the model | +| `mcp_use.llm.tool_call_count` | LLM | Tool calls the model asked for | +| `mcp_use.llm.chunk_count` | LLM | Streamed tokens, when streamed | +| `gen_ai.tool.name`, `gen_ai.tool.call.id` | tool | The tool and the model's tool-call id | +| `mcp_use.tool.error_type` | tool | Error type, when the tool's failure came back as its result | +| `mcp_use.cancelled`, `mcp_use.incomplete` | any | See [Errors and cancellation](#errors-and-cancellation) | + +A value the callback is not given is omitted, never written as 0. With +`capture_content=True` (see [Privacy](#privacy)) these are added: + +| Attribute | Span | Value | +|---|---|---| +| `input.value` | agent | The last user message of the run's input | +| `output.value` | agent | The final answer (the last assistant message without tool calls) | +| `gen_ai.input.messages..message.role` / `.content` / `.tool_calls..tool_call.*` | LLM | The 32 most recent input messages | +| `gen_ai.output.messages..message.*` | LLM | The model's reply, with up to 16 tool calls (name, id, arguments) | +| `input.value`, `output.value` | LLM | The last input message's text and the reply's text | +| `gen_ai.tool.call.arguments`, `input.value` | tool | The arguments, as JSON | +| `gen_ai.tool.call.result`, `output.value` | tool | The tool's result text | + +Spans come from an `fi_instrumentation.FITracer`, so the attributes of +`using_session`, `using_user`, `using_metadata`, `using_tags` and +`using_attributes` (`session.id`, `user.id`, `metadata`, `tag.tags`) are +set on every span of a run started inside them: + +```python +from fi_instrumentation import using_metadata, using_session, using_tags, using_user + +with using_session("session-1"), using_user("user-1"), using_metadata({"tenant": "t1"}), using_tags(["beta"]): + await agent.run("What is 2 + 3?") +``` + +The Python `MCPAgent` has no `set_metadata` or `set_tags` (the TypeScript +agent does); use these context managers instead. + +## Privacy + +Content is off by default. Without `capture_content=True`, spans carry +names, kinds, the model and provider, request parameters, usage, finish +reasons, counts, the tool-call id, statuses and exception types. They do +not carry prompts, messages, model output, tool arguments, tool results, +error messages or stack traces. + +```python +FutureAGICallback( + tracer_provider=trace_provider, + capture_content=True, # prompts, messages, output, tool arguments and results, error text + redact=["my-mcp-server-token"], # extra secret strings to remove +) +``` + +`capture_content` must be a bool and `redact` an iterable of strings; +anything else raises `TypeError`. + +Secrets are removed from every text the callback writes, including names +and ids, with content on or off. They are replaced by `[redacted]`: + +- the strings passed as `redact=`; +- the value (8 characters or longer) of every environment variable whose + name contains `KEY`, `SECRET`, `TOKEN`, `PASSWORD`, `PASSWD`, + `CREDENTIAL` or `AUTH`, read each time a text is written. That covers + provider keys such as `OPENAI_API_KEY` and `FI_API_KEY`/`FI_SECRET_KEY`; +- `Bearer`/`Basic` credentials, `sk-`/`pk-`/`rk-` style keys and + `api_key=`, `token:`, `password=` or `Authorization:` assignments. + +The callback never reads LangChain's serialized model payload (which can +include the model's repr, and so its key) or the invocation parameters +other than the model name. + +`TraceConfig` settings apply. Pass `config=TraceConfig(...)` or set the +environment variables before creating the callback: + +| Setting | Effect | +|---|---| +| `hide_inputs` / `FI_HIDE_INPUTS=true` | `input.value` is `__REDACTED__`; input messages and tool-call arguments (on tool spans and in model replies) are dropped. With `capture_content`, error messages and stack traces have every input text of the run replaced with `__REDACTED__`: the user messages, the messages sent to the model, and each string value of each tool call's arguments. | +| `hide_outputs` / `FI_HIDE_OUTPUTS=true` | `output.value` is `__REDACTED__`; output messages and tool results are dropped. The error text of a tool failure that came back as the tool's result is dropped, and other error text has every output of the run (model replies, their tool-call arguments, tool results) replaced with `__REDACTED__`. | +| `pii_redaction` / `FI_PII_REDACTION=true` | Emails, phone numbers, SSNs, card numbers, IPv4 addresses and `sk-`/`pk-` keys become tokens such as `` in every recorded text: attributes, the error status and the `exception` event. It runs after secrets (and hidden inputs/outputs) are removed and before the size caps. | + +`config` must be a `fi_instrumentation.TraceConfig`; anything else raises +`TypeError`. Other `TraceConfig` fields act only through `FITracer`'s +attribute masking. + +The hide flags match each text verbatim, and also in its Python-repr and +JSON-quoted forms; a copy that was escaped differently, cut or reworded +stays. Texts shorter than 3 characters are only matched in quoted form, +and non-string argument values (numbers, booleans) are not matched. With a +hide flag on, the callback remembers at most 2048 text forms or about 4 +million characters per run; past that it fails closed and records no +content for the rest of that run (no messages, arguments, results, error +messages or stack traces). + +## Errors and cancellation + +A failed tool call is a tool span with status ERROR and one `exception` +event. mcp-use 1.7.1 does not raise for a failed MCP call: it returns a +formatted error to the model as the tool's result +(`mcp_use/agents/adapters/langchain_adapter.py:181-206`). The callback +recognises that result, takes the error type from it +(`mcp_use.tool.error_type`) and counts it in +`mcp_use.agent.tool_error_count`. A tool call LangChain reports as failed +(for example arguments that fail validation) is recorded the same way, +with the exception's type. Either way the model sees the error and the run +usually goes on, so the agent span ends OK when the run returns an answer. + +When the run itself fails (for example the model call raises), the LLM +span and the agent span end with status ERROR and an `exception` event, +and the exception reaches your code unchanged. + +Without `capture_content`, the status description and the event carry +only the exception type. With it they also carry the message (cut to 1 KB) +and the event carries the stack trace (its last 16 KB), cleaned as +described in [Privacy](#privacy). + +Cancelling the task that runs the agent (`asyncio.CancelledError`) or +leaving `stream()` before it ends (`GeneratorExit`) ends the open spans +with status ERROR, description `cancelled` and `mcp_use.cancelled` = +`true`, without an exception event. A span whose own end never came +because its parent ended first is ended with `mcp_use.incomplete` = +`true` and status ERROR, `ended without a result`. + +If the callback itself fails (starting a span, reading a payload, setting +an attribute), it logs at debug level and the agent goes on unchanged. +Inside `fi_instrumentation.suppress_tracing()` it records nothing. + +## Turning it off + +Leave the callback out. `callbacks=[]` is the Python switch for no +callbacks at all: mcp-use's `ObservabilityManager` returns that empty list +as is and does not add its default Langfuse handler +(`mcp_use/agents/observability/callbacks_manager.py:72-74`). With +`callbacks=None`, mcp-use uses its Langfuse handler when Langfuse is +configured. There is no `observe=False` on the Python `MCPAgent` (that is +the TypeScript agent). + +## Other tracing in the same process + +- **traceAI-mcp (`traceai_mcp`)** works at the MCP transport. Version + 0.1.2 records no spans of its own; it carries trace context in the + `_meta` of MCP requests. With both packages installed you get the same + spans as with this one alone: one agent span and one tool span per tool + call, none duplicated (tested). Because this package's spans are not made + current, the context `traceai_mcp` sends is whichever span is current in + your code. mcp-use binds `stdio_client` when it is imported + (`mcp_use/client/task_managers/stdio.py:12`), so run + `MCPInstrumentor().instrument()` before importing `mcp_use` for + `traceai_mcp`'s stdio wrapper to apply. +- **traceAI-langchain** adds its own tracer to every LangChain callback + manager, so an `MCPAgent` run would also produce a full LangChain tree + (graph, nodes, model and tool runs). Use one of the two for an agent. +- **mcp-use's own observability**: the Langfuse handler is used only when + `callbacks` is `None` (see above). Laminar is set up when `mcp_use` is + imported if `LAMINAR_PROJECT_API_KEY` is set and `lmnr` is installed, + whatever `callbacks` holds (`mcp_use/agents/observability/__init__.py:6`, + `laminar.py:19-37`); `MCP_USE_LAMINAR=false` turns it off. +- **mcp-use's usage telemetry** sends anonymized product events to PostHog + and Scarf, not spans; this package does not change it. + `MCP_USE_ANONYMIZED_TELEMETRY=false` turns it off + (`mcp_use/telemetry/telemetry.py:139`). + +## Limits + +- Recorded text is cut on a UTF-8 character boundary after it is cleaned + (secrets, then hidden inputs/outputs, then PII): 4 KB for each content + attribute, 256 bytes for names and ids (model, provider, tool name, + tool-call id, exception type), 1 KB for the error message in the status + and the `exception` event, and 16 KB for the stack trace, keeping its + end. With `pii_redaction`, `FITracer` runs its own PII pass on every + attribute after the cut; a token such as `` can make an + attribute a few bytes longer than its cap. The status and events are not + passed through it. At most 32 messages and 16 tool calls per message are + recorded per LLM span, and 128 chunk events per streamed LLM span. +- One callback tracks at most 10,000 open runs. Past that, the oldest + agent run is ended with `mcp_use.incomplete` and forgotten. +- With `use_server_manager=True`, mcp-use restarts the graph run when the + tool set changes during a run, up to 3 times + (`mcp_use/agents/mcpagent.py:741-876`); each restart is a new agent span. + This path is not covered by the tests. +- With `output_schema`, mcp-use formats the answer after the graph run with + a model call that gets no callbacks (`mcp_use/agents/mcpagent.py:595`), + so that call has no span. +- A remote agent (`MCPAgent(agent_id=...)`) never reads `callbacks` + (`mcp_use/agents/mcpagent.py:117-120`) and is not traced. +- The Python `MCPAgent` has no `flush()`. The provider `register()` + returns exports what is left when the process exits; in a long-running + process call `trace_provider.force_flush()` when you need the spans now. +- The callback runs on the agent's event loop, not in a thread. With + `register(batch=False)`, each span is exported synchronously on that loop + as it ends. +- Passed to a LangChain runnable other than `MCPAgent`, the callback still + names the outermost run `mcp_use.agent`. +- Tested with mcp-use 1.7.1 only. + +## Tests + +From the repository root: + +```bash +env -u PYTHONPATH PYTHONPATH="python/frameworks/mcp-use:python:python/tests" \ + uv run --no-project --python 3.11 \ + --with pytest --with pytest-asyncio --with opentelemetry-api --with opentelemetry-sdk \ + --with opentelemetry-instrumentation --with opentelemetry-exporter-otlp-proto-http \ + --with wrapt --with requests --with jsonschema --with protobuf --with opentelemetry-proto \ + --with 'mcp-use==1.7.1' \ + pytest python/frameworks/mcp-use/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +Nothing calls a real model, MCP server or Future AGI. The LLM is a +scripted LangChain chat model. The MCP server is the FastMCP server in +`tests/_mcp_server.py`, on a 127.0.0.1 port for most tests and over stdio +for the contract test; the example test uses the example's own stdio +calculator server. Those send spans through +`fi_instrumentation.register()` to the shared `harness.Receiver` on +127.0.0.1 and check the `/tracer/v1/traces` path, both auth headers, +`project_type=observe` and that no key or content is exported by default. +The example test runs `examples/agent_with_tools.py` as its own process, +once alone and once with `traceai_mcp` instrumented. The suite turns off +mcp-use's usage telemetry. From 907a2025a3f6b644cfef4a288bc8e1d7bd9aaa46 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 07:51:22 -0700 Subject: [PATCH 11/19] test(mcp-use): packaging, isolation, structured output and fail-closed hiding Packaging: the suite runs on mcp-use 1.7.1; the mcp-use, langchain-core, fi-instrumentation-otel and Python ranges; classifiers equal the tested Pythons; version agrees across pyproject, package and CHANGELOG; README and CHANGELOG claim only the tested versions; the package source imports neither langfuse, lmnr nor traceai_mcp and sets no environment variable. Isolation: importing and using the callback changes no attribute of mcp-use, LangChain or mcp classes and modules and no environment variable; mcp-use usage telemetry is off in the suite. Also pins that the structured-output formatting call has no span and that hide flags fail closed when a run remembers too much text. approved: Nikhil 2026-10-03 blanket Refs: TH-8331 --- .../mcp-use/tests/test_mcp_use_agent.py | 31 +++++- .../mcp-use/tests/test_mcp_use_callback.py | 18 ++- .../mcp-use/tests/test_mcp_use_isolation.py | 96 ++++++++++++++++ .../mcp-use/tests/test_mcp_use_packaging.py | 105 ++++++++++++++++++ 4 files changed, 248 insertions(+), 2 deletions(-) create mode 100644 python/frameworks/mcp-use/tests/test_mcp_use_isolation.py create mode 100644 python/frameworks/mcp-use/tests/test_mcp_use_packaging.py diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_agent.py b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py index 460ecc98c..c5ba34577 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_agent.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py @@ -172,7 +172,7 @@ def test_capture_content_records_content_without_secrets(monkeypatch): def test_mcp_tool_error_fails_the_tool_span_and_the_agent_recovers(): # mcp-use 1.7.1 turns a failed MCP call into a formatted error the LLM - # reads (agents/adapters/langchain_adapter.py:186-205), so the run + # reads (agents/adapters/langchain_adapter.py:181-206), so the run # succeeds; the tool span is ERROR and the agent counts the error. exporter, handler = _traced() script = [tool_call("fail", {"reason": ARG}), answer()] @@ -438,3 +438,32 @@ def add(a: int, b: int) -> str: assert attrs(only(spans, "execute_tool add"))["gen_ai.tool.call.result"] == "SUM-RESULT-5" shapes.append(sorted((span.name, tuple(sorted(attrs(span)))) for span in spans)) assert shapes[0] == shapes[1] + + +def test_the_structured_output_formatting_call_has_no_span(): + # With output_schema, mcp-use 1.7.1 formats the answer after the graph + # run with structured_llm.ainvoke(prompt), passing no callbacks + # (agents/mcpagent.py:595, 886). That call gets no span and does not + # start a second agent span. + from langchain_core.runnables import RunnableLambda + from pydantic import BaseModel + + class Sum(BaseModel): + total: int + + class ChatStructured(ChatFake): + def with_structured_output(self, schema: Any, **kwargs: Any) -> Any: + return RunnableLambda(lambda prompt: Sum(total=5)) + + exporter, handler = _traced() + + async def go() -> Any: + client = mcp_client() + try: + agent = MCPAgent(llm=ChatStructured(script=add_script()), client=client, callbacks=[handler]) + return await agent.run(PROMPT, output_schema=Sum) + finally: + await client.close_all_sessions() + + assert asyncio.run(go()) == Sum(total=5) + _assert_j1_tree(exporter.get_finished_spans()) diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py index 2f495dd48..49c4bdcc9 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py @@ -496,7 +496,7 @@ def test_llm_error_fails_the_llm_and_agent_spans(): def test_mcp_use_formatted_tool_error_is_an_error_span(): # mcp-use 1.7.1 returns a failed MCP call as format_error()'s dict, as - # JSON in the ToolMessage (agents/adapters/langchain_adapter.py:186-205). + # JSON in the ToolMessage (agents/adapters/langchain_adapter.py:181-206). for capture in (False, True): exporter, handler = _handler(capture_content=capture) run = Run(handler) @@ -749,3 +749,19 @@ def test_constructor_validates_its_arguments(): FutureAGICallback(redact="secret-value") # type: ignore[arg-type] with pytest.raises(TypeError, match="strings"): FutureAGICallback(redact=[b"secret-value"]) # type: ignore[list-item] + + +def test_hide_flags_fail_closed_when_a_run_remembers_too_much(monkeypatch): + import traceai_mcp_use._text as text_module + + monkeypatch.setattr(text_module, "MAX_KNOWN_TEXTS", 4) + exporter, handler = _handler(capture_content=True, config=TraceConfig(hide_inputs=True)) + _journey(handler) + _failing_tool(handler, ValueError("tool failed because " + ARG)) + spans = exporter.get_finished_spans() + text = wire(spans) + for marker in CONTENT_MARKERS + (ANSWER,): + assert marker not in text, marker + tool = only(spans, "execute_tool fail") + assert tool.status.description == "ValueError" + assert dict(tool.events[0].attributes) == {"exception.type": "ValueError"} diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_isolation.py b/python/frameworks/mcp-use/tests/test_mcp_use_isolation.py new file mode 100644 index 000000000..eb2664f69 --- /dev/null +++ b/python/frameworks/mcp-use/tests/test_mcp_use_isolation.py @@ -0,0 +1,96 @@ +"""The package only acts where the callback is passed. + +Importing traceai_mcp_use, building FutureAGICallback and running an agent +with it change no attribute of mcp-use, LangChain or mcp classes and +modules and no environment variable. The test suite itself never sends +mcp-use usage telemetry. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap + +import pytest +from _mcp_use_support import ANSWER, add_script, new_provider, run_agent + +from traceai_mcp_use import FutureAGICallback + +pytest.importorskip("uvicorn", reason="the loopback MCP server runs on uvicorn") + +SNAPSHOT = textwrap.dedent( + """ + import importlib, json, os, sys + import _mcp_use_support # telemetry off, as in every test + TARGETS = { + "mcp_use.agents.mcpagent": ["MCPAgent"], + "mcp_use.agents.observability.callbacks_manager": ["ObservabilityManager"], + "mcp_use.agents.adapters.langchain_adapter": ["LangChainAdapter"], + "mcp_use.client.task_managers.stdio": [], + "mcp.client.stdio": [], + "langchain_core.callbacks.manager": ["CallbackManager", "AsyncCallbackManager"], + "langchain_core.callbacks.base": ["BaseCallbackHandler", "BaseCallbackManager"], + "langchain_core.language_models.chat_models": ["BaseChatModel"], + "langchain_core.tools.base": ["BaseTool"], + } + + def snapshot(): + found = {} + for module_name, classes in TARGETS.items(): + module = importlib.import_module(module_name) + for key, value in vars(module).items(): + found[module_name + ":" + key] = id(value) + for class_name in classes: + for key, value in vars(getattr(module, class_name)).items(): + found[module_name + "." + class_name + ":" + key] = id(value) + return found + + before, env_before = snapshot(), dict(os.environ) + import traceai_mcp_use + from uuid import uuid4 + from _mcp_use_support import new_provider + handler = traceai_mcp_use.FutureAGICallback(tracer_provider=new_provider()[1], capture_content=True) + root = uuid4() + handler.on_chain_start({}, {"messages": []}, run_id=root) + handler.on_chain_end({"messages": []}, run_id=root) + after, env_after = snapshot(), dict(os.environ) + changed = sorted(key for key in set(before) | set(after) if before.get(key) != after.get(key)) + print(json.dumps({ + "changed": changed, + "env": sorted(set(env_before.items()) ^ set(env_after.items())), + "modules": sorted(name for name in sys.modules if name.split(".")[0] in ("langfuse", "lmnr", "traceai_mcp")), + })) + """ +) + + +def test_import_and_use_patch_nothing_and_set_no_environment_variable(): + env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path)) + result = subprocess.run( + [sys.executable, "-c", SNAPSHOT], env=env, capture_output=True, timeout=600 + ) + assert result.returncode == 0, result.stderr.decode(errors="replace")[-4000:] + report = json.loads(result.stdout.decode().strip().splitlines()[-1]) + assert report == {"changed": [], "env": [], "modules": []} + + +def test_a_traced_run_leaves_the_agent_callbacks_as_passed(): + from mcp_use import MCPAgent + + exporter, provider = new_provider() + handler = FutureAGICallback(tracer_provider=provider) + before = {key: id(value) for key, value in vars(MCPAgent).items()} + assert run_agent(add_script(), [handler]) == ANSWER + assert {key: id(value) for key, value in vars(MCPAgent).items()} == before + assert len(exporter.get_finished_spans()) == 4 + + +def test_mcp_use_usage_telemetry_is_off_in_this_suite(): + from mcp_use.telemetry.telemetry import Telemetry + + telemetry = Telemetry() + assert telemetry._posthog_client is None + assert telemetry._scarf_client is None diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_packaging.py b/python/frameworks/mcp-use/tests/test_mcp_use_packaging.py new file mode 100644 index 000000000..be85f515f --- /dev/null +++ b/python/frameworks/mcp-use/tests/test_mcp_use_packaging.py @@ -0,0 +1,105 @@ +"""Packaging contract: dependency ranges, the tested versions, Pythons, version.""" + +from __future__ import annotations + +import importlib.metadata +import re +import sys +from pathlib import Path + +from packaging.specifiers import SpecifierSet +from packaging.version import Version + +import traceai_mcp_use + +PACKAGE = Path(__file__).resolve().parents[1] +PYPROJECT = PACKAGE / "pyproject.toml" +README = PACKAGE / "README.md" +CHANGELOG = PACKAGE / "CHANGELOG.md" + +# The architecture's CI pin (mcp-use 1.7.1, commit 1e14ac9d). The README and +# CHANGELOG claim exactly this version; the suite runs on nothing else. +TESTED_MCP_USE = "1.7.1" +TESTED_PYTHONS = ["3.11", "3.12", "3.13"] + + +def _dependency(name: str) -> SpecifierSet: + match = re.search( + r'^{0}\s*=\s*"([^"]+)"'.format(re.escape(name)), PYPROJECT.read_text(), re.M + ) + assert match, name + return SpecifierSet(match.group(1)) + + +def test_the_suite_runs_on_the_tested_mcp_use_version(): + assert importlib.metadata.version("mcp-use") == TESTED_MCP_USE + + +def test_mcp_use_is_a_1x_range_from_the_tested_version(): + spec = _dependency("mcp-use") + assert spec.contains(TESTED_MCP_USE) + for version in ("1.7.2", "1.99.0"): + assert spec.contains(version), version + # 1.7.0 and earlier are not tested; 2.x drops the deprecated paths. + for version in ("1.7.0", "1.6.0", "2.0.0"): + assert not spec.contains(version), version + + +def test_langchain_core_range_admits_the_installed_version(): + spec = _dependency("langchain-core") + assert spec.contains(importlib.metadata.version("langchain-core")) + assert not spec.contains("0.3.80") + assert not spec.contains("2.0.0") + + +def test_fi_instrumentation_floor_is_1_1_0(): + # 0.1.11/0.1.12 export to the old create_otel_span path and the published + # 1.0.0 fails while building its headers; 1.1.0 is the first good release. + spec = _dependency("fi-instrumentation-otel") + assert spec.contains("1.1.0") + for version in ("0.1.11", "0.1.12", "0.1.16", "1.0.0"): + assert not spec.contains(version), version + + +def test_classifiers_and_python_range_match_the_tested_versions(): + # mcp-use requires Python >=3.11 (its Requires-Python), so the floor is + # 3.11, not the 3.9/3.10 floor of the rest of the family. + text = PYPROJECT.read_text() + assert re.findall(r'"Programming Language :: Python :: (3\.\d+)"', text) == TESTED_PYTHONS + python = _dependency("python") + for version in ("3.11.0", "3.12.10", "3.13.1"): + assert python.contains(version), version + for version in ("3.10.14", "3.14.0"): + assert not python.contains(version), version + assert "{0}.{1}".format(*sys.version_info[:2]) in TESTED_PYTHONS + + +def test_version_matches_pyproject_and_changelog(): + version = re.search(r'^version\s*=\s*"([^"]+)"', PYPROJECT.read_text(), re.M).group(1) + assert traceai_mcp_use.__version__ == version == "0.1.0" + assert re.search(r"^## (\S+)", CHANGELOG.read_text(), re.M).group(1) == version + assert re.search(r'^name\s*=\s*"traceAI-mcp-use"', PYPROJECT.read_text(), re.M) + + +def test_readme_and_changelog_claim_only_the_tested_versions(): + for path in (README, CHANGELOG): + text = " ".join(path.read_text().split()) + assert "mcp-use {0}".format(TESTED_MCP_USE) in text, path.name + claimed = set(re.findall(r"mcp-use (\d+\.\d+\.\d+)", text)) + assert claimed == {TESTED_MCP_USE}, (path.name, claimed) + assert "3.11, 3.12 and 3.13" in text, path.name + + +def test_package_source_imports_neither_langfuse_nor_traceai_mcp_nor_patches(): + source = "\n".join(path.read_text() for path in (PACKAGE / "traceai_mcp_use").glob("*.py")) + for forbidden in ( + r"^\s*(from|import)\s+langfuse", + r"^\s*(from|import)\s+lmnr", + r"^\s*(from|import)\s+traceai_mcp\b", + r"^\s*(from|import)\s+wrapt", + r"os\.environ\[[^\]]+\]\s*=", + r"os\.(putenv|environ\.setdefault|environ\.update)", + r"setattr\(", + ): + assert not re.search(forbidden, source, re.M), forbidden + assert Version(traceai_mcp_use.__version__) == Version("0.1.0") From 9969ec0e335cfce1574d8b58d404e40f9e7e7f8c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 07:51:48 -0700 Subject: [PATCH 12/19] test(mcp-use): give slow machines more time to start the server and the tool The loopback server start, the wait for the slow tool to start before cancelling, and the wait for the agent span after leaving stream() early now allow up to two minutes, one minute and one minute. Imports alone take tens of seconds on a loaded laptop; the tests stop waiting as soon as the condition holds. approved: Nikhil 2026-10-03 blanket Refs: TH-8331 --- python/frameworks/mcp-use/tests/_mcp_use_support.py | 2 +- python/frameworks/mcp-use/tests/test_mcp_use_agent.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/frameworks/mcp-use/tests/_mcp_use_support.py b/python/frameworks/mcp-use/tests/_mcp_use_support.py index 8a4c1ae1e..d88b17716 100644 --- a/python/frameworks/mcp-use/tests/_mcp_use_support.py +++ b/python/frameworks/mcp-use/tests/_mcp_use_support.py @@ -217,7 +217,7 @@ def _start() -> str: target=lambda: asyncio.run(server.serve(sockets=[sock])), daemon=True ) thread.start() - deadline = time.monotonic() + 30 + deadline = time.monotonic() + 120 while not server.started: if time.monotonic() > deadline or not thread.is_alive(): raise RuntimeError("the loopback MCP server did not start") diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_agent.py b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py index c5ba34577..633ea321c 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_agent.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py @@ -246,7 +246,7 @@ async def go() -> None: callbacks=[handler, started], ) task = asyncio.ensure_future(agent.run(PROMPT)) - for _ in range(1000): + for _ in range(6000): # up to a minute on a slow machine if started.started.is_set(): break await asyncio.sleep(0.01) @@ -281,7 +281,7 @@ async def go() -> None: async for _ in stream: break # the first item is the (action, observation) of the tool step await stream.aclose() - for _ in range(500): + for _ in range(6000): if spans_named(exporter.get_finished_spans(), AGENT): break await asyncio.sleep(0.01) From af52cf7109239bbdea24aaedd305fb7a091dfa95 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 07:53:19 -0700 Subject: [PATCH 13/19] test(mcp-use): force an internal failure in every callback The other isolation tests fail the tracer and the span, which the inner guards absorb, so the outer guard of each public callback was never reached: a mutation that re-raised from on_tool_end passed the suite. This test makes the run bookkeeping raise and calls every callback. approved: Nikhil 2026-10-03 blanket Refs: TH-8331 --- .../mcp-use/tests/test_mcp_use_callback.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py index 49c4bdcc9..9f51251ef 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py @@ -765,3 +765,37 @@ def test_hide_flags_fail_closed_when_a_run_remembers_too_much(monkeypatch): tool = only(spans, "execute_tool fail") assert tool.status.description == "ValueError" assert dict(tool.events[0].attributes) == {"exception.type": "ValueError"} + + +def test_an_internal_failure_in_any_callback_never_reaches_langchain(monkeypatch): + # Every public callback catches its own failure, past the inner guards + # the other tests exercise: make the bookkeeping itself raise. + exporter, handler = _handler(capture_content=True) + + def boom(*args: Any, **kwargs: Any) -> Any: + raise RuntimeError("internal failure") + + class Exploding(dict): + def get(self, *args: Any, **kwargs: Any) -> Any: + raise RuntimeError("internal failure") + + for name in ("_begin", "_pop", "_finish_error"): + monkeypatch.setattr(handler, name, boom) + monkeypatch.setattr(handler, "_runs", Exploding()) + run_id, parent = uuid4(), uuid4() + reply = _result(AIMessage(content="x")) + handler.on_chain_start({}, {"messages": []}, run_id=run_id) + handler.on_chain_end({}, run_id=run_id) + handler.on_chain_error(RuntimeError(), run_id=run_id) + handler.on_retriever_start({}, "q", run_id=run_id, parent_run_id=parent) + handler.on_retriever_end([], run_id=run_id) + handler.on_retriever_error(RuntimeError(), run_id=run_id) + handler.on_chat_model_start({}, [[HumanMessage(content="x")]], run_id=run_id) + handler.on_llm_start({}, ["x"], run_id=run_id) + handler.on_llm_new_token("t", run_id=run_id) + handler.on_llm_end(reply, run_id=run_id) + handler.on_llm_error(RuntimeError(), run_id=run_id) + handler.on_tool_start({"name": "add"}, "{}", run_id=run_id, inputs={}) + handler.on_tool_end("result", run_id=run_id) + handler.on_tool_error(RuntimeError(), run_id=run_id) + assert exporter.get_finished_spans() == () From e597ad1074436e7798dc86634f5e9ffc9c29e63a Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 07:53:56 -0700 Subject: [PATCH 14/19] test(mcp-use): pin that callbacks run on the event loop thread FutureAGICallback sets run_inline so LangChain calls it on the agent's event loop instead of an executor thread; a mutation back to executor dispatch passed the suite. The README states this behaviour. approved: Nikhil 2026-10-03 blanket Refs: TH-8331 --- .../mcp-use/tests/test_mcp_use_agent.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_agent.py b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py index 633ea321c..3a670db4b 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_agent.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py @@ -467,3 +467,31 @@ async def go() -> Any: assert asyncio.run(go()) == Sum(total=5) _assert_j1_tree(exporter.get_finished_spans()) + + +def test_callbacks_run_on_the_event_loop_thread(): + # run_inline: LangChain would otherwise run each sync callback in an + # executor thread, where a cancelled run can end a parent before its + # child's start callback has finished. + import threading + + exporter, provider = new_provider() + threads: List[int] = [] + + class Recording(FutureAGICallback): + def on_chain_start(self, *args: Any, **kwargs: Any) -> None: + threads.append(threading.get_ident()) + super().on_chain_start(*args, **kwargs) + + def on_chat_model_start(self, *args: Any, **kwargs: Any) -> None: + threads.append(threading.get_ident()) + super().on_chat_model_start(*args, **kwargs) + + def on_tool_start(self, *args: Any, **kwargs: Any) -> None: + threads.append(threading.get_ident()) + super().on_tool_start(*args, **kwargs) + + assert run_agent(add_script(), [Recording(tracer_provider=provider)]) == ANSWER + assert len(threads) >= 4 + assert set(threads) == {threading.get_ident()} + _assert_j1_tree(exporter.get_finished_spans()) From 135405e3f4b16b42d0df0d18b190f6c9604c8d85 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 09:12:12 -0700 Subject: [PATCH 15/19] fix(mcp-use): hide_outputs also hides earlier turns replayed as input MCPAgent keeps memory by default and sends the previous turns' replies and tool results back to the model on the next run (external_history does the same). The callback only scrubbed outputs the current run produced, so with capture_content and hide_outputs, turn 2's LLM span recorded turn 1's tool result and answer in gen_ai.input.messages, and error text could quote them. Under hide_outputs, every assistant- or tool-role input message (any role other than user and system) is now an output: the root run and each LLM call remember its content and tool-call argument strings as outputs before writing anything, and the input messages keep its role but drop its content and arguments. Tested over three turns on one agent via run(), stream(), stream_events() and external_history, with a hide_outputs-off control, in unit tests, and on the export through register(). approved: Nikhil 2026-10-03 blanket Refs: TH-8331 Co-Authored-By: Claude Opus 5.5 --- python/frameworks/mcp-use/CHANGELOG.md | 4 +- python/frameworks/mcp-use/README.md | 19 +++- .../mcp-use/tests/test_mcp_use_agent.py | 97 +++++++++++++++++++ .../mcp-use/tests/test_mcp_use_callback.py | 60 +++++++++++- .../mcp-use/tests/test_mcp_use_contract.py | 57 ++++++++++- .../mcp-use/traceai_mcp_use/_callback.py | 53 ++++++++-- 6 files changed, 271 insertions(+), 19 deletions(-) diff --git a/python/frameworks/mcp-use/CHANGELOG.md b/python/frameworks/mcp-use/CHANGELOG.md index c087657fa..6404c15c5 100644 --- a/python/frameworks/mcp-use/CHANGELOG.md +++ b/python/frameworks/mcp-use/CHANGELOG.md @@ -14,7 +14,9 @@ - Content (prompts, messages, model output, tool arguments and results, error messages and stack traces) is off by default; `capture_content=True` turns it on. `hide_inputs`, `hide_outputs` and `pii_redaction` also apply - to error text. + to error text. Under `hide_outputs`, model replies and tool results that + `MCPAgent` memory or `external_history` sends back on a later turn are + outputs too. - Secrets (values passed in `redact=`, secret-named environment variables, bearer tokens and key-shaped strings) are removed from everything written, names and ids included. Recorded text is size-capped. diff --git a/python/frameworks/mcp-use/README.md b/python/frameworks/mcp-use/README.md index 90924c69d..9c52d9de0 100644 --- a/python/frameworks/mcp-use/README.md +++ b/python/frameworks/mcp-use/README.md @@ -159,13 +159,30 @@ environment variables before creating the callback: | Setting | Effect | |---|---| | `hide_inputs` / `FI_HIDE_INPUTS=true` | `input.value` is `__REDACTED__`; input messages and tool-call arguments (on tool spans and in model replies) are dropped. With `capture_content`, error messages and stack traces have every input text of the run replaced with `__REDACTED__`: the user messages, the messages sent to the model, and each string value of each tool call's arguments. | -| `hide_outputs` / `FI_HIDE_OUTPUTS=true` | `output.value` is `__REDACTED__`; output messages and tool results are dropped. The error text of a tool failure that came back as the tool's result is dropped, and other error text has every output of the run (model replies, their tool-call arguments, tool results) replaced with `__REDACTED__`. | +| `hide_outputs` / `FI_HIDE_OUTPUTS=true` | `output.value` is `__REDACTED__`; output messages and tool results are dropped. Model replies and tool results sent back to the model as input messages (later in the same run, or from earlier turns; see below) keep their role but lose their content and tool-call arguments, and an LLM span whose last input message is one of them has `input.value` `__REDACTED__`. The error text of a tool failure that came back as the tool's result is dropped, and other error text has every output the run has seen (model replies, their tool-call arguments, tool results, earlier turns' included) replaced with `__REDACTED__`. | | `pii_redaction` / `FI_PII_REDACTION=true` | Emails, phone numbers, SSNs, card numbers, IPv4 addresses and `sk-`/`pk-` keys become tokens such as `` in every recorded text: attributes, the error status and the `exception` event. It runs after secrets (and hidden inputs/outputs) are removed and before the size caps. | `config` must be a `fi_instrumentation.TraceConfig`; anything else raises `TypeError`. Other `TraceConfig` fields act only through `FITracer`'s attribute masking. +Multi-turn conversations: `MCPAgent` keeps memory by default +(`memory_enabled=True`) and sends the earlier turns, model replies and tool +results included, back to the model as input on the next `run()`, +`stream()` or `stream_events()` call (`mcp_use/agents/mcpagent.py:76`, +`:724-732`, `:879-880`, `:981-983`); `external_history` does the same. +Each call is its own agent span. Under `hide_outputs`, every assistant- or +tool-role message (any role other than user and system) among a run's +input messages counts as an output: its content and tool-call arguments +are dropped from the input messages and removed from every other text of +that run, error text included. The callback reads them when the run +starts, before it writes any of the run's spans. This is +tested over three turns on one agent through `run()`, `stream()`, +`stream_events()` and `external_history`, and on the export. User messages +of earlier turns are inputs: they are recorded unless `hide_inputs` is on. +The texts a run remembers include its replayed history, so in a long +conversation the per-run limit below is reached sooner. + The hide flags match each text verbatim, and also in its Python-repr and JSON-quoted forms; a copy that was escaped differently, cut or reworded stays. Texts shorter than 3 characters are only matched in quoted form, diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_agent.py b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py index 3a670db4b..8e1c98458 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_agent.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py @@ -30,6 +30,7 @@ add_script, answer, attrs, + drive, mcp_client, new_provider, only, @@ -206,6 +207,102 @@ def test_tool_error_text_is_recorded_with_capture_and_scrubbed_with_hide_inputs( assert ARG not in wire(spans) and "PROMPT-MARKER" not in wire(spans) +SECOND_ANSWER = "SECOND-ANSWER done" +EARLIER_TURN = ("SUM-RESULT", "ANSWER-MARKER") + + +def _quote_the_history(messages: List[Any]) -> Any: + """A model step that fails with an error quoting the earlier turns.""" + earlier = [str(m.content) for m in messages if m.type in ("ai", "tool") and m.content] + raise ValueError("model rejected the history: " + " | ".join(earlier)) + + +def _three_turns(method: str, external: bool, **options: Any) -> List[Any]: + """Three queries on one MCPAgent; returns each turn's spans. + + Turn 1 calls add (SUM-RESULT-5) and answers ANSWER-MARKER. With memory + on (mcp-use's default) or the history passed back as external_history, + turns 2 and 3 send turn 1's messages to the model again. Turn 2 echoes + and answers; in turn 3 the model raises an error quoting the history. + """ + exporter, handler = _traced(capture_content=True, **options) + script = add_script() + [ + tool_call("echo", {"text": "SECOND-ARG"}, "call-2"), + answer(SECOND_ANSWER), + _quote_the_history, + ] + turns: List[Any] = [] + + async def go() -> None: + client = mcp_client() + try: + agent = MCPAgent(llm=ChatFake(script=script), client=client, callbacks=[handler]) + assert agent.memory_enabled + kwargs: Dict[str, Any] = {} + await drive(agent, method, PROMPT) + if external: + kwargs["external_history"] = list(agent.get_conversation_history()) + agent.clear_conversation_history() + assert kwargs.get("external_history") or agent.get_conversation_history() + turns.append(exporter.get_finished_spans()) + exporter.clear() + await drive(agent, method, "SECOND-PROMPT echo it", **kwargs) + turns.append(exporter.get_finished_spans()) + exporter.clear() + with pytest.raises(ValueError): + await drive(agent, method, "THIRD-PROMPT", **kwargs) + turns.append(exporter.get_finished_spans()) + finally: + await client.close_all_sessions() + + asyncio.run(go()) + assert handler._runs == {} + return turns + + +_TURN_MODES = [("run", False), ("stream", False), ("stream_events", False), ("run", True)] + + +@pytest.mark.parametrize("method,external", _TURN_MODES) +def test_hide_outputs_keeps_earlier_turns_out_of_later_turns(method, external): + _, second, third = _three_turns(method, external, config=TraceConfig(hide_outputs=True)) + for spans in (second, third): + text = wire(spans) + for marker in EARLIER_TURN + ("ECHO-RESULT", "SECOND-ANSWER"): + assert marker not in text, marker + assert "SECOND-ARG" not in wire(third) # turn 2's tool-call arguments are model output + # The replayed messages keep their roles and the count; only the + # assistant and tool content is dropped. + replay = sorted(spans_named(second, LLM_SPAN), key=lambda span: span.start_time)[0] + values = attrs(replay) + roles = [values["gen_ai.input.messages.{0}.message.role".format(i)] for i in range(5)] + assert roles == ["system", "user", "assistant", "tool", "assistant"] + assert values["mcp_use.llm.input_message_count"] == 6 + assert values["gen_ai.input.messages.5.message.content"] == "SECOND-PROMPT echo it" + for index in (2, 3, 4): + assert "gen_ai.input.messages.{0}.message.content".format(index) not in values + # Inputs are still recorded, and the error text is kept with the earlier + # outputs taken out of it. + assert attrs(only(third, AGENT))["input.value"] == "THIRD-PROMPT" + for span in (only(third, AGENT), only(third, LLM_SPAN)): + assert span.status.description.startswith("ValueError: model rejected the history: ") + assert REDACTED_VALUE in span.status.description + (event,) = span.events + assert REDACTED_VALUE in event.attributes["exception.message"] + + +@pytest.mark.parametrize("method,external", _TURN_MODES) +def test_without_hide_outputs_later_turns_record_the_earlier_turns(method, external): + # Control for the test above: the same turns carry turn 1's tool result + # and answer as input messages, and the error quotes them. + _, second, third = _three_turns(method, external) + replay = attrs(sorted(spans_named(second, LLM_SPAN), key=lambda span: span.start_time)[0]) + assert replay["gen_ai.input.messages.3.message.content"] == "SUM-RESULT-5" + assert replay["gen_ai.input.messages.4.message.content"] == ANSWER + assert "SUM-RESULT-5" in only(third, LLM_SPAN).status.description + assert ANSWER in only(third, AGENT).status.description + + @pytest.mark.parametrize("retry_on_error", [True, False]) def test_invalid_tool_arguments_fail_the_tool_span(retry_on_error): exporter, handler = _traced() diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py index 9f51251ef..0d9308e5c 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py @@ -11,7 +11,7 @@ import asyncio import json import os -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Sequence from uuid import UUID, uuid4 import pytest @@ -88,10 +88,11 @@ def _result(message: AIMessage) -> LLMResult: class Run: """One agent run, driven callback by callback like LangGraph does.""" - def __init__(self, handler: FutureAGICallback, query: str = PROMPT) -> None: + def __init__(self, handler: FutureAGICallback, query: str = PROMPT, earlier: Sequence[Any] = ()) -> None: self.handler = handler self.root = uuid4() - self.history: List[Any] = [HumanMessage(content=query)] + # ``earlier``: messages of previous turns (MCPAgent memory). + self.history: List[Any] = list(earlier) + [HumanMessage(content=query)] handler.on_chain_start( {}, {"messages": list(self.history)}, run_id=self.root, name="LangGraph" ) @@ -297,6 +298,59 @@ def test_hide_outputs_drops_every_output(): assert attrs(only(spans, AGENT))["input.value"] == PROMPT +def test_hide_outputs_drops_earlier_turns_replayed_as_input(): + # mcp-use's memory (or external_history) sends earlier turns' replies + # and tool results back as input: under hide_outputs they are outputs. + exporter, handler = _handler(capture_content=True, config=TraceConfig(hide_outputs=True)) + earlier = [ + HumanMessage(content="OLD-PROMPT"), + AIMessage( + content="OLD-THINKING", + tool_calls=[{"name": "add", "args": {"note": "OLD-ARG"}, "id": "c0", "type": "tool_call"}], + ), + ToolMessage(content="OLD-RESULT", tool_call_id="c0"), + {"role": "assistant", "content": "OLD-ANSWER"}, + ] + run = Run(handler, earlier=earlier) + node = run.node("model") + llm = uuid4() + handler.on_chat_model_start( + {}, [[SystemMessage(content="You are an MCP agent.")] + run.history], run_id=llm, parent_run_id=node + ) + error = ValueError("cannot use OLD-RESULT, OLD-ANSWER, OLD-ARG or OLD-THINKING") + handler.on_llm_error(error, run_id=llm, parent_run_id=node) + handler.on_chain_error(error, run_id=node, parent_run_id=run.root) + handler.on_chain_error(error, run_id=run.root) + spans = exporter.get_finished_spans() + text = wire(spans) + for marker in ("OLD-RESULT", "OLD-ANSWER", "OLD-ARG", "OLD-THINKING"): + assert marker not in text, marker + llm_span = attrs(only(spans, "chat")) + roles = [llm_span["gen_ai.input.messages.{0}.message.role".format(i)] for i in range(6)] + assert roles == ["system", "user", "assistant", "tool", "assistant", "user"] + assert llm_span["gen_ai.input.messages.1.message.content"] == "OLD-PROMPT" + assert llm_span["gen_ai.input.messages.5.message.content"] == PROMPT + for index in (2, 3, 4): + assert "gen_ai.input.messages.{0}.message.content".format(index) not in llm_span + call = "gen_ai.input.messages.2.message.tool_calls.0.tool_call." + assert llm_span[call + "function.name"] == "add" + assert call + "function.arguments" not in llm_span + assert llm_span["input.value"] == PROMPT + for span in spans: + assert span.status.description == "ValueError: cannot use {0}, {0}, {0} or {0}".format(REDACTED_VALUE) + + +def test_hide_outputs_with_a_replayed_reply_last_redacts_the_llm_input_value(): + exporter, handler = _handler(capture_content=True, config=TraceConfig(hide_outputs=True)) + run = Run(handler) + run.history.append(ToolMessage(content="OLD-RESULT", tool_call_id="c0")) + run.llm(AIMessage(content=ANSWER)) + run.finish() + llm = attrs(only(exporter.get_finished_spans(), LLM_SPAN)) + assert llm["input.value"] == REDACTED_VALUE + assert "OLD-RESULT" not in wire(exporter.get_finished_spans()) + + def test_hide_flags_are_read_from_the_environment(monkeypatch): monkeypatch.setenv("FI_HIDE_INPUTS", "true") monkeypatch.setenv("FI_HIDE_OUTPUTS", "true") diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_contract.py b/python/frameworks/mcp-use/tests/test_mcp_use_contract.py index 4b3a896f1..16944af1b 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_contract.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_contract.py @@ -47,17 +47,22 @@ PROJECT = "mcp-use-contract" -def _journey(receiver: Receiver, monkeypatch, **options: Any) -> List[Dict[str, Any]]: - """J1 (add, then answer) and a failing tool call, through register().""" +def _register(receiver: Receiver, monkeypatch) -> Any: from fi_instrumentation import register from fi_instrumentation.fi_types import ProjectType - from mcp_use import MCPAgent monkeypatch.setenv("FI_BASE_URL", receiver.origin) monkeypatch.setenv("FI_API_KEY", FI_API_KEY) monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET_KEY) monkeypatch.setenv(ENV_SECRET_NAME, ENV_SECRET) - provider = register(project_type=ProjectType.OBSERVE, project_name=PROJECT, verbose=False) + return register(project_type=ProjectType.OBSERVE, project_name=PROJECT, verbose=False) + + +def _journey(receiver: Receiver, monkeypatch, **options: Any) -> List[Dict[str, Any]]: + """J1 (add, then answer) and a failing tool call, through register().""" + from mcp_use import MCPAgent + + provider = _register(receiver, monkeypatch) handler = FutureAGICallback(tracer_provider=provider, **options) async def go() -> None: @@ -169,3 +174,47 @@ def test_opt_in_capture_reaches_the_collector_without_keys(monkeypatch): assert secret not in text, secret agent = _attrs(_trees(spans)[0][0]) assert agent["input.value"] == PROMPT + " [redacted]" + + +def _two_turns(receiver: Receiver, monkeypatch, **options: Any) -> List[Dict[str, Any]]: + """J1, then a second query on the same MCPAgent (memory on, the default).""" + from mcp_use import MCPAgent + + provider = _register(receiver, monkeypatch) + handler = FutureAGICallback(tracer_provider=provider, capture_content=True, **options) + + async def go() -> None: + client = mcp_client("stdio") + try: + agent = MCPAgent( + llm=ChatFake(script=add_script() + [answer("SECOND-ANSWER done")]), + client=client, + callbacks=[handler], + ) + assert await agent.run(PROMPT) == ANSWER + assert await agent.run("SECOND-PROMPT") == "SECOND-ANSWER done" + finally: + await client.close_all_sessions() + + try: + asyncio.run(go()) + assert provider.force_flush(timeout_millis=10_000) + finally: + provider.shutdown() + return receiver.spans() + + +@pytest.mark.parametrize("hide_outputs", [False, True]) +def test_hide_outputs_keeps_the_first_turn_out_of_the_second_turn_export(monkeypatch, hide_outputs): + from fi_instrumentation import TraceConfig + + with Receiver() as receiver: + spans = _two_turns(receiver, monkeypatch, config=TraceConfig(hide_outputs=hide_outputs)) + + _, second = _trees(spans) + assert [span["name"] for span in second] == [AGENT, LLM_SPAN] + assert _attrs(second[0])["input.value"] == "SECOND-PROMPT" + text = json.dumps(second) + for marker in ("SUM-RESULT-5", "ANSWER-MARKER"): + # The control (hide_outputs off) shows the second turn replays them. + assert (marker in text) is not hide_outputs, marker diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py index a8c96463c..abbb98929 100644 --- a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py +++ b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py @@ -196,6 +196,12 @@ def _role(message: Any) -> str: return "unknown" +def _is_output_role(role: str) -> bool: + """A message the model or a tool wrote: assistant, tool, or any role + other than user and system (fail closed for hide_outputs).""" + return role not in ("user", "system") + + def _content(message: Any) -> Optional[str]: """Text of a LangChain message object or a {"role", "content"} dict.""" if isinstance(message, Mapping): @@ -234,6 +240,24 @@ def _strings_in(value: Any, depth: int = 0) -> Iterable[str]: yield from _strings_in(item, depth + 1) +def _replayed_outputs(messages: Iterable[Any]) -> List[Optional[str]]: + """Content and tool-call argument strings of the output-role messages + among a run's input messages. + + MCPAgent keeps memory by default and sends earlier turns (their replies, + tool calls and tool results) back as input on the next run, as does + ``external_history``: for hide_outputs these are outputs. + """ + texts: List[Optional[str]] = [] + for message in messages: + if not _is_output_role(_role(message)): + continue + texts.append(_content(message)) + for call in _tool_calls(message): + texts.extend(_strings_in(call.get("args"))) + return texts + + def _mcp_use_error(output: Any) -> Optional[Mapping[str, Any]]: """The mcp-use formatted error a tool returned, if this output is one.""" candidate: Any = output @@ -610,20 +634,20 @@ def on_chain_start( OPERATION: "invoke_agent", } run, root = self._begin(run_id, parent_run_id, _CHAIN, AGENT_SPAN_NAME, attributes) - if run is None or run.kind != _AGENT or run.span is None: + if run is None or run.kind != _AGENT: return messages = inputs.get("messages") if isinstance(inputs, Mapping) else None - texts = [_content(m) for m in messages or []] if isinstance( - messages, list - ) else [] - self._remember(root, texts, output=False) - if not self._capture: + history = messages if isinstance(messages, list) else [] + self._remember(root, [_content(m) for m in history], output=False) + # Earlier turns' outputs, known before any content of this run is written. + self._remember(root, _replayed_outputs(history), output=True) + if run.span is None or not self._capture: return if self._hide_inputs: self._set(run.span, INPUT_VALUE, REDACTED_VALUE) return query = None - for message in reversed(messages or []) if isinstance(messages, list) else []: + for message in reversed(history): if _role(message) == "user": query = _content(message) break @@ -751,8 +775,13 @@ def _message_attributes( attributes: Dict[str, Any] = {} for index, message in enumerate(list(messages)[-MAX_MESSAGES:]): base = "{0}.{1}.message".format(prefix, index) - attributes[base + ".role"] = _role(message) - text = self._clean(_content(message), root, MAX_VALUE_BYTES) + role = _role(message) + attributes[base + ".role"] = role + # Replies and tool results sent back as input (the run's own, or + # earlier turns' from MCPAgent memory) are outputs: hide_outputs + # drops their content and tool-call arguments. + hidden = self._hide_outputs and _is_output_role(role) + text = None if hidden else self._clean(_content(message), root, MAX_VALUE_BYTES) if text: attributes[base + ".content"] = text for call_index, call in enumerate(_tool_calls(message)[:MAX_TOOL_CALLS]): @@ -765,7 +794,7 @@ def _message_attributes( attributes[call_base + ".id"] = call_id # Tool-call arguments are the tool's input, also when they # appear in the model's output: hide_inputs drops them. - if self._hide_inputs: + if self._hide_inputs or hidden: continue arguments = self._clean(_json(call.get("args")), root, MAX_VALUE_BYTES) if arguments: @@ -791,6 +820,7 @@ def _start_llm( for call in _tool_calls(message): texts.extend(_strings_in(call.get("args"))) self._remember(root, texts, output=False) + self._remember(root, _replayed_outputs(messages), output=True) if not self._capture: return if self._hide_inputs: @@ -799,6 +829,9 @@ def _start_llm( for key, value in self._message_attributes(INPUT_MESSAGES, messages, root).items(): self._set(run.span, key, value) if messages: + if self._hide_outputs and _is_output_role(_role(messages[-1])): + self._set(run.span, INPUT_VALUE, REDACTED_VALUE) + return last = self._clean(_content(messages[-1]), root, MAX_VALUE_BYTES) if last: self._set(run.span, INPUT_VALUE, last) From 4dea77e27a807e59a3a593b45e0bccea39945cd2 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 09:13:38 -0700 Subject: [PATCH 16/19] fix(mcp-use): only mcp-use's own formatted error fails a tool span A tool result that was a JSON object with string "error" plus "details", "stack" and "code" keys was classified as a failed MCP call, so a tool that legitimately returns that shape (an incident lookup, say) got an ERROR span, a tool_error_count increment and no result attribute. mcp-use 1.7.1 adds tool= at every format_error call site that reaches a tool span (langchain_adapter.py:197-205, tool_error_middleware.py:61). The match now also requires a "tool" key equal to the name LangChain reported for the tool run. The server manager's own tools pass no tool key; their failures are now OK spans with the error as the result, as the README states. approved: Nikhil 2026-10-03 blanket Refs: TH-8331 Co-Authored-By: Claude Opus 5.5 --- python/frameworks/mcp-use/CHANGELOG.md | 9 ++++-- python/frameworks/mcp-use/README.md | 11 +++++-- .../mcp-use/tests/test_mcp_use_callback.py | 32 +++++++++++++++++++ .../mcp-use/traceai_mcp_use/_callback.py | 30 ++++++++++++----- 4 files changed, 69 insertions(+), 13 deletions(-) diff --git a/python/frameworks/mcp-use/CHANGELOG.md b/python/frameworks/mcp-use/CHANGELOG.md index 6404c15c5..ace5588c3 100644 --- a/python/frameworks/mcp-use/CHANGELOG.md +++ b/python/frameworks/mcp-use/CHANGELOG.md @@ -20,7 +20,10 @@ - Secrets (values passed in `redact=`, secret-named environment variables, bearer tokens and key-shaped strings) are removed from everything written, names and ids included. Recorded text is size-capped. -- Failed tool calls are ERROR tool spans; a failed run is an ERROR agent - span and the exception reaches the caller unchanged; cancelled runs end - their spans as `cancelled`. Callback failures never reach the agent. +- Failed tool calls are ERROR tool spans, including an MCP failure mcp-use + returns as the tool's result (recognised by its `tool` key matching the + tool, so a tool's own error-shaped result stays a result); a failed run + is an ERROR agent span and the exception reaches the caller unchanged; + cancelled runs end their spans as `cancelled`. Callback failures never + reach the agent. - Tested with mcp-use 1.7.1 on Python 3.11, 3.12 and 3.13. diff --git a/python/frameworks/mcp-use/README.md b/python/frameworks/mcp-use/README.md index 9c52d9de0..11a3e5f05 100644 --- a/python/frameworks/mcp-use/README.md +++ b/python/frameworks/mcp-use/README.md @@ -198,7 +198,10 @@ A failed tool call is a tool span with status ERROR and one `exception` event. mcp-use 1.7.1 does not raise for a failed MCP call: it returns a formatted error to the model as the tool's result (`mcp_use/agents/adapters/langchain_adapter.py:181-206`). The callback -recognises that result, takes the error type from it +recognises that result (an object with mcp-use's `error`, `details`, +`stack` and `code` keys and a `tool` key equal to the tool's name; a tool's +own result with those keys but no matching `tool` is a result), takes the +error type from it (`mcp_use.tool.error_type`) and counts it in `mcp_use.agent.tool_error_count`. A tool call LangChain reports as failed (for example arguments that fail validation) is recorded the same way, @@ -277,7 +280,11 @@ the TypeScript agent). - With `use_server_manager=True`, mcp-use restarts the graph run when the tool set changes during a run, up to 3 times (`mcp_use/agents/mcpagent.py:741-876`); each restart is a new agent span. - This path is not covered by the tests. + The server manager's own tools return their failures as mcp-use errors + without a `tool` key (`mcp_use/agents/managers/tools/connect_server.py:73`, + `disconnect_server.py:39`, `list_servers_tool.py:47`), so those tool spans + end OK with the error as their result. This path is not covered by the + tests. - With `output_schema`, mcp-use formats the answer after the graph run with a model call that gets no callbacks (`mcp_use/agents/mcpagent.py:595`), so that call has no span. diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py index 0d9308e5c..47841eb0f 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py @@ -583,6 +583,38 @@ def test_mcp_use_formatted_tool_error_is_an_error_span(): assert ARG not in wire(spans) +@pytest.mark.parametrize("as_message", [True, False]) +@pytest.mark.parametrize("tool_key", [None, "fail", 42]) +def test_a_tool_result_shaped_like_an_mcp_use_error_is_still_a_result(tool_key, as_message): + # format_error output always carries tool= + # (langchain_adapter.py:197-205, tool_error_middleware.py:61). A tool's + # own result with the error/details/stack/code keys (an incident + # lookup, say) and no matching tool key is a successful call. + exporter, handler = _handler(capture_content=True) + run = Run(handler) + node, tool = run.start_tool("incidents", {"id": "INC-1"}) + body: Dict[str, Any] = { + "error": "DiskFull", + "details": "INCIDENT-MARKER disk full on db-1", + "stack": "n/a", + "code": "E42", + } + if tool_key is not None: + body["tool"] = tool_key + content = json.dumps(body) + output = ToolMessage(content=content, tool_call_id="call-1") if as_message else body + handler.on_tool_end(output, run_id=tool, parent_run_id=node) + run.end_node(node) + run.finish() + spans = exporter.get_finished_spans() + span = only(spans, "execute_tool incidents") + assert span.status.status_code is StatusCode.OK + assert span.events == () + assert "mcp_use.tool.error_type" not in span.attributes + assert span.attributes["gen_ai.tool.call.result"] == content + assert attrs(only(spans, AGENT))["mcp_use.agent.tool_error_count"] == 0 + + def test_tool_message_with_error_status_is_an_error_span(): exporter, handler = _handler() run = Run(handler) diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py index abbb98929..0062d2742 100644 --- a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py +++ b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py @@ -97,7 +97,9 @@ MAX_OPEN_RUNS = 10_000 # mcp-use 1.7.1 returns a failed MCP tool call as this dict -# (mcp_use/errors/error_formatting.py format_error) instead of raising. +# (mcp_use/errors/error_formatting.py format_error) instead of raising, plus +# a "tool" key holding the tool's name (agents/adapters/langchain_adapter.py +# :197-205, agents/middleware/tool_error_middleware.py:61). _MCP_USE_ERROR_KEYS = frozenset(("error", "details", "stack", "code")) _ROLES = {"human": "user", "ai": "assistant", "system": "system", "tool": "tool"} @@ -135,6 +137,8 @@ class _Run: parent: Optional[UUID] span: Optional[Span] = None chunks: int = 0 + # A tool run's name as LangChain reported it (not capped or redacted). + tool_name: Optional[str] = None def _describe(error: BaseException) -> str: @@ -258,8 +262,12 @@ def _replayed_outputs(messages: Iterable[Any]) -> List[Optional[str]]: return texts -def _mcp_use_error(output: Any) -> Optional[Mapping[str, Any]]: - """The mcp-use formatted error a tool returned, if this output is one.""" +def _mcp_use_error(output: Any, tool: Optional[str]) -> Optional[Mapping[str, Any]]: + """The mcp-use formatted error the tool named ``tool`` returned, if this + output is one. A tool's own result with the same keys but no matching + "tool" key is a result, not an error.""" + if not isinstance(tool, str): + return None candidate: Any = output content = getattr(output, "content", None) if content is not None and not isinstance(output, Mapping): @@ -276,6 +284,7 @@ def _mcp_use_error(output: Any) -> Optional[Mapping[str, Any]]: isinstance(candidate, Mapping) and _MCP_USE_ERROR_KEYS.issubset(candidate.keys()) and isinstance(candidate.get("error"), str) + and candidate.get("tool") == tool ): return candidate return None @@ -1007,10 +1016,11 @@ def on_tool_start( **kwargs: Any, ) -> None: try: - tool = None + raw_name = None if isinstance(serialized, Mapping): - tool = serialized.get("name") - tool = self._name(tool or kwargs.get("name")) or "unknown" + raw_name = serialized.get("name") + raw_name = raw_name or kwargs.get("name") + tool = self._name(raw_name) or "unknown" attributes: Dict[str, Any] = { SPAN_KIND: FiSpanKindValues.TOOL.value, OPERATION: "execute_tool", @@ -1022,7 +1032,11 @@ def on_tool_start( run, root = self._begin( run_id, parent_run_id, _TOOL, "execute_tool {0}".format(tool), attributes ) - if run is None or run.span is None: + if run is None: + return + if isinstance(raw_name, str): + run.tool_name = raw_name + if run.span is None: return arguments = _json(inputs) if isinstance(inputs, Mapping) else input_str texts: List[Optional[str]] = [arguments, input_str] @@ -1056,7 +1070,7 @@ def on_tool_end( self._end_incomplete(orphan, None) failure: Optional[_Failure] = None try: - formatted = _mcp_use_error(output) + formatted = _mcp_use_error(output, run.tool_name) if formatted is not None: error_type = str(formatted.get("error")) details = formatted.get("details") From 22f7403e24b9175afa480962d5e86cb982bd58f1 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 09:17:03 -0700 Subject: [PATCH 17/19] perf(mcp-use): scan only the finishing run's own agent run on each end Every run end walked all open runs of the handler, across all concurrent agents, to find open descendants, and again for a finished root's leftovers; evicting the oldest agent run did the same. All of it ran under the lock on the event loop, so the cost of each end grew with total concurrency. Each agent run now keeps the ids of its open runs (an insertion-ordered dict). Ending a run checks only that set, ending the root or evicting it drops exactly that set, and eviction order and the 10,000-run bound are unchanged. A repeated start of an open root keeps its set (setdefault) so its runs are still cleaned up. approved: Nikhil 2026-10-03 blanket Refs: TH-8331 Co-Authored-By: Claude Opus 5.5 --- python/frameworks/mcp-use/README.md | 3 +- .../mcp-use/tests/test_mcp_use_callback.py | 36 +++++++++++++++++ .../mcp-use/traceai_mcp_use/_callback.py | 39 +++++++++++++------ 3 files changed, 65 insertions(+), 13 deletions(-) diff --git a/python/frameworks/mcp-use/README.md b/python/frameworks/mcp-use/README.md index 11a3e5f05..6175c3d82 100644 --- a/python/frameworks/mcp-use/README.md +++ b/python/frameworks/mcp-use/README.md @@ -276,7 +276,8 @@ the TypeScript agent). passed through it. At most 32 messages and 16 tool calls per message are recorded per LLM span, and 128 chunk events per streamed LLM span. - One callback tracks at most 10,000 open runs. Past that, the oldest - agent run is ended with `mcp_use.incomplete` and forgotten. + agent run is ended with `mcp_use.incomplete` and forgotten. Ending a run + looks only at the open runs of its own agent run, not at other agents'. - With `use_server_manager=True`, mcp-use restarts the graph run when the tool set changes during a run, up to 3 times (`mcp_use/agents/mcpagent.py:741-876`); each restart is a new agent span. diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py index 47841eb0f..589fae1e7 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py @@ -708,6 +708,42 @@ def test_open_runs_are_bounded(monkeypatch): assert handler._runs == {} +def test_ending_a_run_only_looks_at_its_own_agent_run(monkeypatch): + # The open-descendant check on each run end covers the runs under the + # same agent run, not every open run of every agent on the handler. + exporter, handler = _handler() + busy = Run(handler) + for _ in range(50): + busy.start_tool("slow", {}) + looked_at: List[UUID] = [] + descends = handler._descends + + def spy(run: Any, ancestor: UUID) -> bool: + looked_at.append(run.root) + return descends(run, ancestor) + + monkeypatch.setattr(handler, "_descends", spy) + journey = _journey(handler) + assert looked_at and set(looked_at) == {journey.root} + assert len(handler._runs) == 101 # busy: the root, 50 tools nodes, 50 tools + exporter.clear() + busy.finish() + spans = exporter.get_finished_spans() + assert [span.name for span in spans].count("execute_tool slow") == 50 + assert all(span.attributes.get("mcp_use.incomplete") for span in spans if span.name != AGENT) + assert handler._runs == {} and len(handler._roots) == 0 + + +def test_a_repeated_start_of_an_open_agent_run_still_cleans_up(): + exporter, handler = _handler() + run = Run(handler) + run.start_tool("slow", {}) + handler.on_chain_start({}, {"messages": []}, run_id=run.root) # same id again + run.finish() + assert handler._runs == {} and len(handler._roots) == 0 + assert only(exporter.get_finished_spans(), "execute_tool slow").attributes["mcp_use.incomplete"] is True + + def test_the_current_span_parents_the_agent_span(): exporter, handler = _handler() tracer = new_provider()[1].get_tracer("test") diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py index 0062d2742..28b3bc7f2 100644 --- a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py +++ b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py @@ -127,6 +127,9 @@ class _Root: llm_calls: int = 0 tool_calls: int = 0 tool_errors: int = 0 + # Ids of the open runs under this agent run, its own included (a dict + # as an insertion-ordered set), so ending a run looks at its own tree only. + runs: Dict[UUID, None] = field(default_factory=dict) @dataclass @@ -445,7 +448,8 @@ def _begin( root_id = run_id if kind == _CHAIN: kind = _AGENT - self._roots[root_id] = _Root() + # setdefault: a repeated start of an open root keeps its index. + self._roots.setdefault(root_id, _Root()) else: root_id = parent.root root = self._roots.get(root_id) @@ -457,6 +461,7 @@ def _begin( root.tool_calls += 1 run = _Run(run_id=run_id, kind=kind, root=root_id, parent=parent_run_id) self._runs[run_id] = run + root.runs[run_id] = None traced_parent = self._traced_ancestor(parent) while len(self._runs) > MAX_OPEN_RUNS and len(self._roots) > 1: oldest = next(iter(self._roots)) @@ -490,11 +495,12 @@ def _traced_ancestor(self, run: Optional[_Run]) -> Optional[_Run]: def _forget_root(self, root_id: UUID) -> List[_Run]: """Drop an agent run and every run under it; caller holds the lock.""" - self._roots.pop(root_id, None) - dropped = [run for run in self._runs.values() if run.root == root_id] - for run in dropped: - self._runs.pop(run.run_id, None) - return [run for run in dropped if run.span is not None] + root = self._roots.pop(root_id, None) + if root is None: + return [] + dropped = [self._runs.pop(run_id, None) for run_id in root.runs] + root.runs.clear() + return [run for run in dropped if run is not None and run.span is not None] def _descends(self, run: _Run, ancestor: UUID) -> bool: parent = run.parent @@ -513,15 +519,24 @@ def _pop(self, run_id: UUID) -> Tuple[Optional[_Run], Optional[_Root], List[_Run run = self._runs.pop(run_id, None) if run is None: return None, None, [] - orphans = [other for other in self._runs.values() if self._descends(other, run_id)] - for orphan in orphans: - self._runs.pop(orphan.run_id, None) root = self._roots.get(run.root) + if root is None: # not expected: a root's runs leave with it + return run, None, [] + root.runs.pop(run_id, None) if run.run_id == run.root: + # The agent run ended: whatever is still open under it. self._roots.pop(run.root, None) - for leftover in [other for other in self._runs.values() if other.root == run.root]: - self._runs.pop(leftover.run_id, None) - orphans.append(leftover) + orphans = [self._runs.pop(other) for other in root.runs if other in self._runs] + root.runs.clear() + else: + orphans = [ + self._runs[other] + for other in root.runs + if other in self._runs and self._descends(self._runs[other], run_id) + ] + for orphan in orphans: + self._runs.pop(orphan.run_id, None) + root.runs.pop(orphan.run_id, None) return run, root, [orphan for orphan in orphans if orphan.span is not None] # ----------------------------------------------------------------- ending From adaee83263c51ced00be2ea007e38656db040f24 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Fri, 16 Oct 2026 22:45:23 -0700 Subject: [PATCH 18/19] fix(mcp-use): follow TraceConfig for content and never export stack traces or local paths Re-verification of TH-8331 (2026-10-16) against a real model and a real stdio MCP server found two privacy problems: - Content used a package-specific switch (capture_content=False by default). Every traceAI instrumentor follows TraceConfig: content is on by default and FI_HIDE_INPUTS / FI_HIDE_OUTPUTS (or TraceConfig) hide it. The capture_content argument is removed; there is no second switch. With either hide flag the error message is dropped (type only), because error text also quotes a server's own data that no hide set knows. - With content on, the export carried mcp-use's format_error "stack" (a full traceback with absolute paths and the user name) in the next LLM span's tool message, exception.stacktrace on exception events, and absolute paths in status descriptions. Now no exception.stacktrace is recorded, tracebacks are cut ([stack trace removed]; inside JSON only the string value, so the rest of format_error's dict survives), absolute local paths become [path], and this machine's user and host name become [user] / [host], in every text the callback writes, in both modes. Tests updated for the new default and extended (stack/path/user/host, either hide flag drops error text, no capture_content argument). 17 of the updated agent/contract/example tests fail on 22f7403. Co-Authored-By: Claude Opus 5.5 --- .../mcp-use/tests/test_mcp_use_agent.py | 59 +++-- .../mcp-use/tests/test_mcp_use_callback.py | 203 +++++++++++++----- .../mcp-use/tests/test_mcp_use_contract.py | 37 +++- .../mcp-use/tests/test_mcp_use_example.py | 5 +- .../mcp-use/tests/test_mcp_use_isolation.py | 2 +- .../mcp-use/traceai_mcp_use/_callback.py | 106 ++++----- .../mcp-use/traceai_mcp_use/_text.py | 107 +++++++-- 7 files changed, 341 insertions(+), 178 deletions(-) diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_agent.py b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py index 8e1c98458..28e7aa4e7 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_agent.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py @@ -41,7 +41,6 @@ wire, ) from fi_instrumentation import ( - REDACTED_VALUE, TraceConfig, suppress_tracing, using_metadata, @@ -63,6 +62,10 @@ def _traced(**kwargs: Any): return exporter, FutureAGICallback(tracer_provider=provider, **kwargs) +# Content off: the standard traceAI hide flags (FI_HIDE_INPUTS / FI_HIDE_OUTPUTS). +HIDDEN = TraceConfig(hide_inputs=True, hide_outputs=True) + + def _assert_j1_tree(spans) -> None: """One agent root; LLM, tool, LLM as its children, in one trace.""" assert sorted(span.name for span in spans) == sorted([AGENT, LLM_SPAN, LLM_SPAN, "execute_tool add"]) @@ -136,14 +139,16 @@ def test_a_streamed_completion_is_one_llm_span(method): assert final_turn.attributes["mcp_use.llm.chunk_count"] >= 5 assert {event.name for event in final_turn.events} == {"mcp_use.llm.chunk"} assert len(final_turn.events) == final_turn.attributes["mcp_use.llm.chunk_count"] - assert "ANSWER-MARKER" not in wire(spans) + # The answer is the LLM span's output, never a chunk event's. + for event in final_turn.events: + assert set(event.attributes) == {"mcp_use.llm.chunk.index"} if method == "run": assert result == ANSWER -def test_content_and_keys_are_not_recorded_by_default(monkeypatch): +def test_content_and_keys_are_not_recorded_with_the_hide_flags(monkeypatch): monkeypatch.setenv(ENV_SECRET_NAME, ENV_SECRET) - exporter, handler = _traced() + exporter, handler = _traced(config=HIDDEN) script = [tool_call("echo", {"text": ARG + " " + ENV_SECRET}), answer()] run_agent(script, [handler], query=PROMPT + " " + ENV_SECRET) spans = exporter.get_finished_spans() @@ -153,9 +158,9 @@ def test_content_and_keys_are_not_recorded_by_default(monkeypatch): assert marker not in text, marker -def test_capture_content_records_content_without_secrets(monkeypatch): +def test_content_is_recorded_by_default_without_secrets(monkeypatch): monkeypatch.setenv(ENV_SECRET_NAME, ENV_SECRET) - exporter, handler = _traced(capture_content=True, redact=[EXPLICIT_SECRET]) + exporter, handler = _traced(redact=[EXPLICIT_SECRET]) leak = "{0} {1} {2}".format(ENV_SECRET, EXPLICIT_SECRET, LLM_KEY) script = [tool_call("echo", {"text": ARG + " " + leak}), answer()] run_agent(script, [handler], query=PROMPT + " " + leak) @@ -175,7 +180,7 @@ def test_mcp_tool_error_fails_the_tool_span_and_the_agent_recovers(): # mcp-use 1.7.1 turns a failed MCP call into a formatted error the LLM # reads (agents/adapters/langchain_adapter.py:181-206), so the run # succeeds; the tool span is ERROR and the agent counts the error. - exporter, handler = _traced() + exporter, handler = _traced(config=HIDDEN) script = [tool_call("fail", {"reason": ARG}), answer()] assert run_agent(script, [handler]) == ANSWER spans = exporter.get_finished_spans() @@ -190,20 +195,29 @@ def test_mcp_tool_error_fails_the_tool_span_and_the_agent_recovers(): assert "tool failed because" not in wire(spans) and ARG not in wire(spans) -def test_tool_error_text_is_recorded_with_capture_and_scrubbed_with_hide_inputs(): - exporter, handler = _traced(capture_content=True) +def test_tool_error_text_is_recorded_by_default_and_dropped_with_hide_inputs(): + exporter, handler = _traced() run_agent([tool_call("fail", {"reason": ARG}), answer()], [handler]) - tool = only(exporter.get_finished_spans(), "execute_tool fail") + spans = exporter.get_finished_spans() + tool = only(spans, "execute_tool fail") assert tool.status.description == "RuntimeError: Error executing tool fail: tool failed because " + ARG - - exporter, handler = _traced(capture_content=True, config=TraceConfig(hide_inputs=True)) + (event,) = tool.events + assert "exception.stacktrace" not in event.attributes + # mcp-use's format_error "stack" reaches the next model call as the + # tool message; the trace is cut there, the rest of the JSON kept. + final = sorted(spans_named(spans, LLM_SPAN), key=lambda span: span.start_time)[-1] + replayed = json.loads(attrs(final)["gen_ai.input.messages.3.message.content"]) + assert replayed["stack"] == "[stack trace removed]" and replayed["tool"] == "fail" + for needle in ("Traceback", 'File "', "site-packages"): + assert needle not in wire(spans), needle + + exporter, handler = _traced(config=TraceConfig(hide_inputs=True)) run_agent([tool_call("fail", {"reason": ARG}), answer()], [handler]) spans = exporter.get_finished_spans() tool = only(spans, "execute_tool fail") - assert tool.status.description == "RuntimeError: Error executing tool fail: tool failed because " + REDACTED_VALUE + assert tool.status.description == "RuntimeError" (event,) = tool.events - assert ARG not in event.attributes["exception.message"] - assert ARG not in event.attributes["exception.stacktrace"] + assert dict(event.attributes) == {"exception.type": "RuntimeError"} assert ARG not in wire(spans) and "PROMPT-MARKER" not in wire(spans) @@ -225,7 +239,7 @@ def _three_turns(method: str, external: bool, **options: Any) -> List[Any]: turns 2 and 3 send turn 1's messages to the model again. Turn 2 echoes and answers; in turn 3 the model raises an error quoting the history. """ - exporter, handler = _traced(capture_content=True, **options) + exporter, handler = _traced(**options) script = add_script() + [ tool_call("echo", {"text": "SECOND-ARG"}, "call-2"), answer(SECOND_ANSWER), @@ -281,14 +295,13 @@ def test_hide_outputs_keeps_earlier_turns_out_of_later_turns(method, external): assert values["gen_ai.input.messages.5.message.content"] == "SECOND-PROMPT echo it" for index in (2, 3, 4): assert "gen_ai.input.messages.{0}.message.content".format(index) not in values - # Inputs are still recorded, and the error text is kept with the earlier - # outputs taken out of it. + # Inputs are still recorded; the error text (it quotes the earlier + # outputs) is dropped, the exception type kept. assert attrs(only(third, AGENT))["input.value"] == "THIRD-PROMPT" for span in (only(third, AGENT), only(third, LLM_SPAN)): - assert span.status.description.startswith("ValueError: model rejected the history: ") - assert REDACTED_VALUE in span.status.description + assert span.status.description == "ValueError" (event,) = span.events - assert REDACTED_VALUE in event.attributes["exception.message"] + assert dict(event.attributes) == {"exception.type": "ValueError"} @pytest.mark.parametrize("method,external", _TURN_MODES) @@ -501,7 +514,7 @@ def get_tracer(self, *args: Any, **kwargs: Any) -> Any: def start_span(self, *args: Any, **kwargs: Any) -> Any: raise RuntimeError("tracer is broken") - handler = FutureAGICallback(tracer_provider=Broken(), capture_content=True) + handler = FutureAGICallback(tracer_provider=Broken()) assert run_agent(add_script(), [handler]) == ANSWER for method in ("stream", "stream_events"): run_agent(add_script(), [handler], method=method) @@ -522,7 +535,7 @@ def add(a: int, b: int) -> str: shapes = [] for mode in ("sync", "async"): - exporter, handler = _traced(capture_content=True) + exporter, handler = _traced() graph = create_agent(ChatFake(script=add_script()), [add]) inputs = {"messages": [("user", PROMPT)]} config: Dict[str, Any] = {"callbacks": [handler]} diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py index 589fae1e7..32f6634d1 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py @@ -9,8 +9,10 @@ from __future__ import annotations import asyncio +import getpass import json import os +import socket from typing import Any, Dict, List, Optional, Sequence from uuid import UUID, uuid4 @@ -53,8 +55,8 @@ from traceai_mcp_use._text import ( MAX_ERROR_BYTES, MAX_NAME_BYTES, - MAX_STACKTRACE_BYTES, MAX_VALUE_BYTES, + scrub_local, ) TOOL_SPAN = "execute_tool add" @@ -169,6 +171,21 @@ def _handler(**kwargs: Any): return exporter, FutureAGICallback(tracer_provider=provider, **kwargs) +# Content off: the standard traceAI hide flags (FI_HIDE_INPUTS / FI_HIDE_OUTPUTS). +HIDDEN = TraceConfig(hide_inputs=True, hide_outputs=True) +_CONTENT_PREFIXES = ("input.", "output.", "gen_ai.input.", "gen_ai.output.") +_CONTENT_KEYS = ("gen_ai.tool.call.arguments", "gen_ai.tool.call.result") + + +def _meta(span: Any) -> Dict[str, Any]: + """A span's attributes without the content ones (messages, arguments, results).""" + return { + key: value + for key, value in attrs(span).items() + if not key.startswith(_CONTENT_PREFIXES) and key not in _CONTENT_KEYS + } + + def test_three_span_kinds_from_a_fake_callback_sequence(): exporter, handler = _handler() _journey(handler) @@ -186,14 +203,14 @@ def test_three_span_kinds_from_a_fake_callback_sequence(): assert parent_id(child) == agent.context.span_id assert child.context.trace_id == agent.context.trace_id - assert attrs(agent) == { + assert _meta(agent) == { "gen_ai.span.kind": "AGENT", "gen_ai.operation.name": "invoke_agent", "mcp_use.agent.llm_call_count": 2, "mcp_use.agent.tool_call_count": 1, "mcp_use.agent.tool_error_count": 0, } - assert attrs(first_llm) == { + assert _meta(first_llm) == { "gen_ai.span.kind": "LLM", "gen_ai.operation.name": "chat", "gen_ai.request.model": MODEL, @@ -211,7 +228,7 @@ def test_three_span_kinds_from_a_fake_callback_sequence(): } assert attrs(second_llm)["mcp_use.llm.input_message_count"] == 4 assert attrs(second_llm)["gen_ai.response.finish_reasons"] == ("stop",) - assert attrs(tool) == { + assert _meta(tool) == { "gen_ai.span.kind": "TOOL", "gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": "add", @@ -222,20 +239,24 @@ def test_three_span_kinds_from_a_fake_callback_sequence(): assert span.events == () -def test_content_is_off_by_default(): - exporter, handler = _handler() +def test_hide_inputs_and_outputs_turn_content_off(): + exporter, handler = _handler(config=HIDDEN) _journey(handler) text = wire(exporter.get_finished_spans()) for marker in CONTENT_MARKERS + (LLM_KEY, "You are an MCP agent"): assert marker not in text, marker for span in exporter.get_finished_spans(): - for key in attrs(span): - assert not key.startswith(("input.", "output.", "gen_ai.input.", "gen_ai.output.")) - assert key not in ("gen_ai.tool.call.arguments", "gen_ai.tool.call.result") + for key, value in attrs(span).items(): + if key in ("input.value", "output.value"): + assert value == REDACTED_VALUE, key + continue + assert not key.startswith(_CONTENT_PREFIXES), key + assert key not in _CONTENT_KEYS, key -def test_capture_content_records_prompt_messages_arguments_and_results(): - exporter, handler = _handler(capture_content=True) +def test_content_is_recorded_by_default(): + # The traceAI TraceConfig convention: content on unless a hide flag is set. + exporter, handler = _handler() _journey(handler) spans = exporter.get_finished_spans() agent = attrs(only(spans, AGENT)) @@ -268,7 +289,7 @@ def test_capture_content_records_prompt_messages_arguments_and_results(): def test_hide_inputs_drops_every_input_including_tool_arguments(): - exporter, handler = _handler(capture_content=True, config=TraceConfig(hide_inputs=True)) + exporter, handler = _handler(config=TraceConfig(hide_inputs=True)) _journey(handler) spans = exporter.get_finished_spans() text = wire(spans) @@ -285,7 +306,7 @@ def test_hide_inputs_drops_every_input_including_tool_arguments(): def test_hide_outputs_drops_every_output(): - exporter, handler = _handler(capture_content=True, config=TraceConfig(hide_outputs=True)) + exporter, handler = _handler(config=TraceConfig(hide_outputs=True)) _journey(handler) spans = exporter.get_finished_spans() text = wire(spans) @@ -301,7 +322,7 @@ def test_hide_outputs_drops_every_output(): def test_hide_outputs_drops_earlier_turns_replayed_as_input(): # mcp-use's memory (or external_history) sends earlier turns' replies # and tool results back as input: under hide_outputs they are outputs. - exporter, handler = _handler(capture_content=True, config=TraceConfig(hide_outputs=True)) + exporter, handler = _handler(config=TraceConfig(hide_outputs=True)) earlier = [ HumanMessage(content="OLD-PROMPT"), AIMessage( @@ -337,11 +358,12 @@ def test_hide_outputs_drops_earlier_turns_replayed_as_input(): assert call + "function.arguments" not in llm_span assert llm_span["input.value"] == PROMPT for span in spans: - assert span.status.description == "ValueError: cannot use {0}, {0}, {0} or {0}".format(REDACTED_VALUE) + # Error text is content: dropped with either hide flag. + assert span.status.description == "ValueError" def test_hide_outputs_with_a_replayed_reply_last_redacts_the_llm_input_value(): - exporter, handler = _handler(capture_content=True, config=TraceConfig(hide_outputs=True)) + exporter, handler = _handler(config=TraceConfig(hide_outputs=True)) run = Run(handler) run.history.append(ToolMessage(content="OLD-RESULT", tool_call_id="c0")) run.llm(AIMessage(content=ANSWER)) @@ -354,7 +376,7 @@ def test_hide_outputs_with_a_replayed_reply_last_redacts_the_llm_input_value(): def test_hide_flags_are_read_from_the_environment(monkeypatch): monkeypatch.setenv("FI_HIDE_INPUTS", "true") monkeypatch.setenv("FI_HIDE_OUTPUTS", "true") - exporter, handler = _handler(capture_content=True) + exporter, handler = _handler() _journey(handler) text = wire(exporter.get_finished_spans()) for marker in CONTENT_MARKERS: @@ -377,38 +399,103 @@ def _failing_tool(handler: FutureAGICallback, error: BaseException, query: str = run.finish() -def test_tool_error_records_type_only_by_default(): +def test_tool_error_records_its_message_by_default_and_never_a_stack_trace(): exporter, handler = _handler() _failing_tool(handler, ValueError("tool failed because " + ARG)) spans = exporter.get_finished_spans() tool = only(spans, "execute_tool fail") assert tool.status.status_code is StatusCode.ERROR - assert tool.status.description == "ValueError" + assert tool.status.description == "ValueError: tool failed because " + ARG (event,) = tool.events assert event.name == "exception" - assert dict(event.attributes) == {"exception.type": "ValueError"} + assert dict(event.attributes) == { + "exception.type": "ValueError", + "exception.message": "tool failed because " + ARG, + } agent = only(spans, AGENT) # The agent recovered (mcp-use returns tool errors to the LLM). assert agent.status.status_code is StatusCode.OK assert attrs(agent)["mcp_use.agent.tool_error_count"] == 1 - assert "tool failed because" not in wire(spans) + assert "Traceback" not in wire(spans) and 'File "' not in wire(spans) -def test_tool_error_text_with_capture_and_hide_inputs_scrubs_the_arguments(): - exporter, handler = _handler(capture_content=True, config=TraceConfig(hide_inputs=True)) - error = ValueError("tool failed because " + ARG + " jane.doe@example.com") - _failing_tool(handler, error) - tool = only(exporter.get_finished_spans(), "execute_tool fail") - (event,) = tool.events - for text in (tool.status.description, event.attributes["exception.message"], event.attributes["exception.stacktrace"]): - assert "tool failed because" in text - assert ARG not in text, text - assert "jane.doe" not in text, text - assert REDACTED_VALUE in text +@pytest.mark.parametrize( + "config", + [TraceConfig(hide_inputs=True), TraceConfig(hide_outputs=True), HIDDEN], + ids=["hide_inputs", "hide_outputs", "both"], +) +def test_either_hide_flag_drops_the_error_text(config): + # Error text quotes tool arguments, tool results and a server's own data. + exporter, handler = _handler(config=config) + _failing_tool(handler, ValueError("tool failed because " + ARG + " SERVER-SIDE-TEXT")) + spans = exporter.get_finished_spans() + tool = only(spans, "execute_tool fail") + assert tool.status.status_code is StatusCode.ERROR + assert tool.status.description == "ValueError" + assert dict(tool.events[0].attributes) == {"exception.type": "ValueError"} + assert "SERVER-SIDE-TEXT" not in wire(spans) and "tool failed because" not in wire(spans) + + +def test_stack_traces_paths_user_and_host_are_never_recorded(): + # mcp-use sends a failed MCP call back to the model as format_error()'s + # dict, whose "stack" is a full traceback; exception text quotes paths. + user, host = getpass.getuser(), socket.gethostname() + home = os.path.expanduser("~") + exporter, handler = _handler() + run = Run(handler) + run.llm(AIMessage(content="", tool_calls=[{"name": "fail", "args": {"topic": "t"}, "id": "c", "type": "tool_call"}])) + node, tool = run.start_tool("fail", {"topic": "t"}, "c") + try: + raise OSError("cannot open {0}/notes/archive.db on {1} as {2}".format(home, host, user)) + except OSError as raised: + handler.on_tool_error(raised, run_id=tool, parent_run_id=node) + run.end_node(node) + formatted = json.dumps( + { + "error": "RuntimeError", + "details": "Error executing tool fail: down at /home/someone/srv/db.sqlite", + "stack": 'Traceback (most recent call last):\n File "{0}/x.py", line 3, in f\nRuntimeError: down'.format(home), + "code": "UNKNOWN", + "tool": "fail", + } + ) + run.history.append(ToolMessage(content=formatted, tool_call_id="c", name="fail")) + run.llm(AIMessage(content="It failed at C:\\Users\\someone\\x.txt")) + run.finish(text="done") + spans = exporter.get_finished_spans() + text = wire(spans) + for needle in ("Traceback", 'File "', "/notes/archive.db", "/home/someone", "C:\\Users", home): + assert needle not in text, needle + if len(user) >= 4 and user.lower() not in ("root", "runner"): + assert user not in text + if len(host.split(".")[0]) >= 4 and host != "localhost": + assert host.split(".")[0] not in text + span = only(spans, "execute_tool fail") + assert "exception.stacktrace" not in span.events[0].attributes + assert span.status.description.startswith("OSError: cannot open [path] on ") + replayed = attrs(spans_named(spans, LLM_SPAN)[1])["gen_ai.input.messages.3.message.content"] + assert json.loads(replayed) == { + "error": "RuntimeError", + "details": "Error executing tool fail: down at [path]", + "stack": "[stack trace removed]", + "code": "UNKNOWN", + "tool": "fail", + } + + +def test_scrub_local_keeps_ordinary_text(): + for text in ( + "see https://example.com/home/docs and /api/v1/items", + "2 + 3 = 5", + "the root cause was a timeout", + ): + assert scrub_local(text) == text + assert scrub_local("x Traceback (most recent call last):\n File \"a.py\"") == "x [stack trace removed]" + assert scrub_local("open /tmp/abc/d.txt, then /var/log/x.") == "open [path], then [path]" def test_pii_redaction_covers_attributes_status_and_exception_event(): - exporter, handler = _handler(capture_content=True, config=TraceConfig(pii_redaction=True)) + exporter, handler = _handler(config=TraceConfig(pii_redaction=True)) _failing_tool(handler, ValueError("mail jane.doe@example.com failed"), query="ask jane.doe@example.com") spans = exporter.get_finished_spans() assert "jane.doe@example.com" not in wire(spans) @@ -420,7 +507,7 @@ def test_pii_redaction_covers_attributes_status_and_exception_event(): def test_secrets_are_removed_from_attributes_status_and_events(monkeypatch): monkeypatch.setenv(ENV_SECRET_NAME, ENV_SECRET) - exporter, handler = _handler(capture_content=True, redact=[EXPLICIT_SECRET]) + exporter, handler = _handler(redact=[EXPLICIT_SECRET]) leaky = "keys {0} {1} {2} Authorization: Bearer abcdefghijklmnop".format( ENV_SECRET, EXPLICIT_SECRET, LLM_KEY ) @@ -442,7 +529,7 @@ def test_secrets_are_removed_from_attributes_status_and_events(monkeypatch): def test_secrets_are_removed_from_names_with_content_off(monkeypatch): monkeypatch.setenv(ENV_SECRET_NAME, ENV_SECRET) - exporter, handler = _handler() + exporter, handler = _handler(config=HIDDEN) run = Run(handler) node, tool = run.start_tool("tool-" + ENV_SECRET, {}, call_id="id-" + ENV_SECRET) handler.on_tool_end("done", run_id=tool, parent_run_id=node) @@ -452,7 +539,7 @@ def test_secrets_are_removed_from_names_with_content_off(monkeypatch): def test_size_caps(): - exporter, handler = _handler(capture_content=True) + exporter, handler = _handler() long_text = "é" * 10_000 # 2 bytes each in UTF-8 run = Run(handler, query=long_text) many = [HumanMessage(content="m{0}".format(index)) for index in range(40)] @@ -464,8 +551,8 @@ def test_size_caps(): run.llm(AIMessage(content=long_text, tool_calls=calls)) long_name = "n" * 1000 node, tool = run.start_tool(long_name, {"text": long_text}) - # A traceback longer than the cap: the cause's 20 KB message comes - # first, the raised exception's line last. The cap keeps the end. + # A chained exception with a long message: no stack trace is recorded, + # only the message, cut at its cap. try: try: raise RuntimeError("c" * 20_000) @@ -500,13 +587,12 @@ def test_size_caps(): message = event.attributes["exception.message"] assert message == "x" * MAX_ERROR_BYTES assert tool_span.status.description == "ValueError: " + "x" * MAX_ERROR_BYTES - stacktrace = event.attributes["exception.stacktrace"] - assert MAX_STACKTRACE_BYTES - 4 <= len(stacktrace.encode("utf-8")) <= MAX_STACKTRACE_BYTES - assert stacktrace.rstrip().endswith("LAST-LINE-END") + assert "exception.stacktrace" not in event.attributes + assert "c" * 100 not in wire(spans) # the cause is not recorded def test_streamed_tokens_are_events_on_one_llm_span_with_capped_count(): - exporter, handler = _handler(capture_content=True) + exporter, handler = _handler() run = Run(handler) node = run.node("model") llm = uuid4() @@ -527,8 +613,9 @@ def test_streamed_tokens_are_events_on_one_llm_span_with_capped_count(): assert "TOKEN-TEXT" not in wire(spans) -def test_llm_error_fails_the_llm_and_agent_spans(): - exporter, handler = _handler() +@pytest.mark.parametrize("hidden", [False, True]) +def test_llm_error_fails_the_llm_and_agent_spans(hidden): + exporter, handler = _handler(**({"config": HIDDEN} if hidden else {})) run = Run(handler) node = run.node("model") llm = uuid4() @@ -541,18 +628,22 @@ def test_llm_error_fails_the_llm_and_agent_spans(): assert sorted(span.name for span in spans) == sorted([AGENT, LLM_SPAN]) for span in spans: assert span.status.status_code is StatusCode.ERROR - assert span.status.description == "ConnectionError" + if hidden: + assert span.status.description == "ConnectionError" + else: + assert span.status.description == "ConnectionError: provider down " + PROMPT assert [event.name for event in span.events] == ["exception"] assert span.events[0].attributes["exception.type"] == "ConnectionError" + assert "exception.stacktrace" not in span.events[0].attributes assert attrs(only(spans, AGENT))["mcp_use.agent.llm_call_count"] == 1 - assert PROMPT not in wire(spans) + assert (PROMPT in wire(spans)) is not hidden def test_mcp_use_formatted_tool_error_is_an_error_span(): # mcp-use 1.7.1 returns a failed MCP call as format_error()'s dict, as # JSON in the ToolMessage (agents/adapters/langchain_adapter.py:181-206). for capture in (False, True): - exporter, handler = _handler(capture_content=capture) + exporter, handler = _handler(**({} if capture else {"config": HIDDEN})) run = Run(handler) node, tool = run.start_tool("fail", {"reason": ARG}) content = json.dumps( @@ -575,9 +666,10 @@ def test_mcp_use_formatted_tool_error_is_an_error_span(): assert event.attributes["exception.type"] == "RuntimeError" assert "gen_ai.tool.call.result" not in span.attributes assert attrs(only(spans, AGENT))["mcp_use.agent.tool_error_count"] == 1 + assert "exception.stacktrace" not in event.attributes + assert "Traceback" not in wire(spans) # format_error's "stack" is never read if capture: assert span.status.description == "RuntimeError: Error executing tool fail: tool failed because " + ARG - assert event.attributes["exception.stacktrace"].endswith("RuntimeError: boom") else: assert span.status.description == "RuntimeError" assert ARG not in wire(spans) @@ -590,7 +682,7 @@ def test_a_tool_result_shaped_like_an_mcp_use_error_is_still_a_result(tool_key, # (langchain_adapter.py:197-205, tool_error_middleware.py:61). A tool's # own result with the error/details/stack/code keys (an incident # lookup, say) and no matching tool key is a successful call. - exporter, handler = _handler(capture_content=True) + exporter, handler = _handler() run = Run(handler) node, tool = run.start_tool("incidents", {"id": "INC-1"}) body: Dict[str, Any] = { @@ -662,7 +754,7 @@ def test_a_run_whose_parent_ended_first_is_ended_as_incomplete(): def test_text_completion_models_get_an_llm_span(): - exporter, handler = _handler(capture_content=True) + exporter, handler = _handler() run = Run(handler) node = run.node("model") llm = uuid4() @@ -828,7 +920,7 @@ def end(self, *args: Any, **kwargs: Any) -> None: "provider", [_Broken(), _BrokenSpans(on_attribute=False), _BrokenSpans(on_attribute=True)] ) def test_tracer_failures_never_reach_the_caller(provider): - handler = FutureAGICallback(tracer_provider=provider, capture_content=True) + handler = FutureAGICallback(tracer_provider=provider) _journey(handler) _failing_tool(handler, ValueError("boom")) run = Run(handler) @@ -842,7 +934,7 @@ def test_tracer_failures_never_reach_the_caller(provider): @pytest.mark.parametrize("capture", [False, True]) def test_malformed_callback_payloads_never_raise(capture): - exporter, handler = _handler(capture_content=capture) + exporter, handler = _handler(**({} if capture else {"config": HIDDEN})) root, child = uuid4(), uuid4() weird: List[Any] = [None, object(), 42, "text", [object()], {"messages": object()}] for value in weird: @@ -865,8 +957,9 @@ def test_malformed_callback_payloads_never_raise(capture): def test_constructor_validates_its_arguments(): with pytest.raises(TypeError, match="TraceConfig"): FutureAGICallback(config={"hide_inputs": True}) # type: ignore[arg-type] + # No package-specific privacy switch: TraceConfig / FI_HIDE_* only. with pytest.raises(TypeError, match="capture_content"): - FutureAGICallback(capture_content="yes") # type: ignore[arg-type] + FutureAGICallback(capture_content=True) # type: ignore[call-arg] with pytest.raises(TypeError, match="not a string"): FutureAGICallback(redact="secret-value") # type: ignore[arg-type] with pytest.raises(TypeError, match="strings"): @@ -877,7 +970,7 @@ def test_hide_flags_fail_closed_when_a_run_remembers_too_much(monkeypatch): import traceai_mcp_use._text as text_module monkeypatch.setattr(text_module, "MAX_KNOWN_TEXTS", 4) - exporter, handler = _handler(capture_content=True, config=TraceConfig(hide_inputs=True)) + exporter, handler = _handler(config=TraceConfig(hide_inputs=True)) _journey(handler) _failing_tool(handler, ValueError("tool failed because " + ARG)) spans = exporter.get_finished_spans() @@ -892,7 +985,7 @@ def test_hide_flags_fail_closed_when_a_run_remembers_too_much(monkeypatch): def test_an_internal_failure_in_any_callback_never_reaches_langchain(monkeypatch): # Every public callback catches its own failure, past the inner guards # the other tests exercise: make the bookkeeping itself raise. - exporter, handler = _handler(capture_content=True) + exporter, handler = _handler() def boom(*args: Any, **kwargs: Any) -> Any: raise RuntimeError("internal failure") diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_contract.py b/python/frameworks/mcp-use/tests/test_mcp_use_contract.py index 16944af1b..a92ce4da2 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_contract.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_contract.py @@ -137,7 +137,8 @@ def test_agent_runs_reach_the_collector_contract(monkeypatch): tool = next(span for span in failed_children if span["name"] == "execute_tool fail") assert tool["parentSpanId"] == failed_agent["spanId"] assert tool["status"]["code"] == "STATUS_CODE_ERROR" - assert tool["status"]["message"] == "RuntimeError" + # Default TraceConfig: the error text is content and is recorded. + assert tool["status"]["message"] == "RuntimeError: Error executing tool fail: tool failed because " + ARG assert [event["name"] for event in tool["events"]] == ["exception"] assert failed_agent["status"]["code"] == "STATUS_CODE_OK" assert int(_attrs(failed_agent)["mcp_use.agent.tool_error_count"]) == 1 @@ -148,7 +149,10 @@ def test_agent_runs_reach_the_collector_contract(monkeypatch): assert set(tools.values()) == {1} -def test_no_key_or_content_is_exported_by_default(monkeypatch): +def test_no_key_or_content_is_exported_with_the_hide_flags(monkeypatch): + # Content off the standard traceAI way: FI_HIDE_INPUTS / FI_HIDE_OUTPUTS. + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + monkeypatch.setenv("FI_HIDE_OUTPUTS", "true") with Receiver() as receiver: spans = _journey(receiver, monkeypatch) @@ -157,21 +161,32 @@ def test_no_key_or_content_is_exported_by_default(monkeypatch): for secret in (FI_API_KEY, FI_SECRET_KEY, LLM_KEY, ENV_SECRET) + CONTENT_MARKERS: assert secret not in text, secret for span in spans: - for key in _attrs(span): - assert not key.startswith(("input.", "output.", "gen_ai.input.", "gen_ai.output.")) - - -def test_opt_in_capture_reaches_the_collector_without_keys(monkeypatch): - # Control for the test above: the same journey with capture_content - # carries the prompt, arguments, results and answer, still without keys. + for key, value in _attrs(span).items(): + if key in ("input.value", "output.value"): + assert value == "__REDACTED__", key + continue + assert not key.startswith(("input.", "output.", "gen_ai.input.", "gen_ai.output.")), key + assert key not in ("gen_ai.tool.call.arguments", "gen_ai.tool.call.result"), key + for needle in ("Traceback", 'File "', "site-packages", "exception.stacktrace"): + assert needle not in text, needle + + +def test_content_reaches_the_collector_by_default_without_keys(monkeypatch): + # Control for the test above: with the default TraceConfig the same + # journey carries the prompt, arguments, results and answer, still + # without keys, stack traces or local paths. + monkeypatch.delenv("FI_HIDE_INPUTS", raising=False) + monkeypatch.delenv("FI_HIDE_OUTPUTS", raising=False) with Receiver() as receiver: - spans = _journey(receiver, monkeypatch, capture_content=True) + spans = _journey(receiver, monkeypatch) text = json.dumps(spans) for marker in ("PROMPT-MARKER", "ANSWER-MARKER", "SUM-RESULT-5", ARG, "tool failed because"): assert marker in text, marker for secret in (FI_API_KEY, FI_SECRET_KEY, LLM_KEY, ENV_SECRET): assert secret not in text, secret + for needle in ("Traceback", 'File "', "site-packages", "exception.stacktrace"): + assert needle not in text, needle agent = _attrs(_trees(spans)[0][0]) assert agent["input.value"] == PROMPT + " [redacted]" @@ -181,7 +196,7 @@ def _two_turns(receiver: Receiver, monkeypatch, **options: Any) -> List[Dict[str from mcp_use import MCPAgent provider = _register(receiver, monkeypatch) - handler = FutureAGICallback(tracer_provider=provider, capture_content=True, **options) + handler = FutureAGICallback(tracer_provider=provider, **options) async def go() -> None: client = mcp_client("stdio") diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_example.py b/python/frameworks/mcp-use/tests/test_mcp_use_example.py index 2ac4891aa..b7e08bd70 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_example.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_example.py @@ -101,8 +101,11 @@ def _check(result: Any, spans: List[Dict[str, Any]], exports: List[Dict[str, Any assert resource["project_type"] == "observe" wire = json.dumps(spans) - for text in (FI_API_KEY, FI_SECRET_KEY, LLM_KEY, "What is 2 + 3", "EXAMPLE-ANSWER"): + for text in (FI_API_KEY, FI_SECRET_KEY, LLM_KEY): assert text not in wire, text + # Content is captured by default (TraceConfig); FI_HIDE_* turns it off. + for text in ("What is 2 + 3", "EXAMPLE-ANSWER"): + assert text in wire, text def test_example_exports_one_agent_tree_at_exit(): diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_isolation.py b/python/frameworks/mcp-use/tests/test_mcp_use_isolation.py index eb2664f69..cc66ebdff 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_isolation.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_isolation.py @@ -52,7 +52,7 @@ def snapshot(): import traceai_mcp_use from uuid import uuid4 from _mcp_use_support import new_provider - handler = traceai_mcp_use.FutureAGICallback(tracer_provider=new_provider()[1], capture_content=True) + handler = traceai_mcp_use.FutureAGICallback(tracer_provider=new_provider()[1]) root = uuid4() handler.on_chain_start({}, {"messages": []}, run_id=root) handler.on_chain_end({"messages": []}, run_id=root) diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py index 28b3bc7f2..b79dabad6 100644 --- a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py +++ b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py @@ -22,7 +22,6 @@ import json import logging import threading -import traceback from collections import OrderedDict from dataclasses import dataclass, field from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple @@ -38,7 +37,6 @@ from traceai_mcp_use._text import ( MAX_ERROR_BYTES, MAX_NAME_BYTES, - MAX_STACKTRACE_BYTES, MAX_VALUE_BYTES, KnownTexts, Secrets, @@ -115,9 +113,6 @@ class _Failure: type_name: str qualified: str message: Optional[str] - stacktrace: Optional[str] - # The message is the tool's own output (an MCP error result). - from_output: bool = False @dataclass @@ -160,14 +155,8 @@ def _qualified(error: BaseException) -> str: def _failure_from_exception(error: BaseException) -> _Failure: - stacktrace: Optional[str] - try: - stacktrace = "".join( - traceback.format_exception(type(error), error, error.__traceback__) - ) - except Exception: - stacktrace = None - return _Failure(type(error).__name__, _qualified(error), _describe(error), stacktrace) + # No stack trace: it carries local file paths and the user name. + return _Failure(type(error).__name__, _qualified(error), _describe(error)) def _is_cancellation(error: BaseException) -> bool: @@ -326,13 +315,14 @@ class FutureAGICallback(BaseCallbackHandler): Args: tracer_provider: the provider spans go to; defaults to the global provider. Pass the one ``fi_instrumentation.register()`` returned. - config: a ``fi_instrumentation.TraceConfig``. ``hide_inputs``, - ``hide_outputs`` and ``pii_redaction`` (or ``FI_HIDE_INPUTS``, - ``FI_HIDE_OUTPUTS``, ``FI_PII_REDACTION``) apply to every text - this handler writes. - capture_content: record prompts, model output, tool arguments, tool - results and error messages. Off by default: spans then carry - names, kinds, model, usage, counts, statuses and exception types. + config: a ``fi_instrumentation.TraceConfig``; by default read from + the environment like every traceAI instrumentor. Content + (prompts, messages, model output, tool arguments and results, + error messages) is recorded unless the config hides it: ``hide_inputs``, + ``hide_outputs`` (``FI_HIDE_INPUTS``, ``FI_HIDE_OUTPUTS``) and + the message/text variants, plus ``pii_redaction`` + (``FI_PII_REDACTION``). Stack traces, local paths, the user + name and the host name are never recorded. redact: secret strings to remove from everything written (for example an LLM key or an MCP server header the environment does not hold). @@ -351,7 +341,6 @@ def __init__( self, tracer_provider: Optional[trace_api.TracerProvider] = None, config: Optional[TraceConfig] = None, - capture_content: bool = False, redact: Iterable[str] = (), ) -> None: super().__init__() @@ -363,10 +352,6 @@ def __init__( type(config).__name__ ) ) - if not isinstance(capture_content, bool): - raise TypeError( - "capture_content must be a bool, got {0}".format(type(capture_content).__name__) - ) if isinstance(redact, str): raise TypeError("redact must be an iterable of strings, not a string") secrets = list(redact) @@ -381,7 +366,6 @@ def __init__( self._tracer = FITracer( trace_api.get_tracer(__name__, __version__, provider), config=config ) - self._capture = capture_content self._hide_inputs = bool(config.hide_inputs) self._hide_outputs = bool(config.hide_outputs) self._pii = bool(config.pii_redaction) @@ -402,12 +386,10 @@ def _scrubs(self, root: Optional[_Root]) -> List[KnownTexts]: scrubs.append(root.outputs) return scrubs - def _clean( - self, text: Optional[str], root: Optional[_Root], limit: int, tail: bool = False - ) -> Optional[str]: + def _clean(self, text: Optional[str], root: Optional[_Root], limit: int) -> Optional[str]: if not isinstance(text, str): return None - return clean(text, self._secrets, limit, self._pii, self._scrubs(root), tail) + return clean(text, self._secrets, limit, self._pii, self._scrubs(root)) def _name(self, text: Any) -> Optional[str]: """An identifier (model, tool name, id): secrets removed, 256-byte cap.""" @@ -416,7 +398,7 @@ def _name(self, text: Any) -> Optional[str]: return cap(self._secrets.redact(text), MAX_NAME_BYTES) def _remember(self, root: Optional[_Root], texts: Iterable[Optional[str]], output: bool) -> None: - if root is None or not self._capture: + if root is None: return if output and not self._hide_outputs: return @@ -590,13 +572,11 @@ def _record_failure(self, span: Span, failure: _Failure, root: Optional[_Root]) type_name = self._name(failure.type_name) or "Error" qualified = self._name(failure.qualified) or type_name message: Optional[str] = None - stacktrace: Optional[str] = None - if self._capture and not (failure.from_output and self._hide_outputs): - # Error text can quote prompts, tool arguments and tool results, - # so it is content: recorded only with capture_content, and with - # every hidden input/output removed from it. + if not (self._hide_inputs or self._hide_outputs): + # Error text quotes prompts, tool arguments and tool results, and + # a server's own data: it is content, dropped when either hide + # flag is on. Stack traces are never recorded. message = self._clean(failure.message, root, MAX_ERROR_BYTES) - stacktrace = self._clean(failure.stacktrace, root, MAX_STACKTRACE_BYTES, tail=True) description = "{0}: {1}".format(type_name, message) if message else type_name try: span.set_status(Status(StatusCode.ERROR, description)) @@ -605,8 +585,6 @@ def _record_failure(self, span: Span, failure: _Failure, root: Optional[_Root]) event: Dict[str, Any] = {"exception.type": qualified} if message: event["exception.message"] = message - if stacktrace: - event["exception.stacktrace"] = stacktrace try: span.add_event("exception", event) except Exception: @@ -665,7 +643,7 @@ def on_chain_start( self._remember(root, [_content(m) for m in history], output=False) # Earlier turns' outputs, known before any content of this run is written. self._remember(root, _replayed_outputs(history), output=True) - if run.span is None or not self._capture: + if run.span is None: return if self._hide_inputs: self._set(run.span, INPUT_VALUE, REDACTED_VALUE) @@ -697,18 +675,17 @@ def on_chain_end( return span = run.span self._agent_counts(span, root) - if self._capture: - answer = None - messages = outputs.get("messages") if isinstance(outputs, Mapping) else None - for message in reversed(messages) if isinstance(messages, list) else []: - if _role(message) == "assistant" and not _tool_calls(message): - answer = _content(message) - break - if answer is not None: - if self._hide_outputs: - self._set(span, OUTPUT_VALUE, REDACTED_VALUE) - else: - self._set(span, OUTPUT_VALUE, self._clean(answer, root, MAX_VALUE_BYTES)) + answer = None + messages = outputs.get("messages") if isinstance(outputs, Mapping) else None + for message in reversed(messages) if isinstance(messages, list) else []: + if _role(message) == "assistant" and not _tool_calls(message): + answer = _content(message) + break + if answer is not None: + if self._hide_outputs: + self._set(span, OUTPUT_VALUE, REDACTED_VALUE) + else: + self._set(span, OUTPUT_VALUE, self._clean(answer, root, MAX_VALUE_BYTES)) self._ok(span) self._end(span) except Exception: @@ -845,8 +822,6 @@ def _start_llm( texts.extend(_strings_in(call.get("args"))) self._remember(root, texts, output=False) self._remember(root, _replayed_outputs(messages), output=True) - if not self._capture: - return if self._hide_inputs: self._set(run.span, INPUT_VALUE, REDACTED_VALUE) return @@ -996,7 +971,7 @@ def _llm_response(self, span: Span, response: Any, root: Optional[_Root]) -> Non self._remember(root, texts + arguments, output=True) # The arguments the model chose are the next tool call's input. self._remember(root, arguments, output=False) - if not self._capture or not messages: + if not messages: return if self._hide_outputs: self._set(span, OUTPUT_VALUE, REDACTED_VALUE) @@ -1058,8 +1033,6 @@ def on_tool_start( if isinstance(inputs, Mapping): texts.extend(_strings_in(inputs)) self._remember(root, texts, output=False) - if not self._capture: - return if self._hide_inputs: self._set(run.span, INPUT_VALUE, REDACTED_VALUE) return @@ -1089,21 +1062,17 @@ def on_tool_end( if formatted is not None: error_type = str(formatted.get("error")) details = formatted.get("details") - stack = formatted.get("stack") + # format_error's "stack" is never read. failure = _Failure( error_type, error_type, details if isinstance(details, str) else None, - stack if isinstance(stack, str) else None, - from_output=True, ) elif getattr(output, "status", None) == "error": failure = _Failure( "ToolException", "langchain_core.tools.ToolException", _tool_output_text(output), - None, - from_output=True, ) except Exception: logger.debug("Could not classify the tool result", exc_info=True) @@ -1119,13 +1088,12 @@ def on_tool_end( self._set(span, TOOL_ERROR_TYPE, self._name(failure.type_name)) self._record_failure(span, failure, root) else: - if self._capture: - if self._hide_outputs: - self._set(span, OUTPUT_VALUE, REDACTED_VALUE) - else: - cleaned = self._clean(text, root, MAX_VALUE_BYTES) - self._set(span, TOOL_RESULT, cleaned) - self._set(span, OUTPUT_VALUE, cleaned) + if self._hide_outputs: + self._set(span, OUTPUT_VALUE, REDACTED_VALUE) + else: + cleaned = self._clean(text, root, MAX_VALUE_BYTES) + self._set(span, TOOL_RESULT, cleaned) + self._set(span, OUTPUT_VALUE, cleaned) self._ok(span) self._end(span) except Exception: diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_text.py b/python/frameworks/mcp-use/traceai_mcp_use/_text.py index cbf3b8254..078e4327e 100644 --- a/python/frameworks/mcp-use/traceai_mcp_use/_text.py +++ b/python/frameworks/mcp-use/traceai_mcp_use/_text.py @@ -1,17 +1,21 @@ """Text cleaning applied to every string the callback writes to a span. Order for every value: remove secrets, then (when a TraceConfig hide flag is -on) remove text the run already saw as an input or output, then PII when -``pii_redaction`` is on, then cut to a UTF-8 byte cap. Secrets go first so a -cap can never leave the start of a key behind; PII goes before the cap so a +on) remove text the run already saw as an input or output, then cut Python +stack traces and replace local paths, the user name and the host name, then +PII when ``pii_redaction`` is on, then cut to a UTF-8 byte cap. Secrets go +first so a cap can never leave the start of a key behind; hidden texts are +matched before the local pass rewrites them; PII goes before the cap so a cut cannot leave part of an email address. """ from __future__ import annotations +import getpass import json import os import re +import socket from typing import Iterable, List, Optional, Sequence, Tuple from fi_instrumentation import REDACTED_VALUE @@ -22,7 +26,6 @@ MAX_VALUE_BYTES = 4096 MAX_NAME_BYTES = 256 MAX_ERROR_BYTES = 1024 -MAX_STACKTRACE_BYTES = 16 * 1024 # A secret shorter than this is not searched for: replacing a 3-character # value everywhere would shred unrelated text. @@ -50,16 +53,84 @@ def cap(value: str, limit: int) -> str: return value.encode("utf-8", "replace")[:limit].decode("utf-8", "ignore") -def cap_tail(value: str, limit: int) -> str: - """Return the longest whole-character suffix of at most ``limit`` UTF-8 bytes. +# --- stack traces, local paths, user and host names are never exported ----- +# +# mcp-use returns a failed MCP tool call to the model as format_error()'s dict +# (mcp_use/errors/error_formatting.py), whose "stack" is traceback.format_exc(): +# absolute file paths, the home directory and so the user name. That text is +# the next LLM call's tool message, and exception messages quote paths too. +# None of it is exported, with content on or hidden. +STACK_TRACE_REMOVED = "[stack trace removed]" +PATH_REMOVED = "[path]" +USER_REMOVED = "[user]" +HOST_REMOVED = "[host]" +_TRACEBACK_HEAD = "Traceback (most recent call last)" +_TRACEBACK = re.compile(r"Traceback \(most recent call last\):.*", re.DOTALL) +_LOCAL_PATH = re.compile( + r"(?|:*?]+" + r"|/(?:Users|home|root|private|var|tmp|opt|usr|srv|mnt|app|workspace|etc|Volumes)/[^\s\"'<>|:*?,;)\]\\]+)" +) +# Account names too generic to replace inside ordinary text. +_GENERIC_NAMES = frozenset( + "root admin administrator user users app ubuntu runner node nobody guest test dev " + "default jovyan vscode daemon localhost".split() +) + - Used for stack traces, whose last lines (the raising frame and the - exception line) matter most. +def _identity_patterns() -> List[Tuple["re.Pattern[str]", str]]: + """Whole-word patterns for this machine's user name and host name.""" + names: List[Tuple[str, str]] = [] + try: + names.append((getpass.getuser(), USER_REMOVED)) + except Exception: # no user database entry (some containers) + pass + try: + host = socket.gethostname() + names.extend(((host, HOST_REMOVED), (host.split(".")[0], HOST_REMOVED))) + except Exception: + pass + patterns: List[Tuple["re.Pattern[str]", str]] = [] + for name, token in names: + if len(name) >= 4 and name.lower() not in _GENERIC_NAMES: + word = re.compile(r"(? object: + if isinstance(value, str): + return _TRACEBACK.sub(STACK_TRACE_REMOVED, value) if _TRACEBACK_HEAD in value else value + if isinstance(value, list): + return [_cut_tracebacks(item) for item in value] + if isinstance(value, dict): + return {key: _cut_tracebacks(item) for key, item in value.items()} + return value + + +def scrub_local(text: str) -> str: + """Cut Python stack traces; replace local paths, the user and the host name. + + A JSON object or array (an mcp-use formatted tool error is one) has the + trace cut inside each string value, so the rest of the JSON survives; + any other text loses everything from ``Traceback (most recent call + last):`` on. """ - encoded = value.encode("utf-8", "replace") - if len(encoded) <= limit: - return value - return encoded[len(encoded) - limit :].decode("utf-8", "ignore") + if _TRACEBACK_HEAD in text: + stripped = text.strip() + if stripped[:1] in ("{", "["): + try: + text = json.dumps(_cut_tracebacks(json.loads(stripped)), ensure_ascii=False) + except ValueError: + pass + if _TRACEBACK_HEAD in text: + text = _TRACEBACK.sub(STACK_TRACE_REMOVED, text) + text = _LOCAL_PATH.sub(PATH_REMOVED, text) + for pattern, token in _IDENTITY: + text = pattern.sub(token, text) + return text class Secrets: @@ -155,13 +226,12 @@ def clean( limit: int, pii: bool = False, scrub: Sequence[KnownTexts] = (), - tail: bool = False, ) -> Optional[str]: - """Secrets, then known inputs/outputs, then PII, then the byte cap. + """Secrets, then known inputs/outputs, then stack traces and local + identity, then PII, then the byte cap. - ``tail`` keeps the end of the text instead of the start. Returns None - when a hide flag's text set overflowed: the caller must then record - nothing rather than an unscrubbed value. + Returns None when a hide flag's text set overflowed: the caller must + then record nothing rather than an unscrubbed value. """ text = secrets.redact(text) for known in scrub: @@ -169,6 +239,7 @@ def clean( if scrubbed is None: return None text = scrubbed + text = scrub_local(text) if pii: text = redact_pii_in_string(text) - return cap_tail(text, limit) if tail else cap(text, limit) + return cap(text, limit) From e2c553f67a389284cab88cb8ce65b9852e41fd89 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Fri, 16 Oct 2026 22:45:23 -0700 Subject: [PATCH 19/19] docs(mcp-use): README and CHANGELOG for the TraceConfig default and mcp-use console logs Privacy section rewritten for content-on-by-default with FI_HIDE_*, what is never recorded (stack traces, local paths, user/host name), the error-text rule, tested versions (mcp-use 1.7.1, latest on 2026-10-16), and mcp-use's own console logging (Logger.set_debug(0) and the "mcp_use" logger level), which is outside the trace. Co-Authored-By: Claude Opus 5.5 --- python/frameworks/mcp-use/CHANGELOG.md | 15 +++-- python/frameworks/mcp-use/README.md | 92 ++++++++++++++++++-------- 2 files changed, 75 insertions(+), 32 deletions(-) diff --git a/python/frameworks/mcp-use/CHANGELOG.md b/python/frameworks/mcp-use/CHANGELOG.md index ace5588c3..e82116599 100644 --- a/python/frameworks/mcp-use/CHANGELOG.md +++ b/python/frameworks/mcp-use/CHANGELOG.md @@ -12,11 +12,16 @@ - Spans come from `FITracer`: `TraceConfig` and `using_session`, `using_user`, `using_metadata` and `using_tags` apply. - Content (prompts, messages, model output, tool arguments and results, - error messages and stack traces) is off by default; `capture_content=True` - turns it on. `hide_inputs`, `hide_outputs` and `pii_redaction` also apply - to error text. Under `hide_outputs`, model replies and tool results that - `MCPAgent` memory or `external_history` sends back on a later turn are - outputs too. + error messages) follows `TraceConfig` like every traceAI instrumentor: + recorded by default, hidden with `FI_HIDE_INPUTS` / `FI_HIDE_OUTPUTS` + (and the message/text variants) or `config=TraceConfig(...)`. There is no + package-specific switch. With either hide flag, error messages are + dropped (exception type only). Under `hide_outputs`, model replies and + tool results that `MCPAgent` memory or `external_history` sends back on a + later turn are outputs too. +- Never recorded: stack traces (no `exception.stacktrace`; the `stack` of + mcp-use's formatted tool error is cut from the tool message the model + gets), absolute local paths, and this machine's user and host name. - Secrets (values passed in `redact=`, secret-named environment variables, bearer tokens and key-shaped strings) are removed from everything written, names and ids included. Recorded text is size-capped. diff --git a/python/frameworks/mcp-use/README.md b/python/frameworks/mcp-use/README.md index 6175c3d82..96330c1bc 100644 --- a/python/frameworks/mcp-use/README.md +++ b/python/frameworks/mcp-use/README.md @@ -21,8 +21,10 @@ pip install traceAI-mcp-use It accepts `mcp-use>=1.7.1,<2`, `langchain-core>=1.0.0,<2` and `fi-instrumentation-otel>=1.1.0`, on Python `>=3.11,<3.14` (mcp-use itself -requires 3.11). The suite runs with mcp-use 1.7.1 on Python 3.11, 3.12 and -3.13. +requires 3.11). The suite runs with mcp-use 1.7.1 (the latest release on +2026-10-16) on Python 3.11, 3.12 and 3.13, and was run end to end against a +real model and a real stdio MCP server with mcp-use 1.7.1, langchain-core +1.6.12, langchain-openai 1.7.2 and langgraph 1.2.16. ## Usage @@ -91,8 +93,9 @@ call (for example an HTTP client span) is not a child of the tool span. | `mcp_use.tool.error_type` | tool | Error type, when the tool's failure came back as its result | | `mcp_use.cancelled`, `mcp_use.incomplete` | any | See [Errors and cancellation](#errors-and-cancellation) | -A value the callback is not given is omitted, never written as 0. With -`capture_content=True` (see [Privacy](#privacy)) these are added: +A value the callback is not given is omitted, never written as 0. Content +is recorded too, unless a `TraceConfig` hide flag turns it off (see +[Privacy](#privacy)): | Attribute | Span | Value | |---|---|---| @@ -121,25 +124,49 @@ agent does); use these context managers instead. ## Privacy -Content is off by default. Without `capture_content=True`, spans carry -names, kinds, the model and provider, request parameters, usage, finish -reasons, counts, the tool-call id, statuses and exception types. They do -not carry prompts, messages, model output, tool arguments, tool results, -error messages or stack traces. +Content follows `TraceConfig`, like every traceAI instrumentor: prompts, +messages, model output, tool arguments, tool results and error messages are +recorded by default, and the standard hide flags turn them off. There is no +separate switch in this package. + +```bash +export FI_HIDE_INPUTS=true # prompts, input messages, tool arguments +export FI_HIDE_OUTPUTS=true # model output, output messages, tool results +``` + +or `FutureAGICallback(tracer_provider=..., config=TraceConfig(hide_inputs=True, hide_outputs=True))`. +With both flags on, spans carry names, kinds, the model and provider, +request parameters, usage, finish reasons, counts, the tool-call id, +statuses and exception types; `input.value` and `output.value` are +`__REDACTED__`. + +Never recorded, whatever the flags: + +- stack traces. The `exception` event has `exception.type` and, when + content is on, `exception.message`; never `exception.stacktrace`. mcp-use + sends a failed tool call back to the model as `format_error()`'s dict, + whose `stack` is a full traceback; in that tool message the trace is + replaced with `[stack trace removed]` and the rest of the JSON is kept. + Any other text loses everything from `Traceback (most recent call last):` + on; +- absolute local paths (`/Users/...`, `/home/...`, `/tmp/...`, `C:\...` + and similar), replaced with `[path]`; +- this machine's user name and host name, replaced with `[user]` and + `[host]` (names of 4 characters or more that are not generic, such as + `root` or `app`). ```python FutureAGICallback( tracer_provider=trace_provider, - capture_content=True, # prompts, messages, output, tool arguments and results, error text - redact=["my-mcp-server-token"], # extra secret strings to remove + config=TraceConfig(hide_inputs=True), # optional; default reads FI_HIDE_* from the environment + redact=["my-mcp-server-token"], # extra secret strings to remove ) ``` -`capture_content` must be a bool and `redact` an iterable of strings; -anything else raises `TypeError`. +`redact` must be an iterable of strings; anything else raises `TypeError`. Secrets are removed from every text the callback writes, including names -and ids, with content on or off. They are replaced by `[redacted]`: +and ids, with content on or hidden. They are replaced by `[redacted]`: - the strings passed as `redact=`; - the value (8 characters or longer) of every environment variable whose @@ -158,14 +185,23 @@ environment variables before creating the callback: | Setting | Effect | |---|---| -| `hide_inputs` / `FI_HIDE_INPUTS=true` | `input.value` is `__REDACTED__`; input messages and tool-call arguments (on tool spans and in model replies) are dropped. With `capture_content`, error messages and stack traces have every input text of the run replaced with `__REDACTED__`: the user messages, the messages sent to the model, and each string value of each tool call's arguments. | -| `hide_outputs` / `FI_HIDE_OUTPUTS=true` | `output.value` is `__REDACTED__`; output messages and tool results are dropped. Model replies and tool results sent back to the model as input messages (later in the same run, or from earlier turns; see below) keep their role but lose their content and tool-call arguments, and an LLM span whose last input message is one of them has `input.value` `__REDACTED__`. The error text of a tool failure that came back as the tool's result is dropped, and other error text has every output the run has seen (model replies, their tool-call arguments, tool results, earlier turns' included) replaced with `__REDACTED__`. | +| `hide_inputs` / `FI_HIDE_INPUTS=true` | `input.value` is `__REDACTED__`; input messages and tool-call arguments (on tool spans and in model replies) are dropped. Every other text of the run has each input it has seen replaced with `__REDACTED__`: the user messages, the messages sent to the model, and each string value of each tool call's arguments. Error messages are dropped: the status and the `exception` event carry the exception type only. | +| `hide_outputs` / `FI_HIDE_OUTPUTS=true` | `output.value` is `__REDACTED__`; output messages and tool results are dropped. Model replies and tool results sent back to the model as input messages (later in the same run, or from earlier turns; see below) keep their role but lose their content and tool-call arguments, and an LLM span whose last input message is one of them has `input.value` `__REDACTED__`. Every other text of the run has each output it has seen (model replies, their tool-call arguments, tool results, earlier turns' included) replaced with `__REDACTED__`. Error messages are dropped: the status and the `exception` event carry the exception type only. | +| `hide_input_messages`, `hide_output_messages`, `hide_input_text`, `hide_output_text` (`FI_HIDE_INPUT_MESSAGES`, ...) | Applied by `FITracer` to the `gen_ai.input.messages.*` / `gen_ai.output.messages.*` attributes, as for every traceAI instrumentor. | | `pii_redaction` / `FI_PII_REDACTION=true` | Emails, phone numbers, SSNs, card numbers, IPv4 addresses and `sk-`/`pk-` keys become tokens such as `` in every recorded text: attributes, the error status and the `exception` event. It runs after secrets (and hidden inputs/outputs) are removed and before the size caps. | `config` must be a `fi_instrumentation.TraceConfig`; anything else raises `TypeError`. Other `TraceConfig` fields act only through `FITracer`'s attribute masking. +mcp-use's own console logging is not part of the trace and is not changed +by these flags: at its default level it prints query previews and tool +results (INFO), and a failed tool call's formatted error with its stack +trace (ERROR, `mcp_use/errors/error_formatting.py`). `Logger.set_debug(0)` +(`from mcp_use import Logger`) hides the INFO lines; +`logging.getLogger("mcp_use").setLevel(logging.CRITICAL)` also hides the +ERROR line. + Multi-turn conversations: `MCPAgent` keeps memory by default (`memory_enabled=True`) and sends the earlier turns, model replies and tool results included, back to the model as input on the next `run()`, @@ -189,8 +225,8 @@ stays. Texts shorter than 3 characters are only matched in quoted form, and non-string argument values (numbers, booleans) are not matched. With a hide flag on, the callback remembers at most 2048 text forms or about 4 million characters per run; past that it fails closed and records no -content for the rest of that run (no messages, arguments, results, error -messages or stack traces). +content for the rest of that run (no messages, arguments, results or +error messages). ## Errors and cancellation @@ -212,10 +248,11 @@ When the run itself fails (for example the model call raises), the LLM span and the agent span end with status ERROR and an `exception` event, and the exception reaches your code unchanged. -Without `capture_content`, the status description and the event carry -only the exception type. With it they also carry the message (cut to 1 KB) -and the event carries the stack trace (its last 16 KB), cleaned as -described in [Privacy](#privacy). +By default the status description is `: ` and the event +carries `exception.type` and `exception.message` (cut to 1 KB, cleaned as +described in [Privacy](#privacy): secrets, local paths, user and host name +removed). With `hide_inputs` or `hide_outputs` both carry the exception type +only. No stack trace is recorded. Cancelling the task that runs the agent (`asyncio.CancelledError`) or leaving `stream()` before it ends (`GeneratorExit`) ends the open spans @@ -268,9 +305,8 @@ the TypeScript agent). - Recorded text is cut on a UTF-8 character boundary after it is cleaned (secrets, then hidden inputs/outputs, then PII): 4 KB for each content attribute, 256 bytes for names and ids (model, provider, tool name, - tool-call id, exception type), 1 KB for the error message in the status - and the `exception` event, and 16 KB for the stack trace, keeping its - end. With `pii_redaction`, `FITracer` runs its own PII pass on every + tool-call id, exception type) and 1 KB for the error message in the + status and the `exception` event. With `pii_redaction`, `FITracer` runs its own PII pass on every attribute after the cut; a token such as `` can make an attribute a few bytes longer than its cap. The status and events are not passed through it. At most 32 messages and 16 tool calls per message are @@ -299,7 +335,7 @@ the TypeScript agent). as it ends. - Passed to a LangChain runnable other than `MCPAgent`, the callback still names the outermost run `mcp_use.agent`. -- Tested with mcp-use 1.7.1 only. +- Tested with mcp-use 1.7.1 only (the latest release on 2026-10-16). ## Tests @@ -322,7 +358,9 @@ for the contract test; the example test uses the example's own stdio calculator server. Those send spans through `fi_instrumentation.register()` to the shared `harness.Receiver` on 127.0.0.1 and check the `/tracer/v1/traces` path, both auth headers, -`project_type=observe` and that no key or content is exported by default. +`project_type=observe`, that content is exported by default and not with +`FI_HIDE_INPUTS`/`FI_HIDE_OUTPUTS`, and that no key, stack trace or local +path is exported either way. The example test runs `examples/agent_with_tools.py` as its own process, once alone and once with `traceai_mcp` instrumented. The suite turns off mcp-use's usage telemetry.