From c0b9e03beece284e59ff9513c1ee185ff45a3def Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 3 Nov 2026 22:43:55 -0800 Subject: [PATCH 1/5] fix(release-tooling): redact custom MCP-use homes and clarify Java planning Resolve the actual POSIX home at redaction time with whole-path boundaries, preserving ordinary URLs, API routes and sibling prefixes. Cover custom mounts in exported spans and document parent-only historical planning separately from strict fresh-deployment verification. Co-Authored-By: Claude Code --- RELEASING.md | 2 +- .../mcp-use/tests/test_mcp_use_callback.py | 33 +++++++++++++++++-- .../mcp-use/traceai_mcp_use/_text.py | 12 +++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index ebf62a0a7..2e18e0fb6 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -90,7 +90,7 @@ Sources: [PyPI publisher setup](https://docs.pypi.org/trusted-publishers/adding- - [ ] Generate a Central Portal user token and add **`MAVEN_CENTRAL_USERNAME`** and **`MAVEN_CENTRAL_PASSWORD`** as repository secrets. These are the Portal token's username/password, not your interactive login password. - [ ] Create a release GPG signing key, publish its **public key** to a public keyserver accessible to Central, and verify the fingerprint. Export the **ASCII-armored private signing key** into **`MAVEN_GPG_PRIVATE_KEY`** and its passphrase into **`MAVEN_GPG_PASSPHRASE`**. Keep the private key out of git and command output. - [ ] Merge the Java release-profile PR: groupId `com.futureagi`, unchanged artifactIds, sources/javadocs, signing, and `central-publishing-maven-plugin` server id **`central`**. setup-java provides Central credentials; the release profile's BC signer reads the private key and passphrase directly from the deployment step's environment (no persistent GnuPG keyring). The publisher runs **`mvn -B -ntp -P release deploy` from `java/`**. -- [ ] The planner skips a version only when **every reactor artifact** has its POM on Central. Central deployments are verified against canonical repository files, not the delayed search index. Check the Portal for a failed/incomplete deployment before retrying; do not replace immutable artifacts. +- [ ] Planning and historical release/tag reconciliation check only the reactor's **parent POM** on Central. If it exists, the version is already published and must not be deployed again; modules added later ship in the next reactor version. Verification after a **fresh deployment** requires **every reactor artifact** to have its POM on Central. Both checks use canonical repository files, not the delayed search index. Check the Portal for a failed/incomplete deployment before retrying; do not replace immutable artifacts. Sources: [namespace verification](https://central.sonatype.org/register/namespace/) and [Central publishing Maven plugin](https://central.sonatype.org/publish/publish-portal-maven/). diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py index 7156f9ecd..299bc28cf 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py @@ -445,11 +445,15 @@ def test_either_hide_flag_drops_the_error_text(config): assert "SERVER-SIDE-TEXT" not in wire(spans) and "tool failed because" not in wire(spans) -def test_stack_traces_paths_user_and_host_are_never_recorded(): +@pytest.mark.parametrize( + "home", ["/home/synthetic", "/work/synthetic", "/data/synthetic", "/custom.mount/synthetic user"] +) +def test_stack_traces_paths_user_and_host_are_never_recorded(monkeypatch, home): # mcp-use sends a failed MCP call back to the model as format_error()'s # dict, whose "stack" is a full traceback; exception text quotes paths. + # HOME is not necessarily under one of the conventional filesystem roots. + monkeypatch.setenv("HOME", home) user, host = getpass.getuser(), socket.gethostname() - home = os.path.expanduser("~") exporter, handler = _handler() run = Run(handler) run.llm(AIMessage(content="", tool_calls=[{"name": "fail", "args": {"topic": "t"}, "id": "c", "type": "tool_call"}])) @@ -492,6 +496,31 @@ def test_stack_traces_paths_user_and_host_are_never_recorded(): } +@pytest.mark.parametrize("home", ["/work/synthetic", "/data/synthetic", "/custom.mount/synthetic user"]) +def test_scrub_local_uses_home_with_path_boundaries(monkeypatch, home): + monkeypatch.setenv("HOME", home) + assert scrub_local("cannot open {0}/notes/archive.db".format(home)) == "cannot open [path]" + assert scrub_local('home="{0}"; retry {0}/'.format(home)) == 'home="[path]"; retry [path]' + for text in ( + "https://example.com" + home + "/notes", + "/api/v1" + home + "/items", + home + "-other/notes", + home + "2/notes", + home + ".bak/notes", + "relative" + home + "/notes", + "see /api/v1/items and /data/v1/query", + ): + assert scrub_local(text) == text + # A changed home is read at call time, not cached from the first import. + monkeypatch.setenv("HOME", home + "2/") + assert scrub_local(home + "2/notes") == "[path]" + + +def test_scrub_local_root_home_does_not_hide_every_api_path(monkeypatch): + monkeypatch.setenv("HOME", "/") + assert scrub_local("GET /api/v1/items") == "GET /api/v1/items" + + def test_scrub_local_keeps_ordinary_text(): for text in ( "see https://example.com/home/docs and /api/v1/items", diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_text.py b/python/frameworks/mcp-use/traceai_mcp_use/_text.py index 078e4327e..5ffacabad 100644 --- a/python/frameworks/mcp-use/traceai_mcp_use/_text.py +++ b/python/frameworks/mcp-use/traceai_mcp_use/_text.py @@ -127,6 +127,18 @@ def scrub_local(text: str) -> str: pass if _TRACEBACK_HEAD in text: text = _TRACEBACK.sub(STACK_TRACE_REMOVED, text) + # POSIX homes can live under arbitrary mounts, not just the roots below. + # Read at call time and escape the exact prefix (including spaces). Match + # whole paths only, never siblings or paths embedded in URLs/API routes. + # An unset or root HOME must not turn this into a catch-all for /api/... + home = os.path.expanduser("~").rstrip("/") + if home.startswith("/"): + text = re.sub( + r"(?|:*?,;)\]\\])(?:/[^\s\"'<>|:*?,;)\]\\]*)?", + PATH_REMOVED, + text, + ) text = _LOCAL_PATH.sub(PATH_REMOVED, text) for pattern, token in _IDENTITY: text = pattern.sub(token, text) From b9ecb5ec10a5a7f756d4d9a4b7785d987f112b0e Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 3 Nov 2026 23:32:48 -0800 Subject: [PATCH 2/5] fix(mcp-use): test supported floor and preserve cache-write usage Raise the published LangChain Core floor to 1.6.12 and test fresh resolutions on Python 3.11, 3.12 and 3.13 without pinning transitive dependencies. Preserve normalized cache-creation token counts through OTLP export, including zero, all privacy combinations and all agent entry points. Co-Authored-By: Claude Code --- .github/workflows/mcp-use-tests.yml | 91 +++++++++++++++++++ python/frameworks/mcp-use/README.md | 9 +- python/frameworks/mcp-use/pyproject.toml | 2 +- .../mcp-use/tests/test_mcp_use_packaging.py | 10 ++ .../mcp-use/tests/test_mcp_use_usage.py | 89 ++++++++++++++++++ .../mcp-use/traceai_mcp_use/_callback.py | 2 + 6 files changed, 200 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/mcp-use-tests.yml create mode 100644 python/frameworks/mcp-use/tests/test_mcp_use_usage.py diff --git a/.github/workflows/mcp-use-tests.yml b/.github/workflows/mcp-use-tests.yml new file mode 100644 index 000000000..b52b02ccf --- /dev/null +++ b/.github/workflows/mcp-use-tests.yml @@ -0,0 +1,91 @@ +name: MCP-use dependency floor + +on: + pull_request: + branches: [dev] + paths: + - 'python/frameworks/mcp-use/**' + - 'python/frameworks/mcp/**' + - 'python/tests/harness/**' + - '.github/workflows/mcp-use-tests.yml' + push: + branches: [dev] + paths: + - 'python/frameworks/mcp-use/**' + - 'python/frameworks/mcp/**' + - 'python/tests/harness/**' + - '.github/workflows/mcp-use-tests.yml' + +permissions: + contents: read + +concurrency: + group: mcp-use-floor-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +defaults: + run: + shell: bash + +env: + MCP_USE_ANONYMIZED_TELEMETRY: 'false' + MCP_USE_LANGFUSE: 'false' + MCP_USE_LAMINAR: 'false' + FI_BASE_URL: http://127.0.0.1:9 + FI_GRPC_URL: http://127.0.0.1:9 + +jobs: + floor: + name: Fresh floor (Python ${{ matrix.python }}) + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + python: ['3.11', '3.12', '3.13'] + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Resolve the declared dependency floors from scratch + run: | + python -m venv .floor-venv + # Pin only our direct dependencies to their advertised minima. Leave + # LangGraph/checkpoint and every other transitive dependency unpinned: + # pip check alone cannot detect an import-broken upstream resolution. + python - <<'PY' > "$RUNNER_TEMP/mcp-use-floor.txt" + import re + import tomllib + from pathlib import Path + + deps = tomllib.loads(Path("python/frameworks/mcp-use/pyproject.toml").read_text())["tool"]["poetry"]["dependencies"] + for name in ("mcp-use", "langchain-core", "fi-instrumentation-otel"): + floor = re.fullmatch(r">=([0-9]+\.[0-9]+\.[0-9]+)(?:,.*)?", deps[name]) + assert floor, f"No explicit supported floor for {name}" + print(f"{name}=={floor[1]}") + PY + .floor-venv/bin/python -m pip install --no-cache-dir \ + -c "$RUNNER_TEMP/mcp-use-floor.txt" ./python/frameworks/mcp-use \ + 'pytest==8.4.2' 'pytest-asyncio==0.24.0' + .floor-venv/bin/python -m pip check + .floor-venv/bin/python -m pip freeze + - name: Import the real client and verify installed floors + run: | + .floor-venv/bin/python - <<'PY' + import os + from importlib.metadata import version + from pathlib import Path + from mcp_use import MCPAgent, MCPClient + from traceai_mcp_use import FutureAGICallback + + for requirement in (Path(os.environ["RUNNER_TEMP"]) / "mcp-use-floor.txt").read_text().splitlines(): + name, expected = requirement.split("==") + assert version(name) == expected, (name, version(name), expected) + PY + - name: Run the full package and real-agent exporter tests + run: | + .floor-venv/bin/python -m pytest python/frameworks/mcp-use/tests \ + -q -p no:cacheprovider --noconftest -o addopts= diff --git a/python/frameworks/mcp-use/README.md b/python/frameworks/mcp-use/README.md index 96330c1bc..6c0c1b30a 100644 --- a/python/frameworks/mcp-use/README.md +++ b/python/frameworks/mcp-use/README.md @@ -19,12 +19,16 @@ calls. See [Other tracing in the same process](#other-tracing-in-the-same-proces pip install traceAI-mcp-use ``` -It accepts `mcp-use>=1.7.1,<2`, `langchain-core>=1.0.0,<2` and +It accepts `mcp-use>=1.7.1,<2`, `langchain-core>=1.6.12,<2` and `fi-instrumentation-otel>=1.1.0`, on Python `>=3.11,<3.14` (mcp-use itself requires 3.11). The suite runs with mcp-use 1.7.1 (the latest release on 2026-10-16) on Python 3.11, 3.12 and 3.13, and was run end to end against a real model and a real stdio MCP server with mcp-use 1.7.1, langchain-core 1.6.12, langchain-openai 1.7.2 and langgraph 1.2.16. +LangChain Core 1.6.12 is the supported floor: older versions can resolve +with newer LangGraph checkpoint packages but fail when importing `MCPAgent`. +The dependency-floor CI job freshly resolves all three declared minimum +versions and runs the full suite on Python 3.11, 3.12 and 3.13. ## Usage @@ -85,7 +89,8 @@ call (for example an HTTP client span) is not a child of the tool span. | `gen_ai.request.model` | LLM | From LangChain's `ls_model_name`, else the invocation `model` / `model_name` / `model_id` | | `gen_ai.provider.name`, `gen_ai.request.temperature`, `gen_ai.request.max_tokens` | LLM | From LangChain's `ls_provider`, `ls_temperature`, `ls_max_tokens` | | `gen_ai.response.model`, `gen_ai.response.finish_reasons` | LLM | From the response metadata | -| `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.usage.total_tokens`, `gen_ai.usage.cache_read_tokens` | LLM | From the message's `usage_metadata` (or the provider's `token_usage`) | +| `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.usage.total_tokens` | LLM | From the message's `usage_metadata` (or the provider's `token_usage`) | +| `gen_ai.usage.cache_read_tokens`, `gen_ai.usage.cache_write_tokens` | LLM | From `usage_metadata.input_token_details.cache_read` / `.cache_creation`, when reported; these do not add to the total token count | | `mcp_use.llm.input_message_count` | LLM | Messages sent to the model | | `mcp_use.llm.tool_call_count` | LLM | Tool calls the model asked for | | `mcp_use.llm.chunk_count` | LLM | Streamed tokens, when streamed | diff --git a/python/frameworks/mcp-use/pyproject.toml b/python/frameworks/mcp-use/pyproject.toml index 46da53b35..68623cc87 100644 --- a/python/frameworks/mcp-use/pyproject.toml +++ b/python/frameworks/mcp-use/pyproject.toml @@ -29,7 +29,7 @@ keywords = ["traceai", "futureagi", "opentelemetry", "tracing", "observability", python = ">=3.11,<3.14" fi-instrumentation-otel = ">=1.1.0" mcp-use = ">=1.7.1,<2" -langchain-core = ">=1.0.0,<2" +langchain-core = ">=1.6.12,<2" [tool.poetry.group.dev.dependencies] pytest = ">=7.0.0" diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_packaging.py b/python/frameworks/mcp-use/tests/test_mcp_use_packaging.py index 91750cc48..ab0c9d3fc 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_packaging.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_packaging.py @@ -52,6 +52,16 @@ def test_langchain_core_range_admits_the_installed_version(): assert not spec.contains("2.0.0") +def test_langchain_core_floor_matches_the_supported_resolution(): + # Older core versions can resolve with current langgraph-checkpoint but + # fail to import MCPAgent (Reviver lacks the allowed_objects argument). + spec = _dependency("langchain-core") + assert spec.contains("1.6.12") + for version in ("1.0.0", "1.6.11"): + assert not spec.contains(version), version + assert "langchain-core>=1.6.12,<2" in README.read_text() + + def test_fi_instrumentation_floor_is_1_1_0(): # 0.1.11/0.1.12 export to the old create_otel_span path and the published # 1.0.0 fails while building its headers; 1.1.0 is the first good release. diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_usage.py b/python/frameworks/mcp-use/tests/test_mcp_use_usage.py new file mode 100644 index 000000000..d742c230a --- /dev/null +++ b/python/frameworks/mcp-use/tests/test_mcp_use_usage.py @@ -0,0 +1,89 @@ +"""Cache cost inputs survive the real agent loop and OTLP HTTP export.""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path +from typing import Optional + +import pytest +from _mcp_use_support import ANSWER, PROMPT, ChatFake, add_script, drive, mcp_client +from fi_instrumentation import TraceConfig, register +from fi_instrumentation.fi_types import ProjectType +from mcp_use import MCPAgent + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) + +from harness import Receiver, _flatten_attributes # noqa: E402 +from traceai_mcp_use import FutureAGICallback # noqa: E402 + + +class CachedModel(ChatFake): + cache_creation: Optional[int] = None + + def _next(self, messages): + details = {"cache_read": 10} + if self.cache_creation is not None: + details["cache_creation"] = self.cache_creation + return super()._next(messages).model_copy( + update={ + "usage_metadata": { + "input_tokens": 100, + "output_tokens": 7, + "total_tokens": 107, + "input_token_details": details, + } + } + ) + + +@pytest.mark.parametrize("method", ["run", "stream", "stream_events"]) +@pytest.mark.parametrize("hide_inputs,hide_outputs", [(False, False), (True, False), (False, True), (True, True)]) +@pytest.mark.parametrize("cache_creation", [None, 0, 20], ids=["absent", "zero", "written"]) +def test_cache_creation_reaches_the_collector(monkeypatch, method, hide_inputs, hide_outputs, cache_creation): + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv("FI_API_KEY", "placeholder-fi-api-key") + monkeypatch.setenv("FI_SECRET_KEY", "placeholder-fi-secret-key") + provider = register(project_type=ProjectType.OBSERVE, project_name="mcp-use-usage", verbose=False) + handler = FutureAGICallback( + tracer_provider=provider, + config=TraceConfig(hide_inputs=hide_inputs, hide_outputs=hide_outputs), + ) + + async def run(): + client = mcp_client() + try: + agent = MCPAgent( + llm=CachedModel(script=add_script(), cache_creation=cache_creation, streaming=method != "run"), + client=client, + callbacks=[handler], + ) + await drive(agent, method) + finally: + await client.close_all_sessions() + + try: + asyncio.run(run()) + assert not handler._runs + assert provider.force_flush(timeout_millis=10_000) + finally: + provider.shutdown() + spans = [_flatten_attributes(span.get("attributes", [])) for span in receiver.spans()] + + assert len(spans) == 4 # agent, LLM, tool, LLM + llms = [span for span in spans if span.get("gen_ai.span.kind") == "LLM"] + assert len(llms) == 2 + for span in llms: + assert int(span["gen_ai.usage.input_tokens"]) == 100 + assert int(span["gen_ai.usage.output_tokens"]) == 7 + # Cache details are subsets of input usage, not extra total tokens. + assert int(span["gen_ai.usage.total_tokens"]) == 107 + assert int(span["gen_ai.usage.cache_read_tokens"]) == 10 + if cache_creation is None: + assert "gen_ai.usage.cache_write_tokens" not in span + else: + assert int(span["gen_ai.usage.cache_write_tokens"]) == cache_creation + assert llms[0]["input.value"] == ("__REDACTED__" if hide_inputs else PROMPT) + assert llms[-1]["output.value"] == ("__REDACTED__" if hide_outputs else ANSWER) diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py index b79dabad6..bbcbab9b7 100644 --- a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py +++ b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py @@ -66,6 +66,7 @@ OUTPUT_TOKENS = SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS TOTAL_TOKENS = SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS CACHE_READ_TOKENS = SpanAttributes.GEN_AI_USAGE_CACHE_READ_TOKENS +CACHE_WRITE_TOKENS = SpanAttributes.GEN_AI_USAGE_CACHE_WRITE_TOKENS TOOL_NAME = SpanAttributes.GEN_AI_TOOL_NAME TOOL_CALL_ID = SpanAttributes.GEN_AI_TOOL_CALL_ID TOOL_ARGUMENTS = SpanAttributes.GEN_AI_TOOL_CALL_ARGUMENTS @@ -955,6 +956,7 @@ def _llm_response(self, span: Span, response: Any, root: Optional[_Root]) -> Non details = usage.get("input_token_details") if isinstance(details, Mapping): self._set(span, CACHE_READ_TOKENS, _int(details.get("cache_read"))) + self._set(span, CACHE_WRITE_TOKENS, _int(details.get("cache_creation"))) elif isinstance(llm_output, Mapping) and isinstance(llm_output.get("token_usage"), Mapping): token_usage = llm_output["token_usage"] self._set(span, INPUT_TOKENS, _int(token_usage.get("prompt_tokens"))) From e0c91de7b28b363ecf60857c22a512294ac083c7 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 4 Nov 2026 00:31:10 -0800 Subject: [PATCH 3/5] fix(mcp-use): retain cache TTL usage and scrub model identifiers Prefer provider cache-write TTL counts over generic cache_creation while preserving both billing tiers and inclusive token totals. Apply the shared privacy cleaner to model and tool identifiers before truncation. Exercise real ChatAnthropic HTTP/SSE responses through MCPAgent, exported usage and hidden-content configurations; keep provider test dependencies compatible with the advertised floor. Co-Authored-By: Claude Code --- .github/workflows/mcp-use-tests.yml | 2 +- python/frameworks/mcp-use/pyproject.toml | 1 + .../mcp-use/tests/test_mcp_use_callback.py | 28 +++ .../mcp-use/tests/test_mcp_use_provider.py | 161 ++++++++++++++++++ .../mcp-use/tests/test_mcp_use_usage.py | 36 ++-- .../mcp-use/traceai_mcp_use/_callback.py | 22 ++- 6 files changed, 233 insertions(+), 17 deletions(-) create mode 100644 python/frameworks/mcp-use/tests/test_mcp_use_provider.py diff --git a/.github/workflows/mcp-use-tests.yml b/.github/workflows/mcp-use-tests.yml index b52b02ccf..fbe37d11e 100644 --- a/.github/workflows/mcp-use-tests.yml +++ b/.github/workflows/mcp-use-tests.yml @@ -69,7 +69,7 @@ jobs: PY .floor-venv/bin/python -m pip install --no-cache-dir \ -c "$RUNNER_TEMP/mcp-use-floor.txt" ./python/frameworks/mcp-use \ - 'pytest==8.4.2' 'pytest-asyncio==0.24.0' + 'pytest==8.4.2' 'pytest-asyncio==0.24.0' 'langchain-anthropic>=1.7.7,<2' .floor-venv/bin/python -m pip check .floor-venv/bin/python -m pip freeze - name: Import the real client and verify installed floors diff --git a/python/frameworks/mcp-use/pyproject.toml b/python/frameworks/mcp-use/pyproject.toml index 68623cc87..6797e315a 100644 --- a/python/frameworks/mcp-use/pyproject.toml +++ b/python/frameworks/mcp-use/pyproject.toml @@ -33,6 +33,7 @@ langchain-core = ">=1.6.12,<2" [tool.poetry.group.dev.dependencies] pytest = ">=7.0.0" +langchain-anthropic = ">=1.7.7,<2" [tool.poetry.urls] Homepage = "https://futureagi.com" diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py index 299bc28cf..eee6bc3c5 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py @@ -992,6 +992,34 @@ def test_malformed_callback_payloads_never_raise(capture): assert handler._runs == {} +@pytest.mark.parametrize("identifier,expected", [ + ("org/model-v1", "org/model-v1"), + ("/work/synthetic-review-user/model.gguf", "[path]"), + ("/work/synthetic-review-user", "[path]"), + ("/work/synthetic-review-user-extra/model.gguf", "/work/synthetic-review-user-extra/model.gguf"), + ("https://models.example/work/synthetic-review-user/model", "https://models.example/work/synthetic-review-user/model"), +]) +def test_identifier_scrub_preserves_model_and_url_boundaries(monkeypatch, identifier, expected): + monkeypatch.setenv("HOME", "/work/synthetic-review-user") + exporter, provider = new_provider() + try: + handler = FutureAGICallback(tracer_provider=provider) + assert handler._name(identifier) == expected + finally: + provider.shutdown() + + +def test_identifier_home_is_scrubbed_before_truncation(monkeypatch): + home = "/data/" + "synthetic-user-" * 30 + monkeypatch.setenv("HOME", home) + exporter, provider = new_provider() + try: + handler = FutureAGICallback(tracer_provider=provider) + assert handler._name(home + "/model.gguf") == "[path]" + finally: + provider.shutdown() + + def test_constructor_validates_its_arguments(): with pytest.raises(TypeError, match="TraceConfig"): FutureAGICallback(config={"hide_inputs": True}) # type: ignore[arg-type] diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_provider.py b/python/frameworks/mcp-use/tests/test_mcp_use_provider.py new file mode 100644 index 000000000..c76f98994 --- /dev/null +++ b/python/frameworks/mcp-use/tests/test_mcp_use_provider.py @@ -0,0 +1,161 @@ +"""Real provider normalization and MCPAgent, with only loopback HTTP/SSE IO.""" + +from __future__ import annotations + +import asyncio +import json +import threading +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest +from _mcp_use_support import drive, mcp_client, new_provider, wire +from fi_instrumentation import TraceConfig +from langchain_anthropic import ChatAnthropic +from mcp_use import MCPAgent + +from traceai_mcp_use import FutureAGICallback + +MODEL = "claude-offline-synthetic-model" +USAGE = { + "input_tokens": 12, + "output_tokens": 3, + "cache_read_input_tokens": 10, + "cache_creation_input_tokens": 50, + "cache_creation": {"ephemeral_5m_input_tokens": 30, "ephemeral_1h_input_tokens": 20}, +} + + +@contextmanager +def provider_server(): + """Exercise the SDK's HTTP and SSE parsers, not a handcrafted AIMessage.""" + requests = [] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + request = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + requests.append(request) + message = { + "id": "msg_offline", + "type": "message", + "role": "assistant", + "model": request["model"], + "content": [{"type": "text", "text": "offline cached answer"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": USAGE, + } + if request.get("stream"): + events = [ + ("message_start", { + "type": "message_start", + "message": dict(message, content=[], stop_reason=None, usage=dict(USAGE, output_tokens=0)), + }), + ("content_block_start", { + "type": "content_block_start", "index": 0, + "content_block": {"type": "text", "text": ""}, + }), + ("content_block_delta", { + "type": "content_block_delta", "index": 0, + "delta": {"type": "text_delta", "text": "offline cached answer"}, + }), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ("message_delta", { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": USAGE, + }), + ("message_stop", {"type": "message_stop"}), + ] + body = "".join( + "event: " + name + "\ndata: " + json.dumps(data) + "\n\n" + for name, data in events + ).encode() + content_type = "text/event-stream" + else: + body = json.dumps(message).encode() + content_type = "application/json" + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield "http://127.0.0.1:" + str(server.server_port), requests + finally: + server.shutdown() + server.server_close() + thread.join(timeout=3) + + +def provider_spans(method, hidden=False, model=MODEL): + exporter, provider = new_provider() + handler = FutureAGICallback( + tracer_provider=provider, + config=TraceConfig(hide_inputs=hidden, hide_outputs=hidden), + ) + try: + with provider_server() as (origin, requests): + llm = ChatAnthropic( + model=model, api_key="synthetic-placeholder-key", base_url=origin, + max_tokens=32, max_retries=0, timeout=10, + streaming=method != "run", + ) + + async def run(): + client = mcp_client() + try: + agent = MCPAgent(llm=llm, client=client, callbacks=[handler]) + await drive(agent, method, query="show usage") + finally: + await client.close_all_sessions() + + asyncio.run(run()) + # Assert that both transports really run, not just the async APIs. + assert len(requests) == 1 + assert bool(requests[0].get("stream")) == (method != "run") + assert not handler._runs and not handler._roots + spans = exporter.get_finished_spans() + assert len(spans) == 2 + return spans + finally: + provider.shutdown() + + +@pytest.mark.parametrize("method", ["run", "stream", "stream_events"]) +@pytest.mark.parametrize("hidden", [False, True]) +def test_real_provider_cache_tiers(method, hidden): + spans = provider_spans(method, hidden) + llm, = [s for s in spans if s.attributes.get("gen_ai.span.kind") == "LLM"] + attrs = llm.attributes + # LangChain already includes cache reads and writes in input/total counts. + assert attrs["gen_ai.usage.input_tokens"] == 72 + assert attrs["gen_ai.usage.output_tokens"] == 3 + assert attrs["gen_ai.usage.total_tokens"] == 75 + assert attrs["gen_ai.usage.cache_read_tokens"] == 10 + assert attrs["gen_ai.usage.cache_write_tokens"] == 50 + # Keep the different billing tiers, not just the aggregate write count. + assert attrs["mcp_use.llm.cache_creation.ephemeral_5m_input_tokens"] == 30 + assert attrs["mcp_use.llm.cache_creation.ephemeral_1h_input_tokens"] == 20 + assert attrs["gen_ai.request.model"] == attrs["gen_ai.response.model"] == MODEL + assert llm.name == "chat " + MODEL + + +@pytest.mark.parametrize("method", ["run", "stream", "stream_events"]) +@pytest.mark.parametrize("home", ["/work/synthetic-review-user", "/data/synthetic-review-user"]) +@pytest.mark.parametrize("hidden", [False, True]) +def test_local_model_identifiers_never_export_home(monkeypatch, method, home, hidden): + monkeypatch.setenv("HOME", home) + spans = provider_spans(method, hidden, model=home + "/models/synthetic-model.gguf") + assert home not in wire(spans) + llm, = [s for s in spans if s.attributes.get("gen_ai.span.kind") == "LLM"] + assert llm.name == "chat [path]" + assert llm.attributes["gen_ai.request.model"] == "[path]" + assert llm.attributes["gen_ai.response.model"] == "[path]" diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_usage.py b/python/frameworks/mcp-use/tests/test_mcp_use_usage.py index d742c230a..fe8250f51 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_usage.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_usage.py @@ -5,7 +5,7 @@ import asyncio import sys from pathlib import Path -from typing import Optional +from typing import Any, Dict import pytest from _mcp_use_support import ANSWER, PROMPT, ChatFake, add_script, drive, mcp_client @@ -20,12 +20,10 @@ class CachedModel(ChatFake): - cache_creation: Optional[int] = None + cache_details: Dict[str, Any] def _next(self, messages): - details = {"cache_read": 10} - if self.cache_creation is not None: - details["cache_creation"] = self.cache_creation + details = {"cache_read": 10, **self.cache_details} return super()._next(messages).model_copy( update={ "usage_metadata": { @@ -40,8 +38,17 @@ def _next(self, messages): @pytest.mark.parametrize("method", ["run", "stream", "stream_events"]) @pytest.mark.parametrize("hide_inputs,hide_outputs", [(False, False), (True, False), (False, True), (True, True)]) -@pytest.mark.parametrize("cache_creation", [None, 0, 20], ids=["absent", "zero", "written"]) -def test_cache_creation_reaches_the_collector(monkeypatch, method, hide_inputs, hide_outputs, cache_creation): +@pytest.mark.parametrize("details,writes,short,long", [ + pytest.param({}, None, None, None, id="absent"), + pytest.param({"cache_creation": 0}, 0, None, None, id="zero"), + pytest.param({"cache_creation": 20}, 20, None, None, id="generic"), + pytest.param({"cache_creation": 0, "ephemeral_5m_input_tokens": 30, "ephemeral_1h_input_tokens": 20}, 50, 30, 20, id="split"), + pytest.param({"cache_creation": 50, "ephemeral_5m_input_tokens": 30, "ephemeral_1h_input_tokens": 20}, 50, 30, 20, id="no-double-count"), + pytest.param({"ephemeral_5m_input_tokens": 30}, 30, 30, None, id="5m-only"), + pytest.param({"ephemeral_1h_input_tokens": 20}, 20, None, 20, id="1h-only"), + pytest.param({"cache_creation": 50, "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, 0, 0, 0, id="zero-tiers-win"), +]) +def test_cache_creation_reaches_the_collector(monkeypatch, method, hide_inputs, hide_outputs, details, writes, short, long): with Receiver() as receiver: monkeypatch.setenv("FI_BASE_URL", receiver.origin) monkeypatch.setenv("FI_API_KEY", "placeholder-fi-api-key") @@ -56,7 +63,7 @@ async def run(): client = mcp_client() try: agent = MCPAgent( - llm=CachedModel(script=add_script(), cache_creation=cache_creation, streaming=method != "run"), + llm=CachedModel(script=add_script(), cache_details=details, streaming=method != "run"), client=client, callbacks=[handler], ) @@ -81,9 +88,14 @@ async def run(): # Cache details are subsets of input usage, not extra total tokens. assert int(span["gen_ai.usage.total_tokens"]) == 107 assert int(span["gen_ai.usage.cache_read_tokens"]) == 10 - if cache_creation is None: - assert "gen_ai.usage.cache_write_tokens" not in span - else: - assert int(span["gen_ai.usage.cache_write_tokens"]) == cache_creation + for key, expected in ( + ("gen_ai.usage.cache_write_tokens", writes), + ("mcp_use.llm.cache_creation.ephemeral_5m_input_tokens", short), + ("mcp_use.llm.cache_creation.ephemeral_1h_input_tokens", long), + ): + if expected is None: + assert key not in span + else: + assert int(span[key]) == expected assert llms[0]["input.value"] == ("__REDACTED__" if hide_inputs else PROMPT) assert llms[-1]["output.value"] == ("__REDACTED__" if hide_outputs else ANSWER) diff --git a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py index bbcbab9b7..08b7e6348 100644 --- a/python/frameworks/mcp-use/traceai_mcp_use/_callback.py +++ b/python/frameworks/mcp-use/traceai_mcp_use/_callback.py @@ -40,7 +40,6 @@ MAX_VALUE_BYTES, KnownTexts, Secrets, - cap, clean, ) from traceai_mcp_use.version import __version__ @@ -67,6 +66,9 @@ TOTAL_TOKENS = SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS CACHE_READ_TOKENS = SpanAttributes.GEN_AI_USAGE_CACHE_READ_TOKENS CACHE_WRITE_TOKENS = SpanAttributes.GEN_AI_USAGE_CACHE_WRITE_TOKENS +# Provider-specific billing tiers are not part of the shared semantic attributes. +CACHE_WRITE_5M_TOKENS = "mcp_use.llm.cache_creation.ephemeral_5m_input_tokens" +CACHE_WRITE_1H_TOKENS = "mcp_use.llm.cache_creation.ephemeral_1h_input_tokens" TOOL_NAME = SpanAttributes.GEN_AI_TOOL_NAME TOOL_CALL_ID = SpanAttributes.GEN_AI_TOOL_CALL_ID TOOL_ARGUMENTS = SpanAttributes.GEN_AI_TOOL_CALL_ARGUMENTS @@ -393,10 +395,10 @@ def _clean(self, text: Optional[str], root: Optional[_Root], limit: int) -> Opti return clean(text, self._secrets, limit, self._pii, self._scrubs(root)) def _name(self, text: Any) -> Optional[str]: - """An identifier (model, tool name, id): secrets removed, 256-byte cap.""" + """An identifier: secrets and local identity removed before the byte cap.""" if not isinstance(text, str) or not text: return None - return cap(self._secrets.redact(text), MAX_NAME_BYTES) + return self._clean(text, None, MAX_NAME_BYTES) def _remember(self, root: Optional[_Root], texts: Iterable[Optional[str]], output: bool) -> None: if root is None: @@ -956,7 +958,19 @@ def _llm_response(self, span: Span, response: Any, root: Optional[_Root]) -> Non details = usage.get("input_token_details") if isinstance(details, Mapping): self._set(span, CACHE_READ_TOKENS, _int(details.get("cache_read"))) - self._set(span, CACHE_WRITE_TOKENS, _int(details.get("cache_creation"))) + short = _int(details.get("ephemeral_5m_input_tokens")) + long = _int(details.get("ephemeral_1h_input_tokens")) + self._set(span, CACHE_WRITE_5M_TOKENS, short) + self._set(span, CACHE_WRITE_1H_TOKENS, long) + # ChatAnthropic reports cache_creation=0 when it supplies the + # TTL breakdown. Prefer the tiers (including explicit zeros), + # but do not add them to LangChain's inclusive input/total. + writes = ( + (short or 0) + (long or 0) + if short is not None or long is not None + else _int(details.get("cache_creation")) + ) + self._set(span, CACHE_WRITE_TOKENS, writes) elif isinstance(llm_output, Mapping) and isinstance(llm_output.get("token_usage"), Mapping): token_usage = llm_output["token_usage"] self._set(span, INPUT_TOKENS, _int(token_usage.get("prompt_tokens"))) From 6bacdb42833fcd05e348f5d6f3ddda2ac44cccf3 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 4 Nov 2026 00:50:49 -0800 Subject: [PATCH 4/5] test(mcp-use): isolate synthetic homes from OAuth storage Use the real stdio MCP fixture in provider regressions so non-root CI workers never try to create a token store under /work or /data. The provider HTTP/SSE requests and home-path assertions remain unchanged. Co-Authored-By: Claude Code --- python/frameworks/mcp-use/tests/test_mcp_use_provider.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_provider.py b/python/frameworks/mcp-use/tests/test_mcp_use_provider.py index c76f98994..49f2b9779 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_provider.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_provider.py @@ -1,4 +1,4 @@ -"""Real provider normalization and MCPAgent, with only loopback HTTP/SSE IO.""" +"""Real provider normalization and MCPAgent, with offline HTTP/SSE and stdio IO.""" from __future__ import annotations @@ -110,7 +110,10 @@ def provider_spans(method, hidden=False, model=MODEL): ) async def run(): - client = mcp_client() + # HTTP MCP creates an OAuth store under HOME even without auth. + # Stdio keeps arbitrary synthetic homes independent of filesystem + # permissions, while the real provider still uses HTTP/SSE. + client = mcp_client("stdio") try: agent = MCPAgent(llm=llm, client=client, callbacks=[handler]) await drive(agent, method, query="show usage") From 7c75ce72343e8ed808dd36bd20baa2ace62d3af1 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 4 Nov 2026 03:02:08 -0800 Subject: [PATCH 5/5] docs(release-tooling): describe per-PR version completion Match complete_release.py: acknowledge the versions changed by each merged release PR, require GitHub releases and registry availability, and retain fail-closed behavior when historical version information is unavailable. Co-Authored-By: Claude Code --- RELEASING.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/RELEASING.md b/RELEASING.md index 2e18e0fb6..aa5aaa714 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -12,7 +12,9 @@ This extends **Tanmay's [#300](https://github.com/future-agi/traceAI/pull/300)** ### Exactly one owner for tags -**Publishers own all tags and GitHub releases.** release-please runs with `skip-github-release: true`; it only authors the release PR. Tags are created after registry availability is verified (Go necessarily creates its tag first, then verifies the proxy). The publisher acknowledges completed merged release PRs by replacing `autorelease: pending` with `autorelease: tagged`. It does this only after every publishable current manifest version has a GitHub release, so missing credentials or a partial release cannot silently unblock the next release. +**Publishers own all tags and GitHub releases.** release-please runs with `skip-github-release: true`; it only authors the release PR. Tags are created after registry availability is verified (Go necessarily creates its tag first, then verifies the proxy). + +The publisher acknowledges each completed merged release PR by replacing `autorelease: pending` with `autorelease: tagged`. Completion is checked against the **versions changed by that release PR**, not every version in the current manifest: each publishable changed version must have a GitHub release and be served by its registry. Later version bumps and unrelated unpublished packages do not hold that PR open. If the PR's changed versions cannot be read, a changed path is missing from the current inventory, or a required release or registry version is unavailable, that PR stays pending. | Ecosystem | Version source | Tag | Delivery | |---|---|---|---|