From a86725717fac29056d9f0646571aa4d7250ae1b7 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:27:05 -0700 Subject: [PATCH 01/10] fix(mistralai): adapt native telemetry for current Python SDK Preserve legacy completions and add embedding coverage, repair stream lifecycle and usage extraction, and exercise both SDK generations through real offline transports. Co-Authored-By: Claude Code --- .github/workflows/mistralai-python-tests.yml | 57 ++ python/frameworks/mistralai/README.md | 101 ++-- .../mistralai/examples/agent_completions.py | 2 +- .../mistralai/examples/chat_completions.py | 2 +- .../examples/chat_completions_async.py | 2 +- .../chat_completions_async_streaming.py | 2 +- .../examples/chat_completions_streaming.py | 2 +- python/frameworks/mistralai/examples/new-1.py | 2 +- .../mistralai/examples/requirements.txt | 3 +- python/frameworks/mistralai/poetry.lock | 290 ++++++---- python/frameworks/mistralai/pyproject.toml | 5 +- python/frameworks/mistralai/tests/test_sdk.py | 533 ++++++++++++++++++ .../frameworks/mistralai/tests/test_wire.py | 205 +++++++ .../mistralai/traceai_mistralai/__init__.py | 43 +- .../traceai_mistralai/_chat_wrapper.py | 13 +- .../traceai_mistralai/_embeddings.py | 32 ++ .../mistralai/traceai_mistralai/_native.py | 240 ++++++++ .../_response_accumulator.py | 19 +- .../mistralai/traceai_mistralai/_stream.py | 58 +- python/tests/test_framework_mistralai.py | 12 +- 20 files changed, 1431 insertions(+), 192 deletions(-) create mode 100644 .github/workflows/mistralai-python-tests.yml create mode 100644 python/frameworks/mistralai/tests/test_sdk.py create mode 100644 python/frameworks/mistralai/tests/test_wire.py create mode 100644 python/frameworks/mistralai/traceai_mistralai/_embeddings.py create mode 100644 python/frameworks/mistralai/traceai_mistralai/_native.py diff --git a/.github/workflows/mistralai-python-tests.yml b/.github/workflows/mistralai-python-tests.yml new file mode 100644 index 000000000..fe7f2adfb --- /dev/null +++ b/.github/workflows/mistralai-python-tests.yml @@ -0,0 +1,57 @@ +name: Mistral Python SDK tests + +on: + pull_request: + branches: [dev, main] + paths: + - 'python/frameworks/mistralai/**' + - 'python/tests/test_framework_mistralai.py' + - '.github/workflows/mistralai-python-tests.yml' + push: + branches: [dev] + paths: + - 'python/frameworks/mistralai/**' + - '.github/workflows/mistralai-python-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + sdk: + name: Mistral ${{ matrix.sdk }} / Python ${{ matrix.python }} + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - {python: '3.10', sdk: '1.1.0'} + - {python: '3.12', sdk: '1.12.4'} + - {python: '3.10', sdk: '3.6.0'} + - {python: '3.13', sdk: '3.6.0'} + env: + FI_BASE_URL: http://127.0.0.1:9 + FI_GRPC_URL: http://127.0.0.1:9 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Build and install the distribution + env: + SDK_VERSION: ${{ matrix.sdk }} + run: | + python -m venv .venv + .venv/bin/pip install build poetry-core pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0 + .venv/bin/python -m build python/frameworks/mistralai --outdir dist + .venv/bin/pip install dist/*.whl "mistralai==$SDK_VERSION" + .venv/bin/pip check + - name: Run SDK, README and OTLP contracts without outbound network + run: | + sudo unshare --net sh -c 'ip link set lo up; exec "$@"' sh \ + "$PWD/.venv/bin/python" -I -m pytest --noconftest \ + python/frameworks/mistralai/tests python/tests/test_framework_mistralai.py \ + -q -o asyncio_default_fixture_loop_scope=function diff --git a/python/frameworks/mistralai/README.md b/python/frameworks/mistralai/README.md index 874bf7d0b..1b5a29d66 100644 --- a/python/frameworks/mistralai/README.md +++ b/python/frameworks/mistralai/README.md @@ -1,67 +1,86 @@ -# MistralAI OpenTelemetry Integration +# Mistral AI tracing -## Overview -This integration provides support for using OpenTelemetry with the MistralAI framework. It enables tracing and monitoring of applications built with MistralAI. +Trace Mistral chat completions, embeddings, and conversations in Future AGI, including model names, messages, token usage, tool calls, and errors. -## Installation +## Install -1. **Install traceAI MistralAI** +Use Python 3.10–3.13 and the current Mistral SDK: -```bash -pip install traceAI-mistralai +```sh +pip install traceAI-mistralai 'mistralai>=3.6,<4' ``` +Set `FI_API_KEY`, `FI_SECRET_KEY`, and `MISTRAL_API_KEY` in your environment. Use your own Mistral API key for model requests. -### Set Environment Variables -Set up your environment variables to authenticate with FutureAGI +## Trace a completion ```python import os -os.environ["FI_API_KEY"] = FI_API_KEY -os.environ["FI_SECRET_KEY"] = FI_SECRET_KEY -os.environ["MISTRAL_API_KEY"] = MISTRAL_API_KEY -``` - -## Quickstart - -### Register Tracer Provider -Set up the trace provider to establish the observability pipeline. The trace provider: - -```python from fi_instrumentation import register from fi_instrumentation.fi_types import ProjectType +from mistralai.client import Mistral +from traceai_mistralai import MistralAIInstrumentor -trace_provider = register( +provider = register( project_type=ProjectType.OBSERVE, - project_name="mistralai_app" + project_name="mistral-app", ) +MistralAIInstrumentor().instrument(tracer_provider=provider) + +try: + with Mistral(api_key=os.environ["MISTRAL_API_KEY"]) as client: + response = client.chat.complete( + model="mistral-small-latest", + messages=[{"role": "user", "content": "Suggest three names for a neighborhood bookstore."}], + ) + print(response.choices[0].message.content) +finally: + provider.shutdown() ``` -### Configure MistralAI Instrumentation -Instrument the MistralAI client to enable telemetry collection. This step ensures that all interactions with the MistralAI SDK are tracked and monitored. +Open **Observe** in Future AGI and select `mistral-app`. The completion appears as an LLM span with the model and returned usage. -```python -from traceai_mistralai import MistralAIInstrumentor +The integration routes the SDK's native OpenTelemetry spans through your provider. Do not add another Mistral instrumentor or configure a second Mistral exporter. Clients created before instrumentation are supported too. Configure tracing once at startup; finish active requests and close streams before uninstrumenting. + +## Stream a response + +Use the stream's context manager so the connection and span close even when you stop reading early: -MistralAIInstrumentor().instrument(tracer_provider=trace_provider) +```python +with client.chat.stream( + model="mistral-small-latest", + messages=[{"role": "user", "content": "Describe a cozy bookstore."}], +) as stream: + for event in stream: + print(event.data.choices[0].delta.content or "", end="") ``` -### Create MistralAI Components -Set up your MistralAI client with built-in observability. +For async requests, use `await client.chat.complete_async(...)` or `async with await client.chat.stream_async(...)`. Keep the client open until the stream is closed. Token usage is available when the service returns it; an early-closed stream may not receive usage. + +## Control captured content + +Messages and tool arguments can contain sensitive data. To keep model names, token counts, timing, and error status without recording input or output content, replace the instrumentor call above with: ```python -from mistralai import Mistral - -client = Mistral(api_key=os.environ["MISTRAL_API_KEY"]) - -if __name__ == "__main__": - response = client.agents.complete( - agent_id="agent_id", - messages=[ - {"role": "user", "content": "plan a vacation for me in Tbilisi"}, - ], - ) - print(response) +from fi_instrumentation import TraceConfig + +MistralAIInstrumentor().instrument( + tracer_provider=provider, + config=TraceConfig(hide_inputs=True, hide_outputs=True), +) ``` +This also hides error details by default. Embedding vectors are not captured. + +## Supported APIs + +- **SDK 3.6–3.x:** native sync/async chat and agent completions, streaming, embeddings, and `beta.conversations.start`, `append`, and `restart`, including their async and streaming variants. Conversation streaming errors and final usage are captured on the native span. +- **SDK 1.1–1.x:** the existing `MistralAIInstrumentor` entry point continues to support chat and agent completions and streams, plus embeddings. Keep `from mistralai import Mistral` for this SDK line. Conversation tracing requires SDK 3.6 or later. +- SDK 1.0, 2.x, and 3.0–3.5 are outside the supported range. Pin one of the ranges above rather than overriding pip's dependency check. + +## Troubleshooting + +- **`cannot import name Mistral`**: use `from mistralai.client import Mistral` with SDK 3.6 or later. The top-level import belongs to SDK 1.x. +- **Missing streaming spans**: fully consume or explicitly close the stream before flushing or shutting down your tracer provider. +- **Duplicate spans**: remove other Mistral SDK instrumentors. This integration adapts native telemetry rather than adding a second completion span. diff --git a/python/frameworks/mistralai/examples/agent_completions.py b/python/frameworks/mistralai/examples/agent_completions.py index 0b2a8b5b2..8798b7c26 100644 --- a/python/frameworks/mistralai/examples/agent_completions.py +++ b/python/frameworks/mistralai/examples/agent_completions.py @@ -8,7 +8,7 @@ EvalTagType, ProjectType, ) -from mistralai import Mistral +from mistralai.client import Mistral from traceai_mistralai import MistralAIInstrumentor # Configure trace provider with custom evaluation tags diff --git a/python/frameworks/mistralai/examples/chat_completions.py b/python/frameworks/mistralai/examples/chat_completions.py index 4a585f6cc..4d236b129 100644 --- a/python/frameworks/mistralai/examples/chat_completions.py +++ b/python/frameworks/mistralai/examples/chat_completions.py @@ -8,7 +8,7 @@ EvalTagType, ProjectType, ) -from mistralai import Mistral +from mistralai.client import Mistral from traceai_mistralai import MistralAIInstrumentor # Configure trace provider with custom evaluation tags diff --git a/python/frameworks/mistralai/examples/chat_completions_async.py b/python/frameworks/mistralai/examples/chat_completions_async.py index 9f2ebe508..5243c973a 100644 --- a/python/frameworks/mistralai/examples/chat_completions_async.py +++ b/python/frameworks/mistralai/examples/chat_completions_async.py @@ -9,7 +9,7 @@ EvalTagType, ProjectType, ) -from mistralai import Mistral +from mistralai.client import Mistral from traceai_mistralai import MistralAIInstrumentor # Configure trace provider with custom evaluation tags diff --git a/python/frameworks/mistralai/examples/chat_completions_async_streaming.py b/python/frameworks/mistralai/examples/chat_completions_async_streaming.py index 836299f56..cc278f64a 100644 --- a/python/frameworks/mistralai/examples/chat_completions_async_streaming.py +++ b/python/frameworks/mistralai/examples/chat_completions_async_streaming.py @@ -9,7 +9,7 @@ EvalTagType, ProjectType, ) -from mistralai import Mistral +from mistralai.client import Mistral from traceai_mistralai import MistralAIInstrumentor # Configure trace provider with custom evaluation tags diff --git a/python/frameworks/mistralai/examples/chat_completions_streaming.py b/python/frameworks/mistralai/examples/chat_completions_streaming.py index 043e4c6fd..8a36a15c2 100644 --- a/python/frameworks/mistralai/examples/chat_completions_streaming.py +++ b/python/frameworks/mistralai/examples/chat_completions_streaming.py @@ -8,7 +8,7 @@ EvalTagType, ProjectType, ) -from mistralai import Mistral +from mistralai.client import Mistral from traceai_mistralai import MistralAIInstrumentor # Configure trace provider with custom evaluation tags diff --git a/python/frameworks/mistralai/examples/new-1.py b/python/frameworks/mistralai/examples/new-1.py index c0f4f4570..3dec50124 100644 --- a/python/frameworks/mistralai/examples/new-1.py +++ b/python/frameworks/mistralai/examples/new-1.py @@ -2,7 +2,7 @@ from fi_instrumentation import register from fi_instrumentation.fi_types import ProjectType -from mistralai import Mistral +from mistralai.client import Mistral from traceai_mistralai import MistralAIInstrumentor trace_provider = register( diff --git a/python/frameworks/mistralai/examples/requirements.txt b/python/frameworks/mistralai/examples/requirements.txt index 3a6f02dd0..9c9b26f5a 100644 --- a/python/frameworks/mistralai/examples/requirements.txt +++ b/python/frameworks/mistralai/examples/requirements.txt @@ -1 +1,2 @@ -mistralai +traceAI-mistralai +mistralai>=3.6,<4 diff --git a/python/frameworks/mistralai/poetry.lock b/python/frameworks/mistralai/poetry.lock index a9e065ce5..932b2035e 100644 --- a/python/frameworks/mistralai/poetry.lock +++ b/python/frameworks/mistralai/poetry.lock @@ -19,6 +19,7 @@ description = "High-level concurrency and networking framework on top of asyncio optional = false python-versions = ">=3.9" groups = ["main"] +markers = "sys_platform != \"emscripten\"" files = [ {file = "anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1"}, {file = "anyio-4.10.0.tar.gz", hash = "sha256:3f3fae35c96039744587aa5b8371e7e8e603c0702999535961dd336026973ba6"}, @@ -187,7 +188,7 @@ description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" groups = ["main"] -markers = "python_version == \"3.10\"" +markers = "sys_platform != \"emscripten\" and python_version == \"3.10\"" files = [ {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, @@ -201,14 +202,14 @@ test = ["pytest (>=6)"] [[package]] name = "fi-instrumentation-otel" -version = "1.1.0" -description = "OpenTelemetry instrumentation" +version = "1.2.2" +description = "OpenTelemetry tracing for AI applications: tracer setup, span helpers, exporters, and semantic conventions for Future AGI." optional = false python-versions = "<3.14,>3.9" groups = ["main"] files = [ - {file = "fi_instrumentation_otel-1.1.0-py3-none-any.whl", hash = "sha256:4fa2bf663e0ce09447aca92ae798cd7b0dee35c5c92c2e033f1d89d158a9e62c"}, - {file = "fi_instrumentation_otel-1.1.0.tar.gz", hash = "sha256:7ec324c3c9242aa87a88cf23a21c11919b8736c6451c5fb50917f592a853b059"}, + {file = "fi_instrumentation_otel-1.2.2-py3-none-any.whl", hash = "sha256:25cf88f93e33aa928d4fae0f631afe8a32565639490862b715869e0633becbaf"}, + {file = "fi_instrumentation_otel-1.2.2.tar.gz", hash = "sha256:b675df47d7a03c506fbd2e4b0420fe4f74d12a6353a88b219bfd97797b276d4c"}, ] [package.dependencies] @@ -251,93 +252,105 @@ description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" optional = false python-versions = ">=3.8" groups = ["main"] +markers = "sys_platform != \"emscripten\"" files = [ {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, ] [[package]] -name = "httpcore" -version = "1.0.9" +name = "httpcore2" +version = "2.14.0" description = "A minimal low-level HTTP client." optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] +markers = "sys_platform != \"emscripten\"" files = [ - {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, - {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, + {file = "httpcore2-2.14.0-py3-none-any.whl", hash = "sha256:4c4f64f4b544afeb778c42c0ffb5941b29e671157e90ed3f6dc787c4bca90cd1"}, + {file = "httpcore2-2.14.0.tar.gz", hash = "sha256:4f57a1a8249fe6af589d935cfd456f1d13bbdcb0f0fc72af9142368fc94811d7"}, ] [package.dependencies] -certifi = "*" h11 = ">=0.16" +truststore = ">=0.10" [package.extras] -asyncio = ["anyio (>=4.0,<5.0)"] +asyncio = ["anyio (>=4.5.0,<5.0)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<1.0)"] +trio = ["trio (>=0.34.0,<1.0)"] [[package]] -name = "httpx" -version = "0.28.1" +name = "httpx2" +version = "2.14.0" description = "The next generation HTTP client." optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, - {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, + {file = "httpx2-2.14.0-py3-none-any.whl", hash = "sha256:9d20c13a10bae1773a0be3b0f1e5beeb4bdd549bc3a2ba18efa32897c65403e4"}, + {file = "httpx2-2.14.0.tar.gz", hash = "sha256:47ad61a3c40282cd8d0dc96311531f00f5bfc71a066c6fe3b36d8f63fb9e4f6e"}, ] [package.dependencies] -anyio = "*" -certifi = "*" -httpcore = "==1.*" -idna = "*" +anyio = {version = ">=4.10", markers = "sys_platform != \"emscripten\""} +httpcore2 = {version = "2.14.0", markers = "sys_platform != \"emscripten\""} +httpx2-jsfetch = {version = "*", markers = "sys_platform == \"emscripten\" and python_version >= \"3.12\""} +idna = ">=3.18" +truststore = {version = ">=0.10", markers = "sys_platform != \"emscripten\""} +typing-extensions = {version = ">=4.5.0", markers = "python_version < \"3.13\""} [package.extras] -brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] -cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.2) ; platform_python_implementation != \"CPython\""] +cli = ["click (>=8.4.2)", "pygments (==2.*)", "rich (>=10,<16)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] -zstd = ["zstandard (>=0.18.0)"] +ws = ["wsproto (>=1.2)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version <= \"3.13\""] [[package]] -name = "idna" -version = "3.7" -description = "Internationalized Domain Names in Applications (IDNA)" +name = "httpx2-jsfetch" +version = "1.0" +description = "httpx2 transports for Emscripten/Pyodide, backed by the JavaScript fetch API." optional = false -python-versions = ">=3.5" +python-versions = ">=3.12" groups = ["main"] +markers = "sys_platform == \"emscripten\" and python_version >= \"3.12\"" files = [ - {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, - {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, + {file = "httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32"}, + {file = "httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60"}, ] [[package]] -name = "importlib-metadata" -version = "8.7.0" -description = "Read metadata from Python packages" +name = "idna" +version = "3.20" +description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd"}, - {file = "importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000"}, + {file = "idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c"}, + {file = "idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44"}, ] -[package.dependencies] -zipp = ">=3.20" +[package.extras] +all = ["coverage (>=7.10.0)", "hypothesis (>=6.141.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.16.0)", "ty (>=0.0.37)"] + +[[package]] +name = "jsonpath-python" +version = "1.1.6" +description = "A lightweight and powerful JSONPath implementation for Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "jsonpath_python-1.1.6-py3-none-any.whl", hash = "sha256:a1c50afd8d3fbbaf47a4873bc890dcb3c15da96f5c020327977d844d8731a2d4"}, + {file = "jsonpath_python-1.1.6.tar.gz", hash = "sha256:dded9932b4ec41fb8726e09c83afa4e6be618f938c2db287cc2a81723c639671"}, +] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] -cover = ["pytest-cov"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -enabler = ["pytest-enabler (>=2.2)"] -perf = ["ipython"] -test = ["flufl.flake8", "importlib_resources (>=1.3) ; python_version < \"3.9\"", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] -type = ["pytest-mypy"] +dev = ["poethepoet", "pytest (>=8.0)", "pytest-benchmark[histogram] (>=4.0)", "pytest-cov (>=5.0)", "ruff (>=0.3)"] [[package]] name = "jsonschema" @@ -378,143 +391,202 @@ referencing = ">=0.31.0" [[package]] name = "mistralai" -version = "1.9.3" +version = "3.6.0" description = "Python Client SDK for the Mistral AI API." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "mistralai-1.9.3-py3-none-any.whl", hash = "sha256:962445e7cebadcbfbcd1daf973e853a832dcf7aba6320468fcf7e2cf5f943aec"}, - {file = "mistralai-1.9.3.tar.gz", hash = "sha256:a69806247ed3a67820ecfc9a68b7dbc0c6120dad5e5c3d507bd57fa388b491b7"}, + {file = "mistralai-3.6.0-py3-none-any.whl", hash = "sha256:709c31236d1c865f756734cb14f2f725fad993d054bc3be2d6aae3e78b4b3efb"}, + {file = "mistralai-3.6.0.tar.gz", hash = "sha256:1cd130da01979d8ecdc374a88bb7a880524b318a2be34b6e69013ca331de9d70"}, ] [package.dependencies] eval-type-backport = ">=0.2.0" -httpx = ">=0.28.1" -pydantic = ">=2.10.3" +httpx2 = ">=2.13.0" +jsonpath-python = ">=1.0.6" +opentelemetry-api = ">=1.33.1,<2.0.0" +opentelemetry-semantic-conventions = ">=0.61b0" +pydantic = ">=2.11.2" python-dateutil = ">=2.8.2" typing-inspection = ">=0.4.0" [package.extras] -agents = ["authlib (>=1.5.2,<2.0)", "griffe (>=1.7.3,<2.0)", "mcp (>=1.0,<2.0) ; python_version >= \"3.10\""] +agents = ["authlib (>=1.8.0,<2.0)", "griffe (>=1.7.3,<2.0)", "mcp (>=2.2.0,<3.0)"] gcp = ["google-auth (>=2.27.0)", "requests (>=2.32.3)"] +mcp = ["mcp (>=2.2.0,<3.0)"] +realtime = ["websockets (>=13.0)"] +telemetry = ["opentelemetry-exporter-otlp-proto-http (>=1.33.1,<2.0.0)", "opentelemetry-sdk (>=1.33.1,<2.0.0)"] +voice-agents = ["livekit (>=1.1,<2.0)", "numpy (>=1.26,<3)", "pyaudio (>=0.2.14,<0.3.0)"] +workflow-payload-compression = ["msgpack (>=1.1.0,<2.0.0)", "zstandard (>=0.25.0,<0.26)"] +workflow-payload-encryption = ["cryptography (>=50.0.0,<51.0.0)"] +workflow-payload-offloading = ["aioboto3 (>=12.4.0,<13.0.0)", "azure-identity (>=1.25.0,<2.0.0)", "azure-storage-blob[aio] (>=12.28.0,<13.0.0)", "gcloud-aio-storage (>=9.3.0,<10.0.0)"] +workflow-payload-offloading-azure = ["azure-identity (>=1.25.0,<2.0.0)", "azure-storage-blob[aio] (>=12.28.0,<13.0.0)"] +workflow-payload-offloading-gcs = ["gcloud-aio-storage (>=9.3.0,<10.0.0)"] +workflow-payload-offloading-s3 = ["aioboto3 (>=12.4.0,<13.0.0)"] [[package]] name = "opentelemetry-api" -version = "1.36.0" +version = "1.45.2" description = "OpenTelemetry Python API" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_api-1.36.0-py3-none-any.whl", hash = "sha256:02f20bcacf666e1333b6b1f04e647dc1d5111f86b8e510238fcc56d7762cda8c"}, - {file = "opentelemetry_api-1.36.0.tar.gz", hash = "sha256:9a72572b9c416d004d492cbc6e61962c0501eaf945ece9b5a0f56597d8348aa0"}, + {file = "opentelemetry_api-1.45.2-py3-none-any.whl", hash = "sha256:2cbd737377e710ee740d261c94e93fa0f08e6d037c18f17513cc0a2d94479a34"}, + {file = "opentelemetry_api-1.45.2.tar.gz", hash = "sha256:d7b0008e21e5fd61cbe7a7f34ca3b6905a78ab677d9bc395e299359c75cda6b4"}, ] [package.dependencies] -importlib-metadata = ">=6.0,<8.8.0" typing-extensions = ">=4.5.0" +[[package]] +name = "opentelemetry-exporter-http-transport" +version = "0.66b2" +description = "OpenTelemetry Exporters HTTP transport" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "opentelemetry_exporter_http_transport-0.66b2-py3-none-any.whl", hash = "sha256:fd5f5b490c099acda1cd7b909d5e8981436bf18cc8c6098d302d114c271720e9"}, + {file = "opentelemetry_exporter_http_transport-0.66b2.tar.gz", hash = "sha256:66d6ed810a515432c8891da9a834464332fd70badeca45bdeeae81b4280c903b"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.15,<2.0" +requests = {version = ">=2.25,<3.0", optional = true, markers = "extra == \"requests\""} + +[package.extras] +requests = ["requests (>=2.25,<3.0)"] +urllib3 = ["urllib3 (>=1.26)"] + +[[package]] +name = "opentelemetry-exporter-otlp-common" +version = "0.66b2" +description = "OpenTelemetry OTLP HTTP export utilities" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "opentelemetry_exporter_otlp_common-0.66b2-py3-none-any.whl", hash = "sha256:ace86efb75489f925f1ef809856ae5f0c7db534e4afdc624530222228b1236a5"}, + {file = "opentelemetry_exporter_otlp_common-0.66b2.tar.gz", hash = "sha256:06f5a9b88638e310efa1f1d206cdf6b75393e112aba4e174a963416192bd1028"}, +] + +[package.dependencies] +opentelemetry-sdk = ">=1.45.2,<1.46.0" + +[package.extras] +http = ["opentelemetry-exporter-http-transport (==0.66b2)"] + [[package]] name = "opentelemetry-exporter-otlp-proto-common" -version = "1.36.0" +version = "1.45.2" description = "OpenTelemetry Protobuf encoding" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_exporter_otlp_proto_common-1.36.0-py3-none-any.whl", hash = "sha256:0fc002a6ed63eac235ada9aa7056e5492e9a71728214a61745f6ad04b923f840"}, - {file = "opentelemetry_exporter_otlp_proto_common-1.36.0.tar.gz", hash = "sha256:6c496ccbcbe26b04653cecadd92f73659b814c6e3579af157d8716e5f9f25cbf"}, + {file = "opentelemetry_exporter_otlp_proto_common-1.45.2-py3-none-any.whl", hash = "sha256:af483b24217028892f017096731b794b196b785025d2e6808874720fd0b05a80"}, + {file = "opentelemetry_exporter_otlp_proto_common-1.45.2.tar.gz", hash = "sha256:f1f2806f3f28ce7938033823e50934fcd91595a39de38e69a28a51e19f40fc6d"}, ] [package.dependencies] -opentelemetry-proto = "1.36.0" +opentelemetry-proto = "1.45.2" [[package]] name = "opentelemetry-exporter-otlp-proto-http" -version = "1.36.0" +version = "1.45.2" description = "OpenTelemetry Collector Protobuf over HTTP Exporter" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_exporter_otlp_proto_http-1.36.0-py3-none-any.whl", hash = "sha256:3d769f68e2267e7abe4527f70deb6f598f40be3ea34c6adc35789bea94a32902"}, - {file = "opentelemetry_exporter_otlp_proto_http-1.36.0.tar.gz", hash = "sha256:dd3637f72f774b9fc9608ab1ac479f8b44d09b6fb5b2f3df68a24ad1da7d356e"}, + {file = "opentelemetry_exporter_otlp_proto_http-1.45.2-py3-none-any.whl", hash = "sha256:6e4d025138829d0ee57946930c2ccad6491ad4d92a958e9c7eea69278fe60cfc"}, + {file = "opentelemetry_exporter_otlp_proto_http-1.45.2.tar.gz", hash = "sha256:5984a3fb46bc4628e4e2b85d4a6e72637e03962aae6cf372791137768a4ad939"}, ] [package.dependencies] googleapis-common-protos = ">=1.52,<2.0" opentelemetry-api = ">=1.15,<2.0" -opentelemetry-exporter-otlp-proto-common = "1.36.0" -opentelemetry-proto = "1.36.0" -opentelemetry-sdk = ">=1.36.0,<1.37.0" +opentelemetry-exporter-http-transport = {version = "0.66b2", extras = ["requests"]} +opentelemetry-exporter-otlp-common = "0.66b2" +opentelemetry-exporter-otlp-proto-common = "1.45.2" +opentelemetry-proto = "1.45.2" +opentelemetry-sdk = ">=1.45.2,<1.46.0" requests = ">=2.7,<3.0" typing-extensions = ">=4.5.0" +[package.extras] +gcp-auth = ["opentelemetry-exporter-credential-provider-gcp (>=0.59b0)"] +requests = ["opentelemetry-exporter-http-transport[requests] (==0.66b2)", "requests (>=2.7,<3.0)"] + [[package]] name = "opentelemetry-instrumentation" -version = "0.57b0" +version = "0.66b2" description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_instrumentation-0.57b0-py3-none-any.whl", hash = "sha256:9109280f44882e07cec2850db28210b90600ae9110b42824d196de357cbddf7e"}, - {file = "opentelemetry_instrumentation-0.57b0.tar.gz", hash = "sha256:f2a30135ba77cdea2b0e1df272f4163c154e978f57214795d72f40befd4fcf05"}, + {file = "opentelemetry_instrumentation-0.66b2-py3-none-any.whl", hash = "sha256:fa9ccfdab147a14af8107d19ef0a7b5015675273c5adced7a1fc38f28b42ec4c"}, + {file = "opentelemetry_instrumentation-0.66b2.tar.gz", hash = "sha256:02b094150e53b76f5906d75e628dbb3833bbc6e62fa109b5c7ba0c081865e923"}, ] [package.dependencies] opentelemetry-api = ">=1.4,<2.0" -opentelemetry-semantic-conventions = "0.57b0" +opentelemetry-semantic-conventions = "0.66b2" packaging = ">=18.0" -wrapt = ">=1.0.0,<2.0.0" +wrapt = ">=1.0.0,<3.0.0" [[package]] name = "opentelemetry-proto" -version = "1.36.0" +version = "1.45.2" description = "OpenTelemetry Python Proto" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_proto-1.36.0-py3-none-any.whl", hash = "sha256:151b3bf73a09f94afc658497cf77d45a565606f62ce0c17acb08cd9937ca206e"}, - {file = "opentelemetry_proto-1.36.0.tar.gz", hash = "sha256:0f10b3c72f74c91e0764a5ec88fd8f1c368ea5d9c64639fb455e2854ef87dd2f"}, + {file = "opentelemetry_proto-1.45.2-py3-none-any.whl", hash = "sha256:03a7ca1b43f89fd5b965d5975902bb2515c0c86e2428b456836e9a6149c6235a"}, + {file = "opentelemetry_proto-1.45.2.tar.gz", hash = "sha256:bd8d23cf770605b0af9b7be2071d26a464fea289d35429abfd2a66226114ad64"}, ] [package.dependencies] -protobuf = ">=5.0,<7.0" +protobuf = ">=5.0,<8.0" [[package]] name = "opentelemetry-sdk" -version = "1.36.0" +version = "1.45.2" description = "OpenTelemetry Python SDK" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_sdk-1.36.0-py3-none-any.whl", hash = "sha256:19fe048b42e98c5c1ffe85b569b7073576ad4ce0bcb6e9b4c6a39e890a6c45fb"}, - {file = "opentelemetry_sdk-1.36.0.tar.gz", hash = "sha256:19c8c81599f51b71670661ff7495c905d8fdf6976e41622d5245b791b06fa581"}, + {file = "opentelemetry_sdk-1.45.2-py3-none-any.whl", hash = "sha256:ff974ea70ee4f1d51e2ec152db37c2b626c7bf82bb3f7d86114fd8704e7199c6"}, + {file = "opentelemetry_sdk-1.45.2.tar.gz", hash = "sha256:7eec4de13dd21ff3a1670dcb0e02079e692736f94d733dfdce5af461d884248d"}, ] [package.dependencies] -opentelemetry-api = "1.36.0" -opentelemetry-semantic-conventions = "0.57b0" +opentelemetry-api = "1.45.2" +opentelemetry-semantic-conventions = "0.66b2" typing-extensions = ">=4.5.0" +[package.extras] +file-configuration = ["opentelemetry-configuration (==0.66b2)"] + [[package]] name = "opentelemetry-semantic-conventions" -version = "0.57b0" +version = "0.66b2" description = "OpenTelemetry Semantic Conventions" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_semantic_conventions-0.57b0-py3-none-any.whl", hash = "sha256:757f7e76293294f124c827e514c2a3144f191ef175b069ce8d1211e1e38e9e78"}, - {file = "opentelemetry_semantic_conventions-0.57b0.tar.gz", hash = "sha256:609a4a79c7891b4620d64c7aac6898f872d790d75f22019913a660756f27ff32"}, + {file = "opentelemetry_semantic_conventions-0.66b2-py3-none-any.whl", hash = "sha256:4cbe61e5f9275d57ab31708446e7938ef4291ebbe6a7fca6f7a629c5acae87d8"}, + {file = "opentelemetry_semantic_conventions-0.66b2.tar.gz", hash = "sha256:7da5f1a5dc4383eb66711d4d3d3e71824d9403ac1559fea021cc103c1a4c4976"}, ] [package.dependencies] -opentelemetry-api = "1.36.0" +opentelemetry-api = "1.45.2" typing-extensions = ">=4.5.0" [[package]] @@ -909,11 +981,25 @@ description = "Sniff out which async library your code is running under" optional = false python-versions = ">=3.7" groups = ["main"] +markers = "sys_platform != \"emscripten\"" files = [ {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, ] +[[package]] +name = "truststore" +version = "0.10.5" +description = "Verify certificates using native system trust stores" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "sys_platform != \"emscripten\"" +files = [ + {file = "truststore-0.10.5-py3-none-any.whl", hash = "sha256:9aaaedaefaf06d8b206278cf8b5012bc897f485a874503501e12d776df78951c"}, + {file = "truststore-0.10.5.tar.gz", hash = "sha256:30d36967ccaded5cbb38d602c433f53600036c79d502f4533a49b60a03bbefcd"}, +] + [[package]] name = "typing-extensions" version = "4.14.1" @@ -1048,27 +1134,7 @@ files = [ {file = "wrapt-1.17.2.tar.gz", hash = "sha256:41388e9d4d1522446fe79d3213196bd9e3b301a336965b9e27ca2788ebd122f3"}, ] -[[package]] -name = "zipp" -version = "3.23.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, - {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, -] - -[package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] -cover = ["pytest-cov"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -enabler = ["pytest-enabler (>=2.2)"] -test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] -type = ["pytest-mypy"] - [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "9e1fa619e2eb0bf07bdebf3c97a1611cb549c467cee5cbc2b4df1b8e4adcf8ee" +content-hash = "d915dc2322aec75e3d8e13184886c1fcabd78072dfda8a6f4385adc5fd79c516" diff --git a/python/frameworks/mistralai/pyproject.toml b/python/frameworks/mistralai/pyproject.toml index 3348f19ad..0bb89e9ca 100644 --- a/python/frameworks/mistralai/pyproject.toml +++ b/python/frameworks/mistralai/pyproject.toml @@ -27,8 +27,9 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.14" -fi-instrumentation-otel = ">=0.1.11" -mistralai = ">=1.0.2,<2" +fi-instrumentation-otel = ">=1.2.2" +mistralai = ">=1.1.0,<4,!=2.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" +packaging = ">=23.2" [tool.poetry.urls] Homepage = "https://futureagi.com" diff --git a/python/frameworks/mistralai/tests/test_sdk.py b/python/frameworks/mistralai/tests/test_sdk.py new file mode 100644 index 000000000..ce533cbd4 --- /dev/null +++ b/python/frameworks/mistralai/tests/test_sdk.py @@ -0,0 +1,533 @@ +"""Real Mistral SDK requests with synthetic in-process HTTP/SSE transports.""" + +import asyncio +import importlib +import json +from importlib.metadata import version + +import pytest +from fi_instrumentation import TraceConfig, suppress_tracing +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +from traceai_mistralai import MistralAIInstrumentor + +NATIVE = int(version("mistralai").split(".")[0]) >= 3 +SDK_ROOT = "mistralai.client" if NATIVE else "mistralai" +Mistral = importlib.import_module(SDK_ROOT).Mistral +httpx = importlib.import_module("httpx2" if NATIVE else "httpx") +MESSAGES = [{"role": "user", "content": "private-input-marker"}] +USAGE = {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3} + + +def chat_response(): + return { + "id": "synthetic", + "object": "chat.completion", + "created": 1, + "model": "mistral-test", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "private-output-marker", + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": { + "name": "weather", + "arguments": '{"city":"test"}', + }, + } + ], + }, + "finish_reason": "stop", + } + ], + "usage": USAGE, + } + + +def embedding_response(): + return { + "id": "synthetic", + "object": "list", + "model": "mistral-embed", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "usage": {"prompt_tokens": 2, "completion_tokens": 0, "total_tokens": 2}, + } + + +def conversation_response(): + return { + "object": "conversation.response", + "conversation_id": "conv-test", + "outputs": [ + { + "type": "message.output", + "role": "assistant", + "content": "private-output-marker", + } + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + } + + +def completion_sse(): + for delta, finish in [ + ({"role": "assistant", "content": "private-output-marker"}, None), + ( + { + "content": "", + "tool_calls": [ + { + "index": 0, + "id": "call-1", + "type": "function", + "function": {"name": "weather", "arguments": '{"city":"test"}'}, + } + ], + }, + "stop", + ), + ]: + yield ( + "data: " + + json.dumps( + { + "id": "synthetic", + "object": "chat.completion.chunk", + "created": 1, + "model": "mistral-test", + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], + "usage": USAGE if finish else None, + } + ) + + "\n\n" + ).encode() + yield b"data: [DONE]\n\n" + + +class SSE(httpx.SyncByteStream, httpx.AsyncByteStream): + def __init__(self, fail=False): + self.fail = fail + self.closed = False + + def __iter__(self): + for chunk in completion_sse(): + yield chunk + if self.fail: + raise RuntimeError("private-stream-error-marker") + + async def __aiter__(self): + for chunk in self: + yield chunk + + def close(self): + self.closed = True + + async def aclose(self): + self.closed = True + + +def conversation_sse(error=False): + events = [ + {"type": "conversation.response.started", "conversation_id": "conv-test"}, + { + "type": "message.output.delta", + "id": "entry-test", + "role": "assistant", + "content": "private-output-marker", + }, + {"type": "conversation.response.done", "usage": USAGE}, + ] + if error: + events[-1] = { + "type": "conversation.response.error", + "message": "private-error-marker", + "code": 400, + } + return [ + f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() + for event in events + ] + + +class ConversationSSE(SSE): + def __iter__(self): + yield from conversation_sse(self.fail) + + +def transport_response(request): + if request.url.path.endswith("embeddings"): + return httpx.Response(200, json=embedding_response()) + if "conversations" in request.url.path: + if json.loads(request.content).get("stream"): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=ConversationSSE(), + ) + return httpx.Response(200, json=conversation_response()) + if json.loads(request.content).get("stream"): + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=SSE() + ) + return httpx.Response(200, json=chat_response()) + + +@pytest.fixture +def tracing(request): + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + config = getattr(request, "param", TraceConfig()) + instrumentor.instrument(tracer_provider=provider, config=config) + assert instrumentor.is_instrumented_by_opentelemetry + yield exporter + instrumentor.uninstrument() + provider.shutdown() + + +def params(resource): + return { + "messages": MESSAGES, + **( + {"model": "mistral-test"} + if resource == "chat" + else {"agent_id": "test-agent"} + ), + } + + +@pytest.mark.parametrize("resource", ["chat", "agents"]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_completions(tracing, resource, streaming, asynchronous): + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(transport_response) + ) as transport: + client = Mistral(api_key="synthetic-test-key", async_client=transport) + method = getattr( + getattr(client, resource), + "stream_async" if streaming else "complete_async", + ) + result = await method(**params(resource)) + if streaming: + chunks = [chunk async for chunk in result] + if hasattr(result, "close"): + await result.close() + return chunks[0].data.choices[0].delta.content + return result.choices[0].message.content + + if asynchronous: + text = asyncio.run(run()) + else: + with httpx.Client( + transport=httpx.MockTransport(transport_response) + ) as transport: + client = Mistral(api_key="synthetic-test-key", client=transport) + result = getattr( + getattr(client, resource), "stream" if streaming else "complete" + )(**params(resource)) + if streaming: + chunks = list(result) + result.close() + text = chunks[0].data.choices[0].delta.content + else: + text = result.choices[0].message.content + assert text == "private-output-marker" + spans = tracing.get_finished_spans() + assert len(spans) == 1 + assert spans[0].attributes["gen_ai.usage.input_tokens"] == 2 + assert spans[0].attributes["gen_ai.usage.output_tokens"] == 1 + assert "weather" in json.dumps(dict(spans[0].attributes)) + assert spans[0].status.status_code == StatusCode.OK + + +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_embeddings(tracing, asynchronous): + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(transport_response) + ) as transport: + return await Mistral( + api_key="synthetic", async_client=transport + ).embeddings.create_async(model="mistral-embed", inputs=["hi"]) + + if asynchronous: + result = asyncio.run(run()) + else: + with httpx.Client( + transport=httpx.MockTransport(transport_response) + ) as transport: + result = Mistral(api_key="synthetic", client=transport).embeddings.create( + model="mistral-embed", inputs=["hi"] + ) + assert result.data[0].embedding == [0.1, 0.2] + spans = tracing.get_finished_spans() + assert len(spans) == 1 + assert spans[0].attributes["gen_ai.span.kind"] == "EMBEDDING" + assert spans[0].attributes["gen_ai.usage.input_tokens"] == 2 + + +@pytest.mark.skipif( + not NATIVE, reason="Native conversation telemetry is supported on SDK >=3.6" +) +@pytest.mark.parametrize("method", ["start", "append", "restart"]) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +def test_conversations(tracing, method, asynchronous, streaming): + kwargs = {"inputs": "private-input-marker"} + if method == "start": + kwargs["model"] = "mistral-test" + else: + kwargs["conversation_id"] = "conv-test" + if method == "restart": + kwargs["from_entry_id"] = "entry-test" + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(transport_response) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + result = await getattr( + client.beta.conversations, + method + ("_stream" if streaming else "") + "_async", + )(**kwargs) + if streaming: + chunks = [chunk async for chunk in result] + await result.close() + assert chunks[-1].data.usage.prompt_tokens == 2 + return chunks[0].data.conversation_id + return result.conversation_id + + if asynchronous: + result = asyncio.run(run()) + else: + with httpx.Client( + transport=httpx.MockTransport(transport_response) + ) as transport: + client = Mistral(api_key="synthetic", client=transport) + result = getattr( + client.beta.conversations, method + ("_stream" if streaming else "") + )(**kwargs) + if streaming: + chunks = list(result) + result.close() + assert chunks[-1].data.usage.prompt_tokens == 2 + result = chunks[0].data.conversation_id + else: + result = result.conversation_id + assert result == "conv-test" + spans = tracing.get_finished_spans() + assert len(spans) == 1 + assert spans[0].attributes["gen_ai.span.kind"] == "AGENT" + assert spans[0].attributes["gen_ai.usage.input_tokens"] == 2 + + +def test_existing_clients_suppression_reenable_and_parent(): + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + with httpx.Client(transport=httpx.MockTransport(transport_response)) as transport: + client = Mistral(api_key="synthetic", client=transport) + original = type(client.chat).complete + instrumentor = MistralAIInstrumentor() + for _ in range(2): + exporter.clear() + instrumentor.instrument(tracer_provider=provider) + instrumentor.instrument(tracer_provider=provider) + if NATIVE: + assert type(client.chat).complete is original + with provider.get_tracer("parent").start_as_current_span( + "parent" + ) as parent: + with suppress_tracing(): + client.chat.complete(**params("chat")) + client.chat.complete(**params("chat")) + spans = exporter.get_finished_spans() + assert len(spans) == 2 + assert spans[0].parent.span_id == parent.get_span_context().span_id + instrumentor.uninstrument() + exporter.clear() + client.chat.complete(**params("chat")) + assert not exporter.get_finished_spans() + provider.shutdown() + + +@pytest.mark.parametrize( + "tracing", [TraceConfig(hide_inputs=True, hide_outputs=True)], indirect=True +) +@pytest.mark.parametrize("error", [False, True]) +def test_privacy_and_errors(tracing, error): + def respond(request): + if error: + return httpx.Response(400, json={"message": "private-error-marker"}) + return transport_response(request) + + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-key-marker", client=transport) + if error: + with pytest.raises(Exception, match="private-error-marker"): + client.chat.complete(**params("chat")) + else: + import inspect + + options = ( + {"http_headers": {"x-private": "private-header-marker"}} + if "http_headers" in inspect.signature(client.chat.complete).parameters + else {} + ) + client.chat.complete(**params("chat"), **options) + spans = tracing.get_finished_spans() + assert len(spans) == 1 + encoded = spans[0].to_json() + for marker in ( + "private-input-marker", + "private-output-marker", + "private-error-marker", + "private-header-marker", + "synthetic-key-marker", + ): + assert marker not in encoded + assert spans[0].status.status_code == (StatusCode.ERROR if error else StatusCode.OK) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("failure", [False, True]) +def test_stream_early_close_and_errors(tracing, asynchronous, failure): + stream = SSE(fail=failure) + + def respond(request): + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=stream + ) + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + result = await Mistral( + api_key="synthetic", async_client=transport + ).chat.stream_async(**params("chat")) + assert not tracing.get_finished_spans() + async with result: + if failure: + with pytest.raises( + RuntimeError, match="private-stream-error-marker" + ): + async for _ in result: + pass + else: + await result.__anext__() + + if asynchronous: + asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + result = Mistral(api_key="synthetic", client=transport).chat.stream( + **params("chat") + ) + assert not tracing.get_finished_spans() + with result: + if failure: + with pytest.raises( + RuntimeError, match="private-stream-error-marker" + ): + list(result) + else: + next(result) + spans = tracing.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code == ( + StatusCode.ERROR if failure else StatusCode.OK + ) + + +@pytest.mark.skipif( + not NATIVE, reason="Native conversation telemetry requires SDK >=3.6" +) +def test_native_conversation_error_event(tracing): + def respond(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=ConversationSSE(fail=True), + ) + + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic", client=transport) + with client.beta.conversations.start_stream( + model="mistral-test", inputs="hello" + ) as result: + chunks = list(result) + assert chunks[-1].data.type == "conversation.response.error" + spans = tracing.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code == StatusCode.ERROR + + +def test_concurrent_async_calls(tracing): + async def run(): + async with httpx.AsyncClient(transport=httpx.MockTransport(transport_response)) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + results = await asyncio.gather(*(client.chat.complete_async(**params("chat")) for _ in range(24))) + assert all(result.choices[0].message.content == "private-output-marker" for result in results) + asyncio.run(run()) + spans = tracing.get_finished_spans() + assert len(spans) == 24 + assert len({span.context.span_id for span in spans}) == 24 + + +@pytest.mark.skipif(not NATIVE, reason="Native provider binding requires SDK >=3.6") +def test_restore_existing_native_provider(): + from mistralai.extra.observability import configure_telemetry + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + other = TracerProvider() + other_exporter = InMemorySpanExporter() + other.add_span_processor(SimpleSpanProcessor(other_exporter)) + with httpx.Client(transport=httpx.MockTransport(transport_response)) as transport: + client = Mistral(api_key="synthetic", client=transport) + configure_telemetry(client, provider=other, redaction=False) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + client.chat.complete(**params("chat")) + assert len(exporter.get_finished_spans()) == 1 + assert not other_exporter.get_finished_spans() + finally: + instrumentor.uninstrument() + client.chat.complete(**params("chat")) + assert len(other_exporter.get_finished_spans()) == 1 + assert len(exporter.get_finished_spans()) == 1 + provider.shutdown() + other.shutdown() + + +def test_sampler_failure_does_not_change_response(caplog): + from opentelemetry.sdk.trace.sampling import Sampler + class BrokenSampler(Sampler): + def should_sample(self, *args, **kwargs): + raise RuntimeError("synthetic telemetry failure") + def get_description(self): + return "broken-test-sampler" + provider = TracerProvider(sampler=BrokenSampler()) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + with httpx.Client(transport=httpx.MockTransport(transport_response)) as transport: + result = Mistral(api_key="synthetic", client=transport).chat.complete(**params("chat")) + assert result.choices[0].message.content == "private-output-marker" + assert caplog.records + finally: + instrumentor.uninstrument() + provider.shutdown() diff --git a/python/frameworks/mistralai/tests/test_wire.py b/python/frameworks/mistralai/tests/test_wire.py new file mode 100644 index 000000000..3fe002573 --- /dev/null +++ b/python/frameworks/mistralai/tests/test_wire.py @@ -0,0 +1,205 @@ +"""Export real SDK spans to a loopback OTLP receiver in both wire encodings.""" + +import gzip +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.request import Request, urlopen + +import pytest +from fi_instrumentation import TraceConfig +from google.protobuf.json_format import MessageToDict +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.exporter.otlp.proto.http import Compression +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, +) +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ( + SimpleSpanProcessor, + SpanExporter, + SpanExportResult, +) + +from traceai_mistralai import MistralAIInstrumentor +from test_sdk import Mistral, httpx, params, transport_response + + +def otlp_json(message): + data = MessageToDict(message) + # OTLP/JSON uses hex IDs, rather than protobuf JSON's base64 bytes. + for resources, encoded_resources in zip( + message.resource_spans, data["resourceSpans"] + ): + for scope, encoded_scope in zip( + resources.scope_spans, encoded_resources["scopeSpans"] + ): + for span, encoded in zip(scope.spans, encoded_scope["spans"]): + encoded["traceId"] = span.trace_id.hex() + encoded["spanId"] = span.span_id.hex() + if span.parent_span_id: + encoded["parentSpanId"] = span.parent_span_id.hex() + return data + + +class JSONExporter(SpanExporter): + """Test-only JSON transport; customers use the standard OTLP exporter.""" + + def __init__(self, endpoint, compressed): + self.endpoint, self.compressed = endpoint, compressed + + def export(self, spans): + body = json.dumps(otlp_json(encode_spans(spans))).encode() + headers = {"Content-Type": "application/json"} + if self.compressed: + body = gzip.compress(body) + headers["Content-Encoding"] = "gzip" + with urlopen( + Request(self.endpoint, data=body, headers=headers), timeout=5 + ) as response: + assert response.status == 200 + return SpanExportResult.SUCCESS + + +@pytest.fixture +def receiver(): + requests = [] + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + if self.headers.get("Content-Encoding") == "gzip": + body = gzip.decompress(body) + if self.headers.get("Content-Type") == "application/json": + data = json.loads(body) + else: + data = otlp_json(ExportTraceServiceRequest.FromString(body)) + requests.append((self.path, data)) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *_): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + yield f"http://127.0.0.1:{server.server_port}/v1/traces", requests + finally: + server.shutdown() + worker.join() + server.server_close() + + +@pytest.mark.parametrize("encoding", ["json", "protobuf"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize("private", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +def test_otlp_wire(receiver, encoding, compressed, private, streaming): + endpoint, requests = receiver + exporter = ( + JSONExporter(endpoint, compressed) + if encoding == "json" + else OTLPSpanExporter( + endpoint=endpoint, + headers={}, + timeout=5, + compression=Compression.Gzip if compressed else Compression.NoCompression, + ) + ) + provider = TracerProvider( + resource=Resource.create({"project_name": "mistral-offline-test"}) + ) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument( + tracer_provider=provider, + config=TraceConfig(hide_inputs=private, hide_outputs=private), + ) + try: + with httpx.Client( + transport=httpx.MockTransport(transport_response) + ) as transport: + client = Mistral(api_key="synthetic-key-marker", client=transport) + with provider.get_tracer("test-parent").start_as_current_span("parent"): + if streaming: + with client.chat.stream(**params("chat")) as stream: + list(stream) + else: + client.chat.complete(**params("chat")) + provider.force_flush() + finally: + instrumentor.uninstrument() + provider.shutdown() + spans = [ + span + for path, data in requests + for resource in data["resourceSpans"] + for scope in resource["scopeSpans"] + for span in scope["spans"] + ] + assert len(spans) == 2 + child, parent = spans + assert child["traceId"] == parent["traceId"] + assert child["parentSpanId"] == parent["spanId"] + assert len(child["traceId"]) == 32 + assert len(child["spanId"]) == 16 + assert all(path == "/v1/traces" for path, _ in requests) + attrs = {a["key"]: a["value"] for a in child["attributes"]} + assert attrs["gen_ai.span.kind"]["stringValue"] == "LLM" + assert int(attrs["gen_ai.usage.input_tokens"]["intValue"]) == 2 + wire = json.dumps(requests) + assert "synthetic-key-marker" not in wire + if private: + assert "private-input-marker" not in wire + assert "private-output-marker" not in wire + else: + assert "private-input-marker" in wire + assert "private-output-marker" in wire + + +def test_readme_example(receiver, monkeypatch): + from pathlib import Path + import re + from test_sdk import NATIVE + + if not NATIVE: + pytest.skip("The quickstart targets the current SDK") + endpoint, requests = receiver + monkeypatch.setenv("FI_BASE_URL", endpoint.rsplit("/v1/traces", 1)[0]) + monkeypatch.setenv("FI_API_KEY", "synthetic-fi-key") + monkeypatch.setenv("FI_SECRET_KEY", "synthetic-fi-secret") + monkeypatch.setenv("MISTRAL_API_KEY", "synthetic-mistral-key") + original_init = Mistral.__init__ + + def init(self, *args, **kwargs): + kwargs["client"] = httpx.Client( + transport=httpx.MockTransport(transport_response) + ) + original_init(self, *args, **kwargs) + + monkeypatch.setattr(Mistral, "__init__", init) + readme = (Path(__file__).parents[1] / "README.md").read_text() + snippet = re.findall(r"```python\n(.*?)```", readme, re.S)[0] + try: + exec(compile(snippet, "README.md", "exec"), {}) + finally: + MistralAIInstrumentor().uninstrument() + spans = [ + span + for _, data in requests + for resource in data["resourceSpans"] + for scope in resource["scopeSpans"] + for span in scope["spans"] + ] + assert len(spans) == 1 + assert requests[0][0] == "/tracer/v1/traces" + encoded = json.dumps(requests) + assert "mistral-app" in encoded + assert "gen_ai.usage.input_tokens" in encoded + assert "synthetic-fi-secret" not in encoded + assert "synthetic-mistral-key" not in encoded diff --git a/python/frameworks/mistralai/traceai_mistralai/__init__.py b/python/frameworks/mistralai/traceai_mistralai/__init__.py index fbf13af1c..6acde35cf 100644 --- a/python/frameworks/mistralai/traceai_mistralai/__init__.py +++ b/python/frameworks/mistralai/traceai_mistralai/__init__.py @@ -4,11 +4,9 @@ from fi_instrumentation import FITracer, TraceConfig from opentelemetry import trace as trace_api from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore -from traceai_mistralai._chat_wrapper import ( - _AsyncChatWrapper, - _AsyncStreamChatWrapper, - _SyncChatWrapper, -) +from importlib.metadata import version as distribution_version + +from packaging.version import Version from traceai_mistralai.package import _instruments from traceai_mistralai.version import __version__ from wrapt import wrap_function_wrapper @@ -26,6 +24,8 @@ class MistralAIInstrumentor(BaseInstrumentor): # type: ignore __slots__ = ( "_tracer", + "_native", + "_embedding_methods", "_original_sync_chat_method", "_original_sync_stream_chat_method", "_original_async_chat_method", @@ -51,6 +51,23 @@ def _instrument(self, **kwargs: Any) -> None: config=config, ) + sdk_version = Version(distribution_version("mistralai")) + if not (Version("1.1.0") <= sdk_version < Version("2") or Version("3.6.0") <= sdk_version < Version("4")): + raise ValueError("traceAI-mistralai supports mistralai >=1.1,<2 or >=3.6,<4") + self._native = None + if sdk_version >= Version("3.6.0"): + from traceai_mistralai._native import _NativeInstrumentation + + self._native = _NativeInstrumentation(tracer_provider, config) + return + + from traceai_mistralai._chat_wrapper import ( + _AsyncChatWrapper, + _AsyncStreamChatWrapper, + _SyncChatWrapper, + ) + from traceai_mistralai._embeddings import _AsyncEmbeddingWrapper, _EmbeddingWrapper + try: import mistralai from mistralai.agents import Agents @@ -124,7 +141,23 @@ def _instrument(self, **kwargs: Any) -> None: ), ) + from mistralai.embeddings import Embeddings + + self._embedding_methods = (Embeddings.create, Embeddings.create_async) + wrap_function_wrapper(module="mistralai.embeddings", name="Embeddings.create", wrapper=_EmbeddingWrapper( + "MistralClient.embeddings", self._tracer, mistralai + )) + wrap_function_wrapper(module="mistralai.embeddings", name="Embeddings.create_async", wrapper=_AsyncEmbeddingWrapper( + "MistralAsyncClient.embeddings", self._tracer, mistralai + )) + def _uninstrument(self, **kwargs: Any) -> None: + if self._native is not None: + self._native.uninstrument() + self._native = None + return + from mistralai.embeddings import Embeddings + Embeddings.create, Embeddings.create_async = self._embedding_methods from mistralai.agents import Agents from mistralai.chat import Chat diff --git a/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py b/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py index 8198eec32..4740a15f8 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py +++ b/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py @@ -22,7 +22,7 @@ FiSpanKindValues, SpanAttributes, ) -from mistralai.utils.eventstreaming import EventStream, EventStreamAsync +from collections.abc import AsyncIterator from opentelemetry import context as context_api from opentelemetry import trace as trace_api from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY @@ -148,6 +148,9 @@ def _parse_args( bound_arguments = bound_signature.arguments request_data: Dict[str, Any] = {} for key, value in bound_arguments.items(): + if key in {"http_headers", "server_url", "retries", "timeout_ms"} or isinstance(value, (Iterator, AsyncIterator)): + # Telemetry must not consume a one-shot message/tool iterable. + continue try: if value is not None: try: @@ -177,18 +180,12 @@ def _finalize_response( chat_completion_type=CompletionEvent, ) - if isinstance(response, EventStreamAsync): + if isinstance(response, AsyncIterator): return _AsyncStream( stream=response, with_span=with_span, response_accumulator=response_accumulator, ) - elif isinstance(response, EventStream): - return _Stream( - stream=response, - with_span=with_span, - response_accumulator=response_accumulator, - ) elif isinstance(response, Iterator): return _Stream( stream=response, diff --git a/python/frameworks/mistralai/traceai_mistralai/_embeddings.py b/python/frameworks/mistralai/traceai_mistralai/_embeddings.py new file mode 100644 index 000000000..68b93bf4a --- /dev/null +++ b/python/frameworks/mistralai/traceai_mistralai/_embeddings.py @@ -0,0 +1,32 @@ +"""Embedding support for the legacy SDK, which has no native OTel hooks.""" + +from fi_instrumentation.fi_types import FiSpanKindValues +from opentelemetry import trace + +from ._chat_wrapper import _AsyncChatWrapper, _SyncChatWrapper + + +class _EmbeddingResponse: + def _get_span_kind(self): + return FiSpanKindValues.EMBEDDING.value + + def _finalize_response(self, response, with_span, request_parameters): + # Never copy vectors into the generic output field: that would bypass + # FI_HIDE_EMBEDDING_VECTORS and create very large attributes. + attributes = dict( + self._response_attributes_extractor.get_attributes_from_response( + response, request_parameters + ) + ) + with_span.finish_tracing( + status=trace.Status(trace.StatusCode.OK), attributes=attributes + ) + return response + + +class _EmbeddingWrapper(_EmbeddingResponse, _SyncChatWrapper): + pass + + +class _AsyncEmbeddingWrapper(_EmbeddingResponse, _AsyncChatWrapper): + pass diff --git a/python/frameworks/mistralai/traceai_mistralai/_native.py b/python/frameworks/mistralai/traceai_mistralai/_native.py new file mode 100644 index 000000000..4bb29f292 --- /dev/null +++ b/python/frameworks/mistralai/traceai_mistralai/_native.py @@ -0,0 +1,240 @@ +"""Route the SDK's own spans through FITracer; never create a second SDK span.""" + +import json +import logging +from threading import RLock +from typing import Any +from weakref import WeakKeyDictionary + +from fi_instrumentation import FITracer, TraceConfig +from opentelemetry import trace +from wrapt import ObjectProxy, wrap_function_wrapper + +logger = logging.getLogger(__name__) +# Fail-open telemetry diagnostics deliberately omit payload-bearing exceptions. + + +class _NativeSpan(ObjectProxy): + """Defer success until end so an SSE read error can still mark ERROR.""" + + def __init__(self, span): + super().__init__(span) + self._self_status = None + + def set_status(self, status, description=None): + if ( + self._self_status is not None + and self._self_status.status_code == trace.StatusCode.ERROR + ): + return + self._self_status = ( + status + if isinstance(status, trace.Status) + else trace.Status(status, description) + ) + + def end(self, *args, **kwargs): + if self._self_status is not None: + self.__wrapped__.set_status(self._self_status) + return self.__wrapped__.end(*args, **kwargs) + + +class _ErrorStream(ObjectProxy): + """Observe read failures before HTTPX closes and ends the native span.""" + + def __init__(self, stream, span): + super().__init__(stream) + self._self_span = span + + def _error(self, error): + if not isinstance(error, GeneratorExit): + try: + self._self_span.record_exception(error) + self._self_span.set_status(trace.Status(trace.StatusCode.ERROR)) + except Exception: + logger.debug("Could not record Mistral stream error") + + def __iter__(self): + try: + yield from self.__wrapped__ + except BaseException as error: + self._error(error) + raise + + async def __aiter__(self): + try: + async for chunk in self.__wrapped__: + yield chunk + except BaseException as error: + self._error(error) + raise + + +class _NativeTracer(FITracer): + def start_span(self, name: str, *args: Any, **kwargs: Any) -> _NativeSpan: + if "embeddings" in name: + kind = "EMBEDDING" + elif "conversations" in name or name.startswith("invoke_agent"): + kind = "AGENT" + elif name.startswith("execute_tool"): + kind = "TOOL" + elif any(part in name for part in ("chat", "completion", "fim")): + kind = "LLM" + else: + kind = "CHAIN" + kwargs.setdefault("fi_span_kind", kind.lower()) + return _NativeSpan(super().start_span(name, *args, **kwargs)) + + +class _NativeProvider(trace.TracerProvider): + def __init__(self, provider: trace.TracerProvider, config: TraceConfig) -> None: + self._provider = provider + self._config = config + + def get_tracer(self, *args: Any, **kwargs: Any) -> _NativeTracer: + return _NativeTracer(self._provider.get_tracer(*args, **kwargs), self._config) + + +class _NativeInstrumentation: + """Adapt the native hook for both existing and subsequently created clients. + + The native hook owns request context and span creation. The adapter routes + those spans and fills HTTP-error and conversation-stream gaps. Do not patch + chat or agent methods alongside this hook: that would duplicate native spans. + """ + + _FIELDS = ( + "tracer_provider", + "tracer", + "tracing_enabled", + "_telemetry_auto_disabled", + ) + + def __init__(self, provider: trace.TracerProvider, config: TraceConfig) -> None: + from mistralai.client._hooks.tracing import TracingHook + + self._provider = _NativeProvider(provider, config) + self._hooks = WeakKeyDictionary() + self._lock = RLock() + from mistralai.extra.observability.otel import TracedResponse + + self._originals = [] + for cls, method, wrapper in ( + (TracingHook, "before_request", self._before_request), + (TracingHook, "after_error", self._after_error), + (TracingHook, "after_success", self._after_success), + (TracedResponse, "_finalize_span", self._finalize_span), + ): + self._originals.append((cls, method, getattr(cls, method))) + wrap_function_wrapper(cls, method, wrapper) + + def _before_request(self, wrapped, instance, args, kwargs): + with self._lock: + if ( + instance not in self._hooks + or instance.tracer_provider is not self._provider + ): + self._hooks[instance] = tuple( + getattr(instance, key) for key in self._FIELDS + ) + instance.tracer_provider = self._provider + # Do not let SDK auto-export configuration create a second destination. + instance._telemetry_auto_disabled = True + return wrapped(*args, **kwargs) + + def _after_error(self, wrapped, instance, args, kwargs): + response = kwargs.get("response", args[1] if len(args) > 1 else None) + if response is not None and response.status_code >= 400: + span = instance._get_span(response) + if isinstance(span, _NativeSpan): + span.set_status(trace.Status(trace.StatusCode.ERROR)) + span.set_attribute("http.response.status_code", response.status_code) + return wrapped(*args, **kwargs) + + def _after_success(self, wrapped, instance, args, kwargs): + response = wrapped(*args, **kwargs) + span = getattr(response, "span", None) + if isinstance(span, _NativeSpan): + response.stream = _ErrorStream(response.stream, span) + return response + + def _finalize_span(self, wrapped, instance, args, kwargs): + span = instance.span + if ( + not isinstance(span, _NativeSpan) + or "conversations" not in instance.operation_id + ): + return wrapped(*args, **kwargs) + # SDK 3.6's native stream finalizer only decodes chat CompletionChunks. + # Adapt conversation SSE attributes on that same span, not a second span. + try: + parts = [] + data_lines = [] + for line in bytes(instance._accumulated_sse).decode( + "utf-8", errors="replace" + ).splitlines() + [""]: + if line.startswith("data:"): + data_lines.append(line[5:].lstrip()) + elif not line and data_lines: + raw, data_lines = "\n".join(data_lines), [] + try: + event = json.loads(raw) + except ValueError: + continue + if not isinstance(event, dict): + continue + event_type = event.get("type") + if event_type == "conversation.response.done": + usage = event.get("usage") or {} + if not isinstance(usage, dict): + continue + for source, target in ( + ("prompt_tokens", "input_tokens"), + ("completion_tokens", "output_tokens"), + ("total_tokens", "total_tokens"), + ): + if isinstance(usage.get(source), int): + span.set_attribute( + "gen_ai.usage." + target, usage[source] + ) + elif event_type == "message.output.delta": + content = event.get("content") + if isinstance(content, str): + parts.append({"type": "text", "content": content}) + elif isinstance(content, dict): + parts.append(content) + elif event_type == "function.call.delta": + parts.append( + { + "type": "tool_call", + "id": event.get("id"), + "name": event.get("name"), + "arguments": event.get("arguments"), + } + ) + elif event_type == "conversation.response.error": + span.set_status(trace.Status(trace.StatusCode.ERROR)) + if parts: + span.set_attribute( + "gen_ai.output.messages", + json.dumps([{"role": "assistant", "parts": parts}]), + ) + except Exception: + logger.debug("Could not extract Mistral conversation stream attributes") + finally: + try: + span.end() + except Exception: + logger.debug("Could not end Mistral conversation span") + instance.span = None + + def uninstrument(self) -> None: + for cls, method, original in reversed(self._originals): + setattr(cls, method, original) + with self._lock: + for hook, values in self._hooks.items(): + # Respect an explicit customer reconfiguration made after ours. + if hook.tracer_provider is self._provider: + for key, value in zip(self._FIELDS, values): + setattr(hook, key, value) + self._hooks.clear() diff --git a/python/frameworks/mistralai/traceai_mistralai/_response_accumulator.py b/python/frameworks/mistralai/traceai_mistralai/_response_accumulator.py index 5bc5a03ce..e3c3034c1 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_response_accumulator.py +++ b/python/frameworks/mistralai/traceai_mistralai/_response_accumulator.py @@ -16,7 +16,6 @@ Type, ) -from fi_instrumentation import safe_json_dumps from fi_instrumentation.fi_types import FiMimeTypeValues, SpanAttributes from opentelemetry.util.types import AttributeValue from traceai_mistralai._utils import _to_dict @@ -44,7 +43,6 @@ class _ChatCompletionAccumulator: "_response_attributes_extractor", "_chat_completion_type", "_content", - "_raw_data", ) def __init__( @@ -59,7 +57,6 @@ def __init__( self._is_null = True self._cached_result: Optional[Dict[str, Any]] = None self._content: List[str] = [] - self._raw_data: List[Dict[str, Any]] = [] self._values = _ValuesAccumulator( data=_ValuesAccumulator( choices=_IndexedAccumulator( @@ -81,7 +78,13 @@ def __init__( def process_chunk(self, chunk: Any) -> None: raw_chunk = _to_dict(chunk) - self._raw_data.append(raw_chunk) + self._is_null = False + self._cached_result = None + accumulated = deepcopy(raw_chunk) + for choice in accumulated.get("data", {}).get("choices", []): + if "delta" in choice: + choice["message"] = choice.pop("delta") + self._values += accumulated if "choices" in raw_chunk.get("data", {}): choices = raw_chunk["data"]["choices"] @@ -99,12 +102,8 @@ def _result(self) -> Optional[Dict[str, Any]]: return self._cached_result def get_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: - # Combine the accumulated content and raw data - output_value = "".join(self._content) - raw_output = safe_json_dumps(self._raw_data) - yield SpanAttributes.OUTPUT_VALUE, output_value - yield SpanAttributes.OUTPUT_VALUE, raw_output - yield SpanAttributes.OUTPUT_MIME_TYPE, FiMimeTypeValues.JSON.value + yield SpanAttributes.OUTPUT_VALUE, "".join(self._content) + yield SpanAttributes.OUTPUT_MIME_TYPE, FiMimeTypeValues.TEXT.value def get_extra_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: if not (result := self._result()): diff --git a/python/frameworks/mistralai/traceai_mistralai/_stream.py b/python/frameworks/mistralai/traceai_mistralai/_stream.py index 7171eff68..2d7bd0cd2 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_stream.py +++ b/python/frameworks/mistralai/traceai_mistralai/_stream.py @@ -1,7 +1,10 @@ +from __future__ import annotations + import logging -from typing import Any, AsyncIterator, Generator, Iterator, Optional, Protocol, Tuple +from typing import TYPE_CHECKING, Any, AsyncIterator, Generator, Iterator, Optional, Protocol, Tuple -from mistralai.models import CompletionEvent +if TYPE_CHECKING: + from mistralai.models import CompletionEvent from opentelemetry import trace as trace_api from opentelemetry.util.types import AttributeValue from traceai_mistralai._utils import _finish_tracing @@ -48,6 +51,28 @@ def __init__( def __iter__(self) -> Iterator[Any]: return self + def __enter__(self): + if hasattr(self.__wrapped__, "__enter__"): + self.__wrapped__.__enter__() + return self + + def close(self): + try: + closer = getattr(self.__wrapped__, "close", None) + if closer is not None: + return closer() + return self.__wrapped__.__exit__(None, None, None) + finally: + if not self._self_is_finished: + self._finish_tracing(trace_api.Status(trace_api.StatusCode.OK)) + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_val is not None: + self._self_with_span.record_exception(exc_val) + self._finish_tracing(trace_api.Status(trace_api.StatusCode.ERROR)) + self.close() + return False + def __next__(self) -> Any: try: chunk = self.__wrapped__.__next__() @@ -56,7 +81,7 @@ def __next__(self) -> Any: status = trace_api.Status(status_code=trace_api.StatusCode.OK) self._finish_tracing(status=status) raise - except Exception as exception: + except BaseException as exception: if not self._self_is_finished: status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, @@ -125,6 +150,31 @@ def __init__( def __aiter__(self) -> AsyncIterator[Any]: return self + async def __aenter__(self): + if hasattr(self.__wrapped__, "__aenter__"): + await self.__wrapped__.__aenter__() + return self + + async def close(self): + try: + closer = getattr(self.__wrapped__, "aclose", None) or getattr(self.__wrapped__, "close", None) + if closer is not None: + return await closer() + return await self.__wrapped__.__aexit__(None, None, None) + finally: + if not self._self_is_finished: + self._finish_tracing(trace_api.Status(trace_api.StatusCode.OK)) + + async def aclose(self): + return await self.close() + + async def __aexit__(self, exc_type, exc_val, exc_tb): + if exc_val is not None: + self._self_with_span.record_exception(exc_val) + self._finish_tracing(trace_api.Status(trace_api.StatusCode.ERROR)) + await self.close() + return False + async def __anext__(self) -> Any: # Pass through mistaken calls if not hasattr(self.__wrapped__, "__anext__"): @@ -137,7 +187,7 @@ async def __anext__(self) -> Any: status = trace_api.Status(status_code=trace_api.StatusCode.OK) self._finish_tracing(status=status) raise - except Exception as exception: + except BaseException as exception: if not self._self_is_finished: status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, diff --git a/python/tests/test_framework_mistralai.py b/python/tests/test_framework_mistralai.py index 9c7237419..6372c4332 100644 --- a/python/tests/test_framework_mistralai.py +++ b/python/tests/test_framework_mistralai.py @@ -1,5 +1,6 @@ import asyncio import json +from importlib.metadata import version from typing import Any, Dict, Iterator, List, Optional from unittest.mock import MagicMock, Mock, patch, AsyncMock, call, ANY from uuid import uuid4 @@ -127,6 +128,8 @@ def sample_response(): return mock_response +@pytest.mark.skipif(int(version("mistralai").split(".")[0]) >= 3, + reason="Legacy method mocks; native lifecycle is tested with the real SDK") class TestMistralAIInstrumentor: """⚡ Test MistralAIInstrumentor lifecycle and basic functionality.""" @@ -153,7 +156,7 @@ def test_instrument_with_defaults(self, tracer_provider): assert isinstance(instrumentor._tracer, FITracer) # Verify all methods were wrapped - assert mock_wrap.call_count == 8 # 4 chat + 4 agents methods + assert mock_wrap.call_count == 10 # chat + agents + embeddings # Verify original methods were stored assert hasattr(instrumentor, '_original_sync_chat_method') @@ -231,7 +234,7 @@ def test_wrapper_registration_calls(self, tracer_provider): instrumentor._instrument(tracer_provider=tracer_provider) # Check that all expected calls were made - assert len(mock_wrap.call_args_list) == 8 + assert len(mock_wrap.call_args_list) == 10 # Check that both chat and agent modules are instrumented modules_called = [] @@ -339,7 +342,8 @@ def test_async_chat_wrapper_call_success(self, fi_tracer, mock_mistralai_module, async def test_call(): # Mock the wrapped function to return a non-stream response - mock_wrapped = AsyncMock(return_value=sample_response) + async def mock_wrapped(**kwargs): + return sample_response with patch.object(wrapper, '_start_as_current_span') as mock_span_ctx: mock_with_span = MagicMock() @@ -679,6 +683,8 @@ def test_span_creation_failure(self, mock_mistralai_module): assert result == "success" +@pytest.mark.skipif(int(version("mistralai").split(".")[0]) >= 3, + reason="Legacy method mocks; native lifecycle is tested with the real SDK") class TestIntegrationScenarios: """🎭 Test integration scenarios and real-world usage patterns.""" From 12803891d82127e0a59222142c2b486bd64f5b5d Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:19:20 -0700 Subject: [PATCH 02/10] fix(mistralai): enforce native message privacy and event semantics Apply existing field policies to native JSON messages at every FITracer boundary, reuse SDK content conversion for conversation streams, and preserve tool invocation IDs. Cover the regressions with real SDK and loopback OTLP contracts. Co-Authored-By: Claude Code --- .../frameworks/mistralai/tests/test_native.py | 345 ++++++++++++++++++ .../mistralai/tests/test_native_privacy.py | 74 ++++ .../mistralai/traceai_mistralai/_native.py | 22 +- .../mistralai/traceai_mistralai/_privacy.py | 85 +++++ 4 files changed, 520 insertions(+), 6 deletions(-) create mode 100644 python/frameworks/mistralai/tests/test_native.py create mode 100644 python/frameworks/mistralai/tests/test_native_privacy.py create mode 100644 python/frameworks/mistralai/traceai_mistralai/_privacy.py diff --git a/python/frameworks/mistralai/tests/test_native.py b/python/frameworks/mistralai/tests/test_native.py new file mode 100644 index 000000000..d68210ea1 --- /dev/null +++ b/python/frameworks/mistralai/tests/test_native.py @@ -0,0 +1,345 @@ +"""Native message privacy and conversation parts through the real SDK and OTLP.""" + +import asyncio +import json + +import pytest +from fi_instrumentation import REDACTED_VALUE, TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor + +from traceai_mistralai import MistralAIInstrumentor +import test_sdk +import test_wire +from test_sdk import ( + NATIVE, + Mistral, + SSE, + USAGE, + conversation_response, + httpx, + transport_response, +) +from test_wire import ( + Compression, + JSONExporter, + OTLPSpanExporter, +) + +tracing = test_sdk.tracing +receiver = test_wire.receiver + +pytestmark = pytest.mark.skipif( + not NATIVE, reason="Native telemetry requires SDK >=3.6" +) + +INPUT_TEXT = "private-input-marker" +OUTPUT_TEXT = "private-output-marker" +IMAGE = "https://example.invalid/private-image-marker" + + +def input_content(): + return [ + {"type": "text", "text": INPUT_TEXT}, + {"type": "image_url", "image_url": {"url": IMAGE}}, + ] + + +class EventSSE(SSE): + def __init__(self, events): + super().__init__() + self.events = events + + def __iter__(self): + for event in self.events: + yield f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() + + +def wire_spans(requests): + return [ + span + for _, data in requests + for resource in data["resourceSpans"] + for scope in resource["scopeSpans"] + for span in scope["spans"] + ] + + +@pytest.mark.parametrize( + "flags", + [ + {}, + {"hide_input_text": True}, + {"hide_output_text": True}, + {"hide_input_images": True}, + {"hide_input_text": True, "hide_output_text": True, "hide_input_images": True}, + ], + ids=["visible", "input-text", "output-text", "input-images", "all-selective"], +) +@pytest.mark.parametrize("encoding", ["json", "protobuf"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize( + "operation", ["chat", "chat-stream", "conversation-stream", "conversation-children"] +) +def test_native_selective_privacy_on_wire( + receiver, flags, encoding, compressed, operation +): + endpoint, requests = receiver + exporter = ( + JSONExporter(endpoint, compressed) + if encoding == "json" + else OTLPSpanExporter( + endpoint=endpoint, + headers={}, + timeout=5, + compression=Compression.Gzip if compressed else Compression.NoCompression, + ) + ) + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**flags)) + + def respond(request): + if operation == "conversation-stream": + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=EventSSE( + [ + { + "type": "message.output.delta", + "id": "entry-message", + "role": "assistant", + "content": {"type": "text", "text": OUTPUT_TEXT}, + }, + { + "type": "function.call.delta", + "id": "entry-function", + "tool_call_id": "actual-tool-call", + "name": "weather", + "arguments": '{"city":"Paris"}', + }, + {"type": "conversation.response.done", "usage": USAGE}, + ] + ), + ) + if operation == "conversation-children": + body = conversation_response() + # Real service timestamps make the native hook emit the message child. + body["outputs"][0].update( + id="message-child", + created_at="2026-10-31T00:00:00Z", + completed_at="2026-10-31T00:00:01Z", + model="mistral-test", + ) + return httpx.Response(200, json=body) + return transport_response(request) + + try: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-credential-marker", client=transport) + if operation.startswith("conversation"): + arguments = { + "model": "mistral-test", + "inputs": [ + { + "type": "message.input", + "role": "user", + "content": input_content(), + } + ], + } + if operation == "conversation-stream": + with client.beta.conversations.start_stream(**arguments) as stream: + chunks = list(stream) + assert chunks[0].data.content.text == OUTPUT_TEXT + assert chunks[1].data.tool_call_id == "actual-tool-call" + else: + result = client.beta.conversations.start(**arguments) + assert result.outputs[0].content == OUTPUT_TEXT + else: + arguments = { + "model": "mistral-test", + "messages": [{"role": "user", "content": input_content()}], + } + if operation == "chat-stream": + with client.chat.stream(**arguments) as stream: + chunks = list(stream) + assert chunks[0].data.choices[0].delta.content == OUTPUT_TEXT + else: + result = client.chat.complete(**arguments) + assert result.choices[0].message.content == OUTPUT_TEXT + provider.force_flush() + finally: + instrumentor.uninstrument() + provider.shutdown() + + spans = wire_spans(requests) + assert len(spans) == (2 if operation == "conversation-children" else 1) + if operation == "conversation-children": + child, parent = spans + assert child["parentSpanId"] == parent["spanId"] + assert child["traceId"] == parent["traceId"] + attrs = {a["key"]: a["value"] for a in spans[-1]["attributes"]} + assert int(attrs["gen_ai.usage.input_tokens"]["intValue"]) == 2 + if operation == "conversation-stream": + parts = json.loads(attrs["gen_ai.output.messages"]["stringValue"])[0]["parts"] + assert parts == [ + { + "type": "text", + "content": REDACTED_VALUE + if flags.get("hide_output_text") + else OUTPUT_TEXT, + }, + { + "type": "tool_call", + "id": "actual-tool-call", + "name": "weather", + "arguments": '{"city":"Paris"}', + }, + ] + wire = json.dumps(requests) + for flag, marker in ( + ("hide_input_text", INPUT_TEXT), + ("hide_output_text", OUTPUT_TEXT), + ("hide_input_images", IMAGE), + ): + assert (marker in wire) is not flags.get(flag, False) + assert "synthetic-credential-marker" not in wire + if operation.startswith("chat"): + # Selective text flags must not discard tool identifiers or arguments. + assert "weather" in wire + assert "call-1" in wire + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize( + "variant", ["string", "text_chunk", "thinking", "image", "tool_call"] +) +def test_conversation_stream_part_conversion(tracing, variant, asynchronous): + content, expected = { + "string": ("chunk-content", {"type": "text", "content": "chunk-content"}), + "text_chunk": ( + {"type": "text", "text": "chunk-content"}, + {"type": "text", "content": "chunk-content"}, + ), + "thinking": ( + { + "type": "thinking", + "thinking": [{"type": "text", "text": "reasoning-content"}], + }, + {"type": "reasoning", "content": "reasoning-content"}, + ), + "image": ( + {"type": "image_url", "image_url": {"url": IMAGE}}, + {"type": "uri", "modality": "image", "uri": IMAGE}, + ), + "tool_call": ( + None, + { + "type": "tool_call", + "id": "actual-tool-call", + "name": "weather", + "arguments": '{"city":"Paris"}', + }, + ), + }[variant] + event = ( + { + "type": "function.call.delta", + "id": "entry-function", + "tool_call_id": "actual-tool-call", + "name": "weather", + "arguments": '{"city":"Paris"}', + } + if variant == "tool_call" + else { + "type": "message.output.delta", + "id": "entry-message", + "role": "assistant", + "content": content, + } + ) + + def respond(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=EventSSE( + [event, {"type": "conversation.response.done", "usage": USAGE}] + ), + ) + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + async with await client.beta.conversations.start_stream_async( + model="mistral-test", inputs="hi" + ) as stream: + return [chunk async for chunk in stream] + + if asynchronous: + chunks = asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-credential-marker", client=transport) + with client.beta.conversations.start_stream( + model="mistral-test", inputs="hi" + ) as stream: + chunks = list(stream) + assert chunks[0].data.type == event["type"] + if variant == "tool_call": + assert chunks[0].data.tool_call_id == "actual-tool-call" + assert chunks[0].data.id == "entry-function" + elif variant == "text_chunk": + assert chunks[0].data.content.text == "chunk-content" + spans = tracing.get_finished_spans() + assert len(spans) == 1 + assert json.loads(spans[0].attributes["gen_ai.output.messages"])[0]["parts"] == [ + expected + ] + + +@pytest.mark.parametrize("direction", ["input", "output"]) +@pytest.mark.parametrize("method", ["creation", "single", "batch"]) +def test_native_privacy_at_attribute_boundaries(direction, method): + from traceai_mistralai._native import _NativeProvider + from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, + ) + + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + config = TraceConfig(**{f"hide_{direction}_text": True}) + tracer = _NativeProvider(provider, config).get_tracer("test") + key = f"gen_ai.{direction}.messages" + original = [ + { + "role": "assistant", + "parts": [{"type": "reasoning", "content": "private-reasoning"}], + } + ] + attributes = {key: json.dumps(original)} + try: + with tracer.start_as_current_span( + "chat", attributes=attributes if method == "creation" else None + ) as span: + if method == "single": + span.set_attribute(key, attributes[key]) + elif method == "batch": + span.set_attributes(attributes) + result = exporter.get_finished_spans() + assert len(result) == 1 + assert "private-reasoning" not in result[0].to_json() + assert ( + json.loads(result[0].attributes[key])[0]["parts"][0]["content"] + == REDACTED_VALUE + ) + assert original[0]["parts"][0]["content"] == "private-reasoning" + assert json.loads(attributes[key]) == original + finally: + provider.shutdown() diff --git a/python/frameworks/mistralai/tests/test_native_privacy.py b/python/frameworks/mistralai/tests/test_native_privacy.py new file mode 100644 index 000000000..9a608dcbf --- /dev/null +++ b/python/frameworks/mistralai/tests/test_native_privacy.py @@ -0,0 +1,74 @@ +"""Boundary cases for the native message privacy adapter.""" + +import json + +import pytest +from fi_instrumentation import REDACTED_VALUE, TraceConfig + +from traceai_mistralai._privacy import _NativeTraceConfig + + +@pytest.mark.parametrize("direction", ["input", "output"]) +@pytest.mark.parametrize( + "value", + [ + "not-json-private-marker", + '{"parts":"private-marker"}', + '[{"parts":["private-marker"]}]', + '[{"parts":null}]', + "null", + 123, + ], +) +def test_unparseable_messages_are_not_forwarded(direction, value): + config = _NativeTraceConfig(TraceConfig(**{f"hide_{direction}_text": True})) + assert config.mask(f"gen_ai.{direction}.messages", value) is None + + +@pytest.mark.parametrize("direction", ["input", "output"]) +def test_whole_direction_privacy_takes_precedence(direction): + key = f"gen_ai.{direction}.messages" + payload = json.dumps( + [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "private-marker"}], + } + ] + ) + for flag in (f"hide_{direction}s", f"hide_{direction}_messages"): + base = TraceConfig(**{flag: True}) + assert _NativeTraceConfig(base).mask(key, payload) == base.mask(key, payload) + + +@pytest.mark.parametrize("hidden", [False, True]) +def test_input_image_limit_and_environment_config(monkeypatch, hidden): + monkeypatch.setenv("FI_HIDE_INPUT_IMAGES", str(hidden).lower()) + config = _NativeTraceConfig(TraceConfig(base64_image_max_length=30)) + payload = [ + { + "role": "user", + "parts": [ + {"type": "text", "content": "visible-text"}, + { + "type": "uri", + "modality": "image", + "uri": "data:image/png;base64," + "A" * 64, + }, + ], + } + ] + result = json.loads(config.mask("gen_ai.input.messages", json.dumps(payload)))[0][ + "parts" + ] + assert result[0] == payload[0]["parts"][0] + if hidden: + assert len(result) == 1 + else: + assert result[1] == {"type": "uri", "modality": "image", "uri": REDACTED_VALUE} + + +def test_unrelated_attributes_and_empty_messages_are_preserved(): + config = _NativeTraceConfig(TraceConfig(hide_input_text=True)) + assert config.mask("gen_ai.request.model", "mistral-test") == "mistral-test" + assert config.mask("gen_ai.input.messages", lambda: "[]") == "[]" diff --git a/python/frameworks/mistralai/traceai_mistralai/_native.py b/python/frameworks/mistralai/traceai_mistralai/_native.py index 4bb29f292..c9aa790ab 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_native.py +++ b/python/frameworks/mistralai/traceai_mistralai/_native.py @@ -7,9 +7,12 @@ from weakref import WeakKeyDictionary from fi_instrumentation import FITracer, TraceConfig +from mistralai.extra.observability.formatting import format_output_message from opentelemetry import trace from wrapt import ObjectProxy, wrap_function_wrapper +from traceai_mistralai._privacy import _NativeTraceConfig + logger = logging.getLogger(__name__) # Fail-open telemetry diagnostics deliberately omit payload-bearing exceptions. @@ -89,7 +92,7 @@ def start_span(self, name: str, *args: Any, **kwargs: Any) -> _NativeSpan: class _NativeProvider(trace.TracerProvider): def __init__(self, provider: trace.TracerProvider, config: TraceConfig) -> None: self._provider = provider - self._config = config + self._config = _NativeTraceConfig(config) def get_tracer(self, *args: Any, **kwargs: Any) -> _NativeTracer: return _NativeTracer(self._provider.get_tracer(*args, **kwargs), self._config) @@ -199,15 +202,22 @@ def _finalize_span(self, wrapped, instance, args, kwargs): ) elif event_type == "message.output.delta": content = event.get("content") - if isinstance(content, str): - parts.append({"type": "text", "content": content}) - elif isinstance(content, dict): - parts.append(content) + if isinstance(content, (str, dict)): + # Reuse the SDK's non-streaming OTel conversion for + # text, reasoning and image chunks, not provider keys. + message = { + "content": [content] + if isinstance(content, dict) + else content + } + parts.extend( + format_output_message({"message": message})["parts"] + ) elif event_type == "function.call.delta": parts.append( { "type": "tool_call", - "id": event.get("id"), + "id": event.get("tool_call_id"), "name": event.get("name"), "arguments": event.get("arguments"), } diff --git a/python/frameworks/mistralai/traceai_mistralai/_privacy.py b/python/frameworks/mistralai/traceai_mistralai/_privacy.py new file mode 100644 index 000000000..f108105e7 --- /dev/null +++ b/python/frameworks/mistralai/traceai_mistralai/_privacy.py @@ -0,0 +1,85 @@ +"""Apply TraceConfig's field policy to the SDK's serialized OTel messages.""" + +import json +import logging +from typing import Callable, Optional, Union + +from fi_instrumentation import REDACTED_VALUE +from fi_instrumentation.fi_types import ( + ImageAttributes, + MessageAttributes, + MessageContentAttributes, + SpanAttributes, +) +from opentelemetry.util.types import AttributeValue +from wrapt import ObjectProxy + +logger = logging.getLogger(__name__) + + +class _NativeTraceConfig(ObjectProxy): + """Keep native JSON spans, using the same policy as flattened FI messages. + + Adapt the config rather than only Span.set_attribute so creation attributes, + batched updates and SDK-created child spans all cross the same boundary. + """ + + def mask( + self, + key: str, + value: Union[AttributeValue, Callable[[], AttributeValue]], + ) -> Optional[AttributeValue]: + value = self.__wrapped__.mask(key, value) + if ( + key + not in ( + SpanAttributes.GEN_AI_INPUT_MESSAGES, + SpanAttributes.GEN_AI_OUTPUT_MESSAGES, + ) + or value is None + or value == REDACTED_VALUE + ): + return value + # Native messages are JSON arrays. Never forward an opaque value whose + # contents cannot be checked against the configured privacy policy. + if not isinstance(value, str): + return None + try: + messages = json.loads(value) + except (ValueError, RecursionError): + # Parsing errors may contain the private payload: log no details. + logger.debug("Discarding malformed Mistral message attributes") + return None + if not isinstance(messages, list): + return None + for message_index, message in enumerate(messages): + if not isinstance(message, dict) or not isinstance( + message.get("parts"), list + ): + return None + parts = [] + for part_index, part in enumerate(message["parts"]): + if not isinstance(part, dict): + return None + prefix = f"{key}.{message_index}.{MessageAttributes.MESSAGE_CONTENTS}.{part_index}" + if part.get("type") in ("text", "reasoning"): + field = "content" + policy_key = ( + f"{prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TEXT}" + ) + elif ( + key == SpanAttributes.GEN_AI_INPUT_MESSAGES + and part.get("type") == "uri" + and part.get("modality") == "image" + ): + field = "uri" + policy_key = f"{prefix}.{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}.{ImageAttributes.IMAGE_URL}" + else: + parts.append(part) + continue + masked = self.__wrapped__.mask(policy_key, part.get(field)) + if masked is not None: + part[field] = masked + parts.append(part) + message["parts"] = parts + return json.dumps(messages) From a705ca5d0b9e35116e58ba136c82bd759f282b01 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:10:28 -0700 Subject: [PATCH 03/10] fix(mistralai): redact tool results and finish cancelled native requests Honor input-text privacy for native tool response payloads and close the SDK-owned span when an async send is cancelled, preserving cancellation and native context handling. Co-Authored-By: Claude Code --- .../frameworks/mistralai/tests/test_native.py | 146 +++++++++++++ .../mistralai/tests/test_native_lifecycle.py | 193 ++++++++++++++++++ .../mistralai/traceai_mistralai/_native.py | 46 ++++- .../mistralai/traceai_mistralai/_privacy.py | 10 +- 4 files changed, 391 insertions(+), 4 deletions(-) create mode 100644 python/frameworks/mistralai/tests/test_native_lifecycle.py diff --git a/python/frameworks/mistralai/tests/test_native.py b/python/frameworks/mistralai/tests/test_native.py index d68210ea1..491d3406c 100644 --- a/python/frameworks/mistralai/tests/test_native.py +++ b/python/frameworks/mistralai/tests/test_native.py @@ -213,6 +213,152 @@ def respond(request): assert "call-1" in wire +@pytest.mark.parametrize("encoding", ["json", "protobuf"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize("operation", ["chat", "chat-chunks", "conversation"]) +@pytest.mark.parametrize( + "policy", ["visible", "hide_input_text", "hide_inputs", "hide_input_messages"] +) +def test_native_tool_result_privacy(receiver, encoding, compressed, operation, policy): + endpoint, requests = receiver + exporter = ( + JSONExporter(endpoint, compressed) + if encoding == "json" + else OTLPSpanExporter( + endpoint=endpoint, + headers={}, + timeout=5, + compression=Compression.Gzip if compressed else Compression.NoCompression, + ) + ) + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument( + tracer_provider=provider, + config=TraceConfig(**({} if policy == "visible" else {policy: True})), + ) + marker = "private-tool-result-marker" + tool_id = "input-tool-call" + + def respond(request): + if operation == "conversation": + body = conversation_response() + body["outputs"] = [] + return httpx.Response(200, json=body) + return transport_response(request) + + try: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic", client=transport) + if operation == "conversation": + result = client.beta.conversations.append( + conversation_id="conv-test", + inputs=[ + { + "type": "function.result", + "tool_call_id": tool_id, + "result": marker, + } + ], + ) + assert result.conversation_id == "conv-test" + else: + result = client.chat.complete( + model="mistral-test", + messages=[ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": tool_id, + "type": "function", + "function": {"name": "lookup", "arguments": "{}"}, + } + ], + }, + { + "role": "tool", + "tool_call_id": tool_id, + "content": [{"type": "text", "text": marker}] + if operation == "chat-chunks" + else marker, + }, + ], + ) + assert result.choices[0].message.content == OUTPUT_TEXT + provider.force_flush() + finally: + instrumentor.uninstrument() + provider.shutdown() + spans = wire_spans(requests) + assert len(spans) == 1 + wire = json.dumps(requests) + assert (marker in wire) == (policy == "visible") + if operation != "conversation": + assert OUTPUT_TEXT in wire + if policy in ("visible", "hide_input_text"): + attrs = {a["key"]: a["value"] for a in spans[0]["attributes"]} + messages = json.loads(attrs["gen_ai.input.messages"]["stringValue"]) + part = messages[-1]["parts"][0] + assert part["type"] == "tool_call_response" + assert part["id"] == tool_id + if policy == "hide_input_text": + assert part["response"] == REDACTED_VALUE + + +@pytest.mark.parametrize("encoding", ["json", "protobuf"]) +@pytest.mark.parametrize("compressed", [False, True]) +def test_cancelled_native_send_on_wire(receiver, encoding, compressed): + endpoint, requests = receiver + exporter = ( + JSONExporter(endpoint, compressed) + if encoding == "json" + else OTLPSpanExporter( + endpoint=endpoint, + headers={}, + timeout=5, + compression=Compression.Gzip if compressed else Compression.NoCompression, + ) + ) + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument( + tracer_provider=provider, + config=TraceConfig(hide_error_details=True), + ) + + async def run(): + async def respond(request): + raise asyncio.CancelledError("private-cancellation-marker") + + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic-key-marker", async_client=transport) + with pytest.raises( + asyncio.CancelledError, match="private-cancellation-marker" + ): + await client.chat.complete_async( + model="mistral-test", + messages=[{"role": "user", "content": "hello"}], + ) + + try: + asyncio.run(run()) + provider.force_flush() + finally: + instrumentor.uninstrument() + provider.shutdown() + spans = wire_spans(requests) + assert len(spans) == 1 + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR" + assert "private-cancellation-marker" not in json.dumps(requests) + assert "synthetic-key-marker" not in json.dumps(requests) + + @pytest.mark.parametrize("asynchronous", [False, True]) @pytest.mark.parametrize( "variant", ["string", "text_chunk", "thinking", "image", "tool_call"] diff --git a/python/frameworks/mistralai/tests/test_native_lifecycle.py b/python/frameworks/mistralai/tests/test_native_lifecycle.py new file mode 100644 index 000000000..522bee9d4 --- /dev/null +++ b/python/frameworks/mistralai/tests/test_native_lifecycle.py @@ -0,0 +1,193 @@ +"""Cancellation and restoration of the SDK's native request-span lifecycle.""" + +import asyncio +import gc +import weakref + +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_mistralai import MistralAIInstrumentor +from test_sdk import NATIVE, Mistral, httpx, params, transport_response + +pytestmark = pytest.mark.skipif( + not NATIVE, reason="Native telemetry requires SDK >=3.6" +) + + +def request(client, operation): + if operation == "embedding": + return client.embeddings.create_async(model="mistral-embed", inputs=["hello"]) + if operation.startswith("conversation"): + method = ( + client.beta.conversations.start_stream_async + if operation.endswith("stream") + else client.beta.conversations.start_async + ) + return method(model="mistral-test", inputs="hello") + method = ( + client.chat.stream_async + if operation == "chat-stream" + else client.chat.complete_async + ) + return method(**params("chat")) + + +@pytest.mark.parametrize( + "operation", + ["chat", "chat-stream", "embedding", "conversation", "conversation-stream"], +) +@pytest.mark.parametrize("timeout", [False, True], ids=["cancel", "timeout"]) +def test_cancelled_native_send_ends_only_its_span(operation, timeout): + provider = TracerProvider() + exporter = InMemorySpanExporter() + started = [] + + class RecordingProcessor(SimpleSpanProcessor): + def on_start(self, span, parent_context=None): + started.append(span) + + provider.add_span_processor(RecordingProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument( + tracer_provider=provider, config=TraceConfig(hide_error_details=True) + ) + + async def run(): + entered = asyncio.Event() + requests = [] + + async def respond(http_request): + requests.append(http_request) + if len(requests) == 1: + entered.set() + await asyncio.Event().wait() + return transport_response(http_request) + + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + task = asyncio.create_task(request(client, operation)) + await asyncio.wait_for(entered.wait(), 5) + # A successful request sharing the client must not be ended by cancellation. + result = await client.chat.complete_async(**params("chat")) + assert result.choices[0].message.content == "private-output-marker" + if timeout: + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(task, 0.01) + else: + task.cancel("private-cancellation-marker") + with pytest.raises( + asyncio.CancelledError, match="private-cancellation-marker" + ): + await task + assert trace.get_current_span() is trace.INVALID_SPAN + assert len(started) == 2 + assert all(not span.is_recording() for span in started) + spans = exporter.get_finished_spans() + assert len(spans) == 2 + assert spans[0].status.status_code == trace.StatusCode.OK + assert spans[1].status.status_code == trace.StatusCode.ERROR + assert spans[1].context.span_id == started[0].context.span_id + assert "private-cancellation-marker" not in spans[1].to_json() + assert requests[0].extensions["_tracing_span"] is None + assert requests[0].extensions["_tracing_span_finished"] is True + assert len(exporter.get_finished_spans()) == 2 + + try: + asyncio.run(run()) + finally: + instrumentor.uninstrument() + provider.shutdown() + + +def test_cancelled_send_preserves_cancellation_if_telemetry_fails(monkeypatch, caplog): + from mistralai.client._hooks.tracing import TracingHook + + caplog.set_level("DEBUG", logger="traceai_mistralai._native") + + def fail(*args, **kwargs): + raise RuntimeError("private-telemetry-marker") + + monkeypatch.setattr(TracingHook, "_finish_request_span_with_error", fail) + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument(tracer_provider=provider) + + async def run(): + async def respond(http_request): + raise asyncio.CancelledError("original-cancellation") + + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + with pytest.raises(asyncio.CancelledError, match="original-cancellation"): + await client.chat.complete_async(**params("chat")) + + try: + asyncio.run(run()) + assert "Could not finish cancelled Mistral request" in caplog.text + assert "private-telemetry-marker" not in caplog.text + assert "original-cancellation" not in caplog.text + finally: + instrumentor.uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize("reconfigured", [False, True]) +def test_native_async_send_restored_without_retaining_clients(reconfigured): + from mistralai.extra.observability import configure_telemetry + + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + other = TracerProvider() + instrumentor = MistralAIInstrumentor() + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(transport_response) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + configure_telemetry(client, provider=other, redaction=False) + await client.chat.complete_async(**params("chat")) + original = transport.send # Includes the SDK's own context/error wrapper. + for _ in range(2): + instrumentor.instrument(tracer_provider=provider) + await client.chat.complete_async(**params("chat")) + assert transport.send is not original + if reconfigured: + + async def replacement(*args, **kwargs): + return await original(*args, **kwargs) + + transport.send = replacement + instrumentor.uninstrument() + if reconfigured: + assert transport.send is replacement + transport.send = original + else: + assert transport.send is original + assert len(exporter.get_finished_spans()) == 2 + # Leave instrumentation on when dropping the client to check weak tracking. + instrumentor.instrument(tracer_provider=provider) + await client.chat.complete_async(**params("chat")) + return weakref.ref(transport) + + try: + client_ref = asyncio.run(run()) + gc.collect() # Run the SDK finalizer, which itself holds the HTTP client. + gc.collect() # Collect cycles released by that finalizer. + assert client_ref() is None + finally: + instrumentor.uninstrument() + provider.shutdown() + other.shutdown() diff --git a/python/frameworks/mistralai/traceai_mistralai/_native.py b/python/frameworks/mistralai/traceai_mistralai/_native.py index c9aa790ab..2c2342e31 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_native.py +++ b/python/frameworks/mistralai/traceai_mistralai/_native.py @@ -1,10 +1,12 @@ """Route the SDK's own spans through FITracer; never create a second SDK span.""" +import asyncio import json import logging +from functools import wraps from threading import RLock from typing import Any -from weakref import WeakKeyDictionary +from weakref import WeakKeyDictionary, ref from fi_instrumentation import FITracer, TraceConfig from mistralai.extra.observability.formatting import format_output_message @@ -118,6 +120,7 @@ def __init__(self, provider: trace.TracerProvider, config: TraceConfig) -> None: self._provider = _NativeProvider(provider, config) self._hooks = WeakKeyDictionary() + self._async_sends = WeakKeyDictionary() self._lock = RLock() from mistralai.extra.observability.otel import TracedResponse @@ -143,7 +146,41 @@ def _before_request(self, wrapped, instance, args, kwargs): instance.tracer_provider = self._provider # Do not let SDK auto-export configuration create a second destination. instance._telemetry_auto_disabled = True - return wrapped(*args, **kwargs) + request = wrapped(*args, **kwargs) + hook_ctx = kwargs.get("hook_ctx", args[0] if args else None) + client = getattr(getattr(hook_ctx, "config", None), "async_client", None) + if client is not None: + self._wrap_async_send(client, instance) + return request + + def _wrap_async_send(self, client, hook): + # SDK 3.6 only catches Exception during send, not CancelledError. Keep + # its context management and finish its existing request span on cancel. + try: + with self._lock: + previous = self._async_sends.get(client) + if previous is not None and client.send is previous(): + return + original = client.send + + @wraps(original) + async def send(request, *args, **kwargs): + try: + return await original(request, *args, **kwargs) + except asyncio.CancelledError as error: + try: + if isinstance(hook._get_request_span(request), _NativeSpan): + hook._finish_request_span_with_error(request, error) + except Exception: + logger.debug("Could not finish cancelled Mistral request") + raise + + client.send = send + # A bound send method retains its client: weak-reference both + # sides so tracking does not keep closed application clients alive. + self._async_sends[client] = ref(send) + except Exception: + logger.debug("Could not wrap Mistral async send") def _after_error(self, wrapped, instance, args, kwargs): response = kwargs.get("response", args[1] if len(args) > 1 else None) @@ -242,6 +279,11 @@ def uninstrument(self) -> None: for cls, method, original in reversed(self._originals): setattr(cls, method, original) with self._lock: + for client, send_ref in self._async_sends.items(): + send = send_ref() + if send is not None and client.send is send: + client.send = send.__wrapped__ + self._async_sends.clear() for hook, values in self._hooks.items(): # Respect an explicit customer reconfiguration made after ours. if hook.tracer_provider is self._provider: diff --git a/python/frameworks/mistralai/traceai_mistralai/_privacy.py b/python/frameworks/mistralai/traceai_mistralai/_privacy.py index f108105e7..14fe7f4b3 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_privacy.py +++ b/python/frameworks/mistralai/traceai_mistralai/_privacy.py @@ -62,8 +62,14 @@ def mask( if not isinstance(part, dict): return None prefix = f"{key}.{message_index}.{MessageAttributes.MESSAGE_CONTENTS}.{part_index}" - if part.get("type") in ("text", "reasoning"): - field = "content" + if part.get("type") in ("text", "reasoning", "tool_call_response"): + # The SDK stores tool-result text/chunks in response rather + # than content. Mask the payload, never the tool-call ID. + field = ( + "response" + if part["type"] == "tool_call_response" + else "content" + ) policy_key = ( f"{prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TEXT}" ) From 40ccf22004625ca190317c53bc12fa924be39e5e Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:20:42 -0700 Subject: [PATCH 04/10] test(mistralai): account for Python 3.10 task cancellation semantics Python 3.10 discards Task.cancel messages at the awaiter even without instrumentation. Assert task cancellation by type while keeping direct-await message checks and span cleanup assertions on all versions. Co-Authored-By: Claude Code --- python/frameworks/mistralai/tests/test_native_lifecycle.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/frameworks/mistralai/tests/test_native_lifecycle.py b/python/frameworks/mistralai/tests/test_native_lifecycle.py index 522bee9d4..ca5bb0c0b 100644 --- a/python/frameworks/mistralai/tests/test_native_lifecycle.py +++ b/python/frameworks/mistralai/tests/test_native_lifecycle.py @@ -82,9 +82,10 @@ async def respond(http_request): await asyncio.wait_for(task, 0.01) else: task.cancel("private-cancellation-marker") - with pytest.raises( - asyncio.CancelledError, match="private-cancellation-marker" - ): + # Python 3.10 drops Task.cancel(msg) at the task-await boundary, + # including without instrumentation. Direct-await tests below + # still require the original cancellation message to survive. + with pytest.raises(asyncio.CancelledError): await task assert trace.get_current_span() is trace.INVALID_SPAN assert len(started) == 2 From a0681af0e5a94bc37a623d0c0e23965e398e728b Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 19:15:59 -0700 Subject: [PATCH 05/10] fix(mistralai): apply image privacy to structured tool results Mask native tool response image chunks with the existing TraceConfig policy, including base64 size limits, while preserving permitted text and call identifiers. Co-Authored-By: Claude Code --- .../mistralai/tests/test_native_privacy.py | 73 ++++++++ .../tests/test_native_tool_images.py | 158 ++++++++++++++++++ .../mistralai/traceai_mistralai/_privacy.py | 29 +++- 3 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 python/frameworks/mistralai/tests/test_native_tool_images.py diff --git a/python/frameworks/mistralai/tests/test_native_privacy.py b/python/frameworks/mistralai/tests/test_native_privacy.py index 9a608dcbf..c8ced6385 100644 --- a/python/frameworks/mistralai/tests/test_native_privacy.py +++ b/python/frameworks/mistralai/tests/test_native_privacy.py @@ -68,6 +68,79 @@ def test_input_image_limit_and_environment_config(monkeypatch, hidden): assert result[1] == {"type": "uri", "modality": "image", "uri": REDACTED_VALUE} +@pytest.mark.parametrize("hidden", [False, True]) +@pytest.mark.parametrize("direction", ["input", "output"]) +def test_tool_image_environment_and_length_boundaries(monkeypatch, hidden, direction): + monkeypatch.setenv("FI_HIDE_INPUT_IMAGES", str(hidden).lower()) + short = "data:image/png;base64," + "A" * 8 + long = short + "A" + url = "https://example.invalid/" + "a" * 80 + config = _NativeTraceConfig(TraceConfig(base64_image_max_length=len(short))) + chunks = [ + {"type": "text", "text": "visible-text"}, + {"type": "image_url", "image_url": short}, + {"type": "image_url", "image_url": {"url": long, "detail": "low"}}, + {"type": "image_url", "image_url": {"url": url}}, + {"type": "text", "text": "trailing-text"}, + ] + payload = json.dumps( + [ + { + "role": "tool", + "parts": [ + {"type": "tool_call_response", "id": "call-1", "response": chunks} + ], + } + ] + ) + result = json.loads(config.mask(f"gen_ai.{direction}.messages", payload))[0][ + "parts" + ][0] + assert result["id"] == "call-1" + if direction == "output": + assert result["response"] == chunks + elif hidden: + assert result["response"] == [chunks[0], chunks[-1]] + else: + assert result["response"] == [ + chunks[0], + chunks[1], + { + "type": "image_url", + "image_url": {"url": REDACTED_VALUE, "detail": "low"}, + }, + chunks[3], + chunks[4], + ] + + +@pytest.mark.parametrize( + "image", [None, 123, {}, {"url": None}, {"url": ["private-image"]}] +) +def test_malformed_tool_images_are_not_forwarded(image): + config = _NativeTraceConfig(TraceConfig(hide_input_images=True)) + payload = json.dumps( + [ + { + "role": "tool", + "parts": [ + { + "type": "tool_call_response", + "id": "call-1", + "response": [ + {"type": "image_url", "image_url": image}, + {"type": "text", "text": "visible-text"}, + ], + } + ], + } + ] + ) + result = json.loads(config.mask("gen_ai.input.messages", payload))[0]["parts"][0] + assert result["id"] == "call-1" + assert result["response"] == [{"type": "text", "text": "visible-text"}] + + def test_unrelated_attributes_and_empty_messages_are_preserved(): config = _NativeTraceConfig(TraceConfig(hide_input_text=True)) assert config.mask("gen_ai.request.model", "mistral-test") == "mistral-test" diff --git a/python/frameworks/mistralai/tests/test_native_tool_images.py b/python/frameworks/mistralai/tests/test_native_tool_images.py new file mode 100644 index 000000000..be0d11c16 --- /dev/null +++ b/python/frameworks/mistralai/tests/test_native_tool_images.py @@ -0,0 +1,158 @@ +"""Structured tool images obey the same policy as native user-message images.""" + +import copy +import json + +import pytest +from fi_instrumentation import REDACTED_VALUE, TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor + +from traceai_mistralai import MistralAIInstrumentor +from test_sdk import NATIVE, Mistral, httpx, transport_response +import test_wire + +receiver = test_wire.receiver +pytestmark = pytest.mark.skipif( + not NATIVE, reason="Native telemetry requires SDK >=3.6" +) + +TEXT = "permitted-tool-result-text" +TOOL_ID = "screenshot-call" +IMAGES = [ + "https://example.invalid/private-tool-screenshot.png", + "data:image/png;base64," + "A" * 128, +] + + +@pytest.mark.parametrize("encoding", ["json", "protobuf"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize("image", IMAGES, ids=["url", "base64"]) +@pytest.mark.parametrize("structured_url", [False, True], ids=["string", "object"]) +@pytest.mark.parametrize( + "policy", + [ + "visible", + "hide_input_images", + "hide_input_text", + "hide_inputs", + "hide_input_messages", + "base64_image_max_length", + ], +) +def test_tool_images_on_wire( + receiver, encoding, compressed, image, structured_url, policy +): + endpoint, requests = receiver + exporter = ( + test_wire.JSONExporter(endpoint, compressed) + if encoding == "json" + else test_wire.OTLPSpanExporter( + endpoint=endpoint, + headers={}, + timeout=5, + compression=( + test_wire.Compression.Gzip + if compressed + else test_wire.Compression.NoCompression + ), + ) + ) + config = TraceConfig( + **( + {} + if policy == "visible" + else {policy: 30 if policy == "base64_image_max_length" else True} + ) + ) + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=config) + content = [ + {"type": "text", "text": TEXT}, + { + "type": "image_url", + "image_url": {"url": image, "detail": "low"} if structured_url else image, + }, + {"type": "text", "text": "trailing-permitted-text"}, + ] + original_content = copy.deepcopy(content) + observed = [] + + def respond(request): + observed.append(json.loads(request.content)) + return transport_response(request) + + try: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + with Mistral( + api_key="synthetic-tool-image-key", client=transport + ) as client: + result = client.chat.complete( + model="mistral-test", + messages=[ + { + "role": "assistant", + "tool_calls": [ + { + "id": TOOL_ID, + "type": "function", + "function": { + "name": "screenshot", + "arguments": "{}", + }, + } + ], + }, + {"role": "tool", "tool_call_id": TOOL_ID, "content": content}, + ], + ) + assert result.choices[0].message.content == "private-output-marker" + provider.force_flush() + finally: + instrumentor.uninstrument() + provider.shutdown() + + # Redaction is telemetry-only, not a mutation of the caller or SDK request. + assert content == original_content + assert observed[0]["messages"][-1]["content"] == original_content + spans = [ + span + for _, data in requests + for resource in data["resourceSpans"] + for scope in resource["scopeSpans"] + for span in scope["spans"] + ] + assert len(spans) == 1 + wire = json.dumps(requests) + assert "synthetic-tool-image-key" not in wire + limited = policy == "base64_image_max_length" and image.startswith("data:") + visible = policy in ("visible", "base64_image_max_length") and not limited + assert (image in wire) == visible + assert (TEXT in wire) == ( + policy in ("visible", "base64_image_max_length", "hide_input_images") + ) + assert "private-output-marker" in wire + if policy in ("hide_inputs", "hide_input_messages"): + assert TOOL_ID not in wire + return + attrs = { + attribute["key"]: attribute["value"] for attribute in spans[0]["attributes"] + } + messages = json.loads(attrs["gen_ai.input.messages"]["stringValue"]) + part = messages[-1]["parts"][0] + assert part["type"] == "tool_call_response" + assert part["id"] == TOOL_ID + if policy == "hide_input_text": + assert part["response"] == REDACTED_VALUE + elif policy == "hide_input_images": + assert part["response"] == [original_content[0], original_content[2]] + else: + expected = copy.deepcopy(original_content) + if limited: + if structured_url: + expected[1]["image_url"]["url"] = REDACTED_VALUE + else: + expected[1]["image_url"] = REDACTED_VALUE + assert part["response"] == expected diff --git a/python/frameworks/mistralai/traceai_mistralai/_privacy.py b/python/frameworks/mistralai/traceai_mistralai/_privacy.py index 14fe7f4b3..f1c6f336d 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_privacy.py +++ b/python/frameworks/mistralai/traceai_mistralai/_privacy.py @@ -2,7 +2,7 @@ import json import logging -from typing import Callable, Optional, Union +from typing import Any, Callable, Optional, Union from fi_instrumentation import REDACTED_VALUE from fi_instrumentation.fi_types import ( @@ -84,8 +84,35 @@ def mask( parts.append(part) continue masked = self.__wrapped__.mask(policy_key, part.get(field)) + if ( + key == SpanAttributes.GEN_AI_INPUT_MESSAGES + and part.get("type") == "tool_call_response" + and isinstance(masked, list) + ): + masked = self._mask_tool_images(masked, prefix) if masked is not None: part[field] = masked parts.append(part) message["parts"] = parts return json.dumps(messages) + + def _mask_tool_images(self, chunks: list[Any], prefix: str) -> list[Any]: + # Unlike user messages, the SDK leaves tool responses as raw content + # chunks, with image_url represented by either a string or a URL object. + result = [] + for index, chunk in enumerate(chunks): + if not isinstance(chunk, dict) or chunk.get("type") != "image_url": + result.append(chunk) + continue + image = chunk.get("image_url") + url = image.get("url") if isinstance(image, dict) else image + if not isinstance(url, str): + continue + policy_key = f"{prefix}.{index}.{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}.{ImageAttributes.IMAGE_URL}" + masked = self.__wrapped__.mask(policy_key, url) + if masked is not None: + chunk["image_url"] = ( + {**image, "url": masked} if isinstance(image, dict) else masked + ) + result.append(chunk) + return result From ffd71bd7e51e76f6cfca5f425c390cecb0943cb6 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 00:55:59 -0800 Subject: [PATCH 06/10] test(mistralai): cover latest native SDK release Keep the legacy and native floors in the offline distribution matrix and exercise Mistral 3.7.0 on Python 3.13. Co-Authored-By: Claude Code --- .github/workflows/mistralai-python-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mistralai-python-tests.yml b/.github/workflows/mistralai-python-tests.yml index fe7f2adfb..43a843293 100644 --- a/.github/workflows/mistralai-python-tests.yml +++ b/.github/workflows/mistralai-python-tests.yml @@ -29,7 +29,7 @@ jobs: - {python: '3.10', sdk: '1.1.0'} - {python: '3.12', sdk: '1.12.4'} - {python: '3.10', sdk: '3.6.0'} - - {python: '3.13', sdk: '3.6.0'} + - {python: '3.13', sdk: '3.7.0'} env: FI_BASE_URL: http://127.0.0.1:9 FI_GRPC_URL: http://127.0.0.1:9 From 212d2d1ab9fd8173ed07498c1395cae93f60b8f3 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 01:50:04 -0800 Subject: [PATCH 07/10] fix(mistralai): protect input privacy and finish native stream errors Redact HTTP URL userinfo, isolate legacy embedding text behind selective privacy, and finish cancelled legacy async requests. Reassemble conversation tool deltas, observe SDK decoder failures on the existing native span, and classify streamed agent completions consistently. Co-Authored-By: Claude Code --- .../mistralai/tests/test_review_edge_cases.py | 345 +++++++++++++++++ .../tests/test_review_regressions.py | 350 ++++++++++++++++++ .../mistralai/traceai_mistralai/__init__.py | 9 +- .../traceai_mistralai/_chat_wrapper.py | 5 +- .../traceai_mistralai/_embeddings.py | 35 +- .../mistralai/traceai_mistralai/_native.py | 93 ++++- .../mistralai/traceai_mistralai/_privacy.py | 23 ++ 7 files changed, 840 insertions(+), 20 deletions(-) create mode 100644 python/frameworks/mistralai/tests/test_review_edge_cases.py create mode 100644 python/frameworks/mistralai/tests/test_review_regressions.py diff --git a/python/frameworks/mistralai/tests/test_review_edge_cases.py b/python/frameworks/mistralai/tests/test_review_edge_cases.py new file mode 100644 index 000000000..a0c92751b --- /dev/null +++ b/python/frameworks/mistralai/tests/test_review_edge_cases.py @@ -0,0 +1,345 @@ +"""Boundary cases for the Mistral privacy and native stream repairs.""" + +import asyncio +import json + +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.trace import StatusCode +from traceai_mistralai import MistralAIInstrumentor +from traceai_mistralai._privacy import _NativeTraceConfig + +from test_review_regressions import ( + CHAT, + EMBED, + NATIVE, + EventStream, + Mistral, + httpx, + tracing, +) + + +@pytest.mark.parametrize("key", ["http.url", "url.full"]) +@pytest.mark.parametrize("lazy", [False, True]) +@pytest.mark.parametrize( + "url, expected", + [ + ( + "https://user:password@proxy.invalid:8443/v1?q=ok#stream", + "https://proxy.invalid:8443/v1?q=ok#stream", + ), + ("https://user%40name:p%40ss@proxy.invalid/v1", "https://proxy.invalid/v1"), + ("http://user:password@[::1]:8080/v1", "http://[::1]:8080/v1"), + ("https://user@proxy.invalid/v1", "https://proxy.invalid/v1"), + ("https://proxy.invalid/v1", "https://proxy.invalid/v1"), + ("https://user:password@[invalid/v1", None), + ], +) +def test_url_mask_removes_only_userinfo(key, lazy, url, expected): + config = _NativeTraceConfig(TraceConfig()) + value = (lambda: url) if lazy else url + assert config.mask(key, value) == expected + + +@pytest.mark.skipif(not NATIVE, reason="Native SDK HTTP attributes") +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("hide_content", [False, True]) +def test_proxy_credentials_not_exported_but_request_unchanged( + asynchronous, hide_content +): + observed = [] + + def respond(request): + observed.append(str(request.url)) + return httpx.Response(200, json=CHAT) + + with tracing(hide_inputs=hide_content, hide_outputs=hide_content) as ( + exporter, + starts, + ): + options = dict( + api_key="synthetic-key", + server_url="https://synthetic-user:synthetic-password@proxy.invalid:8443", + ) + params = dict( + model="mistral-test", messages=[{"role": "user", "content": "hello"}] + ) + if asynchronous: + + async def exercise(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + await Mistral( + async_client=transport, **options + ).chat.complete_async(**params) + + asyncio.run(exercise()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + Mistral(client=transport, **options).chat.complete(**params) + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == len(observed) == 1 + assert "synthetic-user:synthetic-password@" in observed[0] + assert "synthetic-password" not in spans[0].to_json() + assert "synthetic-user" not in spans[0].to_json() + assert ( + spans[0].attributes["http.url"] + == "https://proxy.invalid:8443/v1/chat/completions" + ) + + +@pytest.mark.skipif(NATIVE, reason="Legacy embedding extraction") +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("single", [False, True]) +@pytest.mark.parametrize( + "flag", + [ + None, + "hide_inputs", + "hide_input_messages", + "hide_input_text", + "hide_outputs", + "hide_embedding_vectors", + ], +) +def test_embedding_privacy_preserves_request_and_allowed_text( + asynchronous, single, flag +): + marker = "synthetic-private-embedding-input" + inputs = marker if single else [marker, "second-input"] + observed = [] + + def respond(request): + observed.append(json.loads(request.content)) + return httpx.Response(200, json=EMBED) + + with tracing(**({flag: True} if flag else {})) as (exporter, starts): + params = dict(model="mistral-embed", inputs=inputs) + if asynchronous: + + async def exercise(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + await Mistral( + api_key="synthetic-key", async_client=transport + ).embeddings.create_async(**params) + + asyncio.run(exercise()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + Mistral(api_key="synthetic-key", client=transport).embeddings.create( + **params + ) + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == len(observed) == 1 + assert observed[0]["input"] == inputs + attrs = spans[0].attributes + assert marker not in attrs.get("gen_ai.request.parameters", "") + assert "input.value" not in attrs + assert attrs["gen_ai.span.kind"] == "EMBEDDING" + hidden = flag in ("hide_inputs", "hide_input_messages", "hide_input_text") + assert (marker in spans[0].to_json()) is not hidden + if not hidden: + assert attrs["embedding.embeddings.0.embedding.text"] == marker + + +@pytest.mark.skipif(NATIVE, reason="Legacy embedding cancellation") +@pytest.mark.parametrize("hide_error_details", [False, True]) +def test_cancel_preserves_exception_identity_and_ends_once(hide_error_details): + error = asyncio.CancelledError("synthetic-cancellation-detail") + with tracing(hide_error_details=hide_error_details) as (exporter, starts): + + async def exercise(): + async def respond(request): + raise error + + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + with pytest.raises(asyncio.CancelledError) as caught: + await Mistral( + api_key="synthetic-key", async_client=transport + ).embeddings.create_async(model="mistral-embed", inputs=["hello"]) + assert caught.value is error + + asyncio.run(exercise()) + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == 1 + assert not starts.spans[0].is_recording() + assert spans[0].status.status_code == StatusCode.ERROR + if hide_error_details: + assert "synthetic-cancellation-detail" not in spans[0].to_json() + + +@pytest.mark.skipif(not NATIVE, reason="Native conversation delta reconstruction") +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_interleaved_conversation_calls_are_independent(asynchronous): + events = [{"type": "conversation.response.started", "conversation_id": "conv-test"}] + for index, fragment in [ + (0, '{"city":'), + (1, '{"date":'), + (0, '"Paris"}'), + (1, '"Monday"}'), + ]: + events.append( + { + "type": "function.call.delta", + "id": f"entry-{index}", + "tool_call_id": f"call-{index}", + "output_index": index, + "name": ["weather", "calendar"][index], + "arguments": fragment, + } + ) + events.append( + { + "type": "conversation.response.done", + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + } + ) + + def respond(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=EventStream(events, named=True), + ) + + with tracing() as (exporter, starts): + if asynchronous: + + async def exercise(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic-key", async_client=transport) + async with await client.beta.conversations.start_stream_async( + model="mistral-test", inputs="hello" + ) as stream: + assert len([event async for event in stream]) == 6 + + asyncio.run(exercise()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-key", client=transport) + with client.beta.conversations.start_stream( + model="mistral-test", inputs="hello" + ) as stream: + assert len(list(stream)) == 6 + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == 1 + parts = json.loads(spans[0].attributes["gen_ai.output.messages"])[0]["parts"] + assert parts == [ + { + "type": "tool_call", + "id": "call-0", + "name": "weather", + "arguments": '{"city":"Paris"}', + }, + { + "type": "tool_call", + "id": "call-1", + "name": "calendar", + "arguments": '{"date":"Monday"}', + }, + ] + + +class RawEvents(httpx.SyncByteStream, httpx.AsyncByteStream): + def __init__(self, payload): + self.payload = payload + + def __iter__(self): + yield self.payload + + async def __aiter__(self): + yield self.payload + + +@pytest.mark.skipif(not NATIVE, reason="Native SDK decoder lifetime") +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("hide_error_details", [False, True]) +@pytest.mark.parametrize( + "payload", + [ + b'data: {"error":{"message":"synthetic-private-error"}}', + b'data: {"error":{"message":"synthetic-private-error"}}\n\n', + b"data: \xff\n\n", + ], +) +def test_decode_error_ends_span_even_without_context_manager( + asynchronous, hide_error_details, payload +): + responses = [] + + def respond(request): + response = httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=RawEvents(payload), + ) + responses.append(response) + return response + + with tracing(hide_error_details=hide_error_details) as (exporter, starts): + params = dict( + model="mistral-test", messages=[{"role": "user", "content": "hello"}] + ) + if asynchronous: + + async def exercise(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + stream = await Mistral( + api_key="synthetic-key", async_client=transport + ).chat.stream_async(**params) + with pytest.raises(Exception) as caught: + [event async for event in stream] + assert "Validation" in type(caught.value).__name__ or isinstance( + caught.value, UnicodeDecodeError + ) + assert stream.response.is_closed + await stream.close() + + asyncio.run(exercise()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + stream = Mistral(api_key="synthetic-key", client=transport).chat.stream( + **params + ) + with pytest.raises(Exception) as caught: + list(stream) + assert "Validation" in type(caught.value).__name__ or isinstance( + caught.value, UnicodeDecodeError + ) + assert stream.response.is_closed + stream.close() + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == 1 + assert not starts.spans[0].is_recording() + assert spans[0].status.status_code == StatusCode.ERROR + if hide_error_details: + assert "synthetic-private-error" not in spans[0].to_json() + else: + assert len(spans[0].events) == 1 + + +@pytest.mark.skipif(not NATIVE, reason="Native stream wrapper restoration") +def test_sdk_iteration_methods_restored(): + from mistralai.client.utils.eventstreaming import ( + EventStream as SDKStream, + EventStreamAsync, + ) + + original_sync, original_async = SDKStream.__next__, EventStreamAsync.__anext__ + for _ in range(2): + with tracing(): + assert SDKStream.__next__ is not original_sync + assert EventStreamAsync.__anext__ is not original_async + assert SDKStream.__next__ is original_sync + assert EventStreamAsync.__anext__ is original_async + assert not MistralAIInstrumentor().is_instrumented_by_opentelemetry diff --git a/python/frameworks/mistralai/tests/test_review_regressions.py b/python/frameworks/mistralai/tests/test_review_regressions.py new file mode 100644 index 000000000..22844182c --- /dev/null +++ b/python/frameworks/mistralai/tests/test_review_regressions.py @@ -0,0 +1,350 @@ +"""Regressions exercised through real SDK serialization and synthetic HTTP/SSE.""" + +import asyncio +import importlib +import json +from contextlib import contextmanager +from importlib.metadata import version + +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.sdk.trace import TracerProvider, SpanProcessor +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_mistralai import MistralAIInstrumentor + +NATIVE = version("mistralai").startswith("3.") +Mistral = importlib.import_module("mistralai.client" if NATIVE else "mistralai").Mistral +httpx = importlib.import_module("httpx2" if NATIVE else "httpx") + +CHAT = { + "id": "synthetic", + "object": "chat.completion", + "created": 1, + "model": "mistral-test", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, +} +EMBED = { + "id": "synthetic", + "object": "list", + "model": "mistral-embed", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "usage": {"prompt_tokens": 2, "completion_tokens": 0, "total_tokens": 2}, +} + + +class CaptureStarts(SpanProcessor): + def __init__(self): + self.spans = [] + + def on_start(self, span, parent_context=None): + self.spans.append(span) + + +@contextmanager +def tracing(**flags): + provider = TracerProvider() + exporter = InMemorySpanExporter() + starts = CaptureStarts() + provider.add_span_processor(starts) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**flags)) + try: + yield exporter, starts + finally: + instrumentor.uninstrument() + provider.shutdown() + + +class EventStream(httpx.SyncByteStream, httpx.AsyncByteStream): + def __init__(self, events, named=False): + self.events, self.named = events, named + + def __iter__(self): + for event in self.events: + name = f"event: {event['type']}\n" if self.named else "" + yield (name + "data: " + json.dumps(event) + "\n\n").encode() + + async def __aiter__(self): + for item in self: + yield item + + def close(self): + pass + + async def aclose(self): + pass + + +@pytest.mark.skipif(NATIVE, reason="New legacy embedding path") +@pytest.mark.parametrize( + "flag", ["hide_input_text", "hide_input_messages", "hide_inputs"] +) +def test_legacy_embedding_content_privacy(flag): + marker = "synthetic-private-embedding-input" + with tracing(**{flag: True}) as (exporter, starts): + with httpx.Client( + transport=httpx.MockTransport(lambda r: httpx.Response(200, json=EMBED)) + ) as transport: + client = Mistral(api_key="synthetic-key", client=transport) + client.embeddings.create(model="mistral-embed", inputs=[marker]) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert marker not in spans[0].to_json() + + +@pytest.mark.skipif(NATIVE, reason="New legacy embedding path") +def test_legacy_embedding_cancellation_finishes_span(): + with tracing() as (exporter, starts): + + async def exercise(): + async def respond(request): + raise asyncio.CancelledError("synthetic-cancel") + + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic-key", async_client=transport) + with pytest.raises(asyncio.CancelledError): + await client.embeddings.create_async( + model="mistral-embed", inputs=["hello"] + ) + + asyncio.run(exercise()) + assert len(starts.spans) == 1 + assert not starts.spans[0].is_recording() + assert exporter.get_finished_spans()[0].status.status_code == StatusCode.ERROR + + +@pytest.mark.skipif(not NATIVE, reason="Native adapter") +def test_native_custom_endpoint_redacts_url_credentials(): + with tracing(hide_inputs=True, hide_outputs=True) as (exporter, starts): + with httpx.Client( + transport=httpx.MockTransport(lambda r: httpx.Response(200, json=CHAT)) + ) as transport: + client = Mistral( + api_key="synthetic-key", + client=transport, + server_url="https://synthetic-user:synthetic-password@proxy.invalid", + ) + client.chat.complete( + model="mistral-test", messages=[{"role": "user", "content": "hello"}] + ) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert "synthetic-password" not in spans[0].to_json() + + +@pytest.mark.skipif(not NATIVE, reason="Native conversation aggregation") +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_native_conversation_accumulates_tool_arguments(asynchronous): + events = [ + {"type": "conversation.response.started", "conversation_id": "conv-synthetic"}, + { + "type": "function.call.delta", + "id": "entry-1", + "tool_call_id": "call-1", + "output_index": 0, + "name": "weather", + "arguments": '{"city":', + }, + { + "type": "function.call.delta", + "id": "entry-1", + "tool_call_id": "call-1", + "output_index": 0, + "name": "weather", + "arguments": '"Paris"}', + }, + { + "type": "conversation.response.done", + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + }, + ] + + def respond(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=EventStream(events, named=True), + ) + + with tracing() as (exporter, starts): + if asynchronous: + + async def exercise(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic-key", async_client=transport) + async with await client.beta.conversations.start_stream_async( + model="mistral-test", inputs="hello" + ) as stream: + assert len([event async for event in stream]) == 4 + + asyncio.run(exercise()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-key", client=transport) + with client.beta.conversations.start_stream( + model="mistral-test", inputs="hello" + ) as stream: + assert len(list(stream)) == 4 + spans = exporter.get_finished_spans() + assert len(spans) == 1 + parts = json.loads(spans[0].attributes["gen_ai.output.messages"])[0]["parts"] + assert parts == [ + { + "type": "tool_call", + "id": "call-1", + "name": "weather", + "arguments": '{"city":"Paris"}', + } + ] + + +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_stream_decode_failure_is_error(asynchronous): + def respond(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=EventStream([{"error": {"message": "synthetic-provider-failure"}}]), + ) + + with tracing() as (exporter, starts): + if asynchronous: + + async def exercise(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic-key", async_client=transport) + with pytest.raises(Exception) as raised: + async with await client.chat.stream_async( + model="mistral-test", + messages=[{"role": "user", "content": "hello"}], + ) as stream: + [event async for event in stream] + assert "Validation" in type(raised.value).__name__ + + asyncio.run(exercise()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-key", client=transport) + with pytest.raises(Exception) as raised: + with client.chat.stream( + model="mistral-test", + messages=[{"role": "user", "content": "hello"}], + ) as stream: + list(stream) + assert "Validation" in type(raised.value).__name__ + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code == StatusCode.ERROR + + +@pytest.mark.skipif(not NATIVE, reason="Native kind classification") +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_native_agent_completion_kind(streaming, asynchronous): + chunk = { + "id": "synthetic", + "object": "chat.completion.chunk", + "created": 1, + "model": "mistral-test", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + } + + def respond(request): + if streaming: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=EventStream([chunk]), + ) + return httpx.Response(200, json=CHAT) + + with tracing() as (exporter, starts): + params = dict( + agent_id="synthetic-agent", messages=[{"role": "user", "content": "hello"}] + ) + if asynchronous: + + async def exercise(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic-key", async_client=transport) + if streaming: + async with await client.agents.stream_async(**params) as stream: + [event async for event in stream] + else: + await client.agents.complete_async(**params) + + asyncio.run(exercise()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-key", client=transport) + if streaming: + with client.agents.stream(**params) as stream: + list(stream) + else: + client.agents.complete(**params) + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == 1 + assert spans[0].attributes["gen_ai.span.kind"] == "LLM" + + +@pytest.mark.skipif(not NATIVE, reason="Previously reported native tool image issue") +@pytest.mark.parametrize("structured", [False, True]) +@pytest.mark.parametrize( + "image", + ["https://example.invalid/private-image", "data:image/png;base64," + "A" * 80], +) +def test_previous_tool_image_privacy_fix(structured, image): + chunks = [ + {"type": "text", "text": "allowed-tool-text"}, + { + "type": "image_url", + "image_url": {"url": image, "detail": "low"} if structured else image, + }, + ] + observed = [] + + def respond(request): + observed.append(json.loads(request.content)) + return httpx.Response(200, json=CHAT) + + with tracing(hide_input_images=True) as (exporter, starts): + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-key", client=transport) + client.chat.complete( + model="mistral-test", + messages=[ + {"role": "tool", "tool_call_id": "call-image", "content": chunks} + ], + ) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + wire = spans[0].to_json() + assert image not in wire + assert "allowed-tool-text" in wire + assert "call-image" in wire + assert observed[0]["messages"][0]["content"] == chunks diff --git a/python/frameworks/mistralai/traceai_mistralai/__init__.py b/python/frameworks/mistralai/traceai_mistralai/__init__.py index 6acde35cf..8419de9c5 100644 --- a/python/frameworks/mistralai/traceai_mistralai/__init__.py +++ b/python/frameworks/mistralai/traceai_mistralai/__init__.py @@ -142,13 +142,18 @@ def _instrument(self, **kwargs: Any) -> None: ) from mistralai.embeddings import Embeddings + from traceai_mistralai._privacy import _EmbeddingTraceConfig + embedding_tracer = FITracer( + trace_api.get_tracer(__name__, __version__, tracer_provider), + config=_EmbeddingTraceConfig(config), + ) self._embedding_methods = (Embeddings.create, Embeddings.create_async) wrap_function_wrapper(module="mistralai.embeddings", name="Embeddings.create", wrapper=_EmbeddingWrapper( - "MistralClient.embeddings", self._tracer, mistralai + "MistralClient.embeddings", embedding_tracer, mistralai )) wrap_function_wrapper(module="mistralai.embeddings", name="Embeddings.create_async", wrapper=_AsyncEmbeddingWrapper( - "MistralAsyncClient.embeddings", self._tracer, mistralai + "MistralAsyncClient.embeddings", embedding_tracer, mistralai )) def _uninstrument(self, **kwargs: Any) -> None: diff --git a/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py b/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py index 4740a15f8..27a9e263e 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py +++ b/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py @@ -1,3 +1,4 @@ +import asyncio import json import logging import warnings @@ -292,7 +293,7 @@ async def __call__( ) as with_span: try: response = await wrapped(*args, **kwargs) - except Exception as exception: + except (Exception, asyncio.CancelledError) as exception: with_span.record_exception(exception) status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, @@ -347,7 +348,7 @@ async def __call__( ) as with_span: try: response = await wrapped(*args, **kwargs) - except Exception as exception: + except (Exception, asyncio.CancelledError) as exception: with_span.record_exception(exception) status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, diff --git a/python/frameworks/mistralai/traceai_mistralai/_embeddings.py b/python/frameworks/mistralai/traceai_mistralai/_embeddings.py index 68b93bf4a..7aef33b3d 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_embeddings.py +++ b/python/frameworks/mistralai/traceai_mistralai/_embeddings.py @@ -1,6 +1,11 @@ """Embedding support for the legacy SDK, which has no native OTel hooks.""" -from fi_instrumentation.fi_types import FiSpanKindValues +from fi_instrumentation import safe_json_dumps +from fi_instrumentation.fi_types import ( + EmbeddingAttributes, + FiSpanKindValues, + SpanAttributes, +) from opentelemetry import trace from ._chat_wrapper import _AsyncChatWrapper, _SyncChatWrapper @@ -10,6 +15,34 @@ class _EmbeddingResponse: def _get_span_kind(self): return FiSpanKindValues.EMBEDDING.value + def _get_attributes_from_request(self, request_parameters): + yield SpanAttributes.GEN_AI_SPAN_KIND, self._get_span_kind() + inputs = request_parameters.get("inputs", []) + if isinstance(inputs, str): + inputs = [inputs] + if isinstance(inputs, (list, tuple)): + for index, text in enumerate(inputs): + if isinstance(text, str): + yield ( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{index}." + f"{EmbeddingAttributes.EMBEDDING_TEXT}", + text, + ) + + def _get_extra_attributes_from_request(self, request_parameters): + # Text belongs only in the embedding attributes, where all three input + # privacy flags apply. Do not duplicate it in input.value or parameters. + yield ( + SpanAttributes.GEN_AI_REQUEST_PARAMETERS, + safe_json_dumps( + { + key: value + for key, value in request_parameters.items() + if key != "inputs" + } + ), + ) + def _finalize_response(self, response, with_span, request_parameters): # Never copy vectors into the generic output field: that would bypass # FI_HIDE_EMBEDDING_VECTORS and create very large attributes. diff --git a/python/frameworks/mistralai/traceai_mistralai/_native.py b/python/frameworks/mistralai/traceai_mistralai/_native.py index 2c2342e31..b77d2177a 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_native.py +++ b/python/frameworks/mistralai/traceai_mistralai/_native.py @@ -3,6 +3,7 @@ import asyncio import json import logging +from contextlib import contextmanager from functools import wraps from threading import RLock from typing import Any @@ -25,6 +26,8 @@ class _NativeSpan(ObjectProxy): def __init__(self, span): super().__init__(span) self._self_status = None + self._self_iteration_depth = 0 + self._self_pending_end = None def set_status(self, status, description=None): if ( @@ -38,12 +41,49 @@ def set_status(self, status, description=None): else trace.Status(status, description) ) + @contextmanager + def observe_iteration(self): + # HTTPX can close at EOF before the SDK decodes its buffered last event. + # Delay only the end, so decoding failures still update this same span. + self._self_iteration_depth += 1 + try: + yield + except (StopIteration, StopAsyncIteration, GeneratorExit): + raise + except BaseException as error: + _record_error(self, error) + self.end() + raise + finally: + self._self_iteration_depth -= 1 + if self._self_iteration_depth == 0 and self._self_pending_end is not None: + args, kwargs = self._self_pending_end + self._self_pending_end = None + self.end(*args, **kwargs) + def end(self, *args, **kwargs): + if self._self_iteration_depth: + self._self_pending_end = (args, kwargs) + return if self._self_status is not None: self.__wrapped__.set_status(self._self_status) return self.__wrapped__.end(*args, **kwargs) +def _record_error(span, error): + if isinstance(error, GeneratorExit): + return + try: + if ( + span._self_status is None + or span._self_status.status_code != trace.StatusCode.ERROR + ): + span.record_exception(error) + span.set_status(trace.Status(trace.StatusCode.ERROR)) + except Exception: + logger.debug("Could not record Mistral stream error") + + class _ErrorStream(ObjectProxy): """Observe read failures before HTTPX closes and ends the native span.""" @@ -51,19 +91,11 @@ def __init__(self, stream, span): super().__init__(stream) self._self_span = span - def _error(self, error): - if not isinstance(error, GeneratorExit): - try: - self._self_span.record_exception(error) - self._self_span.set_status(trace.Status(trace.StatusCode.ERROR)) - except Exception: - logger.debug("Could not record Mistral stream error") - def __iter__(self): try: yield from self.__wrapped__ except BaseException as error: - self._error(error) + _record_error(self._self_span, error) raise async def __aiter__(self): @@ -71,7 +103,7 @@ async def __aiter__(self): async for chunk in self.__wrapped__: yield chunk except BaseException as error: - self._error(error) + _record_error(self._self_span, error) raise @@ -83,7 +115,9 @@ def start_span(self, name: str, *args: Any, **kwargs: Any) -> _NativeSpan: kind = "AGENT" elif name.startswith("execute_tool"): kind = "TOOL" - elif any(part in name for part in ("chat", "completion", "fim")): + elif any( + part in name for part in ("chat", "completion", "fim", "stream_agents") + ): kind = "LLM" else: kind = "CHAIN" @@ -123,6 +157,7 @@ def __init__(self, provider: trace.TracerProvider, config: TraceConfig) -> None: self._async_sends = WeakKeyDictionary() self._lock = RLock() from mistralai.extra.observability.otel import TracedResponse + from mistralai.client.utils.eventstreaming import EventStream, EventStreamAsync self._originals = [] for cls, method, wrapper in ( @@ -130,6 +165,8 @@ def __init__(self, provider: trace.TracerProvider, config: TraceConfig) -> None: (TracingHook, "after_error", self._after_error), (TracingHook, "after_success", self._after_success), (TracedResponse, "_finalize_span", self._finalize_span), + (EventStream, "__next__", self._stream_next), + (EventStreamAsync, "__anext__", self._stream_anext), ): self._originals.append((cls, method, getattr(cls, method))) wrap_function_wrapper(cls, method, wrapper) @@ -198,6 +235,20 @@ def _after_success(self, wrapped, instance, args, kwargs): response.stream = _ErrorStream(response.stream, span) return response + def _stream_next(self, wrapped, instance, args, kwargs): + span = getattr(instance.response, "span", None) + if not isinstance(span, _NativeSpan): + return wrapped(*args, **kwargs) + with span.observe_iteration(): + return wrapped(*args, **kwargs) + + async def _stream_anext(self, wrapped, instance, args, kwargs): + span = getattr(instance.response, "span", None) + if not isinstance(span, _NativeSpan): + return await wrapped(*args, **kwargs) + with span.observe_iteration(): + return await wrapped(*args, **kwargs) + def _finalize_span(self, wrapped, instance, args, kwargs): span = instance.span if ( @@ -209,6 +260,7 @@ def _finalize_span(self, wrapped, instance, args, kwargs): # Adapt conversation SSE attributes on that same span, not a second span. try: parts = [] + tool_calls = {} data_lines = [] for line in bytes(instance._accumulated_sse).decode( "utf-8", errors="replace" @@ -251,14 +303,25 @@ def _finalize_span(self, wrapped, instance, args, kwargs): format_output_message({"message": message})["parts"] ) elif event_type == "function.call.delta": - parts.append( - { + # Deltas for different calls may be interleaved. Keep + # the first-seen output order and append each argument + # fragment to its call instead of emitting duplicates. + identity = ( + event.get("output_index"), + event.get("tool_call_id"), + ) + part = tool_calls.get(identity) + if part is None: + part = { "type": "tool_call", "id": event.get("tool_call_id"), "name": event.get("name"), - "arguments": event.get("arguments"), + "arguments": "", } - ) + tool_calls[identity] = part + parts.append(part) + if isinstance(event.get("arguments"), str): + part["arguments"] += event["arguments"] elif event_type == "conversation.response.error": span.set_status(trace.Status(trace.StatusCode.ERROR)) if parts: diff --git a/python/frameworks/mistralai/traceai_mistralai/_privacy.py b/python/frameworks/mistralai/traceai_mistralai/_privacy.py index f1c6f336d..d0015496b 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_privacy.py +++ b/python/frameworks/mistralai/traceai_mistralai/_privacy.py @@ -3,9 +3,11 @@ import json import logging from typing import Any, Callable, Optional, Union +from urllib.parse import urlsplit, urlunsplit from fi_instrumentation import REDACTED_VALUE from fi_instrumentation.fi_types import ( + EmbeddingAttributes, ImageAttributes, MessageAttributes, MessageContentAttributes, @@ -17,6 +19,19 @@ logger = logging.getLogger(__name__) +class _EmbeddingTraceConfig(ObjectProxy): + """Use the input-text policy also on published cores predating its alias.""" + + def mask(self, key, value): + if key.startswith(SpanAttributes.EMBEDDING_EMBEDDINGS + ".") and key.endswith( + "." + EmbeddingAttributes.EMBEDDING_TEXT + ): + # Core 1.2.2 supports selective message privacy but not the embedding + # text alias yet. Preserve that policy without duplicating payloads. + key = f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_CONTENT}" + return self.__wrapped__.mask(key, value) + + class _NativeTraceConfig(ObjectProxy): """Keep native JSON spans, using the same policy as flattened FI messages. @@ -30,6 +45,14 @@ def mask( value: Union[AttributeValue, Callable[[], AttributeValue]], ) -> Optional[AttributeValue]: value = self.__wrapped__.mask(key, value) + if key in ("http.url", "url.full") and isinstance(value, str): + # SDK HTTP metadata is independent of message-hiding flags. Never + # export proxy userinfo, including percent-encoded credentials. + try: + url = urlsplit(value) + return urlunsplit(url._replace(netloc=url.netloc.rsplit("@", 1)[-1])) + except ValueError: + return None if ( key not in ( From 9c63869787d7414911635c92e9b6ad94edfb77b5 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 02:51:06 -0800 Subject: [PATCH 08/10] fix(mistralai): enforce privacy and observe native response decoding Remove unmasked legacy chat payload copies; apply invocation privacy to native parameter keys. Keep existing native spans open through SDK decoding and parse conversation SSE only at CR/LF boundaries. Cover sync/async, streaming, cancellation, concurrency and SDK floor/latest contracts. Co-Authored-By: Claude Code --- .../mistralai/tests/test_review_round2.py | 230 +++++++++++ .../tests/test_review_round2_edges.py | 357 ++++++++++++++++++ python/frameworks/mistralai/tests/test_sdk.py | 5 +- .../traceai_mistralai/_chat_wrapper.py | 38 +- .../mistralai/traceai_mistralai/_native.py | 72 +++- .../mistralai/traceai_mistralai/_privacy.py | 10 +- .../_response_accumulator.py | 15 +- 7 files changed, 674 insertions(+), 53 deletions(-) create mode 100644 python/frameworks/mistralai/tests/test_review_round2.py create mode 100644 python/frameworks/mistralai/tests/test_review_round2_edges.py diff --git a/python/frameworks/mistralai/tests/test_review_round2.py b/python/frameworks/mistralai/tests/test_review_round2.py new file mode 100644 index 000000000..aaa331334 --- /dev/null +++ b/python/frameworks/mistralai/tests/test_review_round2.py @@ -0,0 +1,230 @@ +"""Independent review checks: synthetic requests through installed SDKs only.""" + +import asyncio +import json + +import pytest +from opentelemetry.trace import StatusCode + +from test_review_regressions import tracing, Mistral, httpx, NATIVE, CHAT, EventStream + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("operation", ["chat", "embedding"]) +def test_nonstream_validation_failure_is_error(asynchronous, operation): + def respond(request): + return httpx.Response( + 200, json={"error": {"message": "synthetic-private-error"}} + ) + + def invoke(client): + if operation == "embedding": + method = ( + client.embeddings.create_async + if asynchronous + else client.embeddings.create + ) + return method(model="mistral-embed", inputs=["hello"]) + method = client.chat.complete_async if asynchronous else client.chat.complete + return method( + model="mistral-test", messages=[{"role": "user", "content": "hello"}] + ) + + with tracing(hide_error_details=True) as (exporter, starts): + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + with pytest.raises(Exception) as error: + await invoke( + Mistral(api_key="synthetic", async_client=transport) + ) + assert "Validation" in type(error.value).__name__ + + asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + with pytest.raises(Exception) as error: + invoke(Mistral(api_key="synthetic", client=transport)) + assert "Validation" in type(error.value).__name__ + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == 1 + assert spans[0].status.status_code == StatusCode.ERROR + assert "synthetic-private-error" not in spans[0].to_json() + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize( + "flag,marker", + [ + ("hide_input_text", "synthetic-private-prompt"), + ("hide_input_messages", "synthetic-private-prompt"), + ("hide_output_text", "synthetic-private-answer"), + ("hide_output_messages", "synthetic-private-answer"), + ("hide_inputs", "synthetic-private-prompt"), + ("hide_outputs", "synthetic-private-answer"), + ], +) +def test_selective_chat_privacy(asynchronous, flag, marker): + response = json.loads(json.dumps(CHAT)) + response["choices"][0]["message"]["content"] = "synthetic-private-answer" + + def respond(request): + assert ( + json.loads(request.content)["messages"][0]["content"] + == "synthetic-private-prompt" + ) + return httpx.Response(200, json=response) + + kwargs = dict( + model="mistral-test", + messages=[{"role": "user", "content": "synthetic-private-prompt"}], + ) + with tracing(**{flag: True}) as (exporter, starts): + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + result = await Mistral( + api_key="synthetic", async_client=transport + ).chat.complete_async(**kwargs) + assert ( + result.choices[0].message.content == "synthetic-private-answer" + ) + + asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + result = Mistral(api_key="synthetic", client=transport).chat.complete( + **kwargs + ) + assert result.choices[0].message.content == "synthetic-private-answer" + (span,) = exporter.get_finished_spans() + assert marker not in span.to_json() + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("configured", ["argument", "environment", "visible"]) +def test_hide_invocation_parameters(asynchronous, configured, monkeypatch): + monkeypatch.setenv( + "FI_HIDE_LLM_INVOCATION_PARAMETERS", + "true" if configured == "environment" else "false", + ) + flags = {"hide_llm_invocation_parameters": True} if configured == "argument" else {} + observed = [] + + def respond(request): + observed.append(json.loads(request.content)) + return httpx.Response(200, json=CHAT) + + kwargs = dict( + model="mistral-test", + messages=[{"role": "user", "content": "hello"}], + stop=["synthetic-private-stop"], + temperature=0.731, + ) + with tracing(**flags) as (exporter, starts): + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + await Mistral( + api_key="synthetic", async_client=transport + ).chat.complete_async(**kwargs) + + asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + Mistral(api_key="synthetic", client=transport).chat.complete(**kwargs) + (span,) = exporter.get_finished_spans() + assert observed[0]["stop"] == ["synthetic-private-stop"] + assert observed[0]["temperature"] == 0.731 + assert span.attributes["gen_ai.usage.input_tokens"] == 2 + if NATIVE: + assert span.attributes["gen_ai.request.model"] == "mistral-test" + if configured == "visible": + if NATIVE: + assert span.attributes["gen_ai.request.temperature"] == 0.731 + assert span.attributes["gen_ai.request.stop_sequences"] == ( + "synthetic-private-stop", + ) + else: + params = json.loads(span.attributes["gen_ai.request.parameters"]) + assert params["temperature"] == 0.731 + assert params["stop"] == ["synthetic-private-stop"] + else: + assert "gen_ai.request.temperature" not in span.attributes + assert "synthetic-private-stop" not in span.to_json() + + +@pytest.mark.skipif(not NATIVE, reason="Native conversations") +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("separator", ["\u2028", "\u2029", "\u0085"]) +@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"]) +def test_unicode_conversation_content_preserved(asynchronous, separator, newline): + marker = "first" + separator + "second" + events = [ + { + "type": "message.output.delta", + "id": "msg-1", + "role": "assistant", + "content": marker, + }, + { + "type": "conversation.response.done", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + }, + ] + + class RawEvents(EventStream): + def __iter__(self): + for event in self.events: + yield ( + "event: " + + event["type"] + + newline + + "data: " + + json.dumps(event, ensure_ascii=False) + + newline * 2 + ).encode() + + def respond(request): + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=RawEvents(events) + ) + + with tracing() as (exporter, starts): + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + async with await Mistral( + api_key="synthetic", async_client=transport + ).beta.conversations.start_stream_async( + model="mistral-test", inputs="hello" + ) as stream: + result = [event async for event in stream] + assert result[0].data.content == marker + + asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + with Mistral( + api_key="synthetic", client=transport + ).beta.conversations.start_stream( + model="mistral-test", inputs="hello" + ) as stream: + result = list(stream) + assert result[0].data.content == marker + (span,) = exporter.get_finished_spans() + assert json.loads(span.attributes["gen_ai.output.messages"])[0]["parts"] == [ + {"type": "text", "content": marker} + ] diff --git a/python/frameworks/mistralai/tests/test_review_round2_edges.py b/python/frameworks/mistralai/tests/test_review_round2_edges.py new file mode 100644 index 000000000..f5f26d530 --- /dev/null +++ b/python/frameworks/mistralai/tests/test_review_round2_edges.py @@ -0,0 +1,357 @@ +"""Privacy and decoding boundaries with real SDKs and synthetic transports.""" + +import asyncio +import importlib +import json +from concurrent.futures import ThreadPoolExecutor + +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.trace import StatusCode + +from test_review_regressions import CHAT, Mistral, NATIVE, httpx, tracing +from test_sdk import SSE, chat_response + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("resource", ["chat", "agents"]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("environment", [False, True]) +@pytest.mark.parametrize( + "flag", + [ + "hide_input_text", + "hide_input_messages", + "hide_output_text", + "hide_output_messages", + "visible", + ], +) +def test_chat_privacy_all_paths( + asynchronous, resource, streaming, environment, flag, monkeypatch +): + flags = {} + if flag != "visible": + if environment: + monkeypatch.setenv("FI_" + flag.upper(), "true") + else: + flags[flag] = True + observed = [] + + def respond(request): + observed.append(json.loads(request.content)) + if streaming: + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=SSE() + ) + return httpx.Response(200, json=chat_response()) + + kwargs = {"messages": [{"role": "user", "content": "private-input-marker"}]} + kwargs.update( + {"model": "mistral-test"} + if resource == "chat" + else {"agent_id": "synthetic-agent"} + ) + method = "stream" if streaming else "complete" + with tracing(**flags) as (exporter, starts): + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + result = await getattr( + getattr(client, resource), method + "_async" + )(**kwargs) + if streaming: + chunks = [chunk async for chunk in result] + await result.close() + assert ( + chunks[0].data.choices[0].delta.content + == "private-output-marker" + ) + else: + assert ( + result.choices[0].message.content == "private-output-marker" + ) + + asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic", client=transport) + result = getattr(getattr(client, resource), method)(**kwargs) + if streaming: + chunks = list(result) + result.close() + assert ( + chunks[0].data.choices[0].delta.content + == "private-output-marker" + ) + else: + assert result.choices[0].message.content == "private-output-marker" + assert observed[0]["messages"] == kwargs["messages"] + (span,) = exporter.get_finished_spans() + assert len(starts.spans) == 1 + assert span.status.status_code == StatusCode.OK + assert span.attributes["gen_ai.usage.input_tokens"] == 2 + rendered = span.to_json() + assert ("private-input-marker" in rendered) == ("input" not in flag) + assert ("private-output-marker" in rendered) == ("output" not in flag) + + +@pytest.mark.skipif(not NATIVE, reason="Native SDK response observers") +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize( + "operation", ["chat", "agents", "embedding", "fim", "start", "append", "restart"] +) +@pytest.mark.parametrize("body", ["schema", "json", "content-type"]) +@pytest.mark.parametrize("hide", [False, True]) +def test_native_decoding_boundaries(asynchronous, operation, body, hide): + def respond(request): + if body == "schema": + return httpx.Response( + 200, json={"error": {"message": "synthetic-private-error"}} + ) + return httpx.Response( + 200, + content=b"synthetic-private-error", + headers={ + "content-type": "application/json" if body == "json" else "text/html", + }, + ) + + def invoke(client): + kwargs = {"model": "mistral-test"} + method = "complete" + if operation in ("chat", "agents"): + resource = getattr(client, operation) + kwargs["messages"] = [{"role": "user", "content": "hello"}] + if operation == "agents": + kwargs.pop("model") + kwargs["agent_id"] = "synthetic-agent" + elif operation == "embedding": + resource, method = client.embeddings, "create" + kwargs["inputs"] = ["hello"] + elif operation == "fim": + resource = client.fim + kwargs["prompt"] = "hello" + else: + resource, method = client.beta.conversations, operation + kwargs["inputs"] = "hello" + if operation != "start": + kwargs.pop("model") + kwargs["conversation_id"] = "conv-synthetic" + if operation == "restart": + kwargs["from_entry_id"] = "entry-synthetic" + return getattr(resource, method + ("_async" if asynchronous else ""))(**kwargs) + + with tracing(hide_error_details=hide) as (exporter, starts): + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + with pytest.raises(Exception) as error: + await invoke( + Mistral(api_key="synthetic", async_client=transport) + ) + return error.value + + error = asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + with pytest.raises(Exception) as caught: + invoke(Mistral(api_key="synthetic", client=transport)) + error = caught.value + assert "synthetic-private-error" in error.body + (span,) = exporter.get_finished_spans() + assert len(starts.spans) == 1 + assert not starts.spans[0].is_recording() + assert span.status.status_code == StatusCode.ERROR + assert len(span.events) == 1 + assert ( + span.events[0].attributes["exception.type"].endswith(type(error).__name__) + ) + if hide: + assert "synthetic-private-error" not in span.to_json() + else: + assert span.events[0].attributes["exception.message"] == str(error) + + +@pytest.mark.skipif(not NATIVE, reason="Native SDK response observers") +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_concurrent_decoding_scopes(asynchronous): + def respond(request): + model = json.loads(request.content)["model"] + return httpx.Response(200, json=CHAT if model == "valid" else {"invalid": True}) + + with tracing() as (exporter, starts): + if asynchronous: + + async def run(): + # Yield during transport to interleave requests in one client. + async def async_respond(request): + await asyncio.sleep(0) + return respond(request) + + async with httpx.AsyncClient( + transport=httpx.MockTransport(async_respond) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + return await asyncio.gather( + *[ + client.chat.complete_async( + model=model, + messages=[{"role": "user", "content": "hi"}], + ) + for model in ["valid", "invalid"] * 4 + ], + return_exceptions=True, + ) + + results = asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic", client=transport) + + def call(model): + try: + return client.chat.complete( + model=model, messages=[{"role": "user", "content": "hi"}] + ) + except Exception as error: + return error + + with ThreadPoolExecutor(max_workers=4) as pool: + results = list(pool.map(call, ["valid", "invalid"] * 4)) + assert sum(isinstance(result, Exception) for result in results) == 4 + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == 8 + assert sum(span.status.status_code == StatusCode.ERROR for span in spans) == 4 + assert sum(span.status.status_code == StatusCode.OK for span in spans) == 4 + assert all(not span.is_recording() for span in starts.spans) + + +@pytest.mark.skipif(not NATIVE, reason="Native SDK response observers") +def test_observed_methods_restored(): + methods = [] + for module, cls_name, names in ( + ("chat", "Chat", ("complete", "stream")), + ("agents", "Agents", ("complete", "stream")), + ("embeddings", "Embeddings", ("create",)), + ("fim", "Fim", ("complete", "stream")), + ( + "conversations", + "Conversations", + ( + "start", + "append", + "restart", + "start_stream", + "append_stream", + "restart_stream", + ), + ), + ): + cls = getattr(importlib.import_module("mistralai.client." + module), cls_name) + for name in names: + for suffix in ("", "_async"): + name_with_suffix = name + suffix + methods.append((cls, name_with_suffix, getattr(cls, name_with_suffix))) + for _ in range(2): + with tracing(): + for cls, name, original in methods: + assert getattr(cls, name).__wrapped__ is original + for cls, name, original in methods: + assert getattr(cls, name) is original + + +@pytest.mark.parametrize( + "key", + [ + "choice.count", + "encoding_formats", + "frequency_penalty", + "max_tokens", + "presence_penalty", + "seed", + "stop_sequences", + "temperature", + "top_p", + "top_k", + "future_parameter", + ], +) +def test_native_parameter_policy_drops_lazy_values(key): + from traceai_mistralai._privacy import _NativeTraceConfig + + config = _NativeTraceConfig(TraceConfig(hide_llm_invocation_parameters=True)) + + def sensitive(): + pytest.fail("Hidden parameter value should not be evaluated") + + assert config.mask("gen_ai.request." + key, sensitive) is None + assert config.mask("gen_ai.request.model", "mistral-test") == "mistral-test" + assert config.mask("gen_ai.usage.input_tokens", 2) == 2 + + +@pytest.mark.skipif(not NATIVE, reason="Native SDK response observers") +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("hide", [False, True]) +def test_decoder_exception_identity_and_scope_reset(asynchronous, hide, monkeypatch): + module = importlib.import_module("mistralai.client.chat") + original = module.unmarshal_json_response + failure = ( + asyncio.CancelledError("synthetic-decoder-cancel") + if asynchronous + else RuntimeError("synthetic-decoder-error") + ) + calls = [] + + def decode(*args, **kwargs): + calls.append(True) + if len(calls) == 1: + raise failure + return original(*args, **kwargs) + + monkeypatch.setattr(module, "unmarshal_json_response", decode) + with tracing(hide_error_details=hide) as (exporter, starts): + kwargs = { + "model": "mistral-test", + "messages": [{"role": "user", "content": "hi"}], + } + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda r: httpx.Response(200, json=CHAT) + ) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + with pytest.raises(asyncio.CancelledError) as error: + await client.chat.complete_async(**kwargs) + assert error.value is failure + assert (await client.chat.complete_async(**kwargs)).choices[ + 0 + ].message.content == "OK" + + asyncio.run(run()) + else: + with httpx.Client( + transport=httpx.MockTransport(lambda r: httpx.Response(200, json=CHAT)) + ) as transport: + client = Mistral(api_key="synthetic", client=transport) + with pytest.raises(RuntimeError) as error: + client.chat.complete(**kwargs) + assert error.value is failure + assert client.chat.complete(**kwargs).choices[0].message.content == "OK" + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == 2 + assert spans[0].status.status_code == StatusCode.ERROR + assert spans[1].status.status_code == StatusCode.OK + assert len(spans[0].events) == 1 + assert all(not span.is_recording() for span in starts.spans) + assert ("synthetic-decoder-" in spans[0].to_json()) == (not hide) diff --git a/python/frameworks/mistralai/tests/test_sdk.py b/python/frameworks/mistralai/tests/test_sdk.py index ce533cbd4..96f79dc37 100644 --- a/python/frameworks/mistralai/tests/test_sdk.py +++ b/python/frameworks/mistralai/tests/test_sdk.py @@ -345,7 +345,9 @@ def test_existing_clients_suppression_reenable_and_parent(): instrumentor.instrument(tracer_provider=provider) instrumentor.instrument(tracer_provider=provider) if NATIVE: - assert type(client.chat).complete is original + # A lifecycle observer now holds native spans through decoding; + # it delegates to the original method without creating spans. + assert type(client.chat).complete.__wrapped__ is original with provider.get_tracer("parent").start_as_current_span( "parent" ) as parent: @@ -356,6 +358,7 @@ def test_existing_clients_suppression_reenable_and_parent(): assert len(spans) == 2 assert spans[0].parent.span_id == parent.get_span_context().span_id instrumentor.uninstrument() + assert type(client.chat).complete is original exporter.clear() client.chat.complete(**params("chat")) assert not exporter.get_finished_spans() diff --git a/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py b/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py index 27a9e263e..7c1c0f70b 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py +++ b/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py @@ -1,7 +1,6 @@ import asyncio import json import logging -import warnings from abc import ABC from contextlib import contextmanager from inspect import Signature, signature @@ -19,7 +18,6 @@ from fi_instrumentation import get_attributes_from_context, safe_json_dumps from fi_instrumentation.fi_types import ( - FiMimeTypeValues, FiSpanKindValues, SpanAttributes, ) @@ -38,8 +36,6 @@ from traceai_mistralai._stream import _AsyncStream, _Stream from traceai_mistralai._utils import ( _finish_tracing, - _process_input_messages, - _raw_input, ) from traceai_mistralai._with_span import _WithSpan @@ -108,15 +104,8 @@ def _get_attributes_from_request( request_parameters: Dict[str, Any], ) -> Iterator[Tuple[str, AttributeValue]]: yield SpanAttributes.GEN_AI_SPAN_KIND, self._get_span_kind() - try: - yield from _process_input_messages(request_parameters.get("messages", [])) - yield from _raw_input(request_parameters) - # yield from _as_input_attributes(_io_value_and_type(request_parameters)) - except Exception: - logger.exception( - f"Failed to get input attributes from request parameters of " - f"type {type(request_parameters)}" - ) + # Messages already have policy-aware semantic attributes. Raw payload + # copies bypass selective text/message/image and parameter privacy. def _get_extra_attributes_from_request( self, @@ -388,26 +377,9 @@ def __init__( self._response_attributes_extractor = response_attributes_extractor def get_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: - if hasattr(self._response, "model_dump_json") and callable( - self._response.model_dump_json - ): - try: - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - value = self._response.model_dump_json(exclude_unset=True) - assert isinstance(value, str) - output_value = ( - json.loads(value) - .get("choices", [{}])[0] - .get("message", {}) - .get("content", "") - ) - - yield SpanAttributes.OUTPUT_VALUE, value - yield SpanAttributes.OUTPUT_VALUE, output_value - yield SpanAttributes.OUTPUT_MIME_TYPE, FiMimeTypeValues.JSON.value - except Exception: - logger.exception("Failed to get model dump json") + # Export output only through the semantic message attributes below. + # Generic output.value is not covered by selective message privacy. + yield from () def get_extra_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: yield from self._response_attributes_extractor.get_attributes_from_response( diff --git a/python/frameworks/mistralai/traceai_mistralai/_native.py b/python/frameworks/mistralai/traceai_mistralai/_native.py index b77d2177a..ae3a0b0ac 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_native.py +++ b/python/frameworks/mistralai/traceai_mistralai/_native.py @@ -3,7 +3,9 @@ import asyncio import json import logging -from contextlib import contextmanager +import re +from contextlib import ExitStack, contextmanager +from contextvars import ContextVar from functools import wraps from threading import RLock from typing import Any @@ -139,7 +141,8 @@ class _NativeInstrumentation: The native hook owns request context and span creation. The adapter routes those spans and fills HTTP-error and conversation-stream gaps. Do not patch - chat or agent methods alongside this hook: that would duplicate native spans. + chat or agent methods to create spans alongside this hook. Method observers + below only hold the existing span open through SDK response validation. """ _FIELDS = ( @@ -159,6 +162,7 @@ def __init__(self, provider: trace.TracerProvider, config: TraceConfig) -> None: from mistralai.extra.observability.otel import TracedResponse from mistralai.client.utils.eventstreaming import EventStream, EventStreamAsync + self._response_scope = ContextVar("mistral_response_scope", default=None) self._originals = [] for cls, method, wrapper in ( (TracingHook, "before_request", self._before_request), @@ -171,6 +175,57 @@ def __init__(self, provider: trace.TracerProvider, config: TraceConfig) -> None: self._originals.append((cls, method, getattr(cls, method))) wrap_function_wrapper(cls, method, wrapper) + # The native hook ends buffered responses before generated methods + # deserialize them. Observe those methods without creating any spans. + # A per-call context keeps concurrent sync/async requests independent. + from mistralai.client.agents import Agents + from mistralai.client.chat import Chat + from mistralai.client.conversations import Conversations + from mistralai.client.embeddings import Embeddings + from mistralai.client.fim import Fim + + for cls, methods in ( + (Chat, ("complete", "stream")), + (Agents, ("complete", "stream")), + (Embeddings, ("create",)), + (Fim, ("complete", "stream")), + ( + Conversations, + ( + "start", + "append", + "restart", + "start_stream", + "append_stream", + "restart_stream", + ), + ), + ): + for method in methods: + for name, wrapper in ( + (method, self._call), + (method + "_async", self._call_async), + ): + self._originals.append((cls, name, getattr(cls, name))) + wrap_function_wrapper(cls, name, wrapper) + + @contextmanager + def _call_scope(self): + with ExitStack() as scope: + token = self._response_scope.set(scope) + try: + yield + finally: + self._response_scope.reset(token) + + def _call(self, wrapped, instance, args, kwargs): + with self._call_scope(): + return wrapped(*args, **kwargs) + + async def _call_async(self, wrapped, instance, args, kwargs): + with self._call_scope(): + return await wrapped(*args, **kwargs) + def _before_request(self, wrapped, instance, args, kwargs): with self._lock: if ( @@ -229,6 +284,12 @@ def _after_error(self, wrapped, instance, args, kwargs): return wrapped(*args, **kwargs) def _after_success(self, wrapped, instance, args, kwargs): + response = kwargs.get("response", args[1] if len(args) > 1 else None) + span = instance._get_span(response) + scope = self._response_scope.get() + if isinstance(span, _NativeSpan) and scope is not None: + # Enter before the hook queues end(), leave after SDK validation. + scope.enter_context(span.observe_iteration()) response = wrapped(*args, **kwargs) span = getattr(response, "span", None) if isinstance(span, _NativeSpan): @@ -262,9 +323,10 @@ def _finalize_span(self, wrapped, instance, args, kwargs): parts = [] tool_calls = {} data_lines = [] - for line in bytes(instance._accumulated_sse).decode( - "utf-8", errors="replace" - ).splitlines() + [""]: + # SSE delimiters are CR/LF only. Unicode separators are valid JSON + # string content and must not split a message delta's data field. + payload = bytes(instance._accumulated_sse).decode("utf-8", errors="replace") + for line in re.split(r"\r\n|\r|\n", payload) + [""]: if line.startswith("data:"): data_lines.append(line[5:].lstrip()) elif not line and data_lines: diff --git a/python/frameworks/mistralai/traceai_mistralai/_privacy.py b/python/frameworks/mistralai/traceai_mistralai/_privacy.py index d0015496b..4c62bfe38 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_privacy.py +++ b/python/frameworks/mistralai/traceai_mistralai/_privacy.py @@ -44,7 +44,15 @@ def mask( key: str, value: Union[AttributeValue, Callable[[], AttributeValue]], ) -> Optional[AttributeValue]: - value = self.__wrapped__.mask(key, value) + # Native OTel splits invocation parameters into individual attributes. + # Keep model identity, but apply the core's aggregate parameter policy + # to every other request parameter, including future SDK additions. + policy_key = ( + SpanAttributes.GEN_AI_REQUEST_PARAMETERS + if key.startswith("gen_ai.request.") and key != "gen_ai.request.model" + else key + ) + value = self.__wrapped__.mask(policy_key, value) if key in ("http.url", "url.full") and isinstance(value, str): # SDK HTTP metadata is independent of message-hiding flags. Never # export proxy userinfo, including percent-encoded credentials. diff --git a/python/frameworks/mistralai/traceai_mistralai/_response_accumulator.py b/python/frameworks/mistralai/traceai_mistralai/_response_accumulator.py index e3c3034c1..788ad4a83 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_response_accumulator.py +++ b/python/frameworks/mistralai/traceai_mistralai/_response_accumulator.py @@ -16,7 +16,6 @@ Type, ) -from fi_instrumentation.fi_types import FiMimeTypeValues, SpanAttributes from opentelemetry.util.types import AttributeValue from traceai_mistralai._utils import _to_dict @@ -42,7 +41,6 @@ class _ChatCompletionAccumulator: "_request_parameters", "_response_attributes_extractor", "_chat_completion_type", - "_content", ) def __init__( @@ -56,7 +54,6 @@ def __init__( self._response_attributes_extractor = response_attributes_extractor self._is_null = True self._cached_result: Optional[Dict[str, Any]] = None - self._content: List[str] = [] self._values = _ValuesAccumulator( data=_ValuesAccumulator( choices=_IndexedAccumulator( @@ -86,14 +83,6 @@ def process_chunk(self, chunk: Any) -> None: choice["message"] = choice.pop("delta") self._values += accumulated - if "choices" in raw_chunk.get("data", {}): - choices = raw_chunk["data"]["choices"] - if choices and isinstance(choices, list): - delta = choices[0].get("delta", {}) - content = delta.get("content") - if content is not None: - self._content.append(content) - def _result(self) -> Optional[Dict[str, Any]]: if self._is_null: return None @@ -102,8 +91,8 @@ def _result(self) -> Optional[Dict[str, Any]]: return self._cached_result def get_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: - yield SpanAttributes.OUTPUT_VALUE, "".join(self._content) - yield SpanAttributes.OUTPUT_MIME_TYPE, FiMimeTypeValues.TEXT.value + # The reconstructed semantic messages carry the output and its policy. + yield from () def get_extra_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: if not (result := self._result()): From 18dc2e9e7287aaa6d42bbb0ec5e29b0facea9952 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 13:27:38 -0800 Subject: [PATCH 09/10] fix(mistralai): preserve structured parse and legacy content telemetry Co-Authored-By: Claude Code --- .../mistralai/tests/test_review_round3.py | 312 ++++++++++++++++++ .../mistralai/traceai_mistralai/__init__.py | 45 ++- .../traceai_mistralai/_chat_wrapper.py | 70 +++- .../mistralai/traceai_mistralai/_native.py | 13 +- .../_request_attributes_extractor.py | 43 ++- .../_response_accumulator.py | 12 + python/tests/test_framework_mistralai.py | 5 +- 7 files changed, 475 insertions(+), 25 deletions(-) create mode 100644 python/frameworks/mistralai/tests/test_review_round3.py diff --git a/python/frameworks/mistralai/tests/test_review_round3.py b/python/frameworks/mistralai/tests/test_review_round3.py new file mode 100644 index 000000000..14b80ebd5 --- /dev/null +++ b/python/frameworks/mistralai/tests/test_review_round3.py @@ -0,0 +1,312 @@ +"""Independent real-SDK contracts; synthetic credentials and no vendor calls.""" + +import asyncio +import importlib +import json +from contextlib import contextmanager +from importlib.metadata import version + +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_mistralai import MistralAIInstrumentor + +NATIVE = version("mistralai").startswith("3.") +Mistral = importlib.import_module("mistralai.client" if NATIVE else "mistralai").Mistral +models = importlib.import_module( + "mistralai.client.models" if NATIVE else "mistralai.models" +) +httpx = importlib.import_module("httpx2" if NATIVE else "httpx") +INPUT = "synthetic-input-content-marker" +OUTPUT = "synthetic-output-content-marker" +IMAGE = "https://example.invalid/synthetic-image-content-marker" + + +@contextmanager +def tracing(**flags): + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**flags)) + try: + yield exporter + finally: + instrumentor.uninstrument() + provider.shutdown() + + +def response(content=OUTPUT): + return { + "id": "synthetic", + "object": "chat.completion", + "created": 1, + "model": "mistral-test", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": content}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + } + + +def call(asynchronous, messages, body, captured): + def respond(request): + captured.append(json.loads(request.content)) + return httpx.Response(200, json=body) + + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic-key", async_client=transport) + return await client.chat.complete_async( + model="mistral-test", messages=messages + ) + + return asyncio.run(run()) + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-key", client=transport) + return client.chat.complete(model="mistral-test", messages=messages) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize( + "shape", + ["dict", "model", "typed-chunk", "image-dict", "image-model", "tool-history"], +) +@pytest.mark.parametrize("hide", [False, True]) +def test_supported_input_shapes_preserve_capture(asynchronous, shape, hide): + marker = INPUT + if shape == "dict": + messages = [{"role": "user", "content": INPUT}] + elif shape == "model": + messages = [models.UserMessage(content=INPUT)] + elif shape == "typed-chunk": + messages = [{"role": "user", "content": [models.TextChunk(text=INPUT)]}] + elif shape == "image-dict": + marker = IMAGE + messages = [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": IMAGE}}], + } + ] + elif shape == "image-model": + if NATIVE: + pytest.skip("Legacy typed image model") + marker = IMAGE + messages = [models.UserMessage(content=[models.ImageURLChunk(image_url=IMAGE)])] + else: + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call-synthetic", + "type": "function", + "function": { + "name": "lookup", + "arguments": json.dumps({"query": INPUT}), + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call-synthetic", + "content": "tool-result", + }, + ] + with tracing(**({"hide_inputs": True} if hide else {})) as exporter: + captured = [] + result = call(asynchronous, messages, response(), captured) + assert result.choices[0].message.content == OUTPUT + assert marker in json.dumps(captured) + (span,) = exporter.get_finished_spans() + assert span.status.status_code == StatusCode.OK + assert (marker in span.to_json()) == (not hide), dict(span.attributes) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("shape", ["string", "chunks"]) +@pytest.mark.parametrize("hide", [False, True]) +def test_supported_output_shapes_preserve_capture(asynchronous, shape, hide): + if shape == "chunks" and version("mistralai") == "1.1.0": + pytest.skip("SDK 1.1 response content accepts strings only") + content = OUTPUT if shape == "string" else [{"type": "text", "text": OUTPUT}] + with tracing(**({"hide_output_text": True} if hide else {})) as exporter: + result = call( + asynchronous, [{"role": "user", "content": INPUT}], response(content), [] + ) + assert OUTPUT in result.model_dump_json() + (span,) = exporter.get_finished_spans() + assert span.status.status_code == StatusCode.OK + assert (OUTPUT in span.to_json()) == (not hide), dict(span.attributes) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("policy", ["argument", "environment", "default", "visible"]) +def test_oversized_input_image_policy(asynchronous, policy, monkeypatch): + image = "data:image/png;base64," + "YWJj" * 9000 + flags = {} + if policy == "environment": + monkeypatch.setenv("FI_BASE64_IMAGE_MAX_LENGTH", "32") + elif policy == "argument": + flags["base64_image_max_length"] = 32 + elif policy == "visible": + flags["base64_image_max_length"] = 40000 + with tracing(**flags) as exporter: + captured = [] + result = call( + asynchronous, + [{"role": "user", "content": [{"type": "image_url", "image_url": image}]}], + response(), + captured, + ) + assert result.choices[0].message.content == OUTPUT + assert image in json.dumps(captured) + (span,) = exporter.get_finished_spans() + captured_image = image in span.to_json() + assert captured_image == ( + policy == "visible" + ), "Oversized input image capture does not follow the configured policy" + + +@pytest.mark.skipif( + not hasattr( + importlib.import_module( + "mistralai.client.chat" if NATIVE else "mistralai.chat" + ).Chat, + "parse", + ), + reason="No structured output API on this SDK", +) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("valid", [False, True]) +def test_structured_parse_status(asynchronous, valid): + from pydantic import BaseModel + + class Answer(BaseModel): + city: str + + body = response('{"city":"Paris"}' if valid else "synthetic-invalid-json") + + def respond(request): + return httpx.Response(200, json=body) + + def operation(client): + method = client.chat.parse_async if asynchronous else client.chat.parse + return method( + response_format=Answer, + model="mistral-test", + messages=[{"role": "user", "content": "city"}], + ) + + with tracing() as exporter: + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + if valid: + result = await operation(client) + assert result.choices[0].message.parsed.city == "Paris" + else: + with pytest.raises(json.JSONDecodeError): + await operation(client) + + asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic", client=transport) + if valid: + assert operation(client).choices[0].message.parsed.city == "Paris" + else: + with pytest.raises(json.JSONDecodeError): + operation(client) + (span,) = exporter.get_finished_spans() + assert span.status.status_code == (StatusCode.OK if valid else StatusCode.ERROR) + if not valid: + assert len(span.events) == 1 + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("shape", ["string", "chunks"]) +@pytest.mark.parametrize("hide", [False, True]) +def test_stream_content_shapes(asynchronous, shape, hide): + if shape == "chunks" and version("mistralai") == "1.1.0": + pytest.skip("SDK 1.1 output deltas accept strings only") + content = OUTPUT if shape == "string" else [{"type": "text", "text": OUTPUT}] + event = { + "id": "synthetic", + "object": "chat.completion.chunk", + "created": 1, + "model": "mistral-test", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": content}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + } + + class Stream(httpx.SyncByteStream, httpx.AsyncByteStream): + def __iter__(self): + yield ("data: " + json.dumps(event) + "\n\ndata: [DONE]\n\n").encode() + + async def __aiter__(self): + for b in self: + yield b + + def close(self): + pass + + async def aclose(self): + pass + + def respond(request): + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=Stream() + ) + + with tracing(**({"hide_output_text": True} if hide else {})) as exporter: + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + async with await client.chat.stream_async( + model="mistral-test", + messages=[{"role": "user", "content": INPUT}], + ) as stream: + return [chunk async for chunk in stream] + + chunks = asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic", client=transport) + with client.chat.stream( + model="mistral-test", messages=[{"role": "user", "content": INPUT}] + ) as stream: + chunks = list(stream) + assert OUTPUT in chunks[0].model_dump_json() + (span,) = exporter.get_finished_spans() + assert span.status.status_code == StatusCode.OK + assert span.attributes["gen_ai.usage.input_tokens"] == 2 + assert (OUTPUT in span.to_json()) == (not hide), dict(span.attributes) diff --git a/python/frameworks/mistralai/traceai_mistralai/__init__.py b/python/frameworks/mistralai/traceai_mistralai/__init__.py index 8419de9c5..c72b6139f 100644 --- a/python/frameworks/mistralai/traceai_mistralai/__init__.py +++ b/python/frameworks/mistralai/traceai_mistralai/__init__.py @@ -34,6 +34,7 @@ class MistralAIInstrumentor(BaseInstrumentor): # type: ignore "_original_sync_stream_agent_method", "_original_async_agent_method", "_original_async_stream_agent_method", + "_original_parse_methods", ) def instrumentation_dependencies(self) -> Collection[str]: @@ -52,8 +53,13 @@ def _instrument(self, **kwargs: Any) -> None: ) sdk_version = Version(distribution_version("mistralai")) - if not (Version("1.1.0") <= sdk_version < Version("2") or Version("3.6.0") <= sdk_version < Version("4")): - raise ValueError("traceAI-mistralai supports mistralai >=1.1,<2 or >=3.6,<4") + if not ( + Version("1.1.0") <= sdk_version < Version("2") + or Version("3.6.0") <= sdk_version < Version("4") + ): + raise ValueError( + "traceAI-mistralai supports mistralai >=1.1,<2 or >=3.6,<4" + ) self._native = None if sdk_version >= Version("3.6.0"): from traceai_mistralai._native import _NativeInstrumentation @@ -65,8 +71,13 @@ def _instrument(self, **kwargs: Any) -> None: _AsyncChatWrapper, _AsyncStreamChatWrapper, _SyncChatWrapper, + _sync_parse, + _async_parse, + ) + from traceai_mistralai._embeddings import ( + _AsyncEmbeddingWrapper, + _EmbeddingWrapper, ) - from traceai_mistralai._embeddings import _AsyncEmbeddingWrapper, _EmbeddingWrapper try: import mistralai @@ -77,6 +88,11 @@ def _instrument(self, **kwargs: Any) -> None: "Could not import mistralai. Please install with `pip install mistralai`." ) from err + self._original_parse_methods = () + if hasattr(Chat, "parse") and hasattr(Chat, "parse_async"): + self._original_parse_methods = (Chat.parse, Chat.parse_async) + wrap_function_wrapper(module="mistralai.chat", name="Chat.parse", wrapper=_sync_parse) + wrap_function_wrapper(module="mistralai.chat", name="Chat.parse_async", wrapper=_async_parse) self._original_sync_chat_method = Chat.complete self._original_sync_stream_chat_method = Chat.stream self._original_async_chat_method = Chat.complete_async @@ -149,12 +165,20 @@ def _instrument(self, **kwargs: Any) -> None: config=_EmbeddingTraceConfig(config), ) self._embedding_methods = (Embeddings.create, Embeddings.create_async) - wrap_function_wrapper(module="mistralai.embeddings", name="Embeddings.create", wrapper=_EmbeddingWrapper( - "MistralClient.embeddings", embedding_tracer, mistralai - )) - wrap_function_wrapper(module="mistralai.embeddings", name="Embeddings.create_async", wrapper=_AsyncEmbeddingWrapper( - "MistralAsyncClient.embeddings", embedding_tracer, mistralai - )) + wrap_function_wrapper( + module="mistralai.embeddings", + name="Embeddings.create", + wrapper=_EmbeddingWrapper( + "MistralClient.embeddings", embedding_tracer, mistralai + ), + ) + wrap_function_wrapper( + module="mistralai.embeddings", + name="Embeddings.create_async", + wrapper=_AsyncEmbeddingWrapper( + "MistralAsyncClient.embeddings", embedding_tracer, mistralai + ), + ) def _uninstrument(self, **kwargs: Any) -> None: if self._native is not None: @@ -162,10 +186,13 @@ def _uninstrument(self, **kwargs: Any) -> None: self._native = None return from mistralai.embeddings import Embeddings + Embeddings.create, Embeddings.create_async = self._embedding_methods from mistralai.agents import Agents from mistralai.chat import Chat + if self._original_parse_methods: + Chat.parse, Chat.parse_async = self._original_parse_methods Chat.complete = self._original_sync_chat_method # type: ignore Chat.stream = self._original_sync_stream_chat_method # type: ignore Chat.complete_async = self._original_async_chat_method # type: ignore diff --git a/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py b/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py index 7c1c0f70b..36978a3f9 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py +++ b/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py @@ -3,6 +3,7 @@ import logging from abc import ABC from contextlib import contextmanager +from contextvars import ContextVar from inspect import Signature, signature from types import ModuleType from typing import ( @@ -44,6 +45,58 @@ __all__ = ("_SyncChatWrapper",) +# Structured parse runs a second validation step after complete returns. +# Hold only that request's existing span until the outer parse completes. +_parse_pending = ContextVar("mistral_parse_pending", default=None) + + +def _finish_parse(pending, error=None): + for with_span, response, request_parameters, extractor in pending: + if error is not None: + with_span.record_exception(error) + _finish_tracing( + with_span=with_span, + has_attributes=_ResponseAttributes(response, request_parameters, extractor), + status=trace_api.Status( + status_code=( + trace_api.StatusCode.ERROR + if error is not None + else trace_api.StatusCode.OK + ) + ), + ) + + +def _sync_parse(wrapped, instance, args, kwargs): + pending = [] + token = _parse_pending.set(pending) + try: + result = wrapped(*args, **kwargs) + except Exception as error: + _finish_parse(pending, error) + raise + else: + _finish_parse(pending) + return result + finally: + _parse_pending.reset(token) + + +async def _async_parse(wrapped, instance, args, kwargs): + pending = [] + token = _parse_pending.set(pending) + try: + result = await wrapped(*args, **kwargs) + except (Exception, asyncio.CancelledError) as error: + _finish_parse(pending, error) + raise + else: + _finish_parse(pending) + return result + finally: + _parse_pending.reset(token) + + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) @@ -138,7 +191,12 @@ def _parse_args( bound_arguments = bound_signature.arguments request_data: Dict[str, Any] = {} for key, value in bound_arguments.items(): - if key in {"http_headers", "server_url", "retries", "timeout_ms"} or isinstance(value, (Iterator, AsyncIterator)): + if key in { + "http_headers", + "server_url", + "retries", + "timeout_ms", + } or isinstance(value, (Iterator, AsyncIterator)): # Telemetry must not consume a one-shot message/tool iterable. continue try: @@ -184,6 +242,16 @@ def _finalize_response( ) else: raise TypeError("Response must be an asynchronous iterator") + if (pending := _parse_pending.get()) is not None: + pending.append( + ( + with_span, + response, + request_parameters, + self._response_attributes_extractor, + ) + ) + return response _finish_tracing( status=trace_api.Status(status_code=trace_api.StatusCode.OK), with_span=with_span, diff --git a/python/frameworks/mistralai/traceai_mistralai/_native.py b/python/frameworks/mistralai/traceai_mistralai/_native.py index ae3a0b0ac..d6d19ee9b 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_native.py +++ b/python/frameworks/mistralai/traceai_mistralai/_native.py @@ -185,7 +185,7 @@ def __init__(self, provider: trace.TracerProvider, config: TraceConfig) -> None: from mistralai.client.fim import Fim for cls, methods in ( - (Chat, ("complete", "stream")), + (Chat, ("complete", "stream", "parse")), (Agents, ("complete", "stream")), (Embeddings, ("create",)), (Fim, ("complete", "stream")), @@ -211,6 +211,11 @@ def __init__(self, provider: trace.TracerProvider, config: TraceConfig) -> None: @contextmanager def _call_scope(self): + # chat.parse calls chat.complete and parses its result afterward. Share + # the outer scope so the SDK-owned span stays open through validation. + if (existing := self._response_scope.get()) is not None: + yield + return with ExitStack() as scope: token = self._response_scope.set(scope) try: @@ -357,9 +362,9 @@ def _finalize_span(self, wrapped, instance, args, kwargs): # Reuse the SDK's non-streaming OTel conversion for # text, reasoning and image chunks, not provider keys. message = { - "content": [content] - if isinstance(content, dict) - else content + "content": ( + [content] if isinstance(content, dict) else content + ) } parts.extend( format_output_message({"message": message})["parts"] diff --git a/python/frameworks/mistralai/traceai_mistralai/_request_attributes_extractor.py b/python/frameworks/mistralai/traceai_mistralai/_request_attributes_extractor.py index be4531d05..5b2d9d813 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_request_attributes_extractor.py +++ b/python/frameworks/mistralai/traceai_mistralai/_request_attributes_extractor.py @@ -5,6 +5,7 @@ from fi_instrumentation import safe_json_dumps from fi_instrumentation.fi_types import ( + ImageAttributes, MessageAttributes, MessageContentAttributes, SpanAttributes, @@ -61,7 +62,8 @@ def _get_attributes_from_chat_completion_create_param( def _get_attributes_from_message_param( message: Mapping[str, Any], ) -> Iterator[Tuple[str, AttributeValue]]: - if not hasattr(message, "get"): + message = _message_dict(message) + if not message: return if role := message.get("role"): yield ( @@ -88,11 +90,11 @@ def _get_attributes_from_message_param( MessageAttributes.MESSAGE_FUNCTION_CALL_ARGUMENTS_JSON, function_arguments, ) - if (tool_calls := message.get("tool_calls"),) and isinstance(tool_calls, Iterable): + if (tool_calls := message.get("tool_calls")) and isinstance(tool_calls, Iterable): for index, tool_call in enumerate(tool_calls): - if not hasattr(tool_call, "get"): - continue - if (function := tool_call.get("function")) and hasattr(function, "get"): + tool_call = _message_dict(tool_call) + function = _message_dict(tool_call.get("function")) if tool_call else None + if function: if name := function.get("name"): yield ( f"{MessageAttributes.MESSAGE_TOOL_CALLS}.{index}." @@ -110,15 +112,38 @@ def _get_attributes_from_message_param( def _get_attributes_from_message_content( content: Mapping[str, Any], ) -> Iterator[Tuple[str, AttributeValue]]: - if not hasattr(content, "get"): + content_dict = dict(_message_dict(content)) + if not content_dict: return - content_dict = dict(content) type_ = content_dict.pop("type", None) + if type_ is None and type(content).__module__.startswith("mistralai."): + # SDK 1.1 omits default discriminators on typed content models. + type_ = {"TextChunk": "text", "ImageURLChunk": "image_url"}.get( + type(content).__name__ + ) if type_ == "text": if text := content_dict.get("text"): yield f"{MessageContentAttributes.MESSAGE_CONTENT_TYPE}", "text" yield f"{MessageContentAttributes.MESSAGE_CONTENT_TEXT}", text elif type_ == "image_url": if image := content_dict.get("image_url"): - yield f"{MessageContentAttributes.MESSAGE_CONTENT_TYPE}", "image" - yield f"{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}", image + # The core applies the base64 limit only to semantic image.url leaves. + url = image if isinstance(image, str) else _message_dict(image).get("url") + if isinstance(url, str): + yield MessageContentAttributes.MESSAGE_CONTENT_TYPE, "image" + yield f"{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}.{ImageAttributes.IMAGE_URL}", url + + +def _message_dict(value: Any) -> Mapping[str, Any]: + if isinstance(value, Mapping): + return value + # SDK 1.x exposes Pydantic message and content models. Normalize only + # their known fields, never copy raw models into non-policy attributes. + if callable(getattr(value, "model_dump", None)): + fields = value.model_dump(exclude_none=True) + if hasattr(value, "content"): + # Preserve typed nested chunks: SDK 1.1 drops their discriminator + # during model_dump even though it sends them to the model. + fields["content"] = value.content + return fields + return {} diff --git a/python/frameworks/mistralai/traceai_mistralai/_response_accumulator.py b/python/frameworks/mistralai/traceai_mistralai/_response_accumulator.py index 788ad4a83..dd1765b2c 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_response_accumulator.py +++ b/python/frameworks/mistralai/traceai_mistralai/_response_accumulator.py @@ -139,6 +139,18 @@ def __iadd__(self, values: Optional[Mapping[str, Any]]) -> "_ValuesAccumulator": elif isinstance(self_value, _StringAccumulator): if isinstance(value, str): self_value += value + elif isinstance(value, list): + # Legacy SDK 1.12 accepts typed text chunks in SSE deltas. + # Export the accumulated answer via the semantic output + # attribute, where TraceConfig applies text privacy. + for chunk in value: + chunk = _to_dict(chunk) + if ( + isinstance(chunk, Mapping) + and chunk.get("type") == "text" + and isinstance(chunk.get("text"), str) + ): + self_value += chunk["text"] elif isinstance(self_value, _IndexedAccumulator): if isinstance(value, Iterable): for index, v in enumerate(value): diff --git a/python/tests/test_framework_mistralai.py b/python/tests/test_framework_mistralai.py index 6372c4332..fb25a514b 100644 --- a/python/tests/test_framework_mistralai.py +++ b/python/tests/test_framework_mistralai.py @@ -156,7 +156,7 @@ def test_instrument_with_defaults(self, tracer_provider): assert isinstance(instrumentor._tracer, FITracer) # Verify all methods were wrapped - assert mock_wrap.call_count == 10 # chat + agents + embeddings + assert mock_wrap.call_count == 12 # chat + parse + agents + embeddings # Verify original methods were stored assert hasattr(instrumentor, '_original_sync_chat_method') @@ -234,7 +234,7 @@ def test_wrapper_registration_calls(self, tracer_provider): instrumentor._instrument(tracer_provider=tracer_provider) # Check that all expected calls were made - assert len(mock_wrap.call_args_list) == 10 + assert len(mock_wrap.call_args_list) == 12 # Check that both chat and agent modules are instrumented modules_called = [] @@ -251,6 +251,7 @@ def test_wrapper_registration_calls(self, tracer_provider): # Verify method names expected_names = [ "Chat.complete", "Chat.stream", "Chat.complete_async", "Chat.stream_async", + "Chat.parse", "Chat.parse_async", "Agents.complete", "Agents.stream", "Agents.complete_async", "Agents.stream_async" ] for expected_name in expected_names: From 5ed315037f4454756a8f2fc6c3549a93b9f6e584 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 13:54:24 -0800 Subject: [PATCH 10/10] fix(mistralai): redact query and fragment from native HTTP URLs Co-Authored-By: Claude Code --- .../mistralai/tests/test_review_edge_cases.py | 60 ++++++++++++++++++- .../mistralai/traceai_mistralai/_privacy.py | 11 +++- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/python/frameworks/mistralai/tests/test_review_edge_cases.py b/python/frameworks/mistralai/tests/test_review_edge_cases.py index a0c92751b..a70d8ec15 100644 --- a/python/frameworks/mistralai/tests/test_review_edge_cases.py +++ b/python/frameworks/mistralai/tests/test_review_edge_cases.py @@ -27,7 +27,7 @@ [ ( "https://user:password@proxy.invalid:8443/v1?q=ok#stream", - "https://proxy.invalid:8443/v1?q=ok#stream", + "https://proxy.invalid:8443/v1", ), ("https://user%40name:p%40ss@proxy.invalid/v1", "https://proxy.invalid/v1"), ("http://user:password@[::1]:8080/v1", "http://[::1]:8080/v1"), @@ -36,12 +36,20 @@ ("https://user:password@[invalid/v1", None), ], ) -def test_url_mask_removes_only_userinfo(key, lazy, url, expected): +def test_url_mask_removes_userinfo_query_and_fragment(key, lazy, url, expected): config = _NativeTraceConfig(TraceConfig()) value = (lambda: url) if lazy else url assert config.mask(key, value) == expected +@pytest.mark.parametrize("key", ["http.url", "url.full"]) +def test_url_mask_drops_query_credentials_and_fragment(key): + url = "https://api.mistral.ai/v1/chat/completions?api_key=synthetic-query-credential-458#private-fragment" + assert _NativeTraceConfig(TraceConfig(hide_inputs=True, hide_outputs=True)).mask( + key, url + ) == "https://api.mistral.ai/v1/chat/completions" + + @pytest.mark.skipif(not NATIVE, reason="Native SDK HTTP attributes") @pytest.mark.parametrize("asynchronous", [False, True]) @pytest.mark.parametrize("hide_content", [False, True]) @@ -90,6 +98,54 @@ async def exercise(): ) +@pytest.mark.skipif(not NATIVE, reason="Native SDK HTTP attributes") +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("hide_content", [False, True]) +def test_query_credentials_not_exported_but_request_unchanged( + asynchronous, hide_content +): + marker = "synthetic-query-credential-458" + observed = [] + + def respond(request): + observed.append(str(request.url)) + return httpx.Response(200, json=CHAT) + + with tracing(hide_inputs=hide_content, hide_outputs=hide_content) as ( + exporter, + starts, + ): + params = dict( + model="mistral-test", messages=[{"role": "user", "content": "hello"}] + ) + if asynchronous: + + async def exercise(): + async with httpx.AsyncClient( + params={"api_key": marker}, transport=httpx.MockTransport(respond) + ) as transport: + await Mistral( + api_key="synthetic-key", async_client=transport + ).chat.complete_async(**params) + + asyncio.run(exercise()) + else: + with httpx.Client( + params={"api_key": marker}, transport=httpx.MockTransport(respond) + ) as transport: + Mistral( + api_key="synthetic-key", client=transport + ).chat.complete(**params) + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == len(observed) == 1 + assert marker in observed[0] + assert marker not in spans[0].to_json() + assert ( + spans[0].attributes["http.url"] + == "https://api.mistral.ai/v1/chat/completions" + ) + + @pytest.mark.skipif(NATIVE, reason="Legacy embedding extraction") @pytest.mark.parametrize("asynchronous", [False, True]) @pytest.mark.parametrize("single", [False, True]) diff --git a/python/frameworks/mistralai/traceai_mistralai/_privacy.py b/python/frameworks/mistralai/traceai_mistralai/_privacy.py index 4c62bfe38..b1572f2ae 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_privacy.py +++ b/python/frameworks/mistralai/traceai_mistralai/_privacy.py @@ -55,10 +55,17 @@ def mask( value = self.__wrapped__.mask(policy_key, value) if key in ("http.url", "url.full") and isinstance(value, str): # SDK HTTP metadata is independent of message-hiding flags. Never - # export proxy userinfo, including percent-encoded credentials. + # export userinfo, query parameters, or fragments: custom clients + # can attach credentials to the URL outside the message policy. try: url = urlsplit(value) - return urlunsplit(url._replace(netloc=url.netloc.rsplit("@", 1)[-1])) + return urlunsplit( + url._replace( + netloc=url.netloc.rsplit("@", 1)[-1], + query="", + fragment="", + ) + ) except ValueError: return None if (