diff --git a/.github/workflows/mistralai-python-tests.yml b/.github/workflows/mistralai-python-tests.yml new file mode 100644 index 000000000..43a843293 --- /dev/null +++ b/.github/workflows/mistralai-python-tests.yml @@ -0,0 +1,57 @@ +name: Mistral Python SDK tests + +on: + pull_request: + branches: [dev, main] + paths: + - 'python/frameworks/mistralai/**' + - 'python/tests/test_framework_mistralai.py' + - '.github/workflows/mistralai-python-tests.yml' + push: + branches: [dev] + paths: + - 'python/frameworks/mistralai/**' + - '.github/workflows/mistralai-python-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + sdk: + name: Mistral ${{ matrix.sdk }} / Python ${{ matrix.python }} + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - {python: '3.10', sdk: '1.1.0'} + - {python: '3.12', sdk: '1.12.4'} + - {python: '3.10', sdk: '3.6.0'} + - {python: '3.13', sdk: '3.7.0'} + env: + FI_BASE_URL: http://127.0.0.1:9 + FI_GRPC_URL: http://127.0.0.1:9 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Build and install the distribution + env: + SDK_VERSION: ${{ matrix.sdk }} + run: | + python -m venv .venv + .venv/bin/pip install build poetry-core pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0 + .venv/bin/python -m build python/frameworks/mistralai --outdir dist + .venv/bin/pip install dist/*.whl "mistralai==$SDK_VERSION" + .venv/bin/pip check + - name: Run SDK, README and OTLP contracts without outbound network + run: | + sudo unshare --net sh -c 'ip link set lo up; exec "$@"' sh \ + "$PWD/.venv/bin/python" -I -m pytest --noconftest \ + python/frameworks/mistralai/tests python/tests/test_framework_mistralai.py \ + -q -o asyncio_default_fixture_loop_scope=function diff --git a/python/frameworks/mistralai/README.md b/python/frameworks/mistralai/README.md index 874bf7d0b..1b5a29d66 100644 --- a/python/frameworks/mistralai/README.md +++ b/python/frameworks/mistralai/README.md @@ -1,67 +1,86 @@ -# MistralAI OpenTelemetry Integration +# Mistral AI tracing -## Overview -This integration provides support for using OpenTelemetry with the MistralAI framework. It enables tracing and monitoring of applications built with MistralAI. +Trace Mistral chat completions, embeddings, and conversations in Future AGI, including model names, messages, token usage, tool calls, and errors. -## Installation +## Install -1. **Install traceAI MistralAI** +Use Python 3.10–3.13 and the current Mistral SDK: -```bash -pip install traceAI-mistralai +```sh +pip install traceAI-mistralai 'mistralai>=3.6,<4' ``` +Set `FI_API_KEY`, `FI_SECRET_KEY`, and `MISTRAL_API_KEY` in your environment. Use your own Mistral API key for model requests. -### Set Environment Variables -Set up your environment variables to authenticate with FutureAGI +## Trace a completion ```python import os -os.environ["FI_API_KEY"] = FI_API_KEY -os.environ["FI_SECRET_KEY"] = FI_SECRET_KEY -os.environ["MISTRAL_API_KEY"] = MISTRAL_API_KEY -``` - -## Quickstart - -### Register Tracer Provider -Set up the trace provider to establish the observability pipeline. The trace provider: - -```python from fi_instrumentation import register from fi_instrumentation.fi_types import ProjectType +from mistralai.client import Mistral +from traceai_mistralai import MistralAIInstrumentor -trace_provider = register( +provider = register( project_type=ProjectType.OBSERVE, - project_name="mistralai_app" + project_name="mistral-app", ) +MistralAIInstrumentor().instrument(tracer_provider=provider) + +try: + with Mistral(api_key=os.environ["MISTRAL_API_KEY"]) as client: + response = client.chat.complete( + model="mistral-small-latest", + messages=[{"role": "user", "content": "Suggest three names for a neighborhood bookstore."}], + ) + print(response.choices[0].message.content) +finally: + provider.shutdown() ``` -### Configure MistralAI Instrumentation -Instrument the MistralAI client to enable telemetry collection. This step ensures that all interactions with the MistralAI SDK are tracked and monitored. +Open **Observe** in Future AGI and select `mistral-app`. The completion appears as an LLM span with the model and returned usage. -```python -from traceai_mistralai import MistralAIInstrumentor +The integration routes the SDK's native OpenTelemetry spans through your provider. Do not add another Mistral instrumentor or configure a second Mistral exporter. Clients created before instrumentation are supported too. Configure tracing once at startup; finish active requests and close streams before uninstrumenting. + +## Stream a response + +Use the stream's context manager so the connection and span close even when you stop reading early: -MistralAIInstrumentor().instrument(tracer_provider=trace_provider) +```python +with client.chat.stream( + model="mistral-small-latest", + messages=[{"role": "user", "content": "Describe a cozy bookstore."}], +) as stream: + for event in stream: + print(event.data.choices[0].delta.content or "", end="") ``` -### Create MistralAI Components -Set up your MistralAI client with built-in observability. +For async requests, use `await client.chat.complete_async(...)` or `async with await client.chat.stream_async(...)`. Keep the client open until the stream is closed. Token usage is available when the service returns it; an early-closed stream may not receive usage. + +## Control captured content + +Messages and tool arguments can contain sensitive data. To keep model names, token counts, timing, and error status without recording input or output content, replace the instrumentor call above with: ```python -from mistralai import Mistral - -client = Mistral(api_key=os.environ["MISTRAL_API_KEY"]) - -if __name__ == "__main__": - response = client.agents.complete( - agent_id="agent_id", - messages=[ - {"role": "user", "content": "plan a vacation for me in Tbilisi"}, - ], - ) - print(response) +from fi_instrumentation import TraceConfig + +MistralAIInstrumentor().instrument( + tracer_provider=provider, + config=TraceConfig(hide_inputs=True, hide_outputs=True), +) ``` +This also hides error details by default. Embedding vectors are not captured. + +## Supported APIs + +- **SDK 3.6–3.x:** native sync/async chat and agent completions, streaming, embeddings, and `beta.conversations.start`, `append`, and `restart`, including their async and streaming variants. Conversation streaming errors and final usage are captured on the native span. +- **SDK 1.1–1.x:** the existing `MistralAIInstrumentor` entry point continues to support chat and agent completions and streams, plus embeddings. Keep `from mistralai import Mistral` for this SDK line. Conversation tracing requires SDK 3.6 or later. +- SDK 1.0, 2.x, and 3.0–3.5 are outside the supported range. Pin one of the ranges above rather than overriding pip's dependency check. + +## Troubleshooting + +- **`cannot import name Mistral`**: use `from mistralai.client import Mistral` with SDK 3.6 or later. The top-level import belongs to SDK 1.x. +- **Missing streaming spans**: fully consume or explicitly close the stream before flushing or shutting down your tracer provider. +- **Duplicate spans**: remove other Mistral SDK instrumentors. This integration adapts native telemetry rather than adding a second completion span. diff --git a/python/frameworks/mistralai/examples/agent_completions.py b/python/frameworks/mistralai/examples/agent_completions.py index 0b2a8b5b2..8798b7c26 100644 --- a/python/frameworks/mistralai/examples/agent_completions.py +++ b/python/frameworks/mistralai/examples/agent_completions.py @@ -8,7 +8,7 @@ EvalTagType, ProjectType, ) -from mistralai import Mistral +from mistralai.client import Mistral from traceai_mistralai import MistralAIInstrumentor # Configure trace provider with custom evaluation tags diff --git a/python/frameworks/mistralai/examples/chat_completions.py b/python/frameworks/mistralai/examples/chat_completions.py index 4a585f6cc..4d236b129 100644 --- a/python/frameworks/mistralai/examples/chat_completions.py +++ b/python/frameworks/mistralai/examples/chat_completions.py @@ -8,7 +8,7 @@ EvalTagType, ProjectType, ) -from mistralai import Mistral +from mistralai.client import Mistral from traceai_mistralai import MistralAIInstrumentor # Configure trace provider with custom evaluation tags diff --git a/python/frameworks/mistralai/examples/chat_completions_async.py b/python/frameworks/mistralai/examples/chat_completions_async.py index 9f2ebe508..5243c973a 100644 --- a/python/frameworks/mistralai/examples/chat_completions_async.py +++ b/python/frameworks/mistralai/examples/chat_completions_async.py @@ -9,7 +9,7 @@ EvalTagType, ProjectType, ) -from mistralai import Mistral +from mistralai.client import Mistral from traceai_mistralai import MistralAIInstrumentor # Configure trace provider with custom evaluation tags diff --git a/python/frameworks/mistralai/examples/chat_completions_async_streaming.py b/python/frameworks/mistralai/examples/chat_completions_async_streaming.py index 836299f56..cc278f64a 100644 --- a/python/frameworks/mistralai/examples/chat_completions_async_streaming.py +++ b/python/frameworks/mistralai/examples/chat_completions_async_streaming.py @@ -9,7 +9,7 @@ EvalTagType, ProjectType, ) -from mistralai import Mistral +from mistralai.client import Mistral from traceai_mistralai import MistralAIInstrumentor # Configure trace provider with custom evaluation tags diff --git a/python/frameworks/mistralai/examples/chat_completions_streaming.py b/python/frameworks/mistralai/examples/chat_completions_streaming.py index 043e4c6fd..8a36a15c2 100644 --- a/python/frameworks/mistralai/examples/chat_completions_streaming.py +++ b/python/frameworks/mistralai/examples/chat_completions_streaming.py @@ -8,7 +8,7 @@ EvalTagType, ProjectType, ) -from mistralai import Mistral +from mistralai.client import Mistral from traceai_mistralai import MistralAIInstrumentor # Configure trace provider with custom evaluation tags diff --git a/python/frameworks/mistralai/examples/new-1.py b/python/frameworks/mistralai/examples/new-1.py index c0f4f4570..3dec50124 100644 --- a/python/frameworks/mistralai/examples/new-1.py +++ b/python/frameworks/mistralai/examples/new-1.py @@ -2,7 +2,7 @@ from fi_instrumentation import register from fi_instrumentation.fi_types import ProjectType -from mistralai import Mistral +from mistralai.client import Mistral from traceai_mistralai import MistralAIInstrumentor trace_provider = register( diff --git a/python/frameworks/mistralai/examples/requirements.txt b/python/frameworks/mistralai/examples/requirements.txt index 3a6f02dd0..9c9b26f5a 100644 --- a/python/frameworks/mistralai/examples/requirements.txt +++ b/python/frameworks/mistralai/examples/requirements.txt @@ -1 +1,2 @@ -mistralai +traceAI-mistralai +mistralai>=3.6,<4 diff --git a/python/frameworks/mistralai/poetry.lock b/python/frameworks/mistralai/poetry.lock index a9e065ce5..932b2035e 100644 --- a/python/frameworks/mistralai/poetry.lock +++ b/python/frameworks/mistralai/poetry.lock @@ -19,6 +19,7 @@ description = "High-level concurrency and networking framework on top of asyncio optional = false python-versions = ">=3.9" groups = ["main"] +markers = "sys_platform != \"emscripten\"" files = [ {file = "anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1"}, {file = "anyio-4.10.0.tar.gz", hash = "sha256:3f3fae35c96039744587aa5b8371e7e8e603c0702999535961dd336026973ba6"}, @@ -187,7 +188,7 @@ description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" groups = ["main"] -markers = "python_version == \"3.10\"" +markers = "sys_platform != \"emscripten\" and python_version == \"3.10\"" files = [ {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, @@ -201,14 +202,14 @@ test = ["pytest (>=6)"] [[package]] name = "fi-instrumentation-otel" -version = "1.1.0" -description = "OpenTelemetry instrumentation" +version = "1.2.2" +description = "OpenTelemetry tracing for AI applications: tracer setup, span helpers, exporters, and semantic conventions for Future AGI." optional = false python-versions = "<3.14,>3.9" groups = ["main"] files = [ - {file = "fi_instrumentation_otel-1.1.0-py3-none-any.whl", hash = "sha256:4fa2bf663e0ce09447aca92ae798cd7b0dee35c5c92c2e033f1d89d158a9e62c"}, - {file = "fi_instrumentation_otel-1.1.0.tar.gz", hash = "sha256:7ec324c3c9242aa87a88cf23a21c11919b8736c6451c5fb50917f592a853b059"}, + {file = "fi_instrumentation_otel-1.2.2-py3-none-any.whl", hash = "sha256:25cf88f93e33aa928d4fae0f631afe8a32565639490862b715869e0633becbaf"}, + {file = "fi_instrumentation_otel-1.2.2.tar.gz", hash = "sha256:b675df47d7a03c506fbd2e4b0420fe4f74d12a6353a88b219bfd97797b276d4c"}, ] [package.dependencies] @@ -251,93 +252,105 @@ description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" optional = false python-versions = ">=3.8" groups = ["main"] +markers = "sys_platform != \"emscripten\"" files = [ {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, ] [[package]] -name = "httpcore" -version = "1.0.9" +name = "httpcore2" +version = "2.14.0" description = "A minimal low-level HTTP client." optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] +markers = "sys_platform != \"emscripten\"" files = [ - {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, - {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, + {file = "httpcore2-2.14.0-py3-none-any.whl", hash = "sha256:4c4f64f4b544afeb778c42c0ffb5941b29e671157e90ed3f6dc787c4bca90cd1"}, + {file = "httpcore2-2.14.0.tar.gz", hash = "sha256:4f57a1a8249fe6af589d935cfd456f1d13bbdcb0f0fc72af9142368fc94811d7"}, ] [package.dependencies] -certifi = "*" h11 = ">=0.16" +truststore = ">=0.10" [package.extras] -asyncio = ["anyio (>=4.0,<5.0)"] +asyncio = ["anyio (>=4.5.0,<5.0)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<1.0)"] +trio = ["trio (>=0.34.0,<1.0)"] [[package]] -name = "httpx" -version = "0.28.1" +name = "httpx2" +version = "2.14.0" description = "The next generation HTTP client." optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, - {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, + {file = "httpx2-2.14.0-py3-none-any.whl", hash = "sha256:9d20c13a10bae1773a0be3b0f1e5beeb4bdd549bc3a2ba18efa32897c65403e4"}, + {file = "httpx2-2.14.0.tar.gz", hash = "sha256:47ad61a3c40282cd8d0dc96311531f00f5bfc71a066c6fe3b36d8f63fb9e4f6e"}, ] [package.dependencies] -anyio = "*" -certifi = "*" -httpcore = "==1.*" -idna = "*" +anyio = {version = ">=4.10", markers = "sys_platform != \"emscripten\""} +httpcore2 = {version = "2.14.0", markers = "sys_platform != \"emscripten\""} +httpx2-jsfetch = {version = "*", markers = "sys_platform == \"emscripten\" and python_version >= \"3.12\""} +idna = ">=3.18" +truststore = {version = ">=0.10", markers = "sys_platform != \"emscripten\""} +typing-extensions = {version = ">=4.5.0", markers = "python_version < \"3.13\""} [package.extras] -brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] -cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.2) ; platform_python_implementation != \"CPython\""] +cli = ["click (>=8.4.2)", "pygments (==2.*)", "rich (>=10,<16)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] -zstd = ["zstandard (>=0.18.0)"] +ws = ["wsproto (>=1.2)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version <= \"3.13\""] [[package]] -name = "idna" -version = "3.7" -description = "Internationalized Domain Names in Applications (IDNA)" +name = "httpx2-jsfetch" +version = "1.0" +description = "httpx2 transports for Emscripten/Pyodide, backed by the JavaScript fetch API." optional = false -python-versions = ">=3.5" +python-versions = ">=3.12" groups = ["main"] +markers = "sys_platform == \"emscripten\" and python_version >= \"3.12\"" files = [ - {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, - {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, + {file = "httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32"}, + {file = "httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60"}, ] [[package]] -name = "importlib-metadata" -version = "8.7.0" -description = "Read metadata from Python packages" +name = "idna" +version = "3.20" +description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd"}, - {file = "importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000"}, + {file = "idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c"}, + {file = "idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44"}, ] -[package.dependencies] -zipp = ">=3.20" +[package.extras] +all = ["coverage (>=7.10.0)", "hypothesis (>=6.141.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.16.0)", "ty (>=0.0.37)"] + +[[package]] +name = "jsonpath-python" +version = "1.1.6" +description = "A lightweight and powerful JSONPath implementation for Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "jsonpath_python-1.1.6-py3-none-any.whl", hash = "sha256:a1c50afd8d3fbbaf47a4873bc890dcb3c15da96f5c020327977d844d8731a2d4"}, + {file = "jsonpath_python-1.1.6.tar.gz", hash = "sha256:dded9932b4ec41fb8726e09c83afa4e6be618f938c2db287cc2a81723c639671"}, +] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] -cover = ["pytest-cov"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -enabler = ["pytest-enabler (>=2.2)"] -perf = ["ipython"] -test = ["flufl.flake8", "importlib_resources (>=1.3) ; python_version < \"3.9\"", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] -type = ["pytest-mypy"] +dev = ["poethepoet", "pytest (>=8.0)", "pytest-benchmark[histogram] (>=4.0)", "pytest-cov (>=5.0)", "ruff (>=0.3)"] [[package]] name = "jsonschema" @@ -378,143 +391,202 @@ referencing = ">=0.31.0" [[package]] name = "mistralai" -version = "1.9.3" +version = "3.6.0" description = "Python Client SDK for the Mistral AI API." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "mistralai-1.9.3-py3-none-any.whl", hash = "sha256:962445e7cebadcbfbcd1daf973e853a832dcf7aba6320468fcf7e2cf5f943aec"}, - {file = "mistralai-1.9.3.tar.gz", hash = "sha256:a69806247ed3a67820ecfc9a68b7dbc0c6120dad5e5c3d507bd57fa388b491b7"}, + {file = "mistralai-3.6.0-py3-none-any.whl", hash = "sha256:709c31236d1c865f756734cb14f2f725fad993d054bc3be2d6aae3e78b4b3efb"}, + {file = "mistralai-3.6.0.tar.gz", hash = "sha256:1cd130da01979d8ecdc374a88bb7a880524b318a2be34b6e69013ca331de9d70"}, ] [package.dependencies] eval-type-backport = ">=0.2.0" -httpx = ">=0.28.1" -pydantic = ">=2.10.3" +httpx2 = ">=2.13.0" +jsonpath-python = ">=1.0.6" +opentelemetry-api = ">=1.33.1,<2.0.0" +opentelemetry-semantic-conventions = ">=0.61b0" +pydantic = ">=2.11.2" python-dateutil = ">=2.8.2" typing-inspection = ">=0.4.0" [package.extras] -agents = ["authlib (>=1.5.2,<2.0)", "griffe (>=1.7.3,<2.0)", "mcp (>=1.0,<2.0) ; python_version >= \"3.10\""] +agents = ["authlib (>=1.8.0,<2.0)", "griffe (>=1.7.3,<2.0)", "mcp (>=2.2.0,<3.0)"] gcp = ["google-auth (>=2.27.0)", "requests (>=2.32.3)"] +mcp = ["mcp (>=2.2.0,<3.0)"] +realtime = ["websockets (>=13.0)"] +telemetry = ["opentelemetry-exporter-otlp-proto-http (>=1.33.1,<2.0.0)", "opentelemetry-sdk (>=1.33.1,<2.0.0)"] +voice-agents = ["livekit (>=1.1,<2.0)", "numpy (>=1.26,<3)", "pyaudio (>=0.2.14,<0.3.0)"] +workflow-payload-compression = ["msgpack (>=1.1.0,<2.0.0)", "zstandard (>=0.25.0,<0.26)"] +workflow-payload-encryption = ["cryptography (>=50.0.0,<51.0.0)"] +workflow-payload-offloading = ["aioboto3 (>=12.4.0,<13.0.0)", "azure-identity (>=1.25.0,<2.0.0)", "azure-storage-blob[aio] (>=12.28.0,<13.0.0)", "gcloud-aio-storage (>=9.3.0,<10.0.0)"] +workflow-payload-offloading-azure = ["azure-identity (>=1.25.0,<2.0.0)", "azure-storage-blob[aio] (>=12.28.0,<13.0.0)"] +workflow-payload-offloading-gcs = ["gcloud-aio-storage (>=9.3.0,<10.0.0)"] +workflow-payload-offloading-s3 = ["aioboto3 (>=12.4.0,<13.0.0)"] [[package]] name = "opentelemetry-api" -version = "1.36.0" +version = "1.45.2" description = "OpenTelemetry Python API" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_api-1.36.0-py3-none-any.whl", hash = "sha256:02f20bcacf666e1333b6b1f04e647dc1d5111f86b8e510238fcc56d7762cda8c"}, - {file = "opentelemetry_api-1.36.0.tar.gz", hash = "sha256:9a72572b9c416d004d492cbc6e61962c0501eaf945ece9b5a0f56597d8348aa0"}, + {file = "opentelemetry_api-1.45.2-py3-none-any.whl", hash = "sha256:2cbd737377e710ee740d261c94e93fa0f08e6d037c18f17513cc0a2d94479a34"}, + {file = "opentelemetry_api-1.45.2.tar.gz", hash = "sha256:d7b0008e21e5fd61cbe7a7f34ca3b6905a78ab677d9bc395e299359c75cda6b4"}, ] [package.dependencies] -importlib-metadata = ">=6.0,<8.8.0" typing-extensions = ">=4.5.0" +[[package]] +name = "opentelemetry-exporter-http-transport" +version = "0.66b2" +description = "OpenTelemetry Exporters HTTP transport" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "opentelemetry_exporter_http_transport-0.66b2-py3-none-any.whl", hash = "sha256:fd5f5b490c099acda1cd7b909d5e8981436bf18cc8c6098d302d114c271720e9"}, + {file = "opentelemetry_exporter_http_transport-0.66b2.tar.gz", hash = "sha256:66d6ed810a515432c8891da9a834464332fd70badeca45bdeeae81b4280c903b"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.15,<2.0" +requests = {version = ">=2.25,<3.0", optional = true, markers = "extra == \"requests\""} + +[package.extras] +requests = ["requests (>=2.25,<3.0)"] +urllib3 = ["urllib3 (>=1.26)"] + +[[package]] +name = "opentelemetry-exporter-otlp-common" +version = "0.66b2" +description = "OpenTelemetry OTLP HTTP export utilities" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "opentelemetry_exporter_otlp_common-0.66b2-py3-none-any.whl", hash = "sha256:ace86efb75489f925f1ef809856ae5f0c7db534e4afdc624530222228b1236a5"}, + {file = "opentelemetry_exporter_otlp_common-0.66b2.tar.gz", hash = "sha256:06f5a9b88638e310efa1f1d206cdf6b75393e112aba4e174a963416192bd1028"}, +] + +[package.dependencies] +opentelemetry-sdk = ">=1.45.2,<1.46.0" + +[package.extras] +http = ["opentelemetry-exporter-http-transport (==0.66b2)"] + [[package]] name = "opentelemetry-exporter-otlp-proto-common" -version = "1.36.0" +version = "1.45.2" description = "OpenTelemetry Protobuf encoding" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_exporter_otlp_proto_common-1.36.0-py3-none-any.whl", hash = "sha256:0fc002a6ed63eac235ada9aa7056e5492e9a71728214a61745f6ad04b923f840"}, - {file = "opentelemetry_exporter_otlp_proto_common-1.36.0.tar.gz", hash = "sha256:6c496ccbcbe26b04653cecadd92f73659b814c6e3579af157d8716e5f9f25cbf"}, + {file = "opentelemetry_exporter_otlp_proto_common-1.45.2-py3-none-any.whl", hash = "sha256:af483b24217028892f017096731b794b196b785025d2e6808874720fd0b05a80"}, + {file = "opentelemetry_exporter_otlp_proto_common-1.45.2.tar.gz", hash = "sha256:f1f2806f3f28ce7938033823e50934fcd91595a39de38e69a28a51e19f40fc6d"}, ] [package.dependencies] -opentelemetry-proto = "1.36.0" +opentelemetry-proto = "1.45.2" [[package]] name = "opentelemetry-exporter-otlp-proto-http" -version = "1.36.0" +version = "1.45.2" description = "OpenTelemetry Collector Protobuf over HTTP Exporter" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_exporter_otlp_proto_http-1.36.0-py3-none-any.whl", hash = "sha256:3d769f68e2267e7abe4527f70deb6f598f40be3ea34c6adc35789bea94a32902"}, - {file = "opentelemetry_exporter_otlp_proto_http-1.36.0.tar.gz", hash = "sha256:dd3637f72f774b9fc9608ab1ac479f8b44d09b6fb5b2f3df68a24ad1da7d356e"}, + {file = "opentelemetry_exporter_otlp_proto_http-1.45.2-py3-none-any.whl", hash = "sha256:6e4d025138829d0ee57946930c2ccad6491ad4d92a958e9c7eea69278fe60cfc"}, + {file = "opentelemetry_exporter_otlp_proto_http-1.45.2.tar.gz", hash = "sha256:5984a3fb46bc4628e4e2b85d4a6e72637e03962aae6cf372791137768a4ad939"}, ] [package.dependencies] googleapis-common-protos = ">=1.52,<2.0" opentelemetry-api = ">=1.15,<2.0" -opentelemetry-exporter-otlp-proto-common = "1.36.0" -opentelemetry-proto = "1.36.0" -opentelemetry-sdk = ">=1.36.0,<1.37.0" +opentelemetry-exporter-http-transport = {version = "0.66b2", extras = ["requests"]} +opentelemetry-exporter-otlp-common = "0.66b2" +opentelemetry-exporter-otlp-proto-common = "1.45.2" +opentelemetry-proto = "1.45.2" +opentelemetry-sdk = ">=1.45.2,<1.46.0" requests = ">=2.7,<3.0" typing-extensions = ">=4.5.0" +[package.extras] +gcp-auth = ["opentelemetry-exporter-credential-provider-gcp (>=0.59b0)"] +requests = ["opentelemetry-exporter-http-transport[requests] (==0.66b2)", "requests (>=2.7,<3.0)"] + [[package]] name = "opentelemetry-instrumentation" -version = "0.57b0" +version = "0.66b2" description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_instrumentation-0.57b0-py3-none-any.whl", hash = "sha256:9109280f44882e07cec2850db28210b90600ae9110b42824d196de357cbddf7e"}, - {file = "opentelemetry_instrumentation-0.57b0.tar.gz", hash = "sha256:f2a30135ba77cdea2b0e1df272f4163c154e978f57214795d72f40befd4fcf05"}, + {file = "opentelemetry_instrumentation-0.66b2-py3-none-any.whl", hash = "sha256:fa9ccfdab147a14af8107d19ef0a7b5015675273c5adced7a1fc38f28b42ec4c"}, + {file = "opentelemetry_instrumentation-0.66b2.tar.gz", hash = "sha256:02b094150e53b76f5906d75e628dbb3833bbc6e62fa109b5c7ba0c081865e923"}, ] [package.dependencies] opentelemetry-api = ">=1.4,<2.0" -opentelemetry-semantic-conventions = "0.57b0" +opentelemetry-semantic-conventions = "0.66b2" packaging = ">=18.0" -wrapt = ">=1.0.0,<2.0.0" +wrapt = ">=1.0.0,<3.0.0" [[package]] name = "opentelemetry-proto" -version = "1.36.0" +version = "1.45.2" description = "OpenTelemetry Python Proto" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_proto-1.36.0-py3-none-any.whl", hash = "sha256:151b3bf73a09f94afc658497cf77d45a565606f62ce0c17acb08cd9937ca206e"}, - {file = "opentelemetry_proto-1.36.0.tar.gz", hash = "sha256:0f10b3c72f74c91e0764a5ec88fd8f1c368ea5d9c64639fb455e2854ef87dd2f"}, + {file = "opentelemetry_proto-1.45.2-py3-none-any.whl", hash = "sha256:03a7ca1b43f89fd5b965d5975902bb2515c0c86e2428b456836e9a6149c6235a"}, + {file = "opentelemetry_proto-1.45.2.tar.gz", hash = "sha256:bd8d23cf770605b0af9b7be2071d26a464fea289d35429abfd2a66226114ad64"}, ] [package.dependencies] -protobuf = ">=5.0,<7.0" +protobuf = ">=5.0,<8.0" [[package]] name = "opentelemetry-sdk" -version = "1.36.0" +version = "1.45.2" description = "OpenTelemetry Python SDK" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_sdk-1.36.0-py3-none-any.whl", hash = "sha256:19fe048b42e98c5c1ffe85b569b7073576ad4ce0bcb6e9b4c6a39e890a6c45fb"}, - {file = "opentelemetry_sdk-1.36.0.tar.gz", hash = "sha256:19c8c81599f51b71670661ff7495c905d8fdf6976e41622d5245b791b06fa581"}, + {file = "opentelemetry_sdk-1.45.2-py3-none-any.whl", hash = "sha256:ff974ea70ee4f1d51e2ec152db37c2b626c7bf82bb3f7d86114fd8704e7199c6"}, + {file = "opentelemetry_sdk-1.45.2.tar.gz", hash = "sha256:7eec4de13dd21ff3a1670dcb0e02079e692736f94d733dfdce5af461d884248d"}, ] [package.dependencies] -opentelemetry-api = "1.36.0" -opentelemetry-semantic-conventions = "0.57b0" +opentelemetry-api = "1.45.2" +opentelemetry-semantic-conventions = "0.66b2" typing-extensions = ">=4.5.0" +[package.extras] +file-configuration = ["opentelemetry-configuration (==0.66b2)"] + [[package]] name = "opentelemetry-semantic-conventions" -version = "0.57b0" +version = "0.66b2" description = "OpenTelemetry Semantic Conventions" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_semantic_conventions-0.57b0-py3-none-any.whl", hash = "sha256:757f7e76293294f124c827e514c2a3144f191ef175b069ce8d1211e1e38e9e78"}, - {file = "opentelemetry_semantic_conventions-0.57b0.tar.gz", hash = "sha256:609a4a79c7891b4620d64c7aac6898f872d790d75f22019913a660756f27ff32"}, + {file = "opentelemetry_semantic_conventions-0.66b2-py3-none-any.whl", hash = "sha256:4cbe61e5f9275d57ab31708446e7938ef4291ebbe6a7fca6f7a629c5acae87d8"}, + {file = "opentelemetry_semantic_conventions-0.66b2.tar.gz", hash = "sha256:7da5f1a5dc4383eb66711d4d3d3e71824d9403ac1559fea021cc103c1a4c4976"}, ] [package.dependencies] -opentelemetry-api = "1.36.0" +opentelemetry-api = "1.45.2" typing-extensions = ">=4.5.0" [[package]] @@ -909,11 +981,25 @@ description = "Sniff out which async library your code is running under" optional = false python-versions = ">=3.7" groups = ["main"] +markers = "sys_platform != \"emscripten\"" files = [ {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, ] +[[package]] +name = "truststore" +version = "0.10.5" +description = "Verify certificates using native system trust stores" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "sys_platform != \"emscripten\"" +files = [ + {file = "truststore-0.10.5-py3-none-any.whl", hash = "sha256:9aaaedaefaf06d8b206278cf8b5012bc897f485a874503501e12d776df78951c"}, + {file = "truststore-0.10.5.tar.gz", hash = "sha256:30d36967ccaded5cbb38d602c433f53600036c79d502f4533a49b60a03bbefcd"}, +] + [[package]] name = "typing-extensions" version = "4.14.1" @@ -1048,27 +1134,7 @@ files = [ {file = "wrapt-1.17.2.tar.gz", hash = "sha256:41388e9d4d1522446fe79d3213196bd9e3b301a336965b9e27ca2788ebd122f3"}, ] -[[package]] -name = "zipp" -version = "3.23.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, - {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, -] - -[package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] -cover = ["pytest-cov"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -enabler = ["pytest-enabler (>=2.2)"] -test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] -type = ["pytest-mypy"] - [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "9e1fa619e2eb0bf07bdebf3c97a1611cb549c467cee5cbc2b4df1b8e4adcf8ee" +content-hash = "d915dc2322aec75e3d8e13184886c1fcabd78072dfda8a6f4385adc5fd79c516" diff --git a/python/frameworks/mistralai/pyproject.toml b/python/frameworks/mistralai/pyproject.toml index 3348f19ad..0bb89e9ca 100644 --- a/python/frameworks/mistralai/pyproject.toml +++ b/python/frameworks/mistralai/pyproject.toml @@ -27,8 +27,9 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.14" -fi-instrumentation-otel = ">=0.1.11" -mistralai = ">=1.0.2,<2" +fi-instrumentation-otel = ">=1.2.2" +mistralai = ">=1.1.0,<4,!=2.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" +packaging = ">=23.2" [tool.poetry.urls] Homepage = "https://futureagi.com" diff --git a/python/frameworks/mistralai/tests/test_native.py b/python/frameworks/mistralai/tests/test_native.py new file mode 100644 index 000000000..491d3406c --- /dev/null +++ b/python/frameworks/mistralai/tests/test_native.py @@ -0,0 +1,491 @@ +"""Native message privacy and conversation parts through the real SDK and OTLP.""" + +import asyncio +import json + +import pytest +from fi_instrumentation import REDACTED_VALUE, TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor + +from traceai_mistralai import MistralAIInstrumentor +import test_sdk +import test_wire +from test_sdk import ( + NATIVE, + Mistral, + SSE, + USAGE, + conversation_response, + httpx, + transport_response, +) +from test_wire import ( + Compression, + JSONExporter, + OTLPSpanExporter, +) + +tracing = test_sdk.tracing +receiver = test_wire.receiver + +pytestmark = pytest.mark.skipif( + not NATIVE, reason="Native telemetry requires SDK >=3.6" +) + +INPUT_TEXT = "private-input-marker" +OUTPUT_TEXT = "private-output-marker" +IMAGE = "https://example.invalid/private-image-marker" + + +def input_content(): + return [ + {"type": "text", "text": INPUT_TEXT}, + {"type": "image_url", "image_url": {"url": IMAGE}}, + ] + + +class EventSSE(SSE): + def __init__(self, events): + super().__init__() + self.events = events + + def __iter__(self): + for event in self.events: + yield f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() + + +def wire_spans(requests): + return [ + span + for _, data in requests + for resource in data["resourceSpans"] + for scope in resource["scopeSpans"] + for span in scope["spans"] + ] + + +@pytest.mark.parametrize( + "flags", + [ + {}, + {"hide_input_text": True}, + {"hide_output_text": True}, + {"hide_input_images": True}, + {"hide_input_text": True, "hide_output_text": True, "hide_input_images": True}, + ], + ids=["visible", "input-text", "output-text", "input-images", "all-selective"], +) +@pytest.mark.parametrize("encoding", ["json", "protobuf"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize( + "operation", ["chat", "chat-stream", "conversation-stream", "conversation-children"] +) +def test_native_selective_privacy_on_wire( + receiver, flags, encoding, compressed, operation +): + endpoint, requests = receiver + exporter = ( + JSONExporter(endpoint, compressed) + if encoding == "json" + else OTLPSpanExporter( + endpoint=endpoint, + headers={}, + timeout=5, + compression=Compression.Gzip if compressed else Compression.NoCompression, + ) + ) + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**flags)) + + def respond(request): + if operation == "conversation-stream": + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=EventSSE( + [ + { + "type": "message.output.delta", + "id": "entry-message", + "role": "assistant", + "content": {"type": "text", "text": OUTPUT_TEXT}, + }, + { + "type": "function.call.delta", + "id": "entry-function", + "tool_call_id": "actual-tool-call", + "name": "weather", + "arguments": '{"city":"Paris"}', + }, + {"type": "conversation.response.done", "usage": USAGE}, + ] + ), + ) + if operation == "conversation-children": + body = conversation_response() + # Real service timestamps make the native hook emit the message child. + body["outputs"][0].update( + id="message-child", + created_at="2026-10-31T00:00:00Z", + completed_at="2026-10-31T00:00:01Z", + model="mistral-test", + ) + return httpx.Response(200, json=body) + return transport_response(request) + + try: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-credential-marker", client=transport) + if operation.startswith("conversation"): + arguments = { + "model": "mistral-test", + "inputs": [ + { + "type": "message.input", + "role": "user", + "content": input_content(), + } + ], + } + if operation == "conversation-stream": + with client.beta.conversations.start_stream(**arguments) as stream: + chunks = list(stream) + assert chunks[0].data.content.text == OUTPUT_TEXT + assert chunks[1].data.tool_call_id == "actual-tool-call" + else: + result = client.beta.conversations.start(**arguments) + assert result.outputs[0].content == OUTPUT_TEXT + else: + arguments = { + "model": "mistral-test", + "messages": [{"role": "user", "content": input_content()}], + } + if operation == "chat-stream": + with client.chat.stream(**arguments) as stream: + chunks = list(stream) + assert chunks[0].data.choices[0].delta.content == OUTPUT_TEXT + else: + result = client.chat.complete(**arguments) + assert result.choices[0].message.content == OUTPUT_TEXT + provider.force_flush() + finally: + instrumentor.uninstrument() + provider.shutdown() + + spans = wire_spans(requests) + assert len(spans) == (2 if operation == "conversation-children" else 1) + if operation == "conversation-children": + child, parent = spans + assert child["parentSpanId"] == parent["spanId"] + assert child["traceId"] == parent["traceId"] + attrs = {a["key"]: a["value"] for a in spans[-1]["attributes"]} + assert int(attrs["gen_ai.usage.input_tokens"]["intValue"]) == 2 + if operation == "conversation-stream": + parts = json.loads(attrs["gen_ai.output.messages"]["stringValue"])[0]["parts"] + assert parts == [ + { + "type": "text", + "content": REDACTED_VALUE + if flags.get("hide_output_text") + else OUTPUT_TEXT, + }, + { + "type": "tool_call", + "id": "actual-tool-call", + "name": "weather", + "arguments": '{"city":"Paris"}', + }, + ] + wire = json.dumps(requests) + for flag, marker in ( + ("hide_input_text", INPUT_TEXT), + ("hide_output_text", OUTPUT_TEXT), + ("hide_input_images", IMAGE), + ): + assert (marker in wire) is not flags.get(flag, False) + assert "synthetic-credential-marker" not in wire + if operation.startswith("chat"): + # Selective text flags must not discard tool identifiers or arguments. + assert "weather" in wire + assert "call-1" in wire + + +@pytest.mark.parametrize("encoding", ["json", "protobuf"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize("operation", ["chat", "chat-chunks", "conversation"]) +@pytest.mark.parametrize( + "policy", ["visible", "hide_input_text", "hide_inputs", "hide_input_messages"] +) +def test_native_tool_result_privacy(receiver, encoding, compressed, operation, policy): + endpoint, requests = receiver + exporter = ( + JSONExporter(endpoint, compressed) + if encoding == "json" + else OTLPSpanExporter( + endpoint=endpoint, + headers={}, + timeout=5, + compression=Compression.Gzip if compressed else Compression.NoCompression, + ) + ) + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument( + tracer_provider=provider, + config=TraceConfig(**({} if policy == "visible" else {policy: True})), + ) + marker = "private-tool-result-marker" + tool_id = "input-tool-call" + + def respond(request): + if operation == "conversation": + body = conversation_response() + body["outputs"] = [] + return httpx.Response(200, json=body) + return transport_response(request) + + try: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic", client=transport) + if operation == "conversation": + result = client.beta.conversations.append( + conversation_id="conv-test", + inputs=[ + { + "type": "function.result", + "tool_call_id": tool_id, + "result": marker, + } + ], + ) + assert result.conversation_id == "conv-test" + else: + result = client.chat.complete( + model="mistral-test", + messages=[ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": tool_id, + "type": "function", + "function": {"name": "lookup", "arguments": "{}"}, + } + ], + }, + { + "role": "tool", + "tool_call_id": tool_id, + "content": [{"type": "text", "text": marker}] + if operation == "chat-chunks" + else marker, + }, + ], + ) + assert result.choices[0].message.content == OUTPUT_TEXT + provider.force_flush() + finally: + instrumentor.uninstrument() + provider.shutdown() + spans = wire_spans(requests) + assert len(spans) == 1 + wire = json.dumps(requests) + assert (marker in wire) == (policy == "visible") + if operation != "conversation": + assert OUTPUT_TEXT in wire + if policy in ("visible", "hide_input_text"): + attrs = {a["key"]: a["value"] for a in spans[0]["attributes"]} + messages = json.loads(attrs["gen_ai.input.messages"]["stringValue"]) + part = messages[-1]["parts"][0] + assert part["type"] == "tool_call_response" + assert part["id"] == tool_id + if policy == "hide_input_text": + assert part["response"] == REDACTED_VALUE + + +@pytest.mark.parametrize("encoding", ["json", "protobuf"]) +@pytest.mark.parametrize("compressed", [False, True]) +def test_cancelled_native_send_on_wire(receiver, encoding, compressed): + endpoint, requests = receiver + exporter = ( + JSONExporter(endpoint, compressed) + if encoding == "json" + else OTLPSpanExporter( + endpoint=endpoint, + headers={}, + timeout=5, + compression=Compression.Gzip if compressed else Compression.NoCompression, + ) + ) + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument( + tracer_provider=provider, + config=TraceConfig(hide_error_details=True), + ) + + async def run(): + async def respond(request): + raise asyncio.CancelledError("private-cancellation-marker") + + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic-key-marker", async_client=transport) + with pytest.raises( + asyncio.CancelledError, match="private-cancellation-marker" + ): + await client.chat.complete_async( + model="mistral-test", + messages=[{"role": "user", "content": "hello"}], + ) + + try: + asyncio.run(run()) + provider.force_flush() + finally: + instrumentor.uninstrument() + provider.shutdown() + spans = wire_spans(requests) + assert len(spans) == 1 + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR" + assert "private-cancellation-marker" not in json.dumps(requests) + assert "synthetic-key-marker" not in json.dumps(requests) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize( + "variant", ["string", "text_chunk", "thinking", "image", "tool_call"] +) +def test_conversation_stream_part_conversion(tracing, variant, asynchronous): + content, expected = { + "string": ("chunk-content", {"type": "text", "content": "chunk-content"}), + "text_chunk": ( + {"type": "text", "text": "chunk-content"}, + {"type": "text", "content": "chunk-content"}, + ), + "thinking": ( + { + "type": "thinking", + "thinking": [{"type": "text", "text": "reasoning-content"}], + }, + {"type": "reasoning", "content": "reasoning-content"}, + ), + "image": ( + {"type": "image_url", "image_url": {"url": IMAGE}}, + {"type": "uri", "modality": "image", "uri": IMAGE}, + ), + "tool_call": ( + None, + { + "type": "tool_call", + "id": "actual-tool-call", + "name": "weather", + "arguments": '{"city":"Paris"}', + }, + ), + }[variant] + event = ( + { + "type": "function.call.delta", + "id": "entry-function", + "tool_call_id": "actual-tool-call", + "name": "weather", + "arguments": '{"city":"Paris"}', + } + if variant == "tool_call" + else { + "type": "message.output.delta", + "id": "entry-message", + "role": "assistant", + "content": content, + } + ) + + def respond(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=EventSSE( + [event, {"type": "conversation.response.done", "usage": USAGE}] + ), + ) + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + async with await client.beta.conversations.start_stream_async( + model="mistral-test", inputs="hi" + ) as stream: + return [chunk async for chunk in stream] + + if asynchronous: + chunks = asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-credential-marker", client=transport) + with client.beta.conversations.start_stream( + model="mistral-test", inputs="hi" + ) as stream: + chunks = list(stream) + assert chunks[0].data.type == event["type"] + if variant == "tool_call": + assert chunks[0].data.tool_call_id == "actual-tool-call" + assert chunks[0].data.id == "entry-function" + elif variant == "text_chunk": + assert chunks[0].data.content.text == "chunk-content" + spans = tracing.get_finished_spans() + assert len(spans) == 1 + assert json.loads(spans[0].attributes["gen_ai.output.messages"])[0]["parts"] == [ + expected + ] + + +@pytest.mark.parametrize("direction", ["input", "output"]) +@pytest.mark.parametrize("method", ["creation", "single", "batch"]) +def test_native_privacy_at_attribute_boundaries(direction, method): + from traceai_mistralai._native import _NativeProvider + from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, + ) + + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + config = TraceConfig(**{f"hide_{direction}_text": True}) + tracer = _NativeProvider(provider, config).get_tracer("test") + key = f"gen_ai.{direction}.messages" + original = [ + { + "role": "assistant", + "parts": [{"type": "reasoning", "content": "private-reasoning"}], + } + ] + attributes = {key: json.dumps(original)} + try: + with tracer.start_as_current_span( + "chat", attributes=attributes if method == "creation" else None + ) as span: + if method == "single": + span.set_attribute(key, attributes[key]) + elif method == "batch": + span.set_attributes(attributes) + result = exporter.get_finished_spans() + assert len(result) == 1 + assert "private-reasoning" not in result[0].to_json() + assert ( + json.loads(result[0].attributes[key])[0]["parts"][0]["content"] + == REDACTED_VALUE + ) + assert original[0]["parts"][0]["content"] == "private-reasoning" + assert json.loads(attributes[key]) == original + finally: + provider.shutdown() diff --git a/python/frameworks/mistralai/tests/test_native_lifecycle.py b/python/frameworks/mistralai/tests/test_native_lifecycle.py new file mode 100644 index 000000000..ca5bb0c0b --- /dev/null +++ b/python/frameworks/mistralai/tests/test_native_lifecycle.py @@ -0,0 +1,194 @@ +"""Cancellation and restoration of the SDK's native request-span lifecycle.""" + +import asyncio +import gc +import weakref + +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_mistralai import MistralAIInstrumentor +from test_sdk import NATIVE, Mistral, httpx, params, transport_response + +pytestmark = pytest.mark.skipif( + not NATIVE, reason="Native telemetry requires SDK >=3.6" +) + + +def request(client, operation): + if operation == "embedding": + return client.embeddings.create_async(model="mistral-embed", inputs=["hello"]) + if operation.startswith("conversation"): + method = ( + client.beta.conversations.start_stream_async + if operation.endswith("stream") + else client.beta.conversations.start_async + ) + return method(model="mistral-test", inputs="hello") + method = ( + client.chat.stream_async + if operation == "chat-stream" + else client.chat.complete_async + ) + return method(**params("chat")) + + +@pytest.mark.parametrize( + "operation", + ["chat", "chat-stream", "embedding", "conversation", "conversation-stream"], +) +@pytest.mark.parametrize("timeout", [False, True], ids=["cancel", "timeout"]) +def test_cancelled_native_send_ends_only_its_span(operation, timeout): + provider = TracerProvider() + exporter = InMemorySpanExporter() + started = [] + + class RecordingProcessor(SimpleSpanProcessor): + def on_start(self, span, parent_context=None): + started.append(span) + + provider.add_span_processor(RecordingProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument( + tracer_provider=provider, config=TraceConfig(hide_error_details=True) + ) + + async def run(): + entered = asyncio.Event() + requests = [] + + async def respond(http_request): + requests.append(http_request) + if len(requests) == 1: + entered.set() + await asyncio.Event().wait() + return transport_response(http_request) + + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + task = asyncio.create_task(request(client, operation)) + await asyncio.wait_for(entered.wait(), 5) + # A successful request sharing the client must not be ended by cancellation. + result = await client.chat.complete_async(**params("chat")) + assert result.choices[0].message.content == "private-output-marker" + if timeout: + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(task, 0.01) + else: + task.cancel("private-cancellation-marker") + # Python 3.10 drops Task.cancel(msg) at the task-await boundary, + # including without instrumentation. Direct-await tests below + # still require the original cancellation message to survive. + with pytest.raises(asyncio.CancelledError): + await task + assert trace.get_current_span() is trace.INVALID_SPAN + assert len(started) == 2 + assert all(not span.is_recording() for span in started) + spans = exporter.get_finished_spans() + assert len(spans) == 2 + assert spans[0].status.status_code == trace.StatusCode.OK + assert spans[1].status.status_code == trace.StatusCode.ERROR + assert spans[1].context.span_id == started[0].context.span_id + assert "private-cancellation-marker" not in spans[1].to_json() + assert requests[0].extensions["_tracing_span"] is None + assert requests[0].extensions["_tracing_span_finished"] is True + assert len(exporter.get_finished_spans()) == 2 + + try: + asyncio.run(run()) + finally: + instrumentor.uninstrument() + provider.shutdown() + + +def test_cancelled_send_preserves_cancellation_if_telemetry_fails(monkeypatch, caplog): + from mistralai.client._hooks.tracing import TracingHook + + caplog.set_level("DEBUG", logger="traceai_mistralai._native") + + def fail(*args, **kwargs): + raise RuntimeError("private-telemetry-marker") + + monkeypatch.setattr(TracingHook, "_finish_request_span_with_error", fail) + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument(tracer_provider=provider) + + async def run(): + async def respond(http_request): + raise asyncio.CancelledError("original-cancellation") + + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + with pytest.raises(asyncio.CancelledError, match="original-cancellation"): + await client.chat.complete_async(**params("chat")) + + try: + asyncio.run(run()) + assert "Could not finish cancelled Mistral request" in caplog.text + assert "private-telemetry-marker" not in caplog.text + assert "original-cancellation" not in caplog.text + finally: + instrumentor.uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize("reconfigured", [False, True]) +def test_native_async_send_restored_without_retaining_clients(reconfigured): + from mistralai.extra.observability import configure_telemetry + + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + other = TracerProvider() + instrumentor = MistralAIInstrumentor() + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(transport_response) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + configure_telemetry(client, provider=other, redaction=False) + await client.chat.complete_async(**params("chat")) + original = transport.send # Includes the SDK's own context/error wrapper. + for _ in range(2): + instrumentor.instrument(tracer_provider=provider) + await client.chat.complete_async(**params("chat")) + assert transport.send is not original + if reconfigured: + + async def replacement(*args, **kwargs): + return await original(*args, **kwargs) + + transport.send = replacement + instrumentor.uninstrument() + if reconfigured: + assert transport.send is replacement + transport.send = original + else: + assert transport.send is original + assert len(exporter.get_finished_spans()) == 2 + # Leave instrumentation on when dropping the client to check weak tracking. + instrumentor.instrument(tracer_provider=provider) + await client.chat.complete_async(**params("chat")) + return weakref.ref(transport) + + try: + client_ref = asyncio.run(run()) + gc.collect() # Run the SDK finalizer, which itself holds the HTTP client. + gc.collect() # Collect cycles released by that finalizer. + assert client_ref() is None + finally: + instrumentor.uninstrument() + provider.shutdown() + other.shutdown() diff --git a/python/frameworks/mistralai/tests/test_native_privacy.py b/python/frameworks/mistralai/tests/test_native_privacy.py new file mode 100644 index 000000000..c8ced6385 --- /dev/null +++ b/python/frameworks/mistralai/tests/test_native_privacy.py @@ -0,0 +1,147 @@ +"""Boundary cases for the native message privacy adapter.""" + +import json + +import pytest +from fi_instrumentation import REDACTED_VALUE, TraceConfig + +from traceai_mistralai._privacy import _NativeTraceConfig + + +@pytest.mark.parametrize("direction", ["input", "output"]) +@pytest.mark.parametrize( + "value", + [ + "not-json-private-marker", + '{"parts":"private-marker"}', + '[{"parts":["private-marker"]}]', + '[{"parts":null}]', + "null", + 123, + ], +) +def test_unparseable_messages_are_not_forwarded(direction, value): + config = _NativeTraceConfig(TraceConfig(**{f"hide_{direction}_text": True})) + assert config.mask(f"gen_ai.{direction}.messages", value) is None + + +@pytest.mark.parametrize("direction", ["input", "output"]) +def test_whole_direction_privacy_takes_precedence(direction): + key = f"gen_ai.{direction}.messages" + payload = json.dumps( + [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "private-marker"}], + } + ] + ) + for flag in (f"hide_{direction}s", f"hide_{direction}_messages"): + base = TraceConfig(**{flag: True}) + assert _NativeTraceConfig(base).mask(key, payload) == base.mask(key, payload) + + +@pytest.mark.parametrize("hidden", [False, True]) +def test_input_image_limit_and_environment_config(monkeypatch, hidden): + monkeypatch.setenv("FI_HIDE_INPUT_IMAGES", str(hidden).lower()) + config = _NativeTraceConfig(TraceConfig(base64_image_max_length=30)) + payload = [ + { + "role": "user", + "parts": [ + {"type": "text", "content": "visible-text"}, + { + "type": "uri", + "modality": "image", + "uri": "data:image/png;base64," + "A" * 64, + }, + ], + } + ] + result = json.loads(config.mask("gen_ai.input.messages", json.dumps(payload)))[0][ + "parts" + ] + assert result[0] == payload[0]["parts"][0] + if hidden: + assert len(result) == 1 + else: + assert result[1] == {"type": "uri", "modality": "image", "uri": REDACTED_VALUE} + + +@pytest.mark.parametrize("hidden", [False, True]) +@pytest.mark.parametrize("direction", ["input", "output"]) +def test_tool_image_environment_and_length_boundaries(monkeypatch, hidden, direction): + monkeypatch.setenv("FI_HIDE_INPUT_IMAGES", str(hidden).lower()) + short = "data:image/png;base64," + "A" * 8 + long = short + "A" + url = "https://example.invalid/" + "a" * 80 + config = _NativeTraceConfig(TraceConfig(base64_image_max_length=len(short))) + chunks = [ + {"type": "text", "text": "visible-text"}, + {"type": "image_url", "image_url": short}, + {"type": "image_url", "image_url": {"url": long, "detail": "low"}}, + {"type": "image_url", "image_url": {"url": url}}, + {"type": "text", "text": "trailing-text"}, + ] + payload = json.dumps( + [ + { + "role": "tool", + "parts": [ + {"type": "tool_call_response", "id": "call-1", "response": chunks} + ], + } + ] + ) + result = json.loads(config.mask(f"gen_ai.{direction}.messages", payload))[0][ + "parts" + ][0] + assert result["id"] == "call-1" + if direction == "output": + assert result["response"] == chunks + elif hidden: + assert result["response"] == [chunks[0], chunks[-1]] + else: + assert result["response"] == [ + chunks[0], + chunks[1], + { + "type": "image_url", + "image_url": {"url": REDACTED_VALUE, "detail": "low"}, + }, + chunks[3], + chunks[4], + ] + + +@pytest.mark.parametrize( + "image", [None, 123, {}, {"url": None}, {"url": ["private-image"]}] +) +def test_malformed_tool_images_are_not_forwarded(image): + config = _NativeTraceConfig(TraceConfig(hide_input_images=True)) + payload = json.dumps( + [ + { + "role": "tool", + "parts": [ + { + "type": "tool_call_response", + "id": "call-1", + "response": [ + {"type": "image_url", "image_url": image}, + {"type": "text", "text": "visible-text"}, + ], + } + ], + } + ] + ) + result = json.loads(config.mask("gen_ai.input.messages", payload))[0]["parts"][0] + assert result["id"] == "call-1" + assert result["response"] == [{"type": "text", "text": "visible-text"}] + + +def test_unrelated_attributes_and_empty_messages_are_preserved(): + config = _NativeTraceConfig(TraceConfig(hide_input_text=True)) + assert config.mask("gen_ai.request.model", "mistral-test") == "mistral-test" + assert config.mask("gen_ai.input.messages", lambda: "[]") == "[]" diff --git a/python/frameworks/mistralai/tests/test_native_tool_images.py b/python/frameworks/mistralai/tests/test_native_tool_images.py new file mode 100644 index 000000000..be0d11c16 --- /dev/null +++ b/python/frameworks/mistralai/tests/test_native_tool_images.py @@ -0,0 +1,158 @@ +"""Structured tool images obey the same policy as native user-message images.""" + +import copy +import json + +import pytest +from fi_instrumentation import REDACTED_VALUE, TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor + +from traceai_mistralai import MistralAIInstrumentor +from test_sdk import NATIVE, Mistral, httpx, transport_response +import test_wire + +receiver = test_wire.receiver +pytestmark = pytest.mark.skipif( + not NATIVE, reason="Native telemetry requires SDK >=3.6" +) + +TEXT = "permitted-tool-result-text" +TOOL_ID = "screenshot-call" +IMAGES = [ + "https://example.invalid/private-tool-screenshot.png", + "data:image/png;base64," + "A" * 128, +] + + +@pytest.mark.parametrize("encoding", ["json", "protobuf"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize("image", IMAGES, ids=["url", "base64"]) +@pytest.mark.parametrize("structured_url", [False, True], ids=["string", "object"]) +@pytest.mark.parametrize( + "policy", + [ + "visible", + "hide_input_images", + "hide_input_text", + "hide_inputs", + "hide_input_messages", + "base64_image_max_length", + ], +) +def test_tool_images_on_wire( + receiver, encoding, compressed, image, structured_url, policy +): + endpoint, requests = receiver + exporter = ( + test_wire.JSONExporter(endpoint, compressed) + if encoding == "json" + else test_wire.OTLPSpanExporter( + endpoint=endpoint, + headers={}, + timeout=5, + compression=( + test_wire.Compression.Gzip + if compressed + else test_wire.Compression.NoCompression + ), + ) + ) + config = TraceConfig( + **( + {} + if policy == "visible" + else {policy: 30 if policy == "base64_image_max_length" else True} + ) + ) + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=config) + content = [ + {"type": "text", "text": TEXT}, + { + "type": "image_url", + "image_url": {"url": image, "detail": "low"} if structured_url else image, + }, + {"type": "text", "text": "trailing-permitted-text"}, + ] + original_content = copy.deepcopy(content) + observed = [] + + def respond(request): + observed.append(json.loads(request.content)) + return transport_response(request) + + try: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + with Mistral( + api_key="synthetic-tool-image-key", client=transport + ) as client: + result = client.chat.complete( + model="mistral-test", + messages=[ + { + "role": "assistant", + "tool_calls": [ + { + "id": TOOL_ID, + "type": "function", + "function": { + "name": "screenshot", + "arguments": "{}", + }, + } + ], + }, + {"role": "tool", "tool_call_id": TOOL_ID, "content": content}, + ], + ) + assert result.choices[0].message.content == "private-output-marker" + provider.force_flush() + finally: + instrumentor.uninstrument() + provider.shutdown() + + # Redaction is telemetry-only, not a mutation of the caller or SDK request. + assert content == original_content + assert observed[0]["messages"][-1]["content"] == original_content + spans = [ + span + for _, data in requests + for resource in data["resourceSpans"] + for scope in resource["scopeSpans"] + for span in scope["spans"] + ] + assert len(spans) == 1 + wire = json.dumps(requests) + assert "synthetic-tool-image-key" not in wire + limited = policy == "base64_image_max_length" and image.startswith("data:") + visible = policy in ("visible", "base64_image_max_length") and not limited + assert (image in wire) == visible + assert (TEXT in wire) == ( + policy in ("visible", "base64_image_max_length", "hide_input_images") + ) + assert "private-output-marker" in wire + if policy in ("hide_inputs", "hide_input_messages"): + assert TOOL_ID not in wire + return + attrs = { + attribute["key"]: attribute["value"] for attribute in spans[0]["attributes"] + } + messages = json.loads(attrs["gen_ai.input.messages"]["stringValue"]) + part = messages[-1]["parts"][0] + assert part["type"] == "tool_call_response" + assert part["id"] == TOOL_ID + if policy == "hide_input_text": + assert part["response"] == REDACTED_VALUE + elif policy == "hide_input_images": + assert part["response"] == [original_content[0], original_content[2]] + else: + expected = copy.deepcopy(original_content) + if limited: + if structured_url: + expected[1]["image_url"]["url"] = REDACTED_VALUE + else: + expected[1]["image_url"] = REDACTED_VALUE + assert part["response"] == expected diff --git a/python/frameworks/mistralai/tests/test_review_edge_cases.py b/python/frameworks/mistralai/tests/test_review_edge_cases.py new file mode 100644 index 000000000..a70d8ec15 --- /dev/null +++ b/python/frameworks/mistralai/tests/test_review_edge_cases.py @@ -0,0 +1,401 @@ +"""Boundary cases for the Mistral privacy and native stream repairs.""" + +import asyncio +import json + +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.trace import StatusCode +from traceai_mistralai import MistralAIInstrumentor +from traceai_mistralai._privacy import _NativeTraceConfig + +from test_review_regressions import ( + CHAT, + EMBED, + NATIVE, + EventStream, + Mistral, + httpx, + tracing, +) + + +@pytest.mark.parametrize("key", ["http.url", "url.full"]) +@pytest.mark.parametrize("lazy", [False, True]) +@pytest.mark.parametrize( + "url, expected", + [ + ( + "https://user:password@proxy.invalid:8443/v1?q=ok#stream", + "https://proxy.invalid:8443/v1", + ), + ("https://user%40name:p%40ss@proxy.invalid/v1", "https://proxy.invalid/v1"), + ("http://user:password@[::1]:8080/v1", "http://[::1]:8080/v1"), + ("https://user@proxy.invalid/v1", "https://proxy.invalid/v1"), + ("https://proxy.invalid/v1", "https://proxy.invalid/v1"), + ("https://user:password@[invalid/v1", None), + ], +) +def test_url_mask_removes_userinfo_query_and_fragment(key, lazy, url, expected): + config = _NativeTraceConfig(TraceConfig()) + value = (lambda: url) if lazy else url + assert config.mask(key, value) == expected + + +@pytest.mark.parametrize("key", ["http.url", "url.full"]) +def test_url_mask_drops_query_credentials_and_fragment(key): + url = "https://api.mistral.ai/v1/chat/completions?api_key=synthetic-query-credential-458#private-fragment" + assert _NativeTraceConfig(TraceConfig(hide_inputs=True, hide_outputs=True)).mask( + key, url + ) == "https://api.mistral.ai/v1/chat/completions" + + +@pytest.mark.skipif(not NATIVE, reason="Native SDK HTTP attributes") +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("hide_content", [False, True]) +def test_proxy_credentials_not_exported_but_request_unchanged( + asynchronous, hide_content +): + observed = [] + + def respond(request): + observed.append(str(request.url)) + return httpx.Response(200, json=CHAT) + + with tracing(hide_inputs=hide_content, hide_outputs=hide_content) as ( + exporter, + starts, + ): + options = dict( + api_key="synthetic-key", + server_url="https://synthetic-user:synthetic-password@proxy.invalid:8443", + ) + params = dict( + model="mistral-test", messages=[{"role": "user", "content": "hello"}] + ) + if asynchronous: + + async def exercise(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + await Mistral( + async_client=transport, **options + ).chat.complete_async(**params) + + asyncio.run(exercise()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + Mistral(client=transport, **options).chat.complete(**params) + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == len(observed) == 1 + assert "synthetic-user:synthetic-password@" in observed[0] + assert "synthetic-password" not in spans[0].to_json() + assert "synthetic-user" not in spans[0].to_json() + assert ( + spans[0].attributes["http.url"] + == "https://proxy.invalid:8443/v1/chat/completions" + ) + + +@pytest.mark.skipif(not NATIVE, reason="Native SDK HTTP attributes") +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("hide_content", [False, True]) +def test_query_credentials_not_exported_but_request_unchanged( + asynchronous, hide_content +): + marker = "synthetic-query-credential-458" + observed = [] + + def respond(request): + observed.append(str(request.url)) + return httpx.Response(200, json=CHAT) + + with tracing(hide_inputs=hide_content, hide_outputs=hide_content) as ( + exporter, + starts, + ): + params = dict( + model="mistral-test", messages=[{"role": "user", "content": "hello"}] + ) + if asynchronous: + + async def exercise(): + async with httpx.AsyncClient( + params={"api_key": marker}, transport=httpx.MockTransport(respond) + ) as transport: + await Mistral( + api_key="synthetic-key", async_client=transport + ).chat.complete_async(**params) + + asyncio.run(exercise()) + else: + with httpx.Client( + params={"api_key": marker}, transport=httpx.MockTransport(respond) + ) as transport: + Mistral( + api_key="synthetic-key", client=transport + ).chat.complete(**params) + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == len(observed) == 1 + assert marker in observed[0] + assert marker not in spans[0].to_json() + assert ( + spans[0].attributes["http.url"] + == "https://api.mistral.ai/v1/chat/completions" + ) + + +@pytest.mark.skipif(NATIVE, reason="Legacy embedding extraction") +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("single", [False, True]) +@pytest.mark.parametrize( + "flag", + [ + None, + "hide_inputs", + "hide_input_messages", + "hide_input_text", + "hide_outputs", + "hide_embedding_vectors", + ], +) +def test_embedding_privacy_preserves_request_and_allowed_text( + asynchronous, single, flag +): + marker = "synthetic-private-embedding-input" + inputs = marker if single else [marker, "second-input"] + observed = [] + + def respond(request): + observed.append(json.loads(request.content)) + return httpx.Response(200, json=EMBED) + + with tracing(**({flag: True} if flag else {})) as (exporter, starts): + params = dict(model="mistral-embed", inputs=inputs) + if asynchronous: + + async def exercise(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + await Mistral( + api_key="synthetic-key", async_client=transport + ).embeddings.create_async(**params) + + asyncio.run(exercise()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + Mistral(api_key="synthetic-key", client=transport).embeddings.create( + **params + ) + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == len(observed) == 1 + assert observed[0]["input"] == inputs + attrs = spans[0].attributes + assert marker not in attrs.get("gen_ai.request.parameters", "") + assert "input.value" not in attrs + assert attrs["gen_ai.span.kind"] == "EMBEDDING" + hidden = flag in ("hide_inputs", "hide_input_messages", "hide_input_text") + assert (marker in spans[0].to_json()) is not hidden + if not hidden: + assert attrs["embedding.embeddings.0.embedding.text"] == marker + + +@pytest.mark.skipif(NATIVE, reason="Legacy embedding cancellation") +@pytest.mark.parametrize("hide_error_details", [False, True]) +def test_cancel_preserves_exception_identity_and_ends_once(hide_error_details): + error = asyncio.CancelledError("synthetic-cancellation-detail") + with tracing(hide_error_details=hide_error_details) as (exporter, starts): + + async def exercise(): + async def respond(request): + raise error + + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + with pytest.raises(asyncio.CancelledError) as caught: + await Mistral( + api_key="synthetic-key", async_client=transport + ).embeddings.create_async(model="mistral-embed", inputs=["hello"]) + assert caught.value is error + + asyncio.run(exercise()) + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == 1 + assert not starts.spans[0].is_recording() + assert spans[0].status.status_code == StatusCode.ERROR + if hide_error_details: + assert "synthetic-cancellation-detail" not in spans[0].to_json() + + +@pytest.mark.skipif(not NATIVE, reason="Native conversation delta reconstruction") +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_interleaved_conversation_calls_are_independent(asynchronous): + events = [{"type": "conversation.response.started", "conversation_id": "conv-test"}] + for index, fragment in [ + (0, '{"city":'), + (1, '{"date":'), + (0, '"Paris"}'), + (1, '"Monday"}'), + ]: + events.append( + { + "type": "function.call.delta", + "id": f"entry-{index}", + "tool_call_id": f"call-{index}", + "output_index": index, + "name": ["weather", "calendar"][index], + "arguments": fragment, + } + ) + events.append( + { + "type": "conversation.response.done", + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + } + ) + + def respond(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=EventStream(events, named=True), + ) + + with tracing() as (exporter, starts): + if asynchronous: + + async def exercise(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic-key", async_client=transport) + async with await client.beta.conversations.start_stream_async( + model="mistral-test", inputs="hello" + ) as stream: + assert len([event async for event in stream]) == 6 + + asyncio.run(exercise()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-key", client=transport) + with client.beta.conversations.start_stream( + model="mistral-test", inputs="hello" + ) as stream: + assert len(list(stream)) == 6 + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == 1 + parts = json.loads(spans[0].attributes["gen_ai.output.messages"])[0]["parts"] + assert parts == [ + { + "type": "tool_call", + "id": "call-0", + "name": "weather", + "arguments": '{"city":"Paris"}', + }, + { + "type": "tool_call", + "id": "call-1", + "name": "calendar", + "arguments": '{"date":"Monday"}', + }, + ] + + +class RawEvents(httpx.SyncByteStream, httpx.AsyncByteStream): + def __init__(self, payload): + self.payload = payload + + def __iter__(self): + yield self.payload + + async def __aiter__(self): + yield self.payload + + +@pytest.mark.skipif(not NATIVE, reason="Native SDK decoder lifetime") +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("hide_error_details", [False, True]) +@pytest.mark.parametrize( + "payload", + [ + b'data: {"error":{"message":"synthetic-private-error"}}', + b'data: {"error":{"message":"synthetic-private-error"}}\n\n', + b"data: \xff\n\n", + ], +) +def test_decode_error_ends_span_even_without_context_manager( + asynchronous, hide_error_details, payload +): + responses = [] + + def respond(request): + response = httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=RawEvents(payload), + ) + responses.append(response) + return response + + with tracing(hide_error_details=hide_error_details) as (exporter, starts): + params = dict( + model="mistral-test", messages=[{"role": "user", "content": "hello"}] + ) + if asynchronous: + + async def exercise(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + stream = await Mistral( + api_key="synthetic-key", async_client=transport + ).chat.stream_async(**params) + with pytest.raises(Exception) as caught: + [event async for event in stream] + assert "Validation" in type(caught.value).__name__ or isinstance( + caught.value, UnicodeDecodeError + ) + assert stream.response.is_closed + await stream.close() + + asyncio.run(exercise()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + stream = Mistral(api_key="synthetic-key", client=transport).chat.stream( + **params + ) + with pytest.raises(Exception) as caught: + list(stream) + assert "Validation" in type(caught.value).__name__ or isinstance( + caught.value, UnicodeDecodeError + ) + assert stream.response.is_closed + stream.close() + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == 1 + assert not starts.spans[0].is_recording() + assert spans[0].status.status_code == StatusCode.ERROR + if hide_error_details: + assert "synthetic-private-error" not in spans[0].to_json() + else: + assert len(spans[0].events) == 1 + + +@pytest.mark.skipif(not NATIVE, reason="Native stream wrapper restoration") +def test_sdk_iteration_methods_restored(): + from mistralai.client.utils.eventstreaming import ( + EventStream as SDKStream, + EventStreamAsync, + ) + + original_sync, original_async = SDKStream.__next__, EventStreamAsync.__anext__ + for _ in range(2): + with tracing(): + assert SDKStream.__next__ is not original_sync + assert EventStreamAsync.__anext__ is not original_async + assert SDKStream.__next__ is original_sync + assert EventStreamAsync.__anext__ is original_async + assert not MistralAIInstrumentor().is_instrumented_by_opentelemetry diff --git a/python/frameworks/mistralai/tests/test_review_regressions.py b/python/frameworks/mistralai/tests/test_review_regressions.py new file mode 100644 index 000000000..22844182c --- /dev/null +++ b/python/frameworks/mistralai/tests/test_review_regressions.py @@ -0,0 +1,350 @@ +"""Regressions exercised through real SDK serialization and synthetic HTTP/SSE.""" + +import asyncio +import importlib +import json +from contextlib import contextmanager +from importlib.metadata import version + +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.sdk.trace import TracerProvider, SpanProcessor +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_mistralai import MistralAIInstrumentor + +NATIVE = version("mistralai").startswith("3.") +Mistral = importlib.import_module("mistralai.client" if NATIVE else "mistralai").Mistral +httpx = importlib.import_module("httpx2" if NATIVE else "httpx") + +CHAT = { + "id": "synthetic", + "object": "chat.completion", + "created": 1, + "model": "mistral-test", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, +} +EMBED = { + "id": "synthetic", + "object": "list", + "model": "mistral-embed", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "usage": {"prompt_tokens": 2, "completion_tokens": 0, "total_tokens": 2}, +} + + +class CaptureStarts(SpanProcessor): + def __init__(self): + self.spans = [] + + def on_start(self, span, parent_context=None): + self.spans.append(span) + + +@contextmanager +def tracing(**flags): + provider = TracerProvider() + exporter = InMemorySpanExporter() + starts = CaptureStarts() + provider.add_span_processor(starts) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**flags)) + try: + yield exporter, starts + finally: + instrumentor.uninstrument() + provider.shutdown() + + +class EventStream(httpx.SyncByteStream, httpx.AsyncByteStream): + def __init__(self, events, named=False): + self.events, self.named = events, named + + def __iter__(self): + for event in self.events: + name = f"event: {event['type']}\n" if self.named else "" + yield (name + "data: " + json.dumps(event) + "\n\n").encode() + + async def __aiter__(self): + for item in self: + yield item + + def close(self): + pass + + async def aclose(self): + pass + + +@pytest.mark.skipif(NATIVE, reason="New legacy embedding path") +@pytest.mark.parametrize( + "flag", ["hide_input_text", "hide_input_messages", "hide_inputs"] +) +def test_legacy_embedding_content_privacy(flag): + marker = "synthetic-private-embedding-input" + with tracing(**{flag: True}) as (exporter, starts): + with httpx.Client( + transport=httpx.MockTransport(lambda r: httpx.Response(200, json=EMBED)) + ) as transport: + client = Mistral(api_key="synthetic-key", client=transport) + client.embeddings.create(model="mistral-embed", inputs=[marker]) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert marker not in spans[0].to_json() + + +@pytest.mark.skipif(NATIVE, reason="New legacy embedding path") +def test_legacy_embedding_cancellation_finishes_span(): + with tracing() as (exporter, starts): + + async def exercise(): + async def respond(request): + raise asyncio.CancelledError("synthetic-cancel") + + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic-key", async_client=transport) + with pytest.raises(asyncio.CancelledError): + await client.embeddings.create_async( + model="mistral-embed", inputs=["hello"] + ) + + asyncio.run(exercise()) + assert len(starts.spans) == 1 + assert not starts.spans[0].is_recording() + assert exporter.get_finished_spans()[0].status.status_code == StatusCode.ERROR + + +@pytest.mark.skipif(not NATIVE, reason="Native adapter") +def test_native_custom_endpoint_redacts_url_credentials(): + with tracing(hide_inputs=True, hide_outputs=True) as (exporter, starts): + with httpx.Client( + transport=httpx.MockTransport(lambda r: httpx.Response(200, json=CHAT)) + ) as transport: + client = Mistral( + api_key="synthetic-key", + client=transport, + server_url="https://synthetic-user:synthetic-password@proxy.invalid", + ) + client.chat.complete( + model="mistral-test", messages=[{"role": "user", "content": "hello"}] + ) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert "synthetic-password" not in spans[0].to_json() + + +@pytest.mark.skipif(not NATIVE, reason="Native conversation aggregation") +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_native_conversation_accumulates_tool_arguments(asynchronous): + events = [ + {"type": "conversation.response.started", "conversation_id": "conv-synthetic"}, + { + "type": "function.call.delta", + "id": "entry-1", + "tool_call_id": "call-1", + "output_index": 0, + "name": "weather", + "arguments": '{"city":', + }, + { + "type": "function.call.delta", + "id": "entry-1", + "tool_call_id": "call-1", + "output_index": 0, + "name": "weather", + "arguments": '"Paris"}', + }, + { + "type": "conversation.response.done", + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + }, + ] + + def respond(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=EventStream(events, named=True), + ) + + with tracing() as (exporter, starts): + if asynchronous: + + async def exercise(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic-key", async_client=transport) + async with await client.beta.conversations.start_stream_async( + model="mistral-test", inputs="hello" + ) as stream: + assert len([event async for event in stream]) == 4 + + asyncio.run(exercise()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-key", client=transport) + with client.beta.conversations.start_stream( + model="mistral-test", inputs="hello" + ) as stream: + assert len(list(stream)) == 4 + spans = exporter.get_finished_spans() + assert len(spans) == 1 + parts = json.loads(spans[0].attributes["gen_ai.output.messages"])[0]["parts"] + assert parts == [ + { + "type": "tool_call", + "id": "call-1", + "name": "weather", + "arguments": '{"city":"Paris"}', + } + ] + + +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_stream_decode_failure_is_error(asynchronous): + def respond(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=EventStream([{"error": {"message": "synthetic-provider-failure"}}]), + ) + + with tracing() as (exporter, starts): + if asynchronous: + + async def exercise(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic-key", async_client=transport) + with pytest.raises(Exception) as raised: + async with await client.chat.stream_async( + model="mistral-test", + messages=[{"role": "user", "content": "hello"}], + ) as stream: + [event async for event in stream] + assert "Validation" in type(raised.value).__name__ + + asyncio.run(exercise()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-key", client=transport) + with pytest.raises(Exception) as raised: + with client.chat.stream( + model="mistral-test", + messages=[{"role": "user", "content": "hello"}], + ) as stream: + list(stream) + assert "Validation" in type(raised.value).__name__ + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code == StatusCode.ERROR + + +@pytest.mark.skipif(not NATIVE, reason="Native kind classification") +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_native_agent_completion_kind(streaming, asynchronous): + chunk = { + "id": "synthetic", + "object": "chat.completion.chunk", + "created": 1, + "model": "mistral-test", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + } + + def respond(request): + if streaming: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=EventStream([chunk]), + ) + return httpx.Response(200, json=CHAT) + + with tracing() as (exporter, starts): + params = dict( + agent_id="synthetic-agent", messages=[{"role": "user", "content": "hello"}] + ) + if asynchronous: + + async def exercise(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic-key", async_client=transport) + if streaming: + async with await client.agents.stream_async(**params) as stream: + [event async for event in stream] + else: + await client.agents.complete_async(**params) + + asyncio.run(exercise()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-key", client=transport) + if streaming: + with client.agents.stream(**params) as stream: + list(stream) + else: + client.agents.complete(**params) + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == 1 + assert spans[0].attributes["gen_ai.span.kind"] == "LLM" + + +@pytest.mark.skipif(not NATIVE, reason="Previously reported native tool image issue") +@pytest.mark.parametrize("structured", [False, True]) +@pytest.mark.parametrize( + "image", + ["https://example.invalid/private-image", "data:image/png;base64," + "A" * 80], +) +def test_previous_tool_image_privacy_fix(structured, image): + chunks = [ + {"type": "text", "text": "allowed-tool-text"}, + { + "type": "image_url", + "image_url": {"url": image, "detail": "low"} if structured else image, + }, + ] + observed = [] + + def respond(request): + observed.append(json.loads(request.content)) + return httpx.Response(200, json=CHAT) + + with tracing(hide_input_images=True) as (exporter, starts): + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-key", client=transport) + client.chat.complete( + model="mistral-test", + messages=[ + {"role": "tool", "tool_call_id": "call-image", "content": chunks} + ], + ) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + wire = spans[0].to_json() + assert image not in wire + assert "allowed-tool-text" in wire + assert "call-image" in wire + assert observed[0]["messages"][0]["content"] == chunks diff --git a/python/frameworks/mistralai/tests/test_review_round2.py b/python/frameworks/mistralai/tests/test_review_round2.py new file mode 100644 index 000000000..aaa331334 --- /dev/null +++ b/python/frameworks/mistralai/tests/test_review_round2.py @@ -0,0 +1,230 @@ +"""Independent review checks: synthetic requests through installed SDKs only.""" + +import asyncio +import json + +import pytest +from opentelemetry.trace import StatusCode + +from test_review_regressions import tracing, Mistral, httpx, NATIVE, CHAT, EventStream + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("operation", ["chat", "embedding"]) +def test_nonstream_validation_failure_is_error(asynchronous, operation): + def respond(request): + return httpx.Response( + 200, json={"error": {"message": "synthetic-private-error"}} + ) + + def invoke(client): + if operation == "embedding": + method = ( + client.embeddings.create_async + if asynchronous + else client.embeddings.create + ) + return method(model="mistral-embed", inputs=["hello"]) + method = client.chat.complete_async if asynchronous else client.chat.complete + return method( + model="mistral-test", messages=[{"role": "user", "content": "hello"}] + ) + + with tracing(hide_error_details=True) as (exporter, starts): + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + with pytest.raises(Exception) as error: + await invoke( + Mistral(api_key="synthetic", async_client=transport) + ) + assert "Validation" in type(error.value).__name__ + + asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + with pytest.raises(Exception) as error: + invoke(Mistral(api_key="synthetic", client=transport)) + assert "Validation" in type(error.value).__name__ + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == 1 + assert spans[0].status.status_code == StatusCode.ERROR + assert "synthetic-private-error" not in spans[0].to_json() + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize( + "flag,marker", + [ + ("hide_input_text", "synthetic-private-prompt"), + ("hide_input_messages", "synthetic-private-prompt"), + ("hide_output_text", "synthetic-private-answer"), + ("hide_output_messages", "synthetic-private-answer"), + ("hide_inputs", "synthetic-private-prompt"), + ("hide_outputs", "synthetic-private-answer"), + ], +) +def test_selective_chat_privacy(asynchronous, flag, marker): + response = json.loads(json.dumps(CHAT)) + response["choices"][0]["message"]["content"] = "synthetic-private-answer" + + def respond(request): + assert ( + json.loads(request.content)["messages"][0]["content"] + == "synthetic-private-prompt" + ) + return httpx.Response(200, json=response) + + kwargs = dict( + model="mistral-test", + messages=[{"role": "user", "content": "synthetic-private-prompt"}], + ) + with tracing(**{flag: True}) as (exporter, starts): + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + result = await Mistral( + api_key="synthetic", async_client=transport + ).chat.complete_async(**kwargs) + assert ( + result.choices[0].message.content == "synthetic-private-answer" + ) + + asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + result = Mistral(api_key="synthetic", client=transport).chat.complete( + **kwargs + ) + assert result.choices[0].message.content == "synthetic-private-answer" + (span,) = exporter.get_finished_spans() + assert marker not in span.to_json() + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("configured", ["argument", "environment", "visible"]) +def test_hide_invocation_parameters(asynchronous, configured, monkeypatch): + monkeypatch.setenv( + "FI_HIDE_LLM_INVOCATION_PARAMETERS", + "true" if configured == "environment" else "false", + ) + flags = {"hide_llm_invocation_parameters": True} if configured == "argument" else {} + observed = [] + + def respond(request): + observed.append(json.loads(request.content)) + return httpx.Response(200, json=CHAT) + + kwargs = dict( + model="mistral-test", + messages=[{"role": "user", "content": "hello"}], + stop=["synthetic-private-stop"], + temperature=0.731, + ) + with tracing(**flags) as (exporter, starts): + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + await Mistral( + api_key="synthetic", async_client=transport + ).chat.complete_async(**kwargs) + + asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + Mistral(api_key="synthetic", client=transport).chat.complete(**kwargs) + (span,) = exporter.get_finished_spans() + assert observed[0]["stop"] == ["synthetic-private-stop"] + assert observed[0]["temperature"] == 0.731 + assert span.attributes["gen_ai.usage.input_tokens"] == 2 + if NATIVE: + assert span.attributes["gen_ai.request.model"] == "mistral-test" + if configured == "visible": + if NATIVE: + assert span.attributes["gen_ai.request.temperature"] == 0.731 + assert span.attributes["gen_ai.request.stop_sequences"] == ( + "synthetic-private-stop", + ) + else: + params = json.loads(span.attributes["gen_ai.request.parameters"]) + assert params["temperature"] == 0.731 + assert params["stop"] == ["synthetic-private-stop"] + else: + assert "gen_ai.request.temperature" not in span.attributes + assert "synthetic-private-stop" not in span.to_json() + + +@pytest.mark.skipif(not NATIVE, reason="Native conversations") +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("separator", ["\u2028", "\u2029", "\u0085"]) +@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"]) +def test_unicode_conversation_content_preserved(asynchronous, separator, newline): + marker = "first" + separator + "second" + events = [ + { + "type": "message.output.delta", + "id": "msg-1", + "role": "assistant", + "content": marker, + }, + { + "type": "conversation.response.done", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + }, + ] + + class RawEvents(EventStream): + def __iter__(self): + for event in self.events: + yield ( + "event: " + + event["type"] + + newline + + "data: " + + json.dumps(event, ensure_ascii=False) + + newline * 2 + ).encode() + + def respond(request): + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=RawEvents(events) + ) + + with tracing() as (exporter, starts): + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + async with await Mistral( + api_key="synthetic", async_client=transport + ).beta.conversations.start_stream_async( + model="mistral-test", inputs="hello" + ) as stream: + result = [event async for event in stream] + assert result[0].data.content == marker + + asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + with Mistral( + api_key="synthetic", client=transport + ).beta.conversations.start_stream( + model="mistral-test", inputs="hello" + ) as stream: + result = list(stream) + assert result[0].data.content == marker + (span,) = exporter.get_finished_spans() + assert json.loads(span.attributes["gen_ai.output.messages"])[0]["parts"] == [ + {"type": "text", "content": marker} + ] diff --git a/python/frameworks/mistralai/tests/test_review_round2_edges.py b/python/frameworks/mistralai/tests/test_review_round2_edges.py new file mode 100644 index 000000000..f5f26d530 --- /dev/null +++ b/python/frameworks/mistralai/tests/test_review_round2_edges.py @@ -0,0 +1,357 @@ +"""Privacy and decoding boundaries with real SDKs and synthetic transports.""" + +import asyncio +import importlib +import json +from concurrent.futures import ThreadPoolExecutor + +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.trace import StatusCode + +from test_review_regressions import CHAT, Mistral, NATIVE, httpx, tracing +from test_sdk import SSE, chat_response + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("resource", ["chat", "agents"]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("environment", [False, True]) +@pytest.mark.parametrize( + "flag", + [ + "hide_input_text", + "hide_input_messages", + "hide_output_text", + "hide_output_messages", + "visible", + ], +) +def test_chat_privacy_all_paths( + asynchronous, resource, streaming, environment, flag, monkeypatch +): + flags = {} + if flag != "visible": + if environment: + monkeypatch.setenv("FI_" + flag.upper(), "true") + else: + flags[flag] = True + observed = [] + + def respond(request): + observed.append(json.loads(request.content)) + if streaming: + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=SSE() + ) + return httpx.Response(200, json=chat_response()) + + kwargs = {"messages": [{"role": "user", "content": "private-input-marker"}]} + kwargs.update( + {"model": "mistral-test"} + if resource == "chat" + else {"agent_id": "synthetic-agent"} + ) + method = "stream" if streaming else "complete" + with tracing(**flags) as (exporter, starts): + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + result = await getattr( + getattr(client, resource), method + "_async" + )(**kwargs) + if streaming: + chunks = [chunk async for chunk in result] + await result.close() + assert ( + chunks[0].data.choices[0].delta.content + == "private-output-marker" + ) + else: + assert ( + result.choices[0].message.content == "private-output-marker" + ) + + asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic", client=transport) + result = getattr(getattr(client, resource), method)(**kwargs) + if streaming: + chunks = list(result) + result.close() + assert ( + chunks[0].data.choices[0].delta.content + == "private-output-marker" + ) + else: + assert result.choices[0].message.content == "private-output-marker" + assert observed[0]["messages"] == kwargs["messages"] + (span,) = exporter.get_finished_spans() + assert len(starts.spans) == 1 + assert span.status.status_code == StatusCode.OK + assert span.attributes["gen_ai.usage.input_tokens"] == 2 + rendered = span.to_json() + assert ("private-input-marker" in rendered) == ("input" not in flag) + assert ("private-output-marker" in rendered) == ("output" not in flag) + + +@pytest.mark.skipif(not NATIVE, reason="Native SDK response observers") +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize( + "operation", ["chat", "agents", "embedding", "fim", "start", "append", "restart"] +) +@pytest.mark.parametrize("body", ["schema", "json", "content-type"]) +@pytest.mark.parametrize("hide", [False, True]) +def test_native_decoding_boundaries(asynchronous, operation, body, hide): + def respond(request): + if body == "schema": + return httpx.Response( + 200, json={"error": {"message": "synthetic-private-error"}} + ) + return httpx.Response( + 200, + content=b"synthetic-private-error", + headers={ + "content-type": "application/json" if body == "json" else "text/html", + }, + ) + + def invoke(client): + kwargs = {"model": "mistral-test"} + method = "complete" + if operation in ("chat", "agents"): + resource = getattr(client, operation) + kwargs["messages"] = [{"role": "user", "content": "hello"}] + if operation == "agents": + kwargs.pop("model") + kwargs["agent_id"] = "synthetic-agent" + elif operation == "embedding": + resource, method = client.embeddings, "create" + kwargs["inputs"] = ["hello"] + elif operation == "fim": + resource = client.fim + kwargs["prompt"] = "hello" + else: + resource, method = client.beta.conversations, operation + kwargs["inputs"] = "hello" + if operation != "start": + kwargs.pop("model") + kwargs["conversation_id"] = "conv-synthetic" + if operation == "restart": + kwargs["from_entry_id"] = "entry-synthetic" + return getattr(resource, method + ("_async" if asynchronous else ""))(**kwargs) + + with tracing(hide_error_details=hide) as (exporter, starts): + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + with pytest.raises(Exception) as error: + await invoke( + Mistral(api_key="synthetic", async_client=transport) + ) + return error.value + + error = asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + with pytest.raises(Exception) as caught: + invoke(Mistral(api_key="synthetic", client=transport)) + error = caught.value + assert "synthetic-private-error" in error.body + (span,) = exporter.get_finished_spans() + assert len(starts.spans) == 1 + assert not starts.spans[0].is_recording() + assert span.status.status_code == StatusCode.ERROR + assert len(span.events) == 1 + assert ( + span.events[0].attributes["exception.type"].endswith(type(error).__name__) + ) + if hide: + assert "synthetic-private-error" not in span.to_json() + else: + assert span.events[0].attributes["exception.message"] == str(error) + + +@pytest.mark.skipif(not NATIVE, reason="Native SDK response observers") +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_concurrent_decoding_scopes(asynchronous): + def respond(request): + model = json.loads(request.content)["model"] + return httpx.Response(200, json=CHAT if model == "valid" else {"invalid": True}) + + with tracing() as (exporter, starts): + if asynchronous: + + async def run(): + # Yield during transport to interleave requests in one client. + async def async_respond(request): + await asyncio.sleep(0) + return respond(request) + + async with httpx.AsyncClient( + transport=httpx.MockTransport(async_respond) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + return await asyncio.gather( + *[ + client.chat.complete_async( + model=model, + messages=[{"role": "user", "content": "hi"}], + ) + for model in ["valid", "invalid"] * 4 + ], + return_exceptions=True, + ) + + results = asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic", client=transport) + + def call(model): + try: + return client.chat.complete( + model=model, messages=[{"role": "user", "content": "hi"}] + ) + except Exception as error: + return error + + with ThreadPoolExecutor(max_workers=4) as pool: + results = list(pool.map(call, ["valid", "invalid"] * 4)) + assert sum(isinstance(result, Exception) for result in results) == 4 + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == 8 + assert sum(span.status.status_code == StatusCode.ERROR for span in spans) == 4 + assert sum(span.status.status_code == StatusCode.OK for span in spans) == 4 + assert all(not span.is_recording() for span in starts.spans) + + +@pytest.mark.skipif(not NATIVE, reason="Native SDK response observers") +def test_observed_methods_restored(): + methods = [] + for module, cls_name, names in ( + ("chat", "Chat", ("complete", "stream")), + ("agents", "Agents", ("complete", "stream")), + ("embeddings", "Embeddings", ("create",)), + ("fim", "Fim", ("complete", "stream")), + ( + "conversations", + "Conversations", + ( + "start", + "append", + "restart", + "start_stream", + "append_stream", + "restart_stream", + ), + ), + ): + cls = getattr(importlib.import_module("mistralai.client." + module), cls_name) + for name in names: + for suffix in ("", "_async"): + name_with_suffix = name + suffix + methods.append((cls, name_with_suffix, getattr(cls, name_with_suffix))) + for _ in range(2): + with tracing(): + for cls, name, original in methods: + assert getattr(cls, name).__wrapped__ is original + for cls, name, original in methods: + assert getattr(cls, name) is original + + +@pytest.mark.parametrize( + "key", + [ + "choice.count", + "encoding_formats", + "frequency_penalty", + "max_tokens", + "presence_penalty", + "seed", + "stop_sequences", + "temperature", + "top_p", + "top_k", + "future_parameter", + ], +) +def test_native_parameter_policy_drops_lazy_values(key): + from traceai_mistralai._privacy import _NativeTraceConfig + + config = _NativeTraceConfig(TraceConfig(hide_llm_invocation_parameters=True)) + + def sensitive(): + pytest.fail("Hidden parameter value should not be evaluated") + + assert config.mask("gen_ai.request." + key, sensitive) is None + assert config.mask("gen_ai.request.model", "mistral-test") == "mistral-test" + assert config.mask("gen_ai.usage.input_tokens", 2) == 2 + + +@pytest.mark.skipif(not NATIVE, reason="Native SDK response observers") +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("hide", [False, True]) +def test_decoder_exception_identity_and_scope_reset(asynchronous, hide, monkeypatch): + module = importlib.import_module("mistralai.client.chat") + original = module.unmarshal_json_response + failure = ( + asyncio.CancelledError("synthetic-decoder-cancel") + if asynchronous + else RuntimeError("synthetic-decoder-error") + ) + calls = [] + + def decode(*args, **kwargs): + calls.append(True) + if len(calls) == 1: + raise failure + return original(*args, **kwargs) + + monkeypatch.setattr(module, "unmarshal_json_response", decode) + with tracing(hide_error_details=hide) as (exporter, starts): + kwargs = { + "model": "mistral-test", + "messages": [{"role": "user", "content": "hi"}], + } + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda r: httpx.Response(200, json=CHAT) + ) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + with pytest.raises(asyncio.CancelledError) as error: + await client.chat.complete_async(**kwargs) + assert error.value is failure + assert (await client.chat.complete_async(**kwargs)).choices[ + 0 + ].message.content == "OK" + + asyncio.run(run()) + else: + with httpx.Client( + transport=httpx.MockTransport(lambda r: httpx.Response(200, json=CHAT)) + ) as transport: + client = Mistral(api_key="synthetic", client=transport) + with pytest.raises(RuntimeError) as error: + client.chat.complete(**kwargs) + assert error.value is failure + assert client.chat.complete(**kwargs).choices[0].message.content == "OK" + spans = exporter.get_finished_spans() + assert len(spans) == len(starts.spans) == 2 + assert spans[0].status.status_code == StatusCode.ERROR + assert spans[1].status.status_code == StatusCode.OK + assert len(spans[0].events) == 1 + assert all(not span.is_recording() for span in starts.spans) + assert ("synthetic-decoder-" in spans[0].to_json()) == (not hide) diff --git a/python/frameworks/mistralai/tests/test_review_round3.py b/python/frameworks/mistralai/tests/test_review_round3.py new file mode 100644 index 000000000..14b80ebd5 --- /dev/null +++ b/python/frameworks/mistralai/tests/test_review_round3.py @@ -0,0 +1,312 @@ +"""Independent real-SDK contracts; synthetic credentials and no vendor calls.""" + +import asyncio +import importlib +import json +from contextlib import contextmanager +from importlib.metadata import version + +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_mistralai import MistralAIInstrumentor + +NATIVE = version("mistralai").startswith("3.") +Mistral = importlib.import_module("mistralai.client" if NATIVE else "mistralai").Mistral +models = importlib.import_module( + "mistralai.client.models" if NATIVE else "mistralai.models" +) +httpx = importlib.import_module("httpx2" if NATIVE else "httpx") +INPUT = "synthetic-input-content-marker" +OUTPUT = "synthetic-output-content-marker" +IMAGE = "https://example.invalid/synthetic-image-content-marker" + + +@contextmanager +def tracing(**flags): + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**flags)) + try: + yield exporter + finally: + instrumentor.uninstrument() + provider.shutdown() + + +def response(content=OUTPUT): + return { + "id": "synthetic", + "object": "chat.completion", + "created": 1, + "model": "mistral-test", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": content}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + } + + +def call(asynchronous, messages, body, captured): + def respond(request): + captured.append(json.loads(request.content)) + return httpx.Response(200, json=body) + + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic-key", async_client=transport) + return await client.chat.complete_async( + model="mistral-test", messages=messages + ) + + return asyncio.run(run()) + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-key", client=transport) + return client.chat.complete(model="mistral-test", messages=messages) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize( + "shape", + ["dict", "model", "typed-chunk", "image-dict", "image-model", "tool-history"], +) +@pytest.mark.parametrize("hide", [False, True]) +def test_supported_input_shapes_preserve_capture(asynchronous, shape, hide): + marker = INPUT + if shape == "dict": + messages = [{"role": "user", "content": INPUT}] + elif shape == "model": + messages = [models.UserMessage(content=INPUT)] + elif shape == "typed-chunk": + messages = [{"role": "user", "content": [models.TextChunk(text=INPUT)]}] + elif shape == "image-dict": + marker = IMAGE + messages = [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": IMAGE}}], + } + ] + elif shape == "image-model": + if NATIVE: + pytest.skip("Legacy typed image model") + marker = IMAGE + messages = [models.UserMessage(content=[models.ImageURLChunk(image_url=IMAGE)])] + else: + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call-synthetic", + "type": "function", + "function": { + "name": "lookup", + "arguments": json.dumps({"query": INPUT}), + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call-synthetic", + "content": "tool-result", + }, + ] + with tracing(**({"hide_inputs": True} if hide else {})) as exporter: + captured = [] + result = call(asynchronous, messages, response(), captured) + assert result.choices[0].message.content == OUTPUT + assert marker in json.dumps(captured) + (span,) = exporter.get_finished_spans() + assert span.status.status_code == StatusCode.OK + assert (marker in span.to_json()) == (not hide), dict(span.attributes) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("shape", ["string", "chunks"]) +@pytest.mark.parametrize("hide", [False, True]) +def test_supported_output_shapes_preserve_capture(asynchronous, shape, hide): + if shape == "chunks" and version("mistralai") == "1.1.0": + pytest.skip("SDK 1.1 response content accepts strings only") + content = OUTPUT if shape == "string" else [{"type": "text", "text": OUTPUT}] + with tracing(**({"hide_output_text": True} if hide else {})) as exporter: + result = call( + asynchronous, [{"role": "user", "content": INPUT}], response(content), [] + ) + assert OUTPUT in result.model_dump_json() + (span,) = exporter.get_finished_spans() + assert span.status.status_code == StatusCode.OK + assert (OUTPUT in span.to_json()) == (not hide), dict(span.attributes) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("policy", ["argument", "environment", "default", "visible"]) +def test_oversized_input_image_policy(asynchronous, policy, monkeypatch): + image = "data:image/png;base64," + "YWJj" * 9000 + flags = {} + if policy == "environment": + monkeypatch.setenv("FI_BASE64_IMAGE_MAX_LENGTH", "32") + elif policy == "argument": + flags["base64_image_max_length"] = 32 + elif policy == "visible": + flags["base64_image_max_length"] = 40000 + with tracing(**flags) as exporter: + captured = [] + result = call( + asynchronous, + [{"role": "user", "content": [{"type": "image_url", "image_url": image}]}], + response(), + captured, + ) + assert result.choices[0].message.content == OUTPUT + assert image in json.dumps(captured) + (span,) = exporter.get_finished_spans() + captured_image = image in span.to_json() + assert captured_image == ( + policy == "visible" + ), "Oversized input image capture does not follow the configured policy" + + +@pytest.mark.skipif( + not hasattr( + importlib.import_module( + "mistralai.client.chat" if NATIVE else "mistralai.chat" + ).Chat, + "parse", + ), + reason="No structured output API on this SDK", +) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("valid", [False, True]) +def test_structured_parse_status(asynchronous, valid): + from pydantic import BaseModel + + class Answer(BaseModel): + city: str + + body = response('{"city":"Paris"}' if valid else "synthetic-invalid-json") + + def respond(request): + return httpx.Response(200, json=body) + + def operation(client): + method = client.chat.parse_async if asynchronous else client.chat.parse + return method( + response_format=Answer, + model="mistral-test", + messages=[{"role": "user", "content": "city"}], + ) + + with tracing() as exporter: + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + if valid: + result = await operation(client) + assert result.choices[0].message.parsed.city == "Paris" + else: + with pytest.raises(json.JSONDecodeError): + await operation(client) + + asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic", client=transport) + if valid: + assert operation(client).choices[0].message.parsed.city == "Paris" + else: + with pytest.raises(json.JSONDecodeError): + operation(client) + (span,) = exporter.get_finished_spans() + assert span.status.status_code == (StatusCode.OK if valid else StatusCode.ERROR) + if not valid: + assert len(span.events) == 1 + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("shape", ["string", "chunks"]) +@pytest.mark.parametrize("hide", [False, True]) +def test_stream_content_shapes(asynchronous, shape, hide): + if shape == "chunks" and version("mistralai") == "1.1.0": + pytest.skip("SDK 1.1 output deltas accept strings only") + content = OUTPUT if shape == "string" else [{"type": "text", "text": OUTPUT}] + event = { + "id": "synthetic", + "object": "chat.completion.chunk", + "created": 1, + "model": "mistral-test", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": content}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + } + + class Stream(httpx.SyncByteStream, httpx.AsyncByteStream): + def __iter__(self): + yield ("data: " + json.dumps(event) + "\n\ndata: [DONE]\n\n").encode() + + async def __aiter__(self): + for b in self: + yield b + + def close(self): + pass + + async def aclose(self): + pass + + def respond(request): + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=Stream() + ) + + with tracing(**({"hide_output_text": True} if hide else {})) as exporter: + if asynchronous: + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + async with await client.chat.stream_async( + model="mistral-test", + messages=[{"role": "user", "content": INPUT}], + ) as stream: + return [chunk async for chunk in stream] + + chunks = asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic", client=transport) + with client.chat.stream( + model="mistral-test", messages=[{"role": "user", "content": INPUT}] + ) as stream: + chunks = list(stream) + assert OUTPUT in chunks[0].model_dump_json() + (span,) = exporter.get_finished_spans() + assert span.status.status_code == StatusCode.OK + assert span.attributes["gen_ai.usage.input_tokens"] == 2 + assert (OUTPUT in span.to_json()) == (not hide), dict(span.attributes) diff --git a/python/frameworks/mistralai/tests/test_sdk.py b/python/frameworks/mistralai/tests/test_sdk.py new file mode 100644 index 000000000..96f79dc37 --- /dev/null +++ b/python/frameworks/mistralai/tests/test_sdk.py @@ -0,0 +1,536 @@ +"""Real Mistral SDK requests with synthetic in-process HTTP/SSE transports.""" + +import asyncio +import importlib +import json +from importlib.metadata import version + +import pytest +from fi_instrumentation import TraceConfig, suppress_tracing +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +from traceai_mistralai import MistralAIInstrumentor + +NATIVE = int(version("mistralai").split(".")[0]) >= 3 +SDK_ROOT = "mistralai.client" if NATIVE else "mistralai" +Mistral = importlib.import_module(SDK_ROOT).Mistral +httpx = importlib.import_module("httpx2" if NATIVE else "httpx") +MESSAGES = [{"role": "user", "content": "private-input-marker"}] +USAGE = {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3} + + +def chat_response(): + return { + "id": "synthetic", + "object": "chat.completion", + "created": 1, + "model": "mistral-test", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "private-output-marker", + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": { + "name": "weather", + "arguments": '{"city":"test"}', + }, + } + ], + }, + "finish_reason": "stop", + } + ], + "usage": USAGE, + } + + +def embedding_response(): + return { + "id": "synthetic", + "object": "list", + "model": "mistral-embed", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "usage": {"prompt_tokens": 2, "completion_tokens": 0, "total_tokens": 2}, + } + + +def conversation_response(): + return { + "object": "conversation.response", + "conversation_id": "conv-test", + "outputs": [ + { + "type": "message.output", + "role": "assistant", + "content": "private-output-marker", + } + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + } + + +def completion_sse(): + for delta, finish in [ + ({"role": "assistant", "content": "private-output-marker"}, None), + ( + { + "content": "", + "tool_calls": [ + { + "index": 0, + "id": "call-1", + "type": "function", + "function": {"name": "weather", "arguments": '{"city":"test"}'}, + } + ], + }, + "stop", + ), + ]: + yield ( + "data: " + + json.dumps( + { + "id": "synthetic", + "object": "chat.completion.chunk", + "created": 1, + "model": "mistral-test", + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], + "usage": USAGE if finish else None, + } + ) + + "\n\n" + ).encode() + yield b"data: [DONE]\n\n" + + +class SSE(httpx.SyncByteStream, httpx.AsyncByteStream): + def __init__(self, fail=False): + self.fail = fail + self.closed = False + + def __iter__(self): + for chunk in completion_sse(): + yield chunk + if self.fail: + raise RuntimeError("private-stream-error-marker") + + async def __aiter__(self): + for chunk in self: + yield chunk + + def close(self): + self.closed = True + + async def aclose(self): + self.closed = True + + +def conversation_sse(error=False): + events = [ + {"type": "conversation.response.started", "conversation_id": "conv-test"}, + { + "type": "message.output.delta", + "id": "entry-test", + "role": "assistant", + "content": "private-output-marker", + }, + {"type": "conversation.response.done", "usage": USAGE}, + ] + if error: + events[-1] = { + "type": "conversation.response.error", + "message": "private-error-marker", + "code": 400, + } + return [ + f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() + for event in events + ] + + +class ConversationSSE(SSE): + def __iter__(self): + yield from conversation_sse(self.fail) + + +def transport_response(request): + if request.url.path.endswith("embeddings"): + return httpx.Response(200, json=embedding_response()) + if "conversations" in request.url.path: + if json.loads(request.content).get("stream"): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=ConversationSSE(), + ) + return httpx.Response(200, json=conversation_response()) + if json.loads(request.content).get("stream"): + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=SSE() + ) + return httpx.Response(200, json=chat_response()) + + +@pytest.fixture +def tracing(request): + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + config = getattr(request, "param", TraceConfig()) + instrumentor.instrument(tracer_provider=provider, config=config) + assert instrumentor.is_instrumented_by_opentelemetry + yield exporter + instrumentor.uninstrument() + provider.shutdown() + + +def params(resource): + return { + "messages": MESSAGES, + **( + {"model": "mistral-test"} + if resource == "chat" + else {"agent_id": "test-agent"} + ), + } + + +@pytest.mark.parametrize("resource", ["chat", "agents"]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_completions(tracing, resource, streaming, asynchronous): + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(transport_response) + ) as transport: + client = Mistral(api_key="synthetic-test-key", async_client=transport) + method = getattr( + getattr(client, resource), + "stream_async" if streaming else "complete_async", + ) + result = await method(**params(resource)) + if streaming: + chunks = [chunk async for chunk in result] + if hasattr(result, "close"): + await result.close() + return chunks[0].data.choices[0].delta.content + return result.choices[0].message.content + + if asynchronous: + text = asyncio.run(run()) + else: + with httpx.Client( + transport=httpx.MockTransport(transport_response) + ) as transport: + client = Mistral(api_key="synthetic-test-key", client=transport) + result = getattr( + getattr(client, resource), "stream" if streaming else "complete" + )(**params(resource)) + if streaming: + chunks = list(result) + result.close() + text = chunks[0].data.choices[0].delta.content + else: + text = result.choices[0].message.content + assert text == "private-output-marker" + spans = tracing.get_finished_spans() + assert len(spans) == 1 + assert spans[0].attributes["gen_ai.usage.input_tokens"] == 2 + assert spans[0].attributes["gen_ai.usage.output_tokens"] == 1 + assert "weather" in json.dumps(dict(spans[0].attributes)) + assert spans[0].status.status_code == StatusCode.OK + + +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_embeddings(tracing, asynchronous): + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(transport_response) + ) as transport: + return await Mistral( + api_key="synthetic", async_client=transport + ).embeddings.create_async(model="mistral-embed", inputs=["hi"]) + + if asynchronous: + result = asyncio.run(run()) + else: + with httpx.Client( + transport=httpx.MockTransport(transport_response) + ) as transport: + result = Mistral(api_key="synthetic", client=transport).embeddings.create( + model="mistral-embed", inputs=["hi"] + ) + assert result.data[0].embedding == [0.1, 0.2] + spans = tracing.get_finished_spans() + assert len(spans) == 1 + assert spans[0].attributes["gen_ai.span.kind"] == "EMBEDDING" + assert spans[0].attributes["gen_ai.usage.input_tokens"] == 2 + + +@pytest.mark.skipif( + not NATIVE, reason="Native conversation telemetry is supported on SDK >=3.6" +) +@pytest.mark.parametrize("method", ["start", "append", "restart"]) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +def test_conversations(tracing, method, asynchronous, streaming): + kwargs = {"inputs": "private-input-marker"} + if method == "start": + kwargs["model"] = "mistral-test" + else: + kwargs["conversation_id"] = "conv-test" + if method == "restart": + kwargs["from_entry_id"] = "entry-test" + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(transport_response) + ) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + result = await getattr( + client.beta.conversations, + method + ("_stream" if streaming else "") + "_async", + )(**kwargs) + if streaming: + chunks = [chunk async for chunk in result] + await result.close() + assert chunks[-1].data.usage.prompt_tokens == 2 + return chunks[0].data.conversation_id + return result.conversation_id + + if asynchronous: + result = asyncio.run(run()) + else: + with httpx.Client( + transport=httpx.MockTransport(transport_response) + ) as transport: + client = Mistral(api_key="synthetic", client=transport) + result = getattr( + client.beta.conversations, method + ("_stream" if streaming else "") + )(**kwargs) + if streaming: + chunks = list(result) + result.close() + assert chunks[-1].data.usage.prompt_tokens == 2 + result = chunks[0].data.conversation_id + else: + result = result.conversation_id + assert result == "conv-test" + spans = tracing.get_finished_spans() + assert len(spans) == 1 + assert spans[0].attributes["gen_ai.span.kind"] == "AGENT" + assert spans[0].attributes["gen_ai.usage.input_tokens"] == 2 + + +def test_existing_clients_suppression_reenable_and_parent(): + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + with httpx.Client(transport=httpx.MockTransport(transport_response)) as transport: + client = Mistral(api_key="synthetic", client=transport) + original = type(client.chat).complete + instrumentor = MistralAIInstrumentor() + for _ in range(2): + exporter.clear() + instrumentor.instrument(tracer_provider=provider) + instrumentor.instrument(tracer_provider=provider) + if NATIVE: + # A lifecycle observer now holds native spans through decoding; + # it delegates to the original method without creating spans. + assert type(client.chat).complete.__wrapped__ is original + with provider.get_tracer("parent").start_as_current_span( + "parent" + ) as parent: + with suppress_tracing(): + client.chat.complete(**params("chat")) + client.chat.complete(**params("chat")) + spans = exporter.get_finished_spans() + assert len(spans) == 2 + assert spans[0].parent.span_id == parent.get_span_context().span_id + instrumentor.uninstrument() + assert type(client.chat).complete is original + exporter.clear() + client.chat.complete(**params("chat")) + assert not exporter.get_finished_spans() + provider.shutdown() + + +@pytest.mark.parametrize( + "tracing", [TraceConfig(hide_inputs=True, hide_outputs=True)], indirect=True +) +@pytest.mark.parametrize("error", [False, True]) +def test_privacy_and_errors(tracing, error): + def respond(request): + if error: + return httpx.Response(400, json={"message": "private-error-marker"}) + return transport_response(request) + + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-key-marker", client=transport) + if error: + with pytest.raises(Exception, match="private-error-marker"): + client.chat.complete(**params("chat")) + else: + import inspect + + options = ( + {"http_headers": {"x-private": "private-header-marker"}} + if "http_headers" in inspect.signature(client.chat.complete).parameters + else {} + ) + client.chat.complete(**params("chat"), **options) + spans = tracing.get_finished_spans() + assert len(spans) == 1 + encoded = spans[0].to_json() + for marker in ( + "private-input-marker", + "private-output-marker", + "private-error-marker", + "private-header-marker", + "synthetic-key-marker", + ): + assert marker not in encoded + assert spans[0].status.status_code == (StatusCode.ERROR if error else StatusCode.OK) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("failure", [False, True]) +def test_stream_early_close_and_errors(tracing, asynchronous, failure): + stream = SSE(fail=failure) + + def respond(request): + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=stream + ) + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + result = await Mistral( + api_key="synthetic", async_client=transport + ).chat.stream_async(**params("chat")) + assert not tracing.get_finished_spans() + async with result: + if failure: + with pytest.raises( + RuntimeError, match="private-stream-error-marker" + ): + async for _ in result: + pass + else: + await result.__anext__() + + if asynchronous: + asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + result = Mistral(api_key="synthetic", client=transport).chat.stream( + **params("chat") + ) + assert not tracing.get_finished_spans() + with result: + if failure: + with pytest.raises( + RuntimeError, match="private-stream-error-marker" + ): + list(result) + else: + next(result) + spans = tracing.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code == ( + StatusCode.ERROR if failure else StatusCode.OK + ) + + +@pytest.mark.skipif( + not NATIVE, reason="Native conversation telemetry requires SDK >=3.6" +) +def test_native_conversation_error_event(tracing): + def respond(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=ConversationSSE(fail=True), + ) + + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic", client=transport) + with client.beta.conversations.start_stream( + model="mistral-test", inputs="hello" + ) as result: + chunks = list(result) + assert chunks[-1].data.type == "conversation.response.error" + spans = tracing.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code == StatusCode.ERROR + + +def test_concurrent_async_calls(tracing): + async def run(): + async with httpx.AsyncClient(transport=httpx.MockTransport(transport_response)) as transport: + client = Mistral(api_key="synthetic", async_client=transport) + results = await asyncio.gather(*(client.chat.complete_async(**params("chat")) for _ in range(24))) + assert all(result.choices[0].message.content == "private-output-marker" for result in results) + asyncio.run(run()) + spans = tracing.get_finished_spans() + assert len(spans) == 24 + assert len({span.context.span_id for span in spans}) == 24 + + +@pytest.mark.skipif(not NATIVE, reason="Native provider binding requires SDK >=3.6") +def test_restore_existing_native_provider(): + from mistralai.extra.observability import configure_telemetry + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + other = TracerProvider() + other_exporter = InMemorySpanExporter() + other.add_span_processor(SimpleSpanProcessor(other_exporter)) + with httpx.Client(transport=httpx.MockTransport(transport_response)) as transport: + client = Mistral(api_key="synthetic", client=transport) + configure_telemetry(client, provider=other, redaction=False) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + client.chat.complete(**params("chat")) + assert len(exporter.get_finished_spans()) == 1 + assert not other_exporter.get_finished_spans() + finally: + instrumentor.uninstrument() + client.chat.complete(**params("chat")) + assert len(other_exporter.get_finished_spans()) == 1 + assert len(exporter.get_finished_spans()) == 1 + provider.shutdown() + other.shutdown() + + +def test_sampler_failure_does_not_change_response(caplog): + from opentelemetry.sdk.trace.sampling import Sampler + class BrokenSampler(Sampler): + def should_sample(self, *args, **kwargs): + raise RuntimeError("synthetic telemetry failure") + def get_description(self): + return "broken-test-sampler" + provider = TracerProvider(sampler=BrokenSampler()) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + with httpx.Client(transport=httpx.MockTransport(transport_response)) as transport: + result = Mistral(api_key="synthetic", client=transport).chat.complete(**params("chat")) + assert result.choices[0].message.content == "private-output-marker" + assert caplog.records + finally: + instrumentor.uninstrument() + provider.shutdown() diff --git a/python/frameworks/mistralai/tests/test_wire.py b/python/frameworks/mistralai/tests/test_wire.py new file mode 100644 index 000000000..3fe002573 --- /dev/null +++ b/python/frameworks/mistralai/tests/test_wire.py @@ -0,0 +1,205 @@ +"""Export real SDK spans to a loopback OTLP receiver in both wire encodings.""" + +import gzip +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.request import Request, urlopen + +import pytest +from fi_instrumentation import TraceConfig +from google.protobuf.json_format import MessageToDict +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.exporter.otlp.proto.http import Compression +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, +) +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ( + SimpleSpanProcessor, + SpanExporter, + SpanExportResult, +) + +from traceai_mistralai import MistralAIInstrumentor +from test_sdk import Mistral, httpx, params, transport_response + + +def otlp_json(message): + data = MessageToDict(message) + # OTLP/JSON uses hex IDs, rather than protobuf JSON's base64 bytes. + for resources, encoded_resources in zip( + message.resource_spans, data["resourceSpans"] + ): + for scope, encoded_scope in zip( + resources.scope_spans, encoded_resources["scopeSpans"] + ): + for span, encoded in zip(scope.spans, encoded_scope["spans"]): + encoded["traceId"] = span.trace_id.hex() + encoded["spanId"] = span.span_id.hex() + if span.parent_span_id: + encoded["parentSpanId"] = span.parent_span_id.hex() + return data + + +class JSONExporter(SpanExporter): + """Test-only JSON transport; customers use the standard OTLP exporter.""" + + def __init__(self, endpoint, compressed): + self.endpoint, self.compressed = endpoint, compressed + + def export(self, spans): + body = json.dumps(otlp_json(encode_spans(spans))).encode() + headers = {"Content-Type": "application/json"} + if self.compressed: + body = gzip.compress(body) + headers["Content-Encoding"] = "gzip" + with urlopen( + Request(self.endpoint, data=body, headers=headers), timeout=5 + ) as response: + assert response.status == 200 + return SpanExportResult.SUCCESS + + +@pytest.fixture +def receiver(): + requests = [] + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + if self.headers.get("Content-Encoding") == "gzip": + body = gzip.decompress(body) + if self.headers.get("Content-Type") == "application/json": + data = json.loads(body) + else: + data = otlp_json(ExportTraceServiceRequest.FromString(body)) + requests.append((self.path, data)) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *_): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + yield f"http://127.0.0.1:{server.server_port}/v1/traces", requests + finally: + server.shutdown() + worker.join() + server.server_close() + + +@pytest.mark.parametrize("encoding", ["json", "protobuf"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize("private", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +def test_otlp_wire(receiver, encoding, compressed, private, streaming): + endpoint, requests = receiver + exporter = ( + JSONExporter(endpoint, compressed) + if encoding == "json" + else OTLPSpanExporter( + endpoint=endpoint, + headers={}, + timeout=5, + compression=Compression.Gzip if compressed else Compression.NoCompression, + ) + ) + provider = TracerProvider( + resource=Resource.create({"project_name": "mistral-offline-test"}) + ) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument( + tracer_provider=provider, + config=TraceConfig(hide_inputs=private, hide_outputs=private), + ) + try: + with httpx.Client( + transport=httpx.MockTransport(transport_response) + ) as transport: + client = Mistral(api_key="synthetic-key-marker", client=transport) + with provider.get_tracer("test-parent").start_as_current_span("parent"): + if streaming: + with client.chat.stream(**params("chat")) as stream: + list(stream) + else: + client.chat.complete(**params("chat")) + provider.force_flush() + finally: + instrumentor.uninstrument() + provider.shutdown() + spans = [ + span + for path, data in requests + for resource in data["resourceSpans"] + for scope in resource["scopeSpans"] + for span in scope["spans"] + ] + assert len(spans) == 2 + child, parent = spans + assert child["traceId"] == parent["traceId"] + assert child["parentSpanId"] == parent["spanId"] + assert len(child["traceId"]) == 32 + assert len(child["spanId"]) == 16 + assert all(path == "/v1/traces" for path, _ in requests) + attrs = {a["key"]: a["value"] for a in child["attributes"]} + assert attrs["gen_ai.span.kind"]["stringValue"] == "LLM" + assert int(attrs["gen_ai.usage.input_tokens"]["intValue"]) == 2 + wire = json.dumps(requests) + assert "synthetic-key-marker" not in wire + if private: + assert "private-input-marker" not in wire + assert "private-output-marker" not in wire + else: + assert "private-input-marker" in wire + assert "private-output-marker" in wire + + +def test_readme_example(receiver, monkeypatch): + from pathlib import Path + import re + from test_sdk import NATIVE + + if not NATIVE: + pytest.skip("The quickstart targets the current SDK") + endpoint, requests = receiver + monkeypatch.setenv("FI_BASE_URL", endpoint.rsplit("/v1/traces", 1)[0]) + monkeypatch.setenv("FI_API_KEY", "synthetic-fi-key") + monkeypatch.setenv("FI_SECRET_KEY", "synthetic-fi-secret") + monkeypatch.setenv("MISTRAL_API_KEY", "synthetic-mistral-key") + original_init = Mistral.__init__ + + def init(self, *args, **kwargs): + kwargs["client"] = httpx.Client( + transport=httpx.MockTransport(transport_response) + ) + original_init(self, *args, **kwargs) + + monkeypatch.setattr(Mistral, "__init__", init) + readme = (Path(__file__).parents[1] / "README.md").read_text() + snippet = re.findall(r"```python\n(.*?)```", readme, re.S)[0] + try: + exec(compile(snippet, "README.md", "exec"), {}) + finally: + MistralAIInstrumentor().uninstrument() + spans = [ + span + for _, data in requests + for resource in data["resourceSpans"] + for scope in resource["scopeSpans"] + for span in scope["spans"] + ] + assert len(spans) == 1 + assert requests[0][0] == "/tracer/v1/traces" + encoded = json.dumps(requests) + assert "mistral-app" in encoded + assert "gen_ai.usage.input_tokens" in encoded + assert "synthetic-fi-secret" not in encoded + assert "synthetic-mistral-key" not in encoded diff --git a/python/frameworks/mistralai/traceai_mistralai/__init__.py b/python/frameworks/mistralai/traceai_mistralai/__init__.py index fbf13af1c..c72b6139f 100644 --- a/python/frameworks/mistralai/traceai_mistralai/__init__.py +++ b/python/frameworks/mistralai/traceai_mistralai/__init__.py @@ -4,11 +4,9 @@ from fi_instrumentation import FITracer, TraceConfig from opentelemetry import trace as trace_api from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore -from traceai_mistralai._chat_wrapper import ( - _AsyncChatWrapper, - _AsyncStreamChatWrapper, - _SyncChatWrapper, -) +from importlib.metadata import version as distribution_version + +from packaging.version import Version from traceai_mistralai.package import _instruments from traceai_mistralai.version import __version__ from wrapt import wrap_function_wrapper @@ -26,6 +24,8 @@ class MistralAIInstrumentor(BaseInstrumentor): # type: ignore __slots__ = ( "_tracer", + "_native", + "_embedding_methods", "_original_sync_chat_method", "_original_sync_stream_chat_method", "_original_async_chat_method", @@ -34,6 +34,7 @@ class MistralAIInstrumentor(BaseInstrumentor): # type: ignore "_original_sync_stream_agent_method", "_original_async_agent_method", "_original_async_stream_agent_method", + "_original_parse_methods", ) def instrumentation_dependencies(self) -> Collection[str]: @@ -51,6 +52,33 @@ def _instrument(self, **kwargs: Any) -> None: config=config, ) + sdk_version = Version(distribution_version("mistralai")) + if not ( + Version("1.1.0") <= sdk_version < Version("2") + or Version("3.6.0") <= sdk_version < Version("4") + ): + raise ValueError( + "traceAI-mistralai supports mistralai >=1.1,<2 or >=3.6,<4" + ) + self._native = None + if sdk_version >= Version("3.6.0"): + from traceai_mistralai._native import _NativeInstrumentation + + self._native = _NativeInstrumentation(tracer_provider, config) + return + + from traceai_mistralai._chat_wrapper import ( + _AsyncChatWrapper, + _AsyncStreamChatWrapper, + _SyncChatWrapper, + _sync_parse, + _async_parse, + ) + from traceai_mistralai._embeddings import ( + _AsyncEmbeddingWrapper, + _EmbeddingWrapper, + ) + try: import mistralai from mistralai.agents import Agents @@ -60,6 +88,11 @@ def _instrument(self, **kwargs: Any) -> None: "Could not import mistralai. Please install with `pip install mistralai`." ) from err + self._original_parse_methods = () + if hasattr(Chat, "parse") and hasattr(Chat, "parse_async"): + self._original_parse_methods = (Chat.parse, Chat.parse_async) + wrap_function_wrapper(module="mistralai.chat", name="Chat.parse", wrapper=_sync_parse) + wrap_function_wrapper(module="mistralai.chat", name="Chat.parse_async", wrapper=_async_parse) self._original_sync_chat_method = Chat.complete self._original_sync_stream_chat_method = Chat.stream self._original_async_chat_method = Chat.complete_async @@ -124,10 +157,42 @@ def _instrument(self, **kwargs: Any) -> None: ), ) + from mistralai.embeddings import Embeddings + from traceai_mistralai._privacy import _EmbeddingTraceConfig + + embedding_tracer = FITracer( + trace_api.get_tracer(__name__, __version__, tracer_provider), + config=_EmbeddingTraceConfig(config), + ) + self._embedding_methods = (Embeddings.create, Embeddings.create_async) + wrap_function_wrapper( + module="mistralai.embeddings", + name="Embeddings.create", + wrapper=_EmbeddingWrapper( + "MistralClient.embeddings", embedding_tracer, mistralai + ), + ) + wrap_function_wrapper( + module="mistralai.embeddings", + name="Embeddings.create_async", + wrapper=_AsyncEmbeddingWrapper( + "MistralAsyncClient.embeddings", embedding_tracer, mistralai + ), + ) + def _uninstrument(self, **kwargs: Any) -> None: + if self._native is not None: + self._native.uninstrument() + self._native = None + return + from mistralai.embeddings import Embeddings + + Embeddings.create, Embeddings.create_async = self._embedding_methods from mistralai.agents import Agents from mistralai.chat import Chat + if self._original_parse_methods: + Chat.parse, Chat.parse_async = self._original_parse_methods Chat.complete = self._original_sync_chat_method # type: ignore Chat.stream = self._original_sync_stream_chat_method # type: ignore Chat.complete_async = self._original_async_chat_method # type: ignore diff --git a/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py b/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py index 8198eec32..36978a3f9 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py +++ b/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py @@ -1,8 +1,9 @@ +import asyncio import json import logging -import warnings from abc import ABC from contextlib import contextmanager +from contextvars import ContextVar from inspect import Signature, signature from types import ModuleType from typing import ( @@ -18,11 +19,10 @@ from fi_instrumentation import get_attributes_from_context, safe_json_dumps from fi_instrumentation.fi_types import ( - FiMimeTypeValues, FiSpanKindValues, SpanAttributes, ) -from mistralai.utils.eventstreaming import EventStream, EventStreamAsync +from collections.abc import AsyncIterator from opentelemetry import context as context_api from opentelemetry import trace as trace_api from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY @@ -37,8 +37,6 @@ from traceai_mistralai._stream import _AsyncStream, _Stream from traceai_mistralai._utils import ( _finish_tracing, - _process_input_messages, - _raw_input, ) from traceai_mistralai._with_span import _WithSpan @@ -47,6 +45,58 @@ __all__ = ("_SyncChatWrapper",) +# Structured parse runs a second validation step after complete returns. +# Hold only that request's existing span until the outer parse completes. +_parse_pending = ContextVar("mistral_parse_pending", default=None) + + +def _finish_parse(pending, error=None): + for with_span, response, request_parameters, extractor in pending: + if error is not None: + with_span.record_exception(error) + _finish_tracing( + with_span=with_span, + has_attributes=_ResponseAttributes(response, request_parameters, extractor), + status=trace_api.Status( + status_code=( + trace_api.StatusCode.ERROR + if error is not None + else trace_api.StatusCode.OK + ) + ), + ) + + +def _sync_parse(wrapped, instance, args, kwargs): + pending = [] + token = _parse_pending.set(pending) + try: + result = wrapped(*args, **kwargs) + except Exception as error: + _finish_parse(pending, error) + raise + else: + _finish_parse(pending) + return result + finally: + _parse_pending.reset(token) + + +async def _async_parse(wrapped, instance, args, kwargs): + pending = [] + token = _parse_pending.set(pending) + try: + result = await wrapped(*args, **kwargs) + except (Exception, asyncio.CancelledError) as error: + _finish_parse(pending, error) + raise + else: + _finish_parse(pending) + return result + finally: + _parse_pending.reset(token) + + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) @@ -107,15 +157,8 @@ def _get_attributes_from_request( request_parameters: Dict[str, Any], ) -> Iterator[Tuple[str, AttributeValue]]: yield SpanAttributes.GEN_AI_SPAN_KIND, self._get_span_kind() - try: - yield from _process_input_messages(request_parameters.get("messages", [])) - yield from _raw_input(request_parameters) - # yield from _as_input_attributes(_io_value_and_type(request_parameters)) - except Exception: - logger.exception( - f"Failed to get input attributes from request parameters of " - f"type {type(request_parameters)}" - ) + # Messages already have policy-aware semantic attributes. Raw payload + # copies bypass selective text/message/image and parameter privacy. def _get_extra_attributes_from_request( self, @@ -148,6 +191,14 @@ def _parse_args( bound_arguments = bound_signature.arguments request_data: Dict[str, Any] = {} for key, value in bound_arguments.items(): + if key in { + "http_headers", + "server_url", + "retries", + "timeout_ms", + } or isinstance(value, (Iterator, AsyncIterator)): + # Telemetry must not consume a one-shot message/tool iterable. + continue try: if value is not None: try: @@ -177,18 +228,12 @@ def _finalize_response( chat_completion_type=CompletionEvent, ) - if isinstance(response, EventStreamAsync): + if isinstance(response, AsyncIterator): return _AsyncStream( stream=response, with_span=with_span, response_accumulator=response_accumulator, ) - elif isinstance(response, EventStream): - return _Stream( - stream=response, - with_span=with_span, - response_accumulator=response_accumulator, - ) elif isinstance(response, Iterator): return _Stream( stream=response, @@ -197,6 +242,16 @@ def _finalize_response( ) else: raise TypeError("Response must be an asynchronous iterator") + if (pending := _parse_pending.get()) is not None: + pending.append( + ( + with_span, + response, + request_parameters, + self._response_attributes_extractor, + ) + ) + return response _finish_tracing( status=trace_api.Status(status_code=trace_api.StatusCode.OK), with_span=with_span, @@ -295,7 +350,7 @@ async def __call__( ) as with_span: try: response = await wrapped(*args, **kwargs) - except Exception as exception: + except (Exception, asyncio.CancelledError) as exception: with_span.record_exception(exception) status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, @@ -350,7 +405,7 @@ async def __call__( ) as with_span: try: response = await wrapped(*args, **kwargs) - except Exception as exception: + except (Exception, asyncio.CancelledError) as exception: with_span.record_exception(exception) status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, @@ -390,26 +445,9 @@ def __init__( self._response_attributes_extractor = response_attributes_extractor def get_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: - if hasattr(self._response, "model_dump_json") and callable( - self._response.model_dump_json - ): - try: - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - value = self._response.model_dump_json(exclude_unset=True) - assert isinstance(value, str) - output_value = ( - json.loads(value) - .get("choices", [{}])[0] - .get("message", {}) - .get("content", "") - ) - - yield SpanAttributes.OUTPUT_VALUE, value - yield SpanAttributes.OUTPUT_VALUE, output_value - yield SpanAttributes.OUTPUT_MIME_TYPE, FiMimeTypeValues.JSON.value - except Exception: - logger.exception("Failed to get model dump json") + # Export output only through the semantic message attributes below. + # Generic output.value is not covered by selective message privacy. + yield from () def get_extra_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: yield from self._response_attributes_extractor.get_attributes_from_response( diff --git a/python/frameworks/mistralai/traceai_mistralai/_embeddings.py b/python/frameworks/mistralai/traceai_mistralai/_embeddings.py new file mode 100644 index 000000000..7aef33b3d --- /dev/null +++ b/python/frameworks/mistralai/traceai_mistralai/_embeddings.py @@ -0,0 +1,65 @@ +"""Embedding support for the legacy SDK, which has no native OTel hooks.""" + +from fi_instrumentation import safe_json_dumps +from fi_instrumentation.fi_types import ( + EmbeddingAttributes, + FiSpanKindValues, + SpanAttributes, +) +from opentelemetry import trace + +from ._chat_wrapper import _AsyncChatWrapper, _SyncChatWrapper + + +class _EmbeddingResponse: + def _get_span_kind(self): + return FiSpanKindValues.EMBEDDING.value + + def _get_attributes_from_request(self, request_parameters): + yield SpanAttributes.GEN_AI_SPAN_KIND, self._get_span_kind() + inputs = request_parameters.get("inputs", []) + if isinstance(inputs, str): + inputs = [inputs] + if isinstance(inputs, (list, tuple)): + for index, text in enumerate(inputs): + if isinstance(text, str): + yield ( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{index}." + f"{EmbeddingAttributes.EMBEDDING_TEXT}", + text, + ) + + def _get_extra_attributes_from_request(self, request_parameters): + # Text belongs only in the embedding attributes, where all three input + # privacy flags apply. Do not duplicate it in input.value or parameters. + yield ( + SpanAttributes.GEN_AI_REQUEST_PARAMETERS, + safe_json_dumps( + { + key: value + for key, value in request_parameters.items() + if key != "inputs" + } + ), + ) + + def _finalize_response(self, response, with_span, request_parameters): + # Never copy vectors into the generic output field: that would bypass + # FI_HIDE_EMBEDDING_VECTORS and create very large attributes. + attributes = dict( + self._response_attributes_extractor.get_attributes_from_response( + response, request_parameters + ) + ) + with_span.finish_tracing( + status=trace.Status(trace.StatusCode.OK), attributes=attributes + ) + return response + + +class _EmbeddingWrapper(_EmbeddingResponse, _SyncChatWrapper): + pass + + +class _AsyncEmbeddingWrapper(_EmbeddingResponse, _AsyncChatWrapper): + pass diff --git a/python/frameworks/mistralai/traceai_mistralai/_native.py b/python/frameworks/mistralai/traceai_mistralai/_native.py new file mode 100644 index 000000000..d6d19ee9b --- /dev/null +++ b/python/frameworks/mistralai/traceai_mistralai/_native.py @@ -0,0 +1,422 @@ +"""Route the SDK's own spans through FITracer; never create a second SDK span.""" + +import asyncio +import json +import logging +import re +from contextlib import ExitStack, contextmanager +from contextvars import ContextVar +from functools import wraps +from threading import RLock +from typing import Any +from weakref import WeakKeyDictionary, ref + +from fi_instrumentation import FITracer, TraceConfig +from mistralai.extra.observability.formatting import format_output_message +from opentelemetry import trace +from wrapt import ObjectProxy, wrap_function_wrapper + +from traceai_mistralai._privacy import _NativeTraceConfig + +logger = logging.getLogger(__name__) +# Fail-open telemetry diagnostics deliberately omit payload-bearing exceptions. + + +class _NativeSpan(ObjectProxy): + """Defer success until end so an SSE read error can still mark ERROR.""" + + def __init__(self, span): + super().__init__(span) + self._self_status = None + self._self_iteration_depth = 0 + self._self_pending_end = None + + def set_status(self, status, description=None): + if ( + self._self_status is not None + and self._self_status.status_code == trace.StatusCode.ERROR + ): + return + self._self_status = ( + status + if isinstance(status, trace.Status) + else trace.Status(status, description) + ) + + @contextmanager + def observe_iteration(self): + # HTTPX can close at EOF before the SDK decodes its buffered last event. + # Delay only the end, so decoding failures still update this same span. + self._self_iteration_depth += 1 + try: + yield + except (StopIteration, StopAsyncIteration, GeneratorExit): + raise + except BaseException as error: + _record_error(self, error) + self.end() + raise + finally: + self._self_iteration_depth -= 1 + if self._self_iteration_depth == 0 and self._self_pending_end is not None: + args, kwargs = self._self_pending_end + self._self_pending_end = None + self.end(*args, **kwargs) + + def end(self, *args, **kwargs): + if self._self_iteration_depth: + self._self_pending_end = (args, kwargs) + return + if self._self_status is not None: + self.__wrapped__.set_status(self._self_status) + return self.__wrapped__.end(*args, **kwargs) + + +def _record_error(span, error): + if isinstance(error, GeneratorExit): + return + try: + if ( + span._self_status is None + or span._self_status.status_code != trace.StatusCode.ERROR + ): + span.record_exception(error) + span.set_status(trace.Status(trace.StatusCode.ERROR)) + except Exception: + logger.debug("Could not record Mistral stream error") + + +class _ErrorStream(ObjectProxy): + """Observe read failures before HTTPX closes and ends the native span.""" + + def __init__(self, stream, span): + super().__init__(stream) + self._self_span = span + + def __iter__(self): + try: + yield from self.__wrapped__ + except BaseException as error: + _record_error(self._self_span, error) + raise + + async def __aiter__(self): + try: + async for chunk in self.__wrapped__: + yield chunk + except BaseException as error: + _record_error(self._self_span, error) + raise + + +class _NativeTracer(FITracer): + def start_span(self, name: str, *args: Any, **kwargs: Any) -> _NativeSpan: + if "embeddings" in name: + kind = "EMBEDDING" + elif "conversations" in name or name.startswith("invoke_agent"): + kind = "AGENT" + elif name.startswith("execute_tool"): + kind = "TOOL" + elif any( + part in name for part in ("chat", "completion", "fim", "stream_agents") + ): + kind = "LLM" + else: + kind = "CHAIN" + kwargs.setdefault("fi_span_kind", kind.lower()) + return _NativeSpan(super().start_span(name, *args, **kwargs)) + + +class _NativeProvider(trace.TracerProvider): + def __init__(self, provider: trace.TracerProvider, config: TraceConfig) -> None: + self._provider = provider + self._config = _NativeTraceConfig(config) + + def get_tracer(self, *args: Any, **kwargs: Any) -> _NativeTracer: + return _NativeTracer(self._provider.get_tracer(*args, **kwargs), self._config) + + +class _NativeInstrumentation: + """Adapt the native hook for both existing and subsequently created clients. + + The native hook owns request context and span creation. The adapter routes + those spans and fills HTTP-error and conversation-stream gaps. Do not patch + chat or agent methods to create spans alongside this hook. Method observers + below only hold the existing span open through SDK response validation. + """ + + _FIELDS = ( + "tracer_provider", + "tracer", + "tracing_enabled", + "_telemetry_auto_disabled", + ) + + def __init__(self, provider: trace.TracerProvider, config: TraceConfig) -> None: + from mistralai.client._hooks.tracing import TracingHook + + self._provider = _NativeProvider(provider, config) + self._hooks = WeakKeyDictionary() + self._async_sends = WeakKeyDictionary() + self._lock = RLock() + from mistralai.extra.observability.otel import TracedResponse + from mistralai.client.utils.eventstreaming import EventStream, EventStreamAsync + + self._response_scope = ContextVar("mistral_response_scope", default=None) + self._originals = [] + for cls, method, wrapper in ( + (TracingHook, "before_request", self._before_request), + (TracingHook, "after_error", self._after_error), + (TracingHook, "after_success", self._after_success), + (TracedResponse, "_finalize_span", self._finalize_span), + (EventStream, "__next__", self._stream_next), + (EventStreamAsync, "__anext__", self._stream_anext), + ): + self._originals.append((cls, method, getattr(cls, method))) + wrap_function_wrapper(cls, method, wrapper) + + # The native hook ends buffered responses before generated methods + # deserialize them. Observe those methods without creating any spans. + # A per-call context keeps concurrent sync/async requests independent. + from mistralai.client.agents import Agents + from mistralai.client.chat import Chat + from mistralai.client.conversations import Conversations + from mistralai.client.embeddings import Embeddings + from mistralai.client.fim import Fim + + for cls, methods in ( + (Chat, ("complete", "stream", "parse")), + (Agents, ("complete", "stream")), + (Embeddings, ("create",)), + (Fim, ("complete", "stream")), + ( + Conversations, + ( + "start", + "append", + "restart", + "start_stream", + "append_stream", + "restart_stream", + ), + ), + ): + for method in methods: + for name, wrapper in ( + (method, self._call), + (method + "_async", self._call_async), + ): + self._originals.append((cls, name, getattr(cls, name))) + wrap_function_wrapper(cls, name, wrapper) + + @contextmanager + def _call_scope(self): + # chat.parse calls chat.complete and parses its result afterward. Share + # the outer scope so the SDK-owned span stays open through validation. + if (existing := self._response_scope.get()) is not None: + yield + return + with ExitStack() as scope: + token = self._response_scope.set(scope) + try: + yield + finally: + self._response_scope.reset(token) + + def _call(self, wrapped, instance, args, kwargs): + with self._call_scope(): + return wrapped(*args, **kwargs) + + async def _call_async(self, wrapped, instance, args, kwargs): + with self._call_scope(): + return await wrapped(*args, **kwargs) + + def _before_request(self, wrapped, instance, args, kwargs): + with self._lock: + if ( + instance not in self._hooks + or instance.tracer_provider is not self._provider + ): + self._hooks[instance] = tuple( + getattr(instance, key) for key in self._FIELDS + ) + instance.tracer_provider = self._provider + # Do not let SDK auto-export configuration create a second destination. + instance._telemetry_auto_disabled = True + request = wrapped(*args, **kwargs) + hook_ctx = kwargs.get("hook_ctx", args[0] if args else None) + client = getattr(getattr(hook_ctx, "config", None), "async_client", None) + if client is not None: + self._wrap_async_send(client, instance) + return request + + def _wrap_async_send(self, client, hook): + # SDK 3.6 only catches Exception during send, not CancelledError. Keep + # its context management and finish its existing request span on cancel. + try: + with self._lock: + previous = self._async_sends.get(client) + if previous is not None and client.send is previous(): + return + original = client.send + + @wraps(original) + async def send(request, *args, **kwargs): + try: + return await original(request, *args, **kwargs) + except asyncio.CancelledError as error: + try: + if isinstance(hook._get_request_span(request), _NativeSpan): + hook._finish_request_span_with_error(request, error) + except Exception: + logger.debug("Could not finish cancelled Mistral request") + raise + + client.send = send + # A bound send method retains its client: weak-reference both + # sides so tracking does not keep closed application clients alive. + self._async_sends[client] = ref(send) + except Exception: + logger.debug("Could not wrap Mistral async send") + + def _after_error(self, wrapped, instance, args, kwargs): + response = kwargs.get("response", args[1] if len(args) > 1 else None) + if response is not None and response.status_code >= 400: + span = instance._get_span(response) + if isinstance(span, _NativeSpan): + span.set_status(trace.Status(trace.StatusCode.ERROR)) + span.set_attribute("http.response.status_code", response.status_code) + return wrapped(*args, **kwargs) + + def _after_success(self, wrapped, instance, args, kwargs): + response = kwargs.get("response", args[1] if len(args) > 1 else None) + span = instance._get_span(response) + scope = self._response_scope.get() + if isinstance(span, _NativeSpan) and scope is not None: + # Enter before the hook queues end(), leave after SDK validation. + scope.enter_context(span.observe_iteration()) + response = wrapped(*args, **kwargs) + span = getattr(response, "span", None) + if isinstance(span, _NativeSpan): + response.stream = _ErrorStream(response.stream, span) + return response + + def _stream_next(self, wrapped, instance, args, kwargs): + span = getattr(instance.response, "span", None) + if not isinstance(span, _NativeSpan): + return wrapped(*args, **kwargs) + with span.observe_iteration(): + return wrapped(*args, **kwargs) + + async def _stream_anext(self, wrapped, instance, args, kwargs): + span = getattr(instance.response, "span", None) + if not isinstance(span, _NativeSpan): + return await wrapped(*args, **kwargs) + with span.observe_iteration(): + return await wrapped(*args, **kwargs) + + def _finalize_span(self, wrapped, instance, args, kwargs): + span = instance.span + if ( + not isinstance(span, _NativeSpan) + or "conversations" not in instance.operation_id + ): + return wrapped(*args, **kwargs) + # SDK 3.6's native stream finalizer only decodes chat CompletionChunks. + # Adapt conversation SSE attributes on that same span, not a second span. + try: + parts = [] + tool_calls = {} + data_lines = [] + # SSE delimiters are CR/LF only. Unicode separators are valid JSON + # string content and must not split a message delta's data field. + payload = bytes(instance._accumulated_sse).decode("utf-8", errors="replace") + for line in re.split(r"\r\n|\r|\n", payload) + [""]: + if line.startswith("data:"): + data_lines.append(line[5:].lstrip()) + elif not line and data_lines: + raw, data_lines = "\n".join(data_lines), [] + try: + event = json.loads(raw) + except ValueError: + continue + if not isinstance(event, dict): + continue + event_type = event.get("type") + if event_type == "conversation.response.done": + usage = event.get("usage") or {} + if not isinstance(usage, dict): + continue + for source, target in ( + ("prompt_tokens", "input_tokens"), + ("completion_tokens", "output_tokens"), + ("total_tokens", "total_tokens"), + ): + if isinstance(usage.get(source), int): + span.set_attribute( + "gen_ai.usage." + target, usage[source] + ) + elif event_type == "message.output.delta": + content = event.get("content") + if isinstance(content, (str, dict)): + # Reuse the SDK's non-streaming OTel conversion for + # text, reasoning and image chunks, not provider keys. + message = { + "content": ( + [content] if isinstance(content, dict) else content + ) + } + parts.extend( + format_output_message({"message": message})["parts"] + ) + elif event_type == "function.call.delta": + # Deltas for different calls may be interleaved. Keep + # the first-seen output order and append each argument + # fragment to its call instead of emitting duplicates. + identity = ( + event.get("output_index"), + event.get("tool_call_id"), + ) + part = tool_calls.get(identity) + if part is None: + part = { + "type": "tool_call", + "id": event.get("tool_call_id"), + "name": event.get("name"), + "arguments": "", + } + tool_calls[identity] = part + parts.append(part) + if isinstance(event.get("arguments"), str): + part["arguments"] += event["arguments"] + elif event_type == "conversation.response.error": + span.set_status(trace.Status(trace.StatusCode.ERROR)) + if parts: + span.set_attribute( + "gen_ai.output.messages", + json.dumps([{"role": "assistant", "parts": parts}]), + ) + except Exception: + logger.debug("Could not extract Mistral conversation stream attributes") + finally: + try: + span.end() + except Exception: + logger.debug("Could not end Mistral conversation span") + instance.span = None + + def uninstrument(self) -> None: + for cls, method, original in reversed(self._originals): + setattr(cls, method, original) + with self._lock: + for client, send_ref in self._async_sends.items(): + send = send_ref() + if send is not None and client.send is send: + client.send = send.__wrapped__ + self._async_sends.clear() + for hook, values in self._hooks.items(): + # Respect an explicit customer reconfiguration made after ours. + if hook.tracer_provider is self._provider: + for key, value in zip(self._FIELDS, values): + setattr(hook, key, value) + self._hooks.clear() diff --git a/python/frameworks/mistralai/traceai_mistralai/_privacy.py b/python/frameworks/mistralai/traceai_mistralai/_privacy.py new file mode 100644 index 000000000..b1572f2ae --- /dev/null +++ b/python/frameworks/mistralai/traceai_mistralai/_privacy.py @@ -0,0 +1,156 @@ +"""Apply TraceConfig's field policy to the SDK's serialized OTel messages.""" + +import json +import logging +from typing import Any, Callable, Optional, Union +from urllib.parse import urlsplit, urlunsplit + +from fi_instrumentation import REDACTED_VALUE +from fi_instrumentation.fi_types import ( + EmbeddingAttributes, + ImageAttributes, + MessageAttributes, + MessageContentAttributes, + SpanAttributes, +) +from opentelemetry.util.types import AttributeValue +from wrapt import ObjectProxy + +logger = logging.getLogger(__name__) + + +class _EmbeddingTraceConfig(ObjectProxy): + """Use the input-text policy also on published cores predating its alias.""" + + def mask(self, key, value): + if key.startswith(SpanAttributes.EMBEDDING_EMBEDDINGS + ".") and key.endswith( + "." + EmbeddingAttributes.EMBEDDING_TEXT + ): + # Core 1.2.2 supports selective message privacy but not the embedding + # text alias yet. Preserve that policy without duplicating payloads. + key = f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_CONTENT}" + return self.__wrapped__.mask(key, value) + + +class _NativeTraceConfig(ObjectProxy): + """Keep native JSON spans, using the same policy as flattened FI messages. + + Adapt the config rather than only Span.set_attribute so creation attributes, + batched updates and SDK-created child spans all cross the same boundary. + """ + + def mask( + self, + key: str, + value: Union[AttributeValue, Callable[[], AttributeValue]], + ) -> Optional[AttributeValue]: + # Native OTel splits invocation parameters into individual attributes. + # Keep model identity, but apply the core's aggregate parameter policy + # to every other request parameter, including future SDK additions. + policy_key = ( + SpanAttributes.GEN_AI_REQUEST_PARAMETERS + if key.startswith("gen_ai.request.") and key != "gen_ai.request.model" + else key + ) + value = self.__wrapped__.mask(policy_key, value) + if key in ("http.url", "url.full") and isinstance(value, str): + # SDK HTTP metadata is independent of message-hiding flags. Never + # export userinfo, query parameters, or fragments: custom clients + # can attach credentials to the URL outside the message policy. + try: + url = urlsplit(value) + return urlunsplit( + url._replace( + netloc=url.netloc.rsplit("@", 1)[-1], + query="", + fragment="", + ) + ) + except ValueError: + return None + if ( + key + not in ( + SpanAttributes.GEN_AI_INPUT_MESSAGES, + SpanAttributes.GEN_AI_OUTPUT_MESSAGES, + ) + or value is None + or value == REDACTED_VALUE + ): + return value + # Native messages are JSON arrays. Never forward an opaque value whose + # contents cannot be checked against the configured privacy policy. + if not isinstance(value, str): + return None + try: + messages = json.loads(value) + except (ValueError, RecursionError): + # Parsing errors may contain the private payload: log no details. + logger.debug("Discarding malformed Mistral message attributes") + return None + if not isinstance(messages, list): + return None + for message_index, message in enumerate(messages): + if not isinstance(message, dict) or not isinstance( + message.get("parts"), list + ): + return None + parts = [] + for part_index, part in enumerate(message["parts"]): + if not isinstance(part, dict): + return None + prefix = f"{key}.{message_index}.{MessageAttributes.MESSAGE_CONTENTS}.{part_index}" + if part.get("type") in ("text", "reasoning", "tool_call_response"): + # The SDK stores tool-result text/chunks in response rather + # than content. Mask the payload, never the tool-call ID. + field = ( + "response" + if part["type"] == "tool_call_response" + else "content" + ) + policy_key = ( + f"{prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TEXT}" + ) + elif ( + key == SpanAttributes.GEN_AI_INPUT_MESSAGES + and part.get("type") == "uri" + and part.get("modality") == "image" + ): + field = "uri" + policy_key = f"{prefix}.{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}.{ImageAttributes.IMAGE_URL}" + else: + parts.append(part) + continue + masked = self.__wrapped__.mask(policy_key, part.get(field)) + if ( + key == SpanAttributes.GEN_AI_INPUT_MESSAGES + and part.get("type") == "tool_call_response" + and isinstance(masked, list) + ): + masked = self._mask_tool_images(masked, prefix) + if masked is not None: + part[field] = masked + parts.append(part) + message["parts"] = parts + return json.dumps(messages) + + def _mask_tool_images(self, chunks: list[Any], prefix: str) -> list[Any]: + # Unlike user messages, the SDK leaves tool responses as raw content + # chunks, with image_url represented by either a string or a URL object. + result = [] + for index, chunk in enumerate(chunks): + if not isinstance(chunk, dict) or chunk.get("type") != "image_url": + result.append(chunk) + continue + image = chunk.get("image_url") + url = image.get("url") if isinstance(image, dict) else image + if not isinstance(url, str): + continue + policy_key = f"{prefix}.{index}.{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}.{ImageAttributes.IMAGE_URL}" + masked = self.__wrapped__.mask(policy_key, url) + if masked is not None: + chunk["image_url"] = ( + {**image, "url": masked} if isinstance(image, dict) else masked + ) + result.append(chunk) + return result diff --git a/python/frameworks/mistralai/traceai_mistralai/_request_attributes_extractor.py b/python/frameworks/mistralai/traceai_mistralai/_request_attributes_extractor.py index be4531d05..5b2d9d813 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_request_attributes_extractor.py +++ b/python/frameworks/mistralai/traceai_mistralai/_request_attributes_extractor.py @@ -5,6 +5,7 @@ from fi_instrumentation import safe_json_dumps from fi_instrumentation.fi_types import ( + ImageAttributes, MessageAttributes, MessageContentAttributes, SpanAttributes, @@ -61,7 +62,8 @@ def _get_attributes_from_chat_completion_create_param( def _get_attributes_from_message_param( message: Mapping[str, Any], ) -> Iterator[Tuple[str, AttributeValue]]: - if not hasattr(message, "get"): + message = _message_dict(message) + if not message: return if role := message.get("role"): yield ( @@ -88,11 +90,11 @@ def _get_attributes_from_message_param( MessageAttributes.MESSAGE_FUNCTION_CALL_ARGUMENTS_JSON, function_arguments, ) - if (tool_calls := message.get("tool_calls"),) and isinstance(tool_calls, Iterable): + if (tool_calls := message.get("tool_calls")) and isinstance(tool_calls, Iterable): for index, tool_call in enumerate(tool_calls): - if not hasattr(tool_call, "get"): - continue - if (function := tool_call.get("function")) and hasattr(function, "get"): + tool_call = _message_dict(tool_call) + function = _message_dict(tool_call.get("function")) if tool_call else None + if function: if name := function.get("name"): yield ( f"{MessageAttributes.MESSAGE_TOOL_CALLS}.{index}." @@ -110,15 +112,38 @@ def _get_attributes_from_message_param( def _get_attributes_from_message_content( content: Mapping[str, Any], ) -> Iterator[Tuple[str, AttributeValue]]: - if not hasattr(content, "get"): + content_dict = dict(_message_dict(content)) + if not content_dict: return - content_dict = dict(content) type_ = content_dict.pop("type", None) + if type_ is None and type(content).__module__.startswith("mistralai."): + # SDK 1.1 omits default discriminators on typed content models. + type_ = {"TextChunk": "text", "ImageURLChunk": "image_url"}.get( + type(content).__name__ + ) if type_ == "text": if text := content_dict.get("text"): yield f"{MessageContentAttributes.MESSAGE_CONTENT_TYPE}", "text" yield f"{MessageContentAttributes.MESSAGE_CONTENT_TEXT}", text elif type_ == "image_url": if image := content_dict.get("image_url"): - yield f"{MessageContentAttributes.MESSAGE_CONTENT_TYPE}", "image" - yield f"{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}", image + # The core applies the base64 limit only to semantic image.url leaves. + url = image if isinstance(image, str) else _message_dict(image).get("url") + if isinstance(url, str): + yield MessageContentAttributes.MESSAGE_CONTENT_TYPE, "image" + yield f"{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}.{ImageAttributes.IMAGE_URL}", url + + +def _message_dict(value: Any) -> Mapping[str, Any]: + if isinstance(value, Mapping): + return value + # SDK 1.x exposes Pydantic message and content models. Normalize only + # their known fields, never copy raw models into non-policy attributes. + if callable(getattr(value, "model_dump", None)): + fields = value.model_dump(exclude_none=True) + if hasattr(value, "content"): + # Preserve typed nested chunks: SDK 1.1 drops their discriminator + # during model_dump even though it sends them to the model. + fields["content"] = value.content + return fields + return {} diff --git a/python/frameworks/mistralai/traceai_mistralai/_response_accumulator.py b/python/frameworks/mistralai/traceai_mistralai/_response_accumulator.py index 5bc5a03ce..dd1765b2c 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_response_accumulator.py +++ b/python/frameworks/mistralai/traceai_mistralai/_response_accumulator.py @@ -16,8 +16,6 @@ Type, ) -from fi_instrumentation import safe_json_dumps -from fi_instrumentation.fi_types import FiMimeTypeValues, SpanAttributes from opentelemetry.util.types import AttributeValue from traceai_mistralai._utils import _to_dict @@ -43,8 +41,6 @@ class _ChatCompletionAccumulator: "_request_parameters", "_response_attributes_extractor", "_chat_completion_type", - "_content", - "_raw_data", ) def __init__( @@ -58,8 +54,6 @@ def __init__( self._response_attributes_extractor = response_attributes_extractor self._is_null = True self._cached_result: Optional[Dict[str, Any]] = None - self._content: List[str] = [] - self._raw_data: List[Dict[str, Any]] = [] self._values = _ValuesAccumulator( data=_ValuesAccumulator( choices=_IndexedAccumulator( @@ -81,15 +75,13 @@ def __init__( def process_chunk(self, chunk: Any) -> None: raw_chunk = _to_dict(chunk) - self._raw_data.append(raw_chunk) - - if "choices" in raw_chunk.get("data", {}): - choices = raw_chunk["data"]["choices"] - if choices and isinstance(choices, list): - delta = choices[0].get("delta", {}) - content = delta.get("content") - if content is not None: - self._content.append(content) + self._is_null = False + self._cached_result = None + accumulated = deepcopy(raw_chunk) + for choice in accumulated.get("data", {}).get("choices", []): + if "delta" in choice: + choice["message"] = choice.pop("delta") + self._values += accumulated def _result(self) -> Optional[Dict[str, Any]]: if self._is_null: @@ -99,12 +91,8 @@ def _result(self) -> Optional[Dict[str, Any]]: return self._cached_result def get_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: - # Combine the accumulated content and raw data - output_value = "".join(self._content) - raw_output = safe_json_dumps(self._raw_data) - yield SpanAttributes.OUTPUT_VALUE, output_value - yield SpanAttributes.OUTPUT_VALUE, raw_output - yield SpanAttributes.OUTPUT_MIME_TYPE, FiMimeTypeValues.JSON.value + # The reconstructed semantic messages carry the output and its policy. + yield from () def get_extra_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: if not (result := self._result()): @@ -151,6 +139,18 @@ def __iadd__(self, values: Optional[Mapping[str, Any]]) -> "_ValuesAccumulator": elif isinstance(self_value, _StringAccumulator): if isinstance(value, str): self_value += value + elif isinstance(value, list): + # Legacy SDK 1.12 accepts typed text chunks in SSE deltas. + # Export the accumulated answer via the semantic output + # attribute, where TraceConfig applies text privacy. + for chunk in value: + chunk = _to_dict(chunk) + if ( + isinstance(chunk, Mapping) + and chunk.get("type") == "text" + and isinstance(chunk.get("text"), str) + ): + self_value += chunk["text"] elif isinstance(self_value, _IndexedAccumulator): if isinstance(value, Iterable): for index, v in enumerate(value): diff --git a/python/frameworks/mistralai/traceai_mistralai/_stream.py b/python/frameworks/mistralai/traceai_mistralai/_stream.py index 7171eff68..2d7bd0cd2 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_stream.py +++ b/python/frameworks/mistralai/traceai_mistralai/_stream.py @@ -1,7 +1,10 @@ +from __future__ import annotations + import logging -from typing import Any, AsyncIterator, Generator, Iterator, Optional, Protocol, Tuple +from typing import TYPE_CHECKING, Any, AsyncIterator, Generator, Iterator, Optional, Protocol, Tuple -from mistralai.models import CompletionEvent +if TYPE_CHECKING: + from mistralai.models import CompletionEvent from opentelemetry import trace as trace_api from opentelemetry.util.types import AttributeValue from traceai_mistralai._utils import _finish_tracing @@ -48,6 +51,28 @@ def __init__( def __iter__(self) -> Iterator[Any]: return self + def __enter__(self): + if hasattr(self.__wrapped__, "__enter__"): + self.__wrapped__.__enter__() + return self + + def close(self): + try: + closer = getattr(self.__wrapped__, "close", None) + if closer is not None: + return closer() + return self.__wrapped__.__exit__(None, None, None) + finally: + if not self._self_is_finished: + self._finish_tracing(trace_api.Status(trace_api.StatusCode.OK)) + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_val is not None: + self._self_with_span.record_exception(exc_val) + self._finish_tracing(trace_api.Status(trace_api.StatusCode.ERROR)) + self.close() + return False + def __next__(self) -> Any: try: chunk = self.__wrapped__.__next__() @@ -56,7 +81,7 @@ def __next__(self) -> Any: status = trace_api.Status(status_code=trace_api.StatusCode.OK) self._finish_tracing(status=status) raise - except Exception as exception: + except BaseException as exception: if not self._self_is_finished: status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, @@ -125,6 +150,31 @@ def __init__( def __aiter__(self) -> AsyncIterator[Any]: return self + async def __aenter__(self): + if hasattr(self.__wrapped__, "__aenter__"): + await self.__wrapped__.__aenter__() + return self + + async def close(self): + try: + closer = getattr(self.__wrapped__, "aclose", None) or getattr(self.__wrapped__, "close", None) + if closer is not None: + return await closer() + return await self.__wrapped__.__aexit__(None, None, None) + finally: + if not self._self_is_finished: + self._finish_tracing(trace_api.Status(trace_api.StatusCode.OK)) + + async def aclose(self): + return await self.close() + + async def __aexit__(self, exc_type, exc_val, exc_tb): + if exc_val is not None: + self._self_with_span.record_exception(exc_val) + self._finish_tracing(trace_api.Status(trace_api.StatusCode.ERROR)) + await self.close() + return False + async def __anext__(self) -> Any: # Pass through mistaken calls if not hasattr(self.__wrapped__, "__anext__"): @@ -137,7 +187,7 @@ async def __anext__(self) -> Any: status = trace_api.Status(status_code=trace_api.StatusCode.OK) self._finish_tracing(status=status) raise - except Exception as exception: + except BaseException as exception: if not self._self_is_finished: status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, diff --git a/python/tests/test_framework_mistralai.py b/python/tests/test_framework_mistralai.py index 9c7237419..fb25a514b 100644 --- a/python/tests/test_framework_mistralai.py +++ b/python/tests/test_framework_mistralai.py @@ -1,5 +1,6 @@ import asyncio import json +from importlib.metadata import version from typing import Any, Dict, Iterator, List, Optional from unittest.mock import MagicMock, Mock, patch, AsyncMock, call, ANY from uuid import uuid4 @@ -127,6 +128,8 @@ def sample_response(): return mock_response +@pytest.mark.skipif(int(version("mistralai").split(".")[0]) >= 3, + reason="Legacy method mocks; native lifecycle is tested with the real SDK") class TestMistralAIInstrumentor: """⚡ Test MistralAIInstrumentor lifecycle and basic functionality.""" @@ -153,7 +156,7 @@ def test_instrument_with_defaults(self, tracer_provider): assert isinstance(instrumentor._tracer, FITracer) # Verify all methods were wrapped - assert mock_wrap.call_count == 8 # 4 chat + 4 agents methods + assert mock_wrap.call_count == 12 # chat + parse + agents + embeddings # Verify original methods were stored assert hasattr(instrumentor, '_original_sync_chat_method') @@ -231,7 +234,7 @@ def test_wrapper_registration_calls(self, tracer_provider): instrumentor._instrument(tracer_provider=tracer_provider) # Check that all expected calls were made - assert len(mock_wrap.call_args_list) == 8 + assert len(mock_wrap.call_args_list) == 12 # Check that both chat and agent modules are instrumented modules_called = [] @@ -248,6 +251,7 @@ def test_wrapper_registration_calls(self, tracer_provider): # Verify method names expected_names = [ "Chat.complete", "Chat.stream", "Chat.complete_async", "Chat.stream_async", + "Chat.parse", "Chat.parse_async", "Agents.complete", "Agents.stream", "Agents.complete_async", "Agents.stream_async" ] for expected_name in expected_names: @@ -339,7 +343,8 @@ def test_async_chat_wrapper_call_success(self, fi_tracer, mock_mistralai_module, async def test_call(): # Mock the wrapped function to return a non-stream response - mock_wrapped = AsyncMock(return_value=sample_response) + async def mock_wrapped(**kwargs): + return sample_response with patch.object(wrapper, '_start_as_current_span') as mock_span_ctx: mock_with_span = MagicMock() @@ -679,6 +684,8 @@ def test_span_creation_failure(self, mock_mistralai_module): assert result == "success" +@pytest.mark.skipif(int(version("mistralai").split(".")[0]) >= 3, + reason="Legacy method mocks; native lifecycle is tested with the real SDK") class TestIntegrationScenarios: """🎭 Test integration scenarios and real-world usage patterns."""