diff --git a/.github/workflows/ollama-python-tests.yml b/.github/workflows/ollama-python-tests.yml new file mode 100644 index 000000000..affd5f481 --- /dev/null +++ b/.github/workflows/ollama-python-tests.yml @@ -0,0 +1,48 @@ +name: Ollama Python contracts + +on: + pull_request: + branches: [dev, main] + paths: + - "python/frameworks/ollama/**" + - "python/fi_instrumentation/**" + - "python/pyproject.toml" + - ".github/workflows/ollama-python-tests.yml" + push: + branches: [dev] + paths: + - "python/frameworks/ollama/**" + - "python/fi_instrumentation/**" + - "python/pyproject.toml" + - ".github/workflows/ollama-python-tests.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + contracts: + name: Ollama Python (${{ matrix.ollama }}) + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + ollama: ["0.3.0", "0.6.3"] + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install package and test tools + run: | + python -m venv "$RUNNER_TEMP/ollama-tests" + "$RUNNER_TEMP/ollama-tests/bin/pip" install \ + -e ./python -e ./python/frameworks/ollama \ + "ollama==${{ matrix.ollama }}" "pytest==8.4.2" "pytest-asyncio==0.24.0" + - name: Run real SDK and OTLP contracts with networking disabled + run: | + sudo unshare --net "$RUNNER_TEMP/ollama-tests/bin/python" -I -m pytest \ + python/frameworks/ollama/tests -q -p no:cacheprovider diff --git a/python/frameworks/ollama/README.md b/python/frameworks/ollama/README.md index daa75e8eb..207597405 100644 --- a/python/frameworks/ollama/README.md +++ b/python/frameworks/ollama/README.md @@ -81,10 +81,23 @@ stream = ollama.chat( stream=True ) -for chunk in stream: - print(chunk["message"]["content"], end="", flush=True) +try: + for chunk in stream: + print(chunk["message"]["content"], end="", flush=True) +finally: + stream.close() ``` +Streams keep their span open until exhaustion, an error, or explicit `close()` +(`await stream.aclose()` for async streams). Close streams when stopping early; +final usage is available only when Ollama returns its final usage chunk. + +Streaming telemetry combines content, thinking, and tool calls without retaining +raw chunks. It keeps up to 65,536 characters across those fields and at most 32 +tool calls; `ollama.output_truncated=true` marks a bounded preview. These limits +never truncate the response delivered to your application. `TraceConfig` output +privacy settings apply to both normalized attributes and serialized output. + ### Text Generation ```python @@ -115,12 +128,12 @@ response = ollama.embed( model="nomic-embed-text", input="Hello, world!" ) -print(f"Embedding dimensions: {len(response['embedding'])}") +print(f"Embedding dimensions: {len(response['embeddings'][0])}") # Multiple embeddings (batch) -response = ollama.embeddings( +response = ollama.embed( model="nomic-embed-text", - prompt=["Hello", "World"] + input=["Hello", "World"] ) ``` @@ -204,6 +217,22 @@ OllamaInstrumentor().instrument( ) ``` +### Aggregate privacy values + +With the current core privacy policy, `input.value` and `output.value` are +opaque aggregates. Input hiding (`FI_HIDE_INPUTS`, `FI_HIDE_INPUT_MESSAGES`, +`FI_HIDE_INPUT_TEXT`, or `FI_HIDE_INPUT_IMAGES`) replaces `input.value` with +the literal `__REDACTED__`; output hiding (`FI_HIDE_OUTPUTS`, +`FI_HIDE_OUTPUT_MESSAGES`, or `FI_HIDE_OUTPUT_TEXT`) does the same for +`output.value`. Check for this placeholder before attempting to parse JSON. +The corresponding `TraceConfig` options have the same semantics. + +Structured attributes are masked separately. For example, text-only hiding +retains the message role while redacting message content. Adapter-side +sanitization of detached messages and tool arguments does not exempt the +aggregate from the core policy. Neither layer changes the request sent to +Ollama or the response returned to the application. + ## Captured Attributes ### Request Attributes @@ -257,8 +286,9 @@ query_embedding = ollama.embed( input=query ) -# Search vector database (not shown) -# context = search_vector_db(query_embedding["embedding"]) +# Replace this illustrative context with your vector database's search results. +# ollama.embed returns a batch: search_vector_db(query_embedding["embeddings"][0]) +context = "Machine learning uses data to learn patterns for predictions." # Generate response with context response = ollama.chat( diff --git a/python/frameworks/ollama/tests/test_directional_privacy.py b/python/frameworks/ollama/tests/test_directional_privacy.py new file mode 100644 index 000000000..07a5b8bda --- /dev/null +++ b/python/frameworks/ollama/tests/test_directional_privacy.py @@ -0,0 +1,157 @@ +"""Privacy contracts on the serialized OTLP wire using a real offline Ollama client.""" +import base64 +import json + +import httpx +import ollama +import pytest +from fi_instrumentation import TraceConfig +from fi_instrumentation.instrumentation.config import REDACTED_VALUE +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_ollama import OllamaInstrumentor +from traceai_ollama._request_attributes_extractor import _RequestAttributesExtractor +from traceai_ollama._response_attributes_extractor import _ResponseAttributesExtractor + + +def detached_input(flag, request): + """Adapter sanitization is separate from core's opaque aggregate policy.""" + return dict(_RequestAttributesExtractor( + TraceConfig(**{flag: True}) + ).get_extra_attributes_from_request(request))["input.value"] + + +def detached_output(flag, response, request): + return dict(_ResponseAttributesExtractor( + TraceConfig(**{flag: True}) + ).get_extra_attributes(response, request))["output.value"] + + +PROMPT = "SYNTHETIC_PROMPT_49" +IMAGE = base64.b64encode(b"SYNTHETIC_IMAGE_49").decode() +TOKEN = "SYNTHETIC_TOKEN_49" +ALTERNATE = "SYNTHETIC_ALTERNATE_49" + + +@pytest.fixture +def client_span(): + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = OllamaInstrumentor() + + def run(flag, endpoint, request, response): + exporter.clear() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**{flag: True})) + client = ollama.Client(host="http://offline.invalid", transport=httpx.MockTransport( + lambda req: httpx.Response(200, json=response)), trust_env=False) + try: + getattr(client, endpoint)(model="synthetic-model", **request) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + wire = ExportTraceServiceRequest.FromString(encode_spans(spans).SerializeToString()) + span = wire.resource_spans[0].scope_spans[0].spans[0] + return {item.key: getattr(item.value, item.value.WhichOneof("value")) for item in span.attributes} + finally: + client._client.close() + instrumentor.uninstrument() + + yield run + provider.shutdown() + + +@pytest.mark.parametrize("flag", ["hide_input_text", "hide_input_messages"]) +def test_chat_input_text_not_in_serialized_request(client_span, flag): + messages = [{"role": "user", "content": PROMPT}] + attrs = client_span(flag, "chat", {"messages": messages}, + {"model": "synthetic-model", "message": {"role": "assistant", "content": "ok"}, "done": True}) + assert PROMPT not in json.dumps(attrs) + assert messages[0]["content"] == PROMPT # telemetry must never mutate SDK parameters + assert attrs["input.value"] == REDACTED_VALUE + if flag == "hide_input_text": + assert json.loads(detached_input(flag, {"messages": messages}))["messages"][0]["role"] == "user" + assert attrs["gen_ai.input.messages.0.message.role"] == "user" + + +@pytest.mark.parametrize("flag", ["hide_input_text", "hide_input_messages"]) +def test_generate_and_embed_input_text_not_serialized(client_span, flag): + for endpoint, request, response in [ + ("generate", {"prompt": PROMPT}, {"response": "ok", "done": True}), + ("embed", {"input": PROMPT}, {"embeddings": [[0.1]], "done": True}), + ]: + attrs = client_span(flag, endpoint, request, response) + assert PROMPT not in json.dumps(attrs) + + +def test_chat_input_thinking_and_tool_arguments_hidden(client_span): + messages = [{"role": "user", "content": "ok", "thinking": "SYNTHETIC_INPUT_THINK_83", + "tool_calls": [{"function": {"name": "lookup", "arguments": {"query": "SYNTHETIC_INPUT_TOOL_83"}}}]}] + attrs = client_span("hide_input_text", "chat", {"messages": messages}, + {"model": "synthetic-model", "message": {"role": "assistant", "content": "ok"}, "done": True}) + assert "SYNTHETIC_INPUT_THINK_83" not in json.dumps(attrs) + assert "SYNTHETIC_INPUT_TOOL_83" not in json.dumps(attrs) + assert attrs["input.value"] == REDACTED_VALUE + message = json.loads(detached_input("hide_input_text", {"messages": messages}))["messages"][0] + assert message["role"] == "user" + assert message["tool_calls"][0]["function"]["name"] == "lookup" + assert messages[0]["tool_calls"][0]["function"]["arguments"]["query"] == "SYNTHETIC_INPUT_TOOL_83" + assert messages[0]["thinking"] == "SYNTHETIC_INPUT_THINK_83" + + +def test_chat_none_messages_keeps_span(client_span): + attrs = client_span("hide_input_text", "chat", {"messages": None}, + {"model": "synthetic-model", "message": {"role": "assistant", "content": "ok"}, "done": True}) + assert attrs["gen_ai.provider.name"] == "ollama" + assert attrs["input.value"] == REDACTED_VALUE + assert json.loads(detached_input("hide_input_text", {"messages": None})) == {"messages": []} + + +def test_chat_images_hidden_in_all_attributes(client_span): + messages = [{"role": "user", "content": "ok", "images": [IMAGE]}] + attrs = client_span("hide_input_images", "chat", {"messages": messages}, + {"model": "synthetic-model", "message": {"role": "assistant", "content": "ok"}, "done": True}) + assert IMAGE not in json.dumps(attrs) + assert messages[0]["images"] == [IMAGE] + assert attrs["input.value"] == REDACTED_VALUE + assert "images" not in json.loads(detached_input("hide_input_images", {"messages": messages}))["messages"][0] + assert not any(key.endswith(".images") for key in attrs) + + +def test_generated_tool_arguments_hidden_under_output_text(client_span): + response = {"model": "synthetic-model", "done": True, + "message": {"role": "assistant", "content": "ok", "tool_calls": [ + {"function": {"name": "lookup", "arguments": {"query": "SYNTHETIC_TOOL_ARGUMENT_83"}}}]}} + attrs = client_span("hide_output_text", "chat", {"messages": [{"role": "user", "content": "ok"}]}, response) + assert "SYNTHETIC_TOOL_ARGUMENT_83" not in json.dumps(attrs) + assert attrs["output.value"] == REDACTED_VALUE + output = json.loads(detached_output("hide_output_text", response, {"messages": [{"role": "user", "content": "ok"}]})) + assert output["message"]["tool_calls"][0]["function"]["name"] == "lookup" + assert output["message"]["tool_calls"][0]["function"]["arguments"] == REDACTED_VALUE + assert attrs["gen_ai.output.messages.0.message.tool_calls.0.tool_call.function.name"] == "lookup" + assert response["message"]["tool_calls"][0]["function"]["arguments"]["query"] == "SYNTHETIC_TOOL_ARGUMENT_83" + + +@pytest.mark.parametrize("flag", ["hide_output_text", "hide_output_messages"]) +@pytest.mark.parametrize("endpoint", ["chat", "generate"]) +def test_logprob_tokens_hidden_on_output_wire(client_span, flag, endpoint): + response = {"model": "synthetic-model", "done": True, + "logprobs": [{"token": TOKEN, "logprob": -0.1, + "top_logprobs": [{"token": ALTERNATE, "logprob": -0.2}]}]} + if endpoint == "chat": + response["message"] = {"role": "assistant", "content": "SYNTHETIC_OUTPUT_49"} + request = {"messages": [{"role": "user", "content": "ok"}]} + else: + response["response"] = "SYNTHETIC_OUTPUT_49" + request = {"prompt": "ok"} + attrs = client_span(flag, endpoint, request, response) + output = json.dumps({key: value for key, value in attrs.items() + if key == "output.value" or key.startswith("gen_ai.output.")}) + assert TOKEN not in output + assert ALTERNATE not in output + assert "SYNTHETIC_OUTPUT_49" not in output + assert attrs["output.value"] == REDACTED_VALUE + assert "logprobs" not in json.loads(detached_output(flag, response, request)) diff --git a/python/frameworks/ollama/tests/test_privacy_sdk.py b/python/frameworks/ollama/tests/test_privacy_sdk.py index a6b716056..410effaae 100644 --- a/python/frameworks/ollama/tests/test_privacy_sdk.py +++ b/python/frameworks/ollama/tests/test_privacy_sdk.py @@ -60,7 +60,7 @@ def respond(request): client._client.close() assert len(requests) == 1 assert requests[0][0] == "/api/" + endpoint - received = response.model_dump() + received = response if isinstance(response, dict) else response.model_dump() key = "embeddings" if endpoint == "embed" else "embedding" assert received[key] == body[key] # Redaction must not mutate the SDK result. spans = exporter.get_finished_spans() diff --git a/python/frameworks/ollama/tests/test_readme_examples.py b/python/frameworks/ollama/tests/test_readme_examples.py new file mode 100644 index 000000000..521575bb5 --- /dev/null +++ b/python/frameworks/ollama/tests/test_readme_examples.py @@ -0,0 +1,39 @@ +"""Execute the documented RAG example with a real SDK and offline transport.""" +import json +from pathlib import Path + +import httpx +import ollama + + +def test_rag_example_defines_context(monkeypatch): + readme = (Path(__file__).parents[1] / "README.md").read_text() + example = readme.split("### RAG Pipeline", 1)[1].split("```python", 1)[1].split("```", 1)[0] + requests = [] + + def respond(request): + requests.append((request.url.path, json.loads(request.content))) + if request.url.path == "/api/embed": + return httpx.Response(200, json={"embeddings": [[0.1, 0.2]], "model": "nomic-embed-text"}) + assert request.url.path == "/api/chat" + return httpx.Response(200, json={ + "model": "llama3.2", "done": True, + "message": {"role": "assistant", "content": "Synthetic example response"}, + }) + + client = ollama.Client(host="http://offline.invalid", transport=httpx.MockTransport(respond), trust_env=False) + monkeypatch.setattr(ollama, "embed", client.embed) + monkeypatch.setattr(ollama, "chat", client.chat) + namespace = {} + try: + exec(compile(example, "README.md:RAG Pipeline", "exec"), namespace) + finally: + client._client.close() + assert [path for path, _ in requests] == ["/api/embed", "/api/chat"] + assert namespace["query_embedding"]["embeddings"][0] == [0.1, 0.2] + assert isinstance(namespace["context"], str) and namespace["context"] + assert requests[1][1]["messages"] == [ + {"role": "system", "content": f"Use this context: {namespace['context']}"}, + {"role": "user", "content": namespace["query"]}, + ] + assert namespace["response"]["message"]["content"] == "Synthetic example response" diff --git a/python/frameworks/ollama/tests/test_stream_contract.py b/python/frameworks/ollama/tests/test_stream_contract.py new file mode 100644 index 000000000..99f5f5e98 --- /dev/null +++ b/python/frameworks/ollama/tests/test_stream_contract.py @@ -0,0 +1,488 @@ +"""Real Ollama clients, synthetic HTTP and serialized OTLP; no model service.""" +import asyncio +import inspect +import json + +import httpx +import ollama +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_ollama import OllamaInstrumentor +from traceai_ollama._utils import _to_dict + + +@pytest.fixture +def tracing(): + aliases = {name: getattr(ollama, name) for name in ("chat", "generate", "embed", "embeddings") if hasattr(ollama, name)} + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = OllamaInstrumentor() + + def start(**privacy): + exporter.clear() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**privacy)) + return exporter + + yield start + instrumentor.uninstrument() + # Also isolate the baseline reproduction, whose uninstrument leaves aliases patched. + for name, method in aliases.items(): + setattr(ollama, name, method) + provider.shutdown() + + +def chunks(endpoint): + body = [ + {"model": "offline-model", "done": False}, + {"model": "offline-model", "done": False}, + {"model": "offline-model", "done": True, "prompt_eval_count": 5, "eval_count": 2, "total_duration": 123}, + ] + for index, chunk in enumerate(body): + text = ("hello ", "world", "")[index] + thinking = ("consider ", "carefully", "")[index] + if endpoint == "chat": + chunk["message"] = {"role": "assistant", "content": text, "thinking": thinking} + if index < 2: + chunk["message"]["tool_calls"] = [{"function": {"name": "weather", "arguments": {"city": ("Rome", "Paris")[index]}}}] + else: + chunk.update(response=text, thinking=thinking) + return body + + +def transport(endpoint, streaming=True): + body = chunks(endpoint) + if not streaming: + body = body[0] + body.update(done=True, prompt_eval_count=5, eval_count=2) + return httpx.MockTransport(lambda request: httpx.Response( + 200, content="".join(json.dumps(chunk) + "\n" for chunk in body) if streaming else json.dumps(body), + headers={"content-type": "application/x-ndjson" if streaming else "application/json"}, + )) + + +def parameters(endpoint, stream=True): + return dict(model="offline-model", stream=stream, **({"messages": [{"role": "user", "content": "hello"}]} if endpoint == "chat" else {"prompt": "hello"})) + + +def wire_attributes(exporter): + spans = exporter.get_finished_spans() + assert len(spans) == 1 + encoded = encode_spans(spans).SerializeToString() + decoded = ExportTraceServiceRequest.FromString(encoded) + span = decoded.resource_spans[0].scope_spans[0].spans[0] + return {item.key: getattr(item.value, item.value.WhichOneof("value")) for item in span.attributes} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["chat", "generate"]) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +async def test_real_sdk_return_contract(tracing, endpoint, asynchronous, streaming): + exporter = tracing() + cls = ollama.AsyncClient if asynchronous else ollama.Client + client = cls(host="http://offline.invalid", transport=transport(endpoint, streaming), trust_env=False) + try: + result = getattr(client, endpoint)(**parameters(endpoint, streaming)) + if asynchronous: + result = await result + if streaming: + # An extra coroutine here breaks the SDK's documented single-await contract. + if inspect.iscoroutine(result): + result.close() + pytest.fail("await client call returned a coroutine, not an async iterator") + received = [item async for item in result] if asynchronous else list(result) + assert len(received) == 3 + else: + received = [result] + attrs = wire_attributes(exporter) + assert attrs["gen_ai.usage.input_tokens"] == 5 + assert attrs["gen_ai.usage.output_tokens"] == 2 + assert attrs["gen_ai.usage.total_tokens"] == 7 + assert attrs["gen_ai.provider.name"] == "ollama" + assert attrs["gen_ai.output.messages.0.message.content"] == ("hello world" if streaming else "hello ") + # Old SDKs return dicts; current SDKs return Pydantic models. Neither is mutated. + first = _to_dict(received[0]) + first_output = first["message"] if endpoint == "chat" else first + assert first_output["content" if endpoint == "chat" else "response"] == "hello " + if "thinking" in first_output: + output = json.loads(attrs["output.value"]) + assert (output["message"] if endpoint == "chat" else output)["thinking"] == ("consider carefully" if streaming else "consider ") + if endpoint == "chat": + prefix = "gen_ai.output.messages.0.message.tool_calls.0.tool_call.function" + assert attrs[prefix + ".name"] == "weather" + assert json.loads(attrs[prefix + ".arguments"]) == {"city": "Rome"} + if streaming: + assert json.loads(attrs[prefix.replace("tool_calls.0", "tool_calls.1") + ".arguments"]) == {"city": "Paris"} + finally: + if asynchronous: + await client._client.aclose() + else: + client._client.close() + + +@pytest.mark.parametrize("endpoint", ["chat", "generate", "embed", "embeddings"]) +def test_alias_restoration_and_reinstrumentation(tracing, endpoint): + if not hasattr(ollama, endpoint): + pytest.skip("Endpoint unavailable at the SDK floor") + original = getattr(ollama, endpoint) + original_class = getattr(ollama.Client, endpoint) + original_async = getattr(ollama.AsyncClient, endpoint) + for _ in range(2): + tracing() + assert getattr(ollama, endpoint) is not original + OllamaInstrumentor().uninstrument() + assert getattr(ollama, endpoint) is original + assert getattr(ollama.Client, endpoint) is original_class + assert getattr(ollama.AsyncClient, endpoint) is original_async + + +@pytest.mark.parametrize("endpoint", ["chat", "generate", "embed", "embeddings"]) +def test_module_helpers_export_once_across_reinstrumentation(tracing, monkeypatch, endpoint): + if not hasattr(ollama, endpoint): + pytest.skip("Endpoint unavailable at the SDK floor") + bound_client = getattr(ollama, endpoint).__self__ + if endpoint in ("chat", "generate"): + mock_transport = transport(endpoint) + params = parameters(endpoint) + else: + body = {"embedding": [0.1], "embeddings": [[0.1]]} + mock_transport = httpx.MockTransport(lambda request: httpx.Response(200, json=body)) + params = {"model": "offline-model", "input" if endpoint == "embed" else "prompt": "hello"} + with httpx.Client(transport=mock_transport, base_url="http://offline.invalid", trust_env=False) as client: + monkeypatch.setattr(bound_client, "_client", client) + for _ in range(2): + exporter = tracing() + response = getattr(ollama, endpoint)(**params) + if endpoint in ("chat", "generate"): + assert len(list(response)) == 3 + assert len(exporter.get_finished_spans()) == 1 + OllamaInstrumentor().uninstrument() + response = getattr(ollama, endpoint)(**params) + if endpoint in ("chat", "generate"): + list(response) + assert len(exporter.get_finished_spans()) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["chat", "generate"]) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("consume_first", [False, True]) +async def test_close_ends_span_even_before_iteration(tracing, endpoint, asynchronous, consume_first): + exporter = tracing() + cls = ollama.AsyncClient if asynchronous else ollama.Client + client = cls(host="http://offline.invalid", transport=transport(endpoint), trust_env=False) + try: + stream = getattr(client, endpoint)(**parameters(endpoint)) + if asynchronous: + stream = await stream + if consume_first: + await stream.__anext__() + await stream.aclose() + await stream.aclose() + else: + if consume_first: + next(stream) + stream.close() + stream.close() + assert len(exporter.get_finished_spans()) == 1 + assert exporter.get_finished_spans()[0].status.status_code.name == "UNSET" + finally: + if asynchronous: + await client._client.aclose() + else: + client._client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["chat", "generate"]) +async def test_cancellation_propagates_and_ends_span(tracing, endpoint): + exporter = tracing() + waiting = asyncio.Event() + closed = asyncio.Event() + + class Body(httpx.AsyncByteStream): + async def __aiter__(self): + yield (json.dumps(chunks(endpoint)[0]) + "\n").encode() + waiting.set() + await asyncio.Event().wait() + + async def aclose(self): + closed.set() + + client = ollama.AsyncClient(host="http://offline.invalid", transport=httpx.MockTransport(lambda request: httpx.Response(200, stream=Body())), trust_env=False) + try: + stream = await getattr(client, endpoint)(**parameters(endpoint)) + await stream.__anext__() + task = asyncio.create_task(stream.__anext__()) + await asyncio.wait_for(waiting.wait(), 2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert closed.is_set() + assert len(exporter.get_finished_spans()) == 1 + assert exporter.get_finished_spans()[0].status.status_code.name != "OK" + finally: + await client._client.aclose() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["chat", "generate"]) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("flag", ["hide_outputs", "hide_output_messages", "hide_output_text"]) +async def test_output_privacy_on_otlp_wire(tracing, endpoint, asynchronous, streaming, flag): + exporter = tracing(**{flag: True}) + cls = ollama.AsyncClient if asynchronous else ollama.Client + client = cls(host="http://offline.invalid", transport=transport(endpoint, streaming), trust_env=False) + try: + result = getattr(client, endpoint)(**parameters(endpoint, streaming)) + if asynchronous: + result = await result + if streaming: + received = [item async for item in result] if asynchronous else list(result) + else: + received = [result] + data = _to_dict(received[0]) + message = data["message"] if endpoint == "chat" else data + assert message["content" if endpoint == "chat" else "response"] == "hello " + attrs = wire_attributes(exporter) + # The input contains "hello" as well; inspect output fields only. + output = json.dumps({key: value for key, value in attrs.items() if key == "output.value" or key.startswith("gen_ai.output.")}) + assert "hello" not in output + assert "consider" not in output + assert "Rome" not in output + assert "Paris" not in output + if flag != "hide_output_text": + assert "weather" not in output + elif endpoint == "chat": + assert attrs["gen_ai.output.messages.0.message.tool_calls.0.tool_call.function.name"] == "weather" + assert attrs["gen_ai.usage.total_tokens"] == 7 + finally: + if asynchronous: + await client._client.aclose() + else: + client._client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["chat", "generate"]) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("failure", [False, True]) +async def test_partial_http_stream_closes_and_restores_context(tracing, endpoint, asynchronous, failure): + from opentelemetry import trace + + exporter = tracing(hide_error_details=True) + closed = [] + active = [] + + class Body(httpx.SyncByteStream, httpx.AsyncByteStream): + def __iter__(self): + active.append(trace.get_current_span().get_span_context().span_id) + yield (json.dumps(chunks(endpoint)[0]) + "\n").encode() + if failure: + raise RuntimeError("private-stream-error") + yield (json.dumps(chunks(endpoint)[1]) + "\n").encode() + + async def __aiter__(self): + for item in self: + yield item + + def close(self): + closed.append(True) + + async def aclose(self): + self.close() + + cls = ollama.AsyncClient if asynchronous else ollama.Client + client = cls(host="http://offline.invalid", transport=httpx.MockTransport(lambda request: httpx.Response(200, stream=Body())), trust_env=False) + before = trace.get_current_span() + try: + stream = getattr(client, endpoint)(**parameters(endpoint)) + if asynchronous: + stream = await stream + await stream.__anext__() + else: + next(stream) + assert trace.get_current_span() is before + if failure: + with pytest.raises(RuntimeError, match="private-stream-error"): + if asynchronous: + await stream.__anext__() + else: + next(stream) + if asynchronous: + await stream.aclose() + else: + stream.close() + assert len(closed) == 1 + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert active == [spans[0].context.span_id] + assert spans[0].status.status_code.name == ("ERROR" if failure else "UNSET") + assert b"private-stream-error" not in encode_spans(spans).SerializeToString() + attrs = wire_attributes(exporter) + assert attrs["gen_ai.output.messages.0.message.content"] == "hello " + assert trace.get_current_span() is before + finally: + if asynchronous: + await client._client.aclose() + else: + client._client.close() + + +@pytest.mark.parametrize("endpoint", ["chat", "generate"]) +def test_bounded_preview_keeps_final_usage(tracing, endpoint): + from traceai_ollama._stream import _MAX_OUTPUT_CHARACTERS + + exporter = tracing() + first, _, last = chunks(endpoint) + message = first["message"] if endpoint == "chat" else first + key = "content" if endpoint == "chat" else "response" + message[key] = "x" * 1024 + message.pop("thinking", None) + message.pop("tool_calls", None) + line = json.dumps(first) + "\n" + data = line * 256 + json.dumps(last) + "\n" + client = ollama.Client(host="http://offline.invalid", transport=httpx.MockTransport(lambda request: httpx.Response(200, content=data)), trust_env=False) + try: + result = list(getattr(client, endpoint)(**parameters(endpoint))) + assert len(result) == 257 + assert (_to_dict(result[0])["message"] if endpoint == "chat" else _to_dict(result[0]))[key] == "x" * 1024 + attrs = wire_attributes(exporter) + assert attrs["ollama.output_truncated"] is True + assert len(attrs["gen_ai.output.messages.0.message.content"]) == _MAX_OUTPUT_CHARACTERS + assert attrs["gen_ai.usage.total_tokens"] == 7 + assert attrs["ollama.total_duration_ns"] == 123 + finally: + client._client.close() + + +def test_hidden_stream_does_not_retain_payload(): + from traceai_ollama._stream import _ResponseAccumulator + + accumulator = _ResponseAccumulator(TraceConfig(hide_outputs=True)) + for chunk in chunks("chat"): + accumulator.add(chunk) + assert accumulator._content == [] + assert accumulator._thinking == [] + assert accumulator._tools == [] + assert accumulator.response()["eval_count"] == 2 + + +def test_tool_budget_preserves_valid_json_and_detaches_arguments(): + from traceai_ollama._stream import _MAX_TOOL_CALLS, _ResponseAccumulator + + accumulator = _ResponseAccumulator(TraceConfig()) + chunk = chunks("chat")[0] + for _ in range(1000): + accumulator.add(chunk) + assert accumulator.truncated + tools = accumulator.response()["message"]["tool_calls"] + assert len(tools) == _MAX_TOOL_CALLS + chunk["message"]["tool_calls"][0]["function"]["arguments"]["city"] = "changed" + assert tools[0]["function"]["arguments"]["city"] == "Rome" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("endpoint", ["chat", "generate"]) +async def test_positional_stream_argument(tracing, asynchronous, endpoint): + exporter = tracing() + cls = ollama.AsyncClient if asynchronous else ollama.Client + # SDK releases have different signatures; bind the public signature first. + signature = inspect.signature(getattr(cls, endpoint)) + bound = signature.bind(None, **parameters(endpoint)) + bound.apply_defaults() + client = cls(host="http://offline.invalid", transport=transport(endpoint), trust_env=False) + try: + result = getattr(client, endpoint)(*bound.args[1:], **bound.kwargs) + if asynchronous: + result = await result + assert len([item async for item in result]) == 3 + else: + assert len(list(result)) == 3 + assert wire_attributes(exporter)["gen_ai.usage.total_tokens"] == 7 + finally: + if asynchronous: + await client._client.aclose() + else: + client._client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +async def test_suppression_preserves_native_stream(tracing, asynchronous): + import opentelemetry.context as context + + exporter = tracing() + cls = ollama.AsyncClient if asynchronous else ollama.Client + client = cls(host="http://offline.invalid", transport=transport("chat"), trust_env=False) + token = context.attach(context.set_value(context._SUPPRESS_INSTRUMENTATION_KEY, True)) + try: + stream = client.chat(**parameters("chat")) + if asynchronous: + stream = await stream + assert len([chunk async for chunk in stream]) == 3 + else: + assert len(list(stream)) == 3 + assert exporter.get_finished_spans() == () + finally: + context.detach(token) + if asynchronous: + await client._client.aclose() + else: + client._client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +async def test_empty_stream_finishes_once(tracing, asynchronous): + exporter = tracing() + cls = ollama.AsyncClient if asynchronous else ollama.Client + client = cls(host="http://offline.invalid", transport=httpx.MockTransport(lambda request: httpx.Response(200, content=b"")), trust_env=False) + try: + stream = client.chat(**parameters("chat")) + if asynchronous: + stream = await stream + assert [chunk async for chunk in stream] == [] + await stream.aclose() + else: + assert list(stream) == [] + stream.close() + assert len(exporter.get_finished_spans()) == 1 + assert exporter.get_finished_spans()[0].status.status_code.name == "OK" + finally: + if asynchronous: + await client._client.aclose() + else: + client._client.close() + + +def test_optional_protect_restored(tracing, monkeypatch): + import sys + from types import ModuleType + import traceai_ollama + + class Protect: + def protect(self): + return "protected" + + original = Protect.protect + fi_module = ModuleType("fi") + eval_module = ModuleType("fi.evals") + eval_module.Protect = Protect + fi_module.evals = eval_module + monkeypatch.setitem(sys.modules, "fi", fi_module) + monkeypatch.setitem(sys.modules, "fi.evals", eval_module) + monkeypatch.setattr(traceai_ollama, "Protect", Protect) + tracing() + assert Protect.protect is not original + OllamaInstrumentor().uninstrument() + assert Protect.protect is original diff --git a/python/frameworks/ollama/tests/test_typed_tool_privacy.py b/python/frameworks/ollama/tests/test_typed_tool_privacy.py new file mode 100644 index 000000000..097a0dae2 --- /dev/null +++ b/python/frameworks/ollama/tests/test_typed_tool_privacy.py @@ -0,0 +1,111 @@ +"""Typed SDK tool calls obey input-text privacy without changing the request.""" +import copy +import json + +import httpx +import ollama +import pytest +from fi_instrumentation import TraceConfig +from fi_instrumentation.instrumentation.config import REDACTED_VALUE +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_ollama import OllamaInstrumentor +from traceai_ollama._request_attributes_extractor import _RequestAttributesExtractor + + +MARKER = "SYNTHETIC_TYPED_TOOL_ARGUMENT_426" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("shape", ["message", "tool_call", "function"]) +@pytest.mark.parametrize("hidden", [False, True]) +@pytest.mark.parametrize("configuration", ["environment", "explicit"]) +async def test_typed_tool_input_text_privacy( + monkeypatch, asynchronous, stream, shape, hidden, configuration +): + message = ollama.Message( + role="assistant", + content="previous tool invocation", + tool_calls=[{"function": {"name": "lookup", "arguments": {"query": MARKER}}}], + ) + # Current SDKs return models; the SDK floor represents these as dictionaries. + # Also exercise mappings containing either a typed call or a typed function. + if shape != "message": + message = dict(message) + if shape == "function": + message["tool_calls"] = [dict(message["tool_calls"][0])] + before = copy.deepcopy(message) + requests = [] + response = { + "model": "synthetic-model", "done": True, + "message": {"role": "assistant", "content": "complete"}, + } + + def respond(request): + requests.append(json.loads(request.content)) + if stream: + return httpx.Response(200, content=json.dumps(response) + "\n") + return httpx.Response(200, json=response) + + monkeypatch.setenv("FI_HIDE_INPUT_TEXT", str(hidden).lower()) + config = None + if configuration == "explicit": + monkeypatch.setenv("FI_HIDE_INPUT_TEXT", str(not hidden).lower()) + config = TraceConfig(hide_input_text=hidden) + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = OllamaInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=config) + client_type = ollama.AsyncClient if asynchronous else ollama.Client + client = client_type( + host="http://offline.invalid", transport=httpx.MockTransport(respond), trust_env=False + ) + try: + result = client.chat(model="synthetic-model", messages=[message], stream=stream) + if asynchronous: + result = await result + if stream: + result = [chunk async for chunk in result][-1] + elif stream: + result = list(result)[-1] + received = result if isinstance(result, dict) else result.model_dump() + assert received["message"]["content"] == "complete" + assert len(requests) == 1 + assert requests[0]["messages"][0]["tool_calls"][0]["function"]["arguments"] == { + "query": MARKER + } + assert message == before + spans = exporter.get_finished_spans() + assert len(spans) == 1 + # Scan the complete protobuf, including resource fields and events. + wire = encode_spans(spans).SerializeToString() + attributes = dict(spans[0].attributes) + if hidden: + assert MARKER.encode() not in wire + # Core hides opaque aggregates in full. Independently retain the + # adapter's detached-copy assertions so core masking cannot conceal + # a regression in typed tool/function normalization. + assert attributes["input.value"] == REDACTED_VALUE + detached = dict(_RequestAttributesExtractor( + TraceConfig(hide_input_text=True) + ).get_extra_attributes_from_request({"messages": [message]})) + assert MARKER not in detached["input.value"] + tool = json.loads(detached["input.value"])["messages"][0]["tool_calls"][0] + assert tool["function"]["name"] == "lookup" + assert tool["function"]["arguments"] == REDACTED_VALUE + else: + assert MARKER.encode() in wire + assert MARKER in attributes["input.value"] + finally: + if asynchronous: + await client._client.aclose() + else: + client._client.close() + instrumentor.uninstrument() + provider.shutdown() diff --git a/python/frameworks/ollama/traceai_ollama/__init__.py b/python/frameworks/ollama/traceai_ollama/__init__.py index 2743dbe08..8a8ab2d54 100644 --- a/python/frameworks/ollama/traceai_ollama/__init__.py +++ b/python/frameworks/ollama/traceai_ollama/__init__.py @@ -44,6 +44,7 @@ class OllamaInstrumentor(BaseInstrumentor): "_original_async_embed", "_original_embeddings", "_original_async_embeddings", + "_original_module_functions", "_tracer", ) @@ -80,7 +81,7 @@ def _instrument(self, **kwargs: Any) -> None: wrapper=_GenerateWrapper(tracer=self._tracer), ) - # Ollama uses 'embed' for single and 'embeddings' for batch + # 'embed' accepts one or many inputs; 'embeddings' is the legacy endpoint. if hasattr(ollama_module.Client, "embed"): self._original_embed = ollama_module.Client.embed wrap_function_wrapper( @@ -128,7 +129,12 @@ def _instrument(self, **kwargs: Any) -> None: wrapper=_AsyncEmbedWrapper(tracer=self._tracer), ) - # Wrap module-level functions + # Module helpers are bound aliases, not lookups of Client methods. + self._original_module_functions = { + name: getattr(ollama_module, name) + for name in ("chat", "generate", "embed", "embeddings") + if hasattr(ollama_module, name) + } if hasattr(ollama_module, "chat"): wrap_function_wrapper( module="ollama", @@ -187,3 +193,8 @@ def _uninstrument(self, **kwargs: Any) -> None: ollama_module.AsyncClient.embed = self._original_async_embed if hasattr(self, "_original_async_embeddings"): ollama_module.AsyncClient.embeddings = self._original_async_embeddings + for name, original in self._original_module_functions.items(): + setattr(ollama_module, name, original) + self._original_module_functions.clear() + if Protect is not None and self._original_protect is not None: + Protect.protect = self._original_protect diff --git a/python/frameworks/ollama/traceai_ollama/_request_attributes_extractor.py b/python/frameworks/ollama/traceai_ollama/_request_attributes_extractor.py index 4dbe19304..8da84c3ec 100644 --- a/python/frameworks/ollama/traceai_ollama/_request_attributes_extractor.py +++ b/python/frameworks/ollama/traceai_ollama/_request_attributes_extractor.py @@ -1,7 +1,8 @@ import logging -from typing import Any, Dict, Iterator, List, Mapping, Tuple +from typing import Any, Dict, Iterator, List, Mapping, Optional, Tuple -from fi_instrumentation import safe_json_dumps +from fi_instrumentation import TraceConfig, safe_json_dumps +from fi_instrumentation.instrumentation.config import REDACTED_VALUE from fi_instrumentation.fi_types import ( FiMimeTypeValues, FiSpanKindValues, @@ -10,6 +11,8 @@ ) from opentelemetry.util.types import AttributeValue +from traceai_ollama._utils import _redact_tool_calls + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) @@ -17,6 +20,9 @@ class _RequestAttributesExtractor: """Extract span attributes from Ollama request parameters.""" + def __init__(self, config: Optional[TraceConfig] = None) -> None: + self._config = config if config is not None else TraceConfig() + def get_attributes_from_request( self, request_parameters: Mapping[str, Any] ) -> Iterator[Tuple[str, AttributeValue]]: @@ -54,20 +60,41 @@ def get_extra_attributes_from_request( # For chat endpoint if "messages" in request_parameters: - messages = system_messages + list(request_parameters.get("messages", [])) + messages = system_messages + list(request_parameters.get("messages") or ()) yield from self._extract_messages(messages) - yield SpanAttributes.INPUT_VALUE, safe_json_dumps({"messages": messages}) + if self._config.hide_input_messages: + yield SpanAttributes.INPUT_VALUE, REDACTED_VALUE + else: + # Serialize a detached telemetry copy; the SDK still receives its + # original messages and image payloads without modification. + serialized = [] + for message in messages: + item = dict(message) + if self._config.hide_input_text: + for key in ("content", "thinking"): + if key in item: + item[key] = REDACTED_VALUE + if "tool_calls" in item: + item["tool_calls"] = _redact_tool_calls(item["tool_calls"]) + if self._config.hide_input_images: + item.pop("images", None) + serialized.append(item) + yield SpanAttributes.INPUT_VALUE, safe_json_dumps({"messages": serialized}) # For generate endpoint elif "prompt" in request_parameters: prompt = request_parameters.get("prompt", "") yield from self._extract_messages(system_messages + [{"role": "user", "content": prompt}]) - yield SpanAttributes.INPUT_VALUE, prompt + yield SpanAttributes.INPUT_VALUE, REDACTED_VALUE if ( + self._config.hide_input_text or self._config.hide_input_messages + ) else prompt # For embed endpoint elif "input" in request_parameters: input_text = request_parameters.get("input", "") - if isinstance(input_text, list): + if self._config.hide_input_text or self._config.hide_input_messages: + yield SpanAttributes.INPUT_VALUE, REDACTED_VALUE + elif isinstance(input_text, list): yield SpanAttributes.INPUT_VALUE, safe_json_dumps(input_text) else: yield SpanAttributes.INPUT_VALUE, str(input_text) @@ -84,8 +111,8 @@ def _extract_messages( yield f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.{i}.{MessageAttributes.MESSAGE_CONTENT}", content # Handle images in multimodal messages - if "images" in message: - images = message.get("images", []) + if "images" in message and not self._config.hide_input_images: + images = message.get("images") or [] yield f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.{i}.images", safe_json_dumps(images[:3]) # Limit to 3 def _extract_invocation_parameters( diff --git a/python/frameworks/ollama/traceai_ollama/_response_attributes_extractor.py b/python/frameworks/ollama/traceai_ollama/_response_attributes_extractor.py index be337745b..4f2b76f39 100644 --- a/python/frameworks/ollama/traceai_ollama/_response_attributes_extractor.py +++ b/python/frameworks/ollama/traceai_ollama/_response_attributes_extractor.py @@ -6,10 +6,14 @@ from fi_instrumentation.fi_types import ( EmbeddingAttributes, MessageAttributes, + MessageContentAttributes, SpanAttributes, + ToolCallAttributes, ) from opentelemetry.util.types import AttributeValue +from traceai_ollama._utils import _redact_tool_calls + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) @@ -56,13 +60,13 @@ def get_extra_attributes( message = response["message"] yield f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_ROLE}", message.get("role", "assistant") yield f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_CONTENT}", message.get("content", "") - yield SpanAttributes.OUTPUT_VALUE, message.get("content", "") + yield from self._message_details(message) # Generate response elif "response" in response: yield f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_ROLE}", "assistant" yield f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_CONTENT}", response["response"] - yield SpanAttributes.OUTPUT_VALUE, response["response"] + yield from self._message_details(response) # Embedding response elif "embedding" in response: @@ -77,14 +81,31 @@ def get_extra_attributes( yield f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{i}.{EmbeddingAttributes.EMBEDDING_VECTOR}", safe_json_dumps(emb[:10]) yield SpanAttributes.OUTPUT_VALUE, f"[{len(embeddings)} embeddings]" - # Core masks dedicated vector keys, but cannot inspect serialized JSON. - # Redact only the copy used for telemetry, never the SDK response itself. - output = response + # Core masks dedicated attributes but cannot inspect serialized JSON. + # Apply directional privacy to a telemetry copy, not to SDK responses. + output = dict(response) + if self._config.hide_output_messages or self._config.hide_output_text: + # Logprob tokens (including top alternatives) reveal generated text + # even when the visible response/message content has been masked. + output.pop("logprobs", None) if self._config.hide_embedding_vectors: - output = { - key: REDACTED_VALUE if key in ("embedding", "embeddings") else value - for key, value in response.items() - } + for key in ("embedding", "embeddings"): + if key in output: + output[key] = REDACTED_VALUE + if self._config.hide_output_messages: + for key in ("message", "response", "thinking"): + if key in output: + output[key] = REDACTED_VALUE + elif self._config.hide_output_text: + if isinstance(output.get("message"), dict): + output["message"] = { + key: REDACTED_VALUE if key in ("content", "thinking") else + _redact_tool_calls(value) if key == "tool_calls" else value + for key, value in output["message"].items() if key != "logprobs" + } + for key in ("response", "thinking"): + if key in output: + output[key] = REDACTED_VALUE yield SpanAttributes.OUTPUT_VALUE, safe_json_dumps(output) # Performance metrics (Ollama-specific) @@ -104,3 +125,20 @@ def get_extra_attributes( # Done reason if "done_reason" in response: yield "ollama.done_reason", response["done_reason"] + + def _message_details(self, message: Mapping[str, Any]) -> Iterator[Tuple[str, AttributeValue]]: + prefix = f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.0" + if thinking := message.get("thinking"): + content_prefix = f"{prefix}.{MessageAttributes.MESSAGE_CONTENTS}.0" + yield f"{content_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TYPE}", "thinking" + yield f"{content_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TEXT}", thinking + for index, tool in enumerate(message.get("tool_calls") or ()): + function = tool.get("function") or {} + tool_prefix = f"{prefix}.{MessageAttributes.MESSAGE_TOOL_CALLS}.{index}" + if tool.get("id"): + yield f"{tool_prefix}.{ToolCallAttributes.TOOL_CALL_ID}", tool["id"] + if function.get("name"): + yield f"{tool_prefix}.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}", function["name"] + if "arguments" in function: + arguments = REDACTED_VALUE if self._config.hide_output_text else function["arguments"] + yield f"{tool_prefix}.{ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON}", safe_json_dumps(arguments) diff --git a/python/frameworks/ollama/traceai_ollama/_stream.py b/python/frameworks/ollama/traceai_ollama/_stream.py new file mode 100644 index 000000000..a56315ef1 --- /dev/null +++ b/python/frameworks/ollama/traceai_ollama/_stream.py @@ -0,0 +1,210 @@ +"""Bounded telemetry and explicit lifecycle for Ollama's lazy iterators.""" +import json +import logging +from typing import Any, AsyncIterator, ContextManager, Dict, Iterator, Mapping, Optional + +from fi_instrumentation import TraceConfig, safe_json_dumps +from opentelemetry import trace + +from traceai_ollama._response_attributes_extractor import _ResponseAttributesExtractor +from traceai_ollama._utils import _finish_tracing, _to_dict +from traceai_ollama._with_span import _WithSpan + +logger = logging.getLogger(__name__) + +# A stream may be arbitrarily long. Keep a preview, not every raw chunk, and +# continue collecting usage after the preview fills. Limits never affect users. +_MAX_OUTPUT_CHARACTERS = 64 * 1024 +_MAX_TOOL_CALLS = 32 +_METADATA_KEYS = ( + "model", "created_at", "done", "done_reason", "total_duration", + "load_duration", "prompt_eval_count", "prompt_eval_duration", + "eval_count", "eval_duration", +) + + +class _ResponseAccumulator: + def __init__(self, config: TraceConfig) -> None: + self._config = config + self._remaining = _MAX_OUTPUT_CHARACTERS + self._metadata: Dict[str, Any] = {} + self._content: list[str] = [] + self._thinking: list[str] = [] + self._tools: list[Dict[str, Any]] = [] + self._chat = False + self.truncated = False + + def _text(self, value: Any, target: list) -> None: + if not isinstance(value, str) or not value: + return + if self._config.hide_output_text: + return + preview = value[:self._remaining] + if preview: + target.append(preview) + self._remaining -= len(preview) + self.truncated |= len(preview) < len(value) + + def add(self, chunk: Any) -> None: + data = _to_dict(chunk) + for key in _METADATA_KEYS: + value = data.get(key) + if isinstance(value, (bool, int, float)): + self._metadata[key] = value + elif isinstance(value, str): + self._metadata[key] = value[:1024] + self._chat |= "message" in data + if self._config.hide_outputs or self._config.hide_output_messages: + return + message = data.get("message") or data + self._text(message.get("content" if self._chat else "response"), self._content) + self._text(message.get("thinking"), self._thinking) + for tool in message.get("tool_calls") or (): + if len(self._tools) >= _MAX_TOOL_CALLS or not self._remaining: + self.truncated = True + break + serialized = safe_json_dumps(tool) + if len(serialized) > self._remaining: + self.truncated = True + continue + self._remaining -= len(serialized) + # Keep a detached, bounded copy; callers still receive the original. + self._tools.append(json.loads(serialized)) + + def response(self) -> Dict[str, Any]: + result = dict(self._metadata) + message = {"content" if self._chat else "response": "".join(self._content)} + if self._thinking: + message["thinking"] = "".join(self._thinking) + if self._tools: + message["tool_calls"] = list(self._tools) + if self._chat: + result["message"] = dict(role="assistant", **message) + else: + result.update(message) + return result + + +class _StreamState: + def __init__( + self, span: _WithSpan, extractor: _ResponseAttributesExtractor, + parameters: Mapping[str, Any], config: TraceConfig, + ) -> None: + self.span = span + self._extractor = extractor + self._parameters = parameters + self._accumulator = _ResponseAccumulator(config) + self.finished = False + + def current_span(self) -> ContextManager[trace.Span]: + # Scope only SDK advancement/close, never application code between yields. + return trace.use_span( + self.span._span, end_on_exit=False, + record_exception=False, set_status_on_exception=False, + ) + + def accept(self, chunk: Any) -> None: + try: + self._accumulator.add(chunk) + except Exception: + logger.exception("Failed to collect Ollama stream attributes") + + def finish(self, error: Optional[BaseException] = None, complete: bool = False) -> None: + if self.finished: + return + self.finished = True + status = trace.Status(trace.StatusCode.OK if complete else trace.StatusCode.UNSET) + try: + if error is not None: + self.span.record_exception(error) + status = trace.Status(trace.StatusCode.ERROR, f"{type(error).__name__}: {error}") + response = self._accumulator.response() + if self._accumulator.truncated: + self.span._span.set_attribute("ollama.output_truncated", True) + _finish_tracing( + status=status, with_span=self.span, + attributes=self._extractor.get_attributes(response, is_streaming=True), + extra_attributes=self._extractor.get_extra_attributes( + response, self._parameters, is_streaming=True, + ), + ) + except Exception: + logger.exception("Failed to finalize Ollama stream") + finally: + self.span.finish_tracing(status=status) + + def __del__(self) -> None: + # Best effort for abandoned streams. Explicit close/aclose is deterministic. + self.finish() + + +class _SyncStream(Iterator[Any]): + def __init__(self, response: Iterator[Any], state: _StreamState) -> None: + self._response = iter(response) + self._state = state + + def __iter__(self) -> "_SyncStream": + return self + + def __next__(self) -> Any: + if self._state.finished: + raise StopIteration + try: + with self._state.current_span(): + chunk = next(self._response) + except StopIteration: + self._state.finish(complete=True) + raise + except BaseException as error: + self._state.finish(error=error) + raise + self._state.accept(chunk) + return chunk + + def close(self) -> None: + try: + with self._state.current_span(): + close = getattr(self._response, "close", None) + if close is not None: + close() + except BaseException as error: + self._state.finish(error=error) + raise + finally: + self._state.finish() + + +class _AsyncStream(AsyncIterator[Any]): + def __init__(self, response: AsyncIterator[Any], state: _StreamState) -> None: + self._response = response.__aiter__() + self._state = state + + def __aiter__(self) -> "_AsyncStream": + return self + + async def __anext__(self) -> Any: + if self._state.finished: + raise StopAsyncIteration + try: + with self._state.current_span(): + chunk = await self._response.__anext__() + except StopAsyncIteration: + self._state.finish(complete=True) + raise + except BaseException as error: + self._state.finish(error=error) + raise + self._state.accept(chunk) + return chunk + + async def aclose(self) -> None: + try: + with self._state.current_span(): + close = getattr(self._response, "aclose", None) + if close is not None: + await close() + except BaseException as error: + self._state.finish(error=error) + raise + finally: + self._state.finish() diff --git a/python/frameworks/ollama/traceai_ollama/_utils.py b/python/frameworks/ollama/traceai_ollama/_utils.py index f4300bcdb..354ee4812 100644 --- a/python/frameworks/ollama/traceai_ollama/_utils.py +++ b/python/frameworks/ollama/traceai_ollama/_utils.py @@ -2,6 +2,7 @@ from typing import Any, Dict, Iterator, Mapping, Optional, Tuple from fi_instrumentation import safe_json_dumps +from fi_instrumentation.instrumentation.config import REDACTED_VALUE from fi_instrumentation.fi_types import SpanAttributes from opentelemetry import trace as trace_api from opentelemetry.util.types import AttributeValue @@ -27,6 +28,25 @@ def _to_dict(obj: Any) -> Dict[str, Any]: return {"value": str(obj)} +def _redact_tool_calls(tool_calls: Any) -> Any: + """Mask argument payloads in a detached telemetry copy, preserving tool names.""" + if not isinstance(tool_calls, (list, tuple)): + return tool_calls + redacted = [] + for tool in tool_calls: + # SDK models are iterable but are not Mappings. Normalize both levels + # before redaction, including a mapping containing a typed function. + item = dict(tool) if isinstance(tool, Mapping) else dict(_to_dict(tool)) + if item.get("function") is not None: + value = item["function"] + function = dict(value) if isinstance(value, Mapping) else dict(_to_dict(value)) + if "arguments" in function: + function["arguments"] = REDACTED_VALUE + item["function"] = function + redacted.append(item) + return redacted + + def _flatten( mapping: Mapping[str, Any], prefix: str = "" ) -> Iterator[Tuple[str, AttributeValue]]: diff --git a/python/frameworks/ollama/traceai_ollama/_with_span.py b/python/frameworks/ollama/traceai_ollama/_with_span.py index a51f0025e..1e60b7342 100644 --- a/python/frameworks/ollama/traceai_ollama/_with_span.py +++ b/python/frameworks/ollama/traceai_ollama/_with_span.py @@ -18,7 +18,7 @@ def __init__( self._context_attributes = context_attributes or {} self._extra_attributes = extra_attributes or {} - def record_exception(self, exception: Exception) -> None: + def record_exception(self, exception: BaseException) -> None: """Record an exception on the span.""" if self._span.is_recording(): self._span.record_exception(exception) diff --git a/python/frameworks/ollama/traceai_ollama/_wrappers.py b/python/frameworks/ollama/traceai_ollama/_wrappers.py index b5f33c22e..e2f2a8582 100644 --- a/python/frameworks/ollama/traceai_ollama/_wrappers.py +++ b/python/frameworks/ollama/traceai_ollama/_wrappers.py @@ -1,18 +1,17 @@ -import json import logging from abc import ABC from contextlib import contextmanager -from typing import Any, Callable, Dict, Iterable, Iterator, List, Mapping, Tuple +from typing import Any, Callable, Dict, Iterable, Iterator, Mapping, Tuple import opentelemetry.context as context_api -from fi_instrumentation import TraceConfig, get_attributes_from_context, safe_json_dumps -from fi_instrumentation.fi_types import SpanAttributes +from fi_instrumentation import TraceConfig, get_attributes_from_context from opentelemetry import trace as trace_api from opentelemetry.trace import INVALID_SPAN from opentelemetry.util.types import AttributeValue from traceai_ollama._request_attributes_extractor import _RequestAttributesExtractor from traceai_ollama._response_attributes_extractor import _ResponseAttributesExtractor +from traceai_ollama._stream import _AsyncStream, _StreamState, _SyncStream from traceai_ollama._utils import _finish_tracing, _to_dict from traceai_ollama._with_span import _WithSpan @@ -62,7 +61,7 @@ class _ChatWrapper(_WithTracer): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self._request_extractor = _RequestAttributesExtractor() + self._request_extractor = _RequestAttributesExtractor(config=self._config) self._response_extractor = _ResponseAttributesExtractor(config=self._config) def __call__( @@ -83,7 +82,6 @@ def __call__( request_parameters["messages"] = args[1] span_name = f"ollama.chat" - streaming = request_parameters.get("stream", False) with self._start_as_current_span( span_name=span_name, @@ -93,7 +91,7 @@ def __call__( ) as span: try: response = wrapped(*args, **kwargs) - except Exception as exception: + except BaseException as exception: span.record_exception(exception) status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, @@ -102,7 +100,7 @@ def __call__( span.finish_tracing(status=status) raise - if streaming and isinstance(response, Iterable): + if isinstance(response, Iterator): return self._handle_streaming_response(span, response, request_parameters) else: return self._handle_response(span, response, request_parameters) @@ -127,66 +125,12 @@ def _handle_response( return response def _handle_streaming_response( - self, span: _WithSpan, response: Iterable, request_parameters: Dict[str, Any] - ) -> Iterable: - """Handle streaming response.""" - content_parts: List[str] = [] - raw_chunks: List[Dict[str, Any]] = [] - final_response: Dict[str, Any] = {} - - def streaming_wrapper(): - nonlocal final_response - try: - for chunk in response: - chunk_dict = _to_dict(chunk) - raw_chunks.append(chunk_dict) - - # Accumulate content - if "message" in chunk_dict: - content = chunk_dict["message"].get("content", "") - if content: - content_parts.append(content) - - # Capture final chunk with done=True - if chunk_dict.get("done", False): - final_response = chunk_dict - - yield chunk - - except Exception as exception: - span.record_exception(exception) - status = trace_api.Status( - status_code=trace_api.StatusCode.ERROR, - description=f"{type(exception).__name__}: {exception}", - ) - span.finish_tracing(status=status) - raise - else: - try: - # Build final response for attribute extraction - final_response["message"] = { - "role": "assistant", - "content": "".join(content_parts) - } - span._span.set_attribute(SpanAttributes.OUTPUT_VALUE, safe_json_dumps(raw_chunks)) - span._span.set_attribute(SpanAttributes.OUTPUT_VALUE, "".join(content_parts)) - - _finish_tracing( - status=trace_api.Status(status_code=trace_api.StatusCode.OK), - with_span=span, - attributes=self._response_extractor.get_attributes(final_response, is_streaming=True), - extra_attributes=self._response_extractor.get_extra_attributes( - final_response, request_parameters, is_streaming=True - ), - ) - except Exception: - logger.exception("Failed to finalize streaming response") - span.finish_tracing() - finally: - if span._span.is_recording(): - span.finish_tracing() - - return streaming_wrapper() + self, span: _WithSpan, response: Any, request_parameters: Dict[str, Any] + ) -> Any: + return _SyncStream( + response, + _StreamState(span, self._response_extractor, request_parameters, self._config), + ) class _AsyncChatWrapper(_WithTracer): @@ -194,7 +138,7 @@ class _AsyncChatWrapper(_WithTracer): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self._request_extractor = _RequestAttributesExtractor() + self._request_extractor = _RequestAttributesExtractor(config=self._config) self._response_extractor = _ResponseAttributesExtractor(config=self._config) async def __call__( @@ -214,7 +158,6 @@ async def __call__( request_parameters["messages"] = args[1] span_name = f"ollama.chat" - streaming = request_parameters.get("stream", False) with self._start_as_current_span( span_name=span_name, @@ -224,7 +167,7 @@ async def __call__( ) as span: try: response = await wrapped(*args, **kwargs) - except Exception as exception: + except BaseException as exception: span.record_exception(exception) status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, @@ -233,7 +176,7 @@ async def __call__( span.finish_tracing(status=status) raise - if streaming and hasattr(response, "__aiter__"): + if hasattr(response, "__aiter__"): return self._handle_async_streaming_response(span, response, request_parameters) else: return self._handle_response(span, response, request_parameters) @@ -257,64 +200,13 @@ def _handle_response( span.finish_tracing() return response - async def _handle_async_streaming_response( + def _handle_async_streaming_response( self, span: _WithSpan, response: Any, request_parameters: Dict[str, Any] - ): - """Handle async streaming response.""" - content_parts: List[str] = [] - raw_chunks: List[Dict[str, Any]] = [] - final_response: Dict[str, Any] = {} - - async def streaming_wrapper(): - nonlocal final_response - try: - async for chunk in response: - chunk_dict = _to_dict(chunk) - raw_chunks.append(chunk_dict) - - if "message" in chunk_dict: - content = chunk_dict["message"].get("content", "") - if content: - content_parts.append(content) - - if chunk_dict.get("done", False): - final_response = chunk_dict - - yield chunk - - except Exception as exception: - span.record_exception(exception) - status = trace_api.Status( - status_code=trace_api.StatusCode.ERROR, - description=f"{type(exception).__name__}: {exception}", - ) - span.finish_tracing(status=status) - raise - else: - try: - final_response["message"] = { - "role": "assistant", - "content": "".join(content_parts) - } - span._span.set_attribute(SpanAttributes.OUTPUT_VALUE, safe_json_dumps(raw_chunks)) - span._span.set_attribute(SpanAttributes.OUTPUT_VALUE, "".join(content_parts)) - - _finish_tracing( - status=trace_api.Status(status_code=trace_api.StatusCode.OK), - with_span=span, - attributes=self._response_extractor.get_attributes(final_response, is_streaming=True), - extra_attributes=self._response_extractor.get_extra_attributes( - final_response, request_parameters, is_streaming=True - ), - ) - except Exception: - logger.exception("Failed to finalize streaming response") - span.finish_tracing() - finally: - if span._span.is_recording(): - span.finish_tracing() - - return streaming_wrapper() + ) -> Any: + return _AsyncStream( + response, + _StreamState(span, self._response_extractor, request_parameters, self._config), + ) class _GenerateWrapper(_WithTracer): @@ -322,7 +214,7 @@ class _GenerateWrapper(_WithTracer): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self._request_extractor = _RequestAttributesExtractor() + self._request_extractor = _RequestAttributesExtractor(config=self._config) self._response_extractor = _ResponseAttributesExtractor(config=self._config) def __call__( @@ -342,7 +234,6 @@ def __call__( request_parameters["prompt"] = args[1] span_name = f"ollama.generate" - streaming = request_parameters.get("stream", False) with self._start_as_current_span( span_name=span_name, @@ -352,7 +243,7 @@ def __call__( ) as span: try: response = wrapped(*args, **kwargs) - except Exception as exception: + except BaseException as exception: span.record_exception(exception) status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, @@ -361,7 +252,7 @@ def __call__( span.finish_tracing(status=status) raise - if streaming and isinstance(response, Iterable): + if isinstance(response, Iterator): return self._handle_streaming_response(span, response, request_parameters) else: return self._handle_response(span, response, request_parameters) @@ -386,60 +277,12 @@ def _handle_response( return response def _handle_streaming_response( - self, span: _WithSpan, response: Iterable, request_parameters: Dict[str, Any] - ) -> Iterable: - """Handle streaming response.""" - content_parts: List[str] = [] - raw_chunks: List[Dict[str, Any]] = [] - final_response: Dict[str, Any] = {} - - def streaming_wrapper(): - nonlocal final_response - try: - for chunk in response: - chunk_dict = _to_dict(chunk) - raw_chunks.append(chunk_dict) - - if "response" in chunk_dict: - content = chunk_dict.get("response", "") - if content: - content_parts.append(content) - - if chunk_dict.get("done", False): - final_response = chunk_dict - - yield chunk - - except Exception as exception: - span.record_exception(exception) - status = trace_api.Status( - status_code=trace_api.StatusCode.ERROR, - description=f"{type(exception).__name__}: {exception}", - ) - span.finish_tracing(status=status) - raise - else: - try: - final_response["response"] = "".join(content_parts) - span._span.set_attribute(SpanAttributes.OUTPUT_VALUE, safe_json_dumps(raw_chunks)) - span._span.set_attribute(SpanAttributes.OUTPUT_VALUE, "".join(content_parts)) - - _finish_tracing( - status=trace_api.Status(status_code=trace_api.StatusCode.OK), - with_span=span, - attributes=self._response_extractor.get_attributes(final_response, is_streaming=True), - extra_attributes=self._response_extractor.get_extra_attributes( - final_response, request_parameters, is_streaming=True - ), - ) - except Exception: - logger.exception("Failed to finalize streaming response") - span.finish_tracing() - finally: - if span._span.is_recording(): - span.finish_tracing() - - return streaming_wrapper() + self, span: _WithSpan, response: Any, request_parameters: Dict[str, Any] + ) -> Any: + return _SyncStream( + response, + _StreamState(span, self._response_extractor, request_parameters, self._config), + ) class _AsyncGenerateWrapper(_WithTracer): @@ -447,7 +290,7 @@ class _AsyncGenerateWrapper(_WithTracer): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self._request_extractor = _RequestAttributesExtractor() + self._request_extractor = _RequestAttributesExtractor(config=self._config) self._response_extractor = _ResponseAttributesExtractor(config=self._config) async def __call__( @@ -467,7 +310,6 @@ async def __call__( request_parameters["prompt"] = args[1] span_name = f"ollama.generate" - streaming = request_parameters.get("stream", False) with self._start_as_current_span( span_name=span_name, @@ -477,7 +319,7 @@ async def __call__( ) as span: try: response = await wrapped(*args, **kwargs) - except Exception as exception: + except BaseException as exception: span.record_exception(exception) status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, @@ -486,7 +328,7 @@ async def __call__( span.finish_tracing(status=status) raise - if streaming and hasattr(response, "__aiter__"): + if hasattr(response, "__aiter__"): return self._handle_async_streaming_response(span, response, request_parameters) else: return self._handle_response(span, response, request_parameters) @@ -510,61 +352,13 @@ def _handle_response( span.finish_tracing() return response - async def _handle_async_streaming_response( + def _handle_async_streaming_response( self, span: _WithSpan, response: Any, request_parameters: Dict[str, Any] - ): - """Handle async streaming response.""" - content_parts: List[str] = [] - raw_chunks: List[Dict[str, Any]] = [] - final_response: Dict[str, Any] = {} - - async def streaming_wrapper(): - nonlocal final_response - try: - async for chunk in response: - chunk_dict = _to_dict(chunk) - raw_chunks.append(chunk_dict) - - if "response" in chunk_dict: - content = chunk_dict.get("response", "") - if content: - content_parts.append(content) - - if chunk_dict.get("done", False): - final_response = chunk_dict - - yield chunk - - except Exception as exception: - span.record_exception(exception) - status = trace_api.Status( - status_code=trace_api.StatusCode.ERROR, - description=f"{type(exception).__name__}: {exception}", - ) - span.finish_tracing(status=status) - raise - else: - try: - final_response["response"] = "".join(content_parts) - span._span.set_attribute(SpanAttributes.OUTPUT_VALUE, safe_json_dumps(raw_chunks)) - span._span.set_attribute(SpanAttributes.OUTPUT_VALUE, "".join(content_parts)) - - _finish_tracing( - status=trace_api.Status(status_code=trace_api.StatusCode.OK), - with_span=span, - attributes=self._response_extractor.get_attributes(final_response, is_streaming=True), - extra_attributes=self._response_extractor.get_extra_attributes( - final_response, request_parameters, is_streaming=True - ), - ) - except Exception: - logger.exception("Failed to finalize streaming response") - span.finish_tracing() - finally: - if span._span.is_recording(): - span.finish_tracing() - - return streaming_wrapper() + ) -> Any: + return _AsyncStream( + response, + _StreamState(span, self._response_extractor, request_parameters, self._config), + ) class _EmbedWrapper(_WithTracer): @@ -572,7 +366,7 @@ class _EmbedWrapper(_WithTracer): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self._request_extractor = _RequestAttributesExtractor() + self._request_extractor = _RequestAttributesExtractor(config=self._config) self._response_extractor = _ResponseAttributesExtractor(config=self._config) def __call__( @@ -602,7 +396,7 @@ def __call__( ) as span: try: response = wrapped(*args, **kwargs) - except Exception as exception: + except BaseException as exception: span.record_exception(exception) status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, @@ -632,7 +426,7 @@ class _AsyncEmbedWrapper(_WithTracer): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self._request_extractor = _RequestAttributesExtractor() + self._request_extractor = _RequestAttributesExtractor(config=self._config) self._response_extractor = _ResponseAttributesExtractor(config=self._config) async def __call__( @@ -662,7 +456,7 @@ async def __call__( ) as span: try: response = await wrapped(*args, **kwargs) - except Exception as exception: + except BaseException as exception: span.record_exception(exception) status = trace_api.Status( status_code=trace_api.StatusCode.ERROR,