Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions .github/workflows/ollama-python-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
name: Ollama Python contracts

on:
pull_request:
branches: [dev, main]
paths:
- "python/frameworks/ollama/**"
- "python/fi_instrumentation/**"
- "python/pyproject.toml"
- ".github/workflows/ollama-python-tests.yml"
push:
branches: [dev]
paths:
- "python/frameworks/ollama/**"
- "python/fi_instrumentation/**"
- "python/pyproject.toml"
- ".github/workflows/ollama-python-tests.yml"
workflow_dispatch:

permissions:
contents: read

jobs:
contracts:
name: Ollama Python (${{ matrix.ollama }})
runs-on: ubuntu-latest
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
ollama: ["0.3.0", "0.6.3"]
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install package and test tools
run: |
python -m venv "$RUNNER_TEMP/ollama-tests"
"$RUNNER_TEMP/ollama-tests/bin/pip" install \
-e ./python -e ./python/frameworks/ollama \
"ollama==${{ matrix.ollama }}" "pytest==8.4.2" "pytest-asyncio==0.24.0"
- name: Run real SDK and OTLP contracts with networking disabled
run: |
sudo unshare --net "$RUNNER_TEMP/ollama-tests/bin/python" -I -m pytest \
python/frameworks/ollama/tests -q -p no:cacheprovider
44 changes: 37 additions & 7 deletions python/frameworks/ollama/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,10 +81,23 @@ stream = ollama.chat(
stream=True
)

for chunk in stream:
print(chunk["message"]["content"], end="", flush=True)
try:
for chunk in stream:
print(chunk["message"]["content"], end="", flush=True)
finally:
stream.close()
```

Streams keep their span open until exhaustion, an error, or explicit `close()`
(`await stream.aclose()` for async streams). Close streams when stopping early;
final usage is available only when Ollama returns its final usage chunk.

Streaming telemetry combines content, thinking, and tool calls without retaining
raw chunks. It keeps up to 65,536 characters across those fields and at most 32
tool calls; `ollama.output_truncated=true` marks a bounded preview. These limits
never truncate the response delivered to your application. `TraceConfig` output
privacy settings apply to both normalized attributes and serialized output.

### Text Generation

```python
Expand Down Expand Up @@ -115,12 +128,12 @@ response = ollama.embed(
model="nomic-embed-text",
input="Hello, world!"
)
print(f"Embedding dimensions: {len(response['embedding'])}")
print(f"Embedding dimensions: {len(response['embeddings'][0])}")

# Multiple embeddings (batch)
response = ollama.embeddings(
response = ollama.embed(
model="nomic-embed-text",
prompt=["Hello", "World"]
input=["Hello", "World"]
)
```

Expand Down Expand Up @@ -204,6 +217,22 @@ OllamaInstrumentor().instrument(
)
```

### Aggregate privacy values

With the current core privacy policy, `input.value` and `output.value` are
opaque aggregates. Input hiding (`FI_HIDE_INPUTS`, `FI_HIDE_INPUT_MESSAGES`,
`FI_HIDE_INPUT_TEXT`, or `FI_HIDE_INPUT_IMAGES`) replaces `input.value` with
the literal `__REDACTED__`; output hiding (`FI_HIDE_OUTPUTS`,
`FI_HIDE_OUTPUT_MESSAGES`, or `FI_HIDE_OUTPUT_TEXT`) does the same for
`output.value`. Check for this placeholder before attempting to parse JSON.
The corresponding `TraceConfig` options have the same semantics.

Structured attributes are masked separately. For example, text-only hiding
retains the message role while redacting message content. Adapter-side
sanitization of detached messages and tool arguments does not exempt the
aggregate from the core policy. Neither layer changes the request sent to
Ollama or the response returned to the application.

## Captured Attributes

### Request Attributes
Expand Down Expand Up @@ -257,8 +286,9 @@ query_embedding = ollama.embed(
input=query
)

# Search vector database (not shown)
# context = search_vector_db(query_embedding["embedding"])
# Replace this illustrative context with your vector database's search results.
# ollama.embed returns a batch: search_vector_db(query_embedding["embeddings"][0])
context = "Machine learning uses data to learn patterns for predictions."

# Generate response with context
response = ollama.chat(
Expand Down
157 changes: 157 additions & 0 deletions python/frameworks/ollama/tests/test_directional_privacy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
"""Privacy contracts on the serialized OTLP wire using a real offline Ollama client."""
import base64
import json

import httpx
import ollama
import pytest
from fi_instrumentation import TraceConfig
from fi_instrumentation.instrumentation.config import REDACTED_VALUE
from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter

from traceai_ollama import OllamaInstrumentor
from traceai_ollama._request_attributes_extractor import _RequestAttributesExtractor
from traceai_ollama._response_attributes_extractor import _ResponseAttributesExtractor


def detached_input(flag, request):
"""Adapter sanitization is separate from core's opaque aggregate policy."""
return dict(_RequestAttributesExtractor(
TraceConfig(**{flag: True})
).get_extra_attributes_from_request(request))["input.value"]


def detached_output(flag, response, request):
return dict(_ResponseAttributesExtractor(
TraceConfig(**{flag: True})
).get_extra_attributes(response, request))["output.value"]


PROMPT = "SYNTHETIC_PROMPT_49"
IMAGE = base64.b64encode(b"SYNTHETIC_IMAGE_49").decode()
TOKEN = "SYNTHETIC_TOKEN_49"
ALTERNATE = "SYNTHETIC_ALTERNATE_49"


@pytest.fixture
def client_span():
provider = TracerProvider()
exporter = InMemorySpanExporter()
provider.add_span_processor(SimpleSpanProcessor(exporter))
instrumentor = OllamaInstrumentor()

def run(flag, endpoint, request, response):
exporter.clear()
instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**{flag: True}))
client = ollama.Client(host="http://offline.invalid", transport=httpx.MockTransport(
lambda req: httpx.Response(200, json=response)), trust_env=False)
try:
getattr(client, endpoint)(model="synthetic-model", **request)
spans = exporter.get_finished_spans()
assert len(spans) == 1
wire = ExportTraceServiceRequest.FromString(encode_spans(spans).SerializeToString())
span = wire.resource_spans[0].scope_spans[0].spans[0]
return {item.key: getattr(item.value, item.value.WhichOneof("value")) for item in span.attributes}
finally:
client._client.close()
instrumentor.uninstrument()

yield run
provider.shutdown()


@pytest.mark.parametrize("flag", ["hide_input_text", "hide_input_messages"])
def test_chat_input_text_not_in_serialized_request(client_span, flag):
messages = [{"role": "user", "content": PROMPT}]
attrs = client_span(flag, "chat", {"messages": messages},
{"model": "synthetic-model", "message": {"role": "assistant", "content": "ok"}, "done": True})
assert PROMPT not in json.dumps(attrs)
assert messages[0]["content"] == PROMPT # telemetry must never mutate SDK parameters
assert attrs["input.value"] == REDACTED_VALUE
if flag == "hide_input_text":
assert json.loads(detached_input(flag, {"messages": messages}))["messages"][0]["role"] == "user"
assert attrs["gen_ai.input.messages.0.message.role"] == "user"


@pytest.mark.parametrize("flag", ["hide_input_text", "hide_input_messages"])
def test_generate_and_embed_input_text_not_serialized(client_span, flag):
for endpoint, request, response in [
("generate", {"prompt": PROMPT}, {"response": "ok", "done": True}),
("embed", {"input": PROMPT}, {"embeddings": [[0.1]], "done": True}),
]:
attrs = client_span(flag, endpoint, request, response)
assert PROMPT not in json.dumps(attrs)


def test_chat_input_thinking_and_tool_arguments_hidden(client_span):
messages = [{"role": "user", "content": "ok", "thinking": "SYNTHETIC_INPUT_THINK_83",
"tool_calls": [{"function": {"name": "lookup", "arguments": {"query": "SYNTHETIC_INPUT_TOOL_83"}}}]}]
attrs = client_span("hide_input_text", "chat", {"messages": messages},
{"model": "synthetic-model", "message": {"role": "assistant", "content": "ok"}, "done": True})
assert "SYNTHETIC_INPUT_THINK_83" not in json.dumps(attrs)
assert "SYNTHETIC_INPUT_TOOL_83" not in json.dumps(attrs)
assert attrs["input.value"] == REDACTED_VALUE
message = json.loads(detached_input("hide_input_text", {"messages": messages}))["messages"][0]
assert message["role"] == "user"
assert message["tool_calls"][0]["function"]["name"] == "lookup"
assert messages[0]["tool_calls"][0]["function"]["arguments"]["query"] == "SYNTHETIC_INPUT_TOOL_83"
assert messages[0]["thinking"] == "SYNTHETIC_INPUT_THINK_83"


def test_chat_none_messages_keeps_span(client_span):
attrs = client_span("hide_input_text", "chat", {"messages": None},
{"model": "synthetic-model", "message": {"role": "assistant", "content": "ok"}, "done": True})
assert attrs["gen_ai.provider.name"] == "ollama"
assert attrs["input.value"] == REDACTED_VALUE
assert json.loads(detached_input("hide_input_text", {"messages": None})) == {"messages": []}


def test_chat_images_hidden_in_all_attributes(client_span):
messages = [{"role": "user", "content": "ok", "images": [IMAGE]}]
attrs = client_span("hide_input_images", "chat", {"messages": messages},
{"model": "synthetic-model", "message": {"role": "assistant", "content": "ok"}, "done": True})
assert IMAGE not in json.dumps(attrs)
assert messages[0]["images"] == [IMAGE]
assert attrs["input.value"] == REDACTED_VALUE
assert "images" not in json.loads(detached_input("hide_input_images", {"messages": messages}))["messages"][0]
assert not any(key.endswith(".images") for key in attrs)


def test_generated_tool_arguments_hidden_under_output_text(client_span):
response = {"model": "synthetic-model", "done": True,
"message": {"role": "assistant", "content": "ok", "tool_calls": [
{"function": {"name": "lookup", "arguments": {"query": "SYNTHETIC_TOOL_ARGUMENT_83"}}}]}}
attrs = client_span("hide_output_text", "chat", {"messages": [{"role": "user", "content": "ok"}]}, response)
assert "SYNTHETIC_TOOL_ARGUMENT_83" not in json.dumps(attrs)
assert attrs["output.value"] == REDACTED_VALUE
output = json.loads(detached_output("hide_output_text", response, {"messages": [{"role": "user", "content": "ok"}]}))
assert output["message"]["tool_calls"][0]["function"]["name"] == "lookup"
assert output["message"]["tool_calls"][0]["function"]["arguments"] == REDACTED_VALUE
assert attrs["gen_ai.output.messages.0.message.tool_calls.0.tool_call.function.name"] == "lookup"
assert response["message"]["tool_calls"][0]["function"]["arguments"]["query"] == "SYNTHETIC_TOOL_ARGUMENT_83"


@pytest.mark.parametrize("flag", ["hide_output_text", "hide_output_messages"])
@pytest.mark.parametrize("endpoint", ["chat", "generate"])
def test_logprob_tokens_hidden_on_output_wire(client_span, flag, endpoint):
response = {"model": "synthetic-model", "done": True,
"logprobs": [{"token": TOKEN, "logprob": -0.1,
"top_logprobs": [{"token": ALTERNATE, "logprob": -0.2}]}]}
if endpoint == "chat":
response["message"] = {"role": "assistant", "content": "SYNTHETIC_OUTPUT_49"}
request = {"messages": [{"role": "user", "content": "ok"}]}
else:
response["response"] = "SYNTHETIC_OUTPUT_49"
request = {"prompt": "ok"}
attrs = client_span(flag, endpoint, request, response)
output = json.dumps({key: value for key, value in attrs.items()
if key == "output.value" or key.startswith("gen_ai.output.")})
assert TOKEN not in output
assert ALTERNATE not in output
assert "SYNTHETIC_OUTPUT_49" not in output
assert attrs["output.value"] == REDACTED_VALUE
assert "logprobs" not in json.loads(detached_output(flag, response, request))
2 changes: 1 addition & 1 deletion python/frameworks/ollama/tests/test_privacy_sdk.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ def respond(request):
client._client.close()
assert len(requests) == 1
assert requests[0][0] == "/api/" + endpoint
received = response.model_dump()
received = response if isinstance(response, dict) else response.model_dump()
key = "embeddings" if endpoint == "embed" else "embedding"
assert received[key] == body[key] # Redaction must not mutate the SDK result.
spans = exporter.get_finished_spans()
Expand Down
39 changes: 39 additions & 0 deletions python/frameworks/ollama/tests/test_readme_examples.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
"""Execute the documented RAG example with a real SDK and offline transport."""
import json
from pathlib import Path

import httpx
import ollama


def test_rag_example_defines_context(monkeypatch):
readme = (Path(__file__).parents[1] / "README.md").read_text()
example = readme.split("### RAG Pipeline", 1)[1].split("```python", 1)[1].split("```", 1)[0]
requests = []

def respond(request):
requests.append((request.url.path, json.loads(request.content)))
if request.url.path == "/api/embed":
return httpx.Response(200, json={"embeddings": [[0.1, 0.2]], "model": "nomic-embed-text"})
assert request.url.path == "/api/chat"
return httpx.Response(200, json={
"model": "llama3.2", "done": True,
"message": {"role": "assistant", "content": "Synthetic example response"},
})

client = ollama.Client(host="http://offline.invalid", transport=httpx.MockTransport(respond), trust_env=False)
monkeypatch.setattr(ollama, "embed", client.embed)
monkeypatch.setattr(ollama, "chat", client.chat)
namespace = {}
try:
exec(compile(example, "README.md:RAG Pipeline", "exec"), namespace)
finally:
client._client.close()
assert [path for path, _ in requests] == ["/api/embed", "/api/chat"]
assert namespace["query_embedding"]["embeddings"][0] == [0.1, 0.2]
assert isinstance(namespace["context"], str) and namespace["context"]
assert requests[1][1]["messages"] == [
{"role": "system", "content": f"Use this context: {namespace['context']}"},
{"role": "user", "content": namespace["query"]},
]
assert namespace["response"]["message"]["content"] == "Synthetic example response"
Loading
Loading