From 8161c3ceb5b4aadb2fa9f6382b57d4941be45cf0 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 09:38:26 -0700 Subject: [PATCH 1/2] fix(openai): attribute Ollama and explicitly configured clients Recognize only the official Ollama host automatically. Add a per-client provider override for local and self-hosted compatible servers without changing requests or installing another instrumentor. Co-Authored-By: Claude Code --- .github/workflows/ollama-provider-tests.yml | 52 ++ python/frameworks/openai/README.md | 36 ++ .../openai/tests/test_ollama_provider.py | 465 ++++++++++++++++++ .../openai/traceai_openai/__init__.py | 1 + .../openai/traceai_openai/_client_provider.py | 39 ++ .../openai/traceai_openai/_providers.py | 1 + .../openai/traceai_openai/_request.py | 11 +- 7 files changed, 600 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/ollama-provider-tests.yml create mode 100644 python/frameworks/openai/tests/test_ollama_provider.py create mode 100644 python/frameworks/openai/traceai_openai/_client_provider.py diff --git a/.github/workflows/ollama-provider-tests.yml b/.github/workflows/ollama-provider-tests.yml new file mode 100644 index 000000000..f877b4ddd --- /dev/null +++ b/.github/workflows/ollama-provider-tests.yml @@ -0,0 +1,52 @@ +name: Ollama provider tests + +on: + pull_request: + branches: [main, dev] + paths: + - 'python/frameworks/openai/**' + - 'python/fi_instrumentation/**' + - 'python/pyproject.toml' + - '.github/workflows/ollama-provider-tests.yml' + push: + branches: [dev] + paths: + - 'python/frameworks/openai/**' + - 'python/fi_instrumentation/**' + - 'python/pyproject.toml' + - '.github/workflows/ollama-provider-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + provider: + name: Ollama provider (OpenAI ${{ matrix.openai }}) + runs-on: ubuntu-24.04 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + openai: ['1.69.0', '3.42.0'] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.11' + - name: Install SDK and test dependencies + env: + OPENAI_VERSION: ${{ matrix.openai }} + run: >- + python -m pip install -e python -e python/frameworks/openai + "openai==$OPENAI_VERSION" 'pytest==8.4.2' 'pytest-socket==0.7.0' + - name: Test attribution over localhost OTLP + env: + FI_BASE_URL: http://127.0.0.1:9 + FI_GRPC_URL: http://127.0.0.1:9 + run: >- + python -m pytest python/frameworks/openai/tests/test_ollama_provider.py + -q -o addopts='' -p no:cacheprovider --disable-socket + --allow-hosts=127.0.0.1 --allow-unix-socket diff --git a/python/frameworks/openai/README.md b/python/frameworks/openai/README.md index 13dc6fc3a..9969ece2f 100644 --- a/python/frameworks/openai/README.md +++ b/python/frameworks/openai/README.md @@ -23,6 +23,42 @@ os.environ["FI_SECRET_KEY"] = FI_SECRET_KEY os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY ``` +## Ollama and self-hosted OpenAI-compatible clients + +Keep using this package when your application calls Ollama through the OpenAI +SDK. The official `https://ollama.com/v1` endpoint is attributed to `ollama` +automatically. A local address can host many providers, so set the provider +explicitly for each local or self-hosted client: + +```python +from openai import OpenAI +from traceai_openai import OpenAIInstrumentor, set_client_provider + +# Use the trace provider configured in the quickstart below. +OpenAIInstrumentor().instrument(tracer_provider=trace_provider) + +client = OpenAI(base_url="http://localhost:11434/v1", api_key="ollama") +set_client_provider(client, provider="ollama") +``` + +The override controls `gen_ai.provider.name` on sync and async requests, +including streams, embeddings, Responses and failures. It changes tracing +metadata only; it adds no spans and sends no provider setting to the server. +Use a lowercase provider identifier (1–64 characters, starting with a letter; +letters, digits, `.`, `_` and `-` are accepted), not a URL or secret. + +Set the override before making requests. It takes precedence over host detection; +`set_client_provider(client, provider=None)` restores host detection. Other +clients are unaffected. A client returned by `copy()` or `with_options()` is a +separate instance: set its override explicitly too, especially if its endpoint +changes. Unconfigured loopback, proxy and unknown hosts retain the historical +`openai` label; the instrumentor never assumes a local server is Ollama. + +For applications using the native `ollama` Python client instead, use +`traceai-ollama`. Do not install both instrumentors for the same API call. +See [Ollama's OpenAI compatibility guide](https://docs.ollama.com/api/openai-compatibility) +for supported endpoints and models. + ## Quickstart ### Register Tracer Provider diff --git a/python/frameworks/openai/tests/test_ollama_provider.py b/python/frameworks/openai/tests/test_ollama_provider.py new file mode 100644 index 000000000..89db7e112 --- /dev/null +++ b/python/frameworks/openai/tests/test_ollama_provider.py @@ -0,0 +1,465 @@ +"""Offline provider contracts using real SDKs and a localhost OTLP receiver.""" + +import asyncio +import gc +import json +import threading +import weakref +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import openai +import pytest +import requests +import traceai_openai +from fi_instrumentation import TraceConfig +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, +) +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from traceai_openai import OpenAIInstrumentor +from traceai_openai._providers import provider_from_host + +# OpenAI 3.x uses httpx2; never substitute a different transport implementation. +if int(openai.__version__.split(".")[0]) >= 3: + import httpx2 as httpx +else: + import httpx + +MODEL = "local-model" +PRIVATE_INPUT = "private-input-sentinel" +PRIVATE_OUTPUT = "private-output-sentinel" +FAKE_KEY = "synthetic-key-not-a-credential" + + +@pytest.fixture(scope="module") +def collector(): + bodies = [] + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + assert self.path == "/v1/traces" + bodies.append(self.rfile.read(int(self.headers["Content-Length"]))) + self.send_response(200) + self.end_headers() + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield server.server_port, bodies + server.shutdown() + server.server_close() + thread.join() + + +@pytest.fixture +def wire(collector): + port, bodies = collector + bodies.clear() + session = requests.Session() + session.trust_env = False + provider = TracerProvider() + provider.add_span_processor( + SimpleSpanProcessor( + OTLPSpanExporter( + endpoint=f"http://127.0.0.1:{port}/v1/traces", + session=session, + ) + ) + ) + instrumentor = OpenAIInstrumentor() + instrumentor.instrument( + tracer_provider=provider, + config=TraceConfig( + hide_inputs=True, + hide_outputs=True, + hide_embedding_vectors=True, + hide_llm_invocation_parameters=True, + ), + ) + yield bodies + instrumentor.uninstrument() + provider.shutdown() + session.close() + + +def spans_on_wire(bodies): + return [ + span + for body in bodies + for resource in ExportTraceServiceRequest.FromString(body).resource_spans + for scope in resource.scope_spans + for span in scope.spans + ] + + +def attributes(span): + return {entry.key: entry.value.string_value for entry in span.attributes} + + +def set_provider(client, provider): + # Attribute lookup lets the unchanged dev revision demonstrate the missing API. + traceai_openai.set_client_provider(client, provider=provider) + + +def handler_for(operation): + def handler(request): + # Provider attribution must stay out of vendor request headers and bodies. + assert not any( + "traceai" in name or "provider" in name for name in request.headers + ) + assert "provider" not in json.loads(request.content) + if operation.startswith("error-"): + return httpx.Response(401, json={"error": {"message": "Unauthorized"}}) + if operation == "embeddings": + return httpx.Response( + 200, + json={ + "object": "list", + "model": MODEL, + "data": [ + {"object": "embedding", "index": 0, "embedding": [0.25, -0.5]} + ], + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + }, + ) + if operation.startswith("responses"): + response = { + "id": "r1", + "object": "response", + "created_at": 1, + "model": MODEL, + "status": "completed", + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "output": [ + { + "id": "m1", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": PRIVATE_OUTPUT, + "annotations": [], + } + ], + } + ], + "usage": {"input_tokens": 2, "output_tokens": 1, "total_tokens": 3}, + } + if operation == "responses": + return httpx.Response(200, json=response) + events = [ + { + "type": "response.created", + "sequence_number": 0, + "response": {**response, "status": "in_progress", "output": []}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "m1", + "output_index": 0, + "content_index": 0, + "delta": PRIVATE_OUTPUT, + }, + { + "type": "response.completed", + "sequence_number": 2, + "response": response, + }, + ] + body = "".join( + f"event: {e['type']}\ndata: {json.dumps(e)}\n\n" for e in events + ) + elif operation == "chat-stream": + event = { + "id": "c1", + "object": "chat.completion.chunk", + "created": 1, + "model": MODEL, + "choices": [ + { + "index": 0, + "delta": {"content": PRIVATE_OUTPUT}, + "finish_reason": "stop", + } + ], + } + body = f"data: {json.dumps(event)}\n\ndata: [DONE]\n\n" + else: + return httpx.Response( + 200, + json={ + "id": "c1", + "object": "chat.completion", + "created": 1, + "model": MODEL, + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": PRIVATE_OUTPUT}, + } + ], + "usage": { + "prompt_tokens": 2, + "completion_tokens": 1, + "total_tokens": 3, + }, + }, + ) + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, content=body + ) + + return handler + + +def make_client(base_url, operation="chat", use_async=False): + cls = openai.AsyncOpenAI if use_async else openai.OpenAI + http_cls = httpx.AsyncClient if use_async else httpx.Client + return cls( + api_key=FAKE_KEY, + base_url=base_url, + max_retries=0, + http_client=http_cls( + transport=httpx.MockTransport(handler_for(operation)), trust_env=False + ), + ) + + +def call(client, operation): + if "embeddings" in operation: + return client.embeddings.create( + model=MODEL, input=PRIVATE_INPUT, encoding_format="float" + ) + if "responses" in operation: + return client.responses.create( + model=MODEL, input=PRIVATE_INPUT, stream=operation.endswith("-stream") + ) + return client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": PRIVATE_INPUT}], + stream=operation.endswith("-stream"), + ) + + +def assert_response(response, operation): + if operation == "embeddings": + assert response.data[0].embedding == [0.25, -0.5] + elif operation == "responses": + assert response.output_text == PRIVATE_OUTPUT + else: + assert response.choices[0].message.content == PRIVATE_OUTPUT + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize( + "operation", + [ + "chat", + "chat-stream", + "embeddings", + "responses", + "responses-stream", + "error-chat", + "error-embeddings", + "error-responses", + ], +) +@pytest.mark.parametrize( + "base_url,override,expected", + [ + ("https://ollama.com/v1", None, "ollama"), + ("http://localhost:11434/v1", "ollama", "ollama"), + ("http://127.0.0.1:11434/v1", "ollama", "ollama"), + ("http://localhost:11434/v1", None, "openai"), + ("https://ollama.com.example/v1", None, "openai"), + ], + ids=[ + "official", + "localhost-override", + "loopback-override", + "unknown-local", + "lookalike", + ], +) +def test_provider_on_wire(wire, use_async, operation, base_url, override, expected): + async def run_async(): + async with make_client(base_url, operation, use_async=True) as client: + if override: + set_provider(client, override) + response = await call(client, operation) + if operation.endswith("-stream"): + assert not spans_on_wire(wire) + chunks = [chunk async for chunk in response] + assert chunks + else: + assert_response(response, operation) + + def run_sync(): + with make_client(base_url, operation) as client: + if override: + set_provider(client, override) + response = call(client, operation) + if operation.endswith("-stream"): + assert not spans_on_wire(wire) + assert list(response) + else: + assert_response(response, operation) + + def run(): + return asyncio.run(run_async()) if use_async else run_sync() + + if operation.startswith("error-"): + with pytest.raises(openai.AuthenticationError): + run() + else: + run() + spans = spans_on_wire(wire) + assert len(spans) == 1 + attrs = attributes(spans[0]) + assert attrs["gen_ai.provider.name"] == expected + assert attrs["gen_ai.request.model"] == MODEL + assert spans[0].status.code == (2 if operation.startswith("error-") else 1) + assert spans[0].end_time_unix_nano >= spans[0].start_time_unix_nano + payload = b"".join(wire) + for sentinel in (PRIVATE_INPUT, PRIVATE_OUTPUT, FAKE_KEY): + assert sentinel.encode() not in payload + + +@pytest.mark.parametrize( + "host,expected", + [ + ("OLLAMA.COM.", "ollama"), + ("ollama.com", "ollama"), + ("localhost", "openai"), + ("127.0.0.1", "openai"), + ("::1", "openai"), + ("api.ollama.com", "openai"), + ("notollama.com", "openai"), + ("ollama.com.evil.example", "openai"), + (None, "openai"), + ], +) +def test_exact_official_host_only(host, expected): + assert provider_from_host(host) == expected + + +def test_override_precedence_clear_and_client_isolation(wire): + with ( + make_client("https://api.openai.com/v1") as a, + make_client("https://api.openai.com/v1") as b, + ): + set_provider(a, "ollama") + call(a, "chat") + call(b, "chat") + set_provider(a, "vllm") + call(a, "chat") + set_provider(a, None) + call(a, "chat") + assert [attributes(s)["gen_ai.provider.name"] for s in spans_on_wire(wire)] == [ + "ollama", + "openai", + "vllm", + "openai", + ] + + +def test_async_clients_do_not_share_overrides(wire): + async def run(): + async with ( + make_client("http://localhost:11434/v1", use_async=True) as a, + make_client("http://localhost:11434/v1", use_async=True) as b, + ): + set_provider(a, "ollama") + set_provider(b, "vllm") + await asyncio.gather(call(a, "chat"), call(b, "chat")) + + asyncio.run(run()) + assert sorted( + attributes(s)["gen_ai.provider.name"] for s in spans_on_wire(wire) + ) == [ + "ollama", + "vllm", + ] + + +def test_copy_requires_explicit_override(wire): + with make_client("http://localhost:11434/v1") as original: + set_provider(original, "ollama") + copied = original.with_options() + call(copied, "chat") + set_provider(copied, "ollama") + call(copied, "chat") + assert [attributes(s)["gen_ai.provider.name"] for s in spans_on_wire(wire)] == [ + "openai", + "ollama", + ] + + +@pytest.mark.parametrize( + "invalid", ["", "Ollama", " ollama", "https://ollama.com", "x" * 65, 123] +) +def test_invalid_override_does_not_replace_existing_value(wire, invalid): + with make_client("http://localhost:11434/v1") as client: + set_provider(client, "ollama") + with pytest.raises(ValueError, match="provider"): + set_provider(client, invalid) + call(client, "chat") + assert attributes(spans_on_wire(wire)[0])["gen_ai.provider.name"] == "ollama" + + +def test_override_requires_an_openai_client(): + with pytest.raises(TypeError, match="client"): + set_provider(object(), "ollama") + + +@pytest.mark.parametrize("provider", ["o", "x" * 64, "self_hosted.v1-2"]) +def test_valid_identifier_boundaries(wire, provider): + with make_client("http://localhost:11434/v1") as client: + set_provider(client, None) # Clearing an absent override is safe. + set_provider(client, provider) + set_provider(client, provider) # Repeated configuration is idempotent. + call(client, "chat") + assert attributes(spans_on_wire(wire)[0])["gen_ai.provider.name"] == provider + + +def test_unhashable_client_subclass_keeps_existing_tracing(wire): + class Client(openai.OpenAI): + def __eq__(self, other): + return self is other + + with Client( + base_url="http://localhost:11434/v1", + api_key=FAKE_KEY, + http_client=httpx.Client( + transport=httpx.MockTransport(handler_for("chat")), + trust_env=False, + ), + ) as client: + call(client, "chat") + set_provider(client, "ollama") + call(client, "chat") + assert [attributes(s)["gen_ai.provider.name"] for s in spans_on_wire(wire)] == [ + "openai", + "ollama", + ] + + +def test_override_does_not_keep_client_alive(): + client = make_client("http://localhost:11434/v1") + set_provider(client, "ollama") + ref = weakref.ref(client) + client.close() + del client + gc.collect() + assert ref() is None diff --git a/python/frameworks/openai/traceai_openai/__init__.py b/python/frameworks/openai/traceai_openai/__init__.py index 0747c8745..891928712 100644 --- a/python/frameworks/openai/traceai_openai/__init__.py +++ b/python/frameworks/openai/traceai_openai/__init__.py @@ -14,6 +14,7 @@ from fi_instrumentation.instrumentation._protect_wrapper import GuardrailProtectWrapper from opentelemetry import trace as trace_api from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore +from traceai_openai._client_provider import set_client_provider as set_client_provider from traceai_openai._request import _AsyncRequest, _Request from traceai_openai.package import _instruments from traceai_openai.version import __version__ diff --git a/python/frameworks/openai/traceai_openai/_client_provider.py b/python/frameworks/openai/traceai_openai/_client_provider.py new file mode 100644 index 000000000..c1b66fdd6 --- /dev/null +++ b/python/frameworks/openai/traceai_openai/_client_provider.py @@ -0,0 +1,39 @@ +"""Explicit provider attribution for individual OpenAI-compatible SDK clients.""" + +import re +from typing import Optional, Union + +from openai import AsyncOpenAI, OpenAI + +_Client = Union[OpenAI, AsyncOpenAI] +_PROVIDER_ATTRIBUTE = "_traceai_provider_name" +_PROVIDER_NAME = re.compile(r"[a-z][a-z0-9_.-]{0,63}") + + +def set_client_provider(client: _Client, *, provider: Optional[str]) -> None: + """Set ``gen_ai.provider.name`` for requests made by this client instance. + + Use a lowercase provider identifier, such as ``ollama`` or ``vllm``, not a + URL or credential. ``None`` clears the override and restores host detection. + The override takes precedence over host detection, is never sent to the + server, and does not keep the client alive. Set it before starting requests. + + This does not install instrumentation: call ``OpenAIInstrumentor.instrument`` + as usual. Copies made with ``copy`` or ``with_options`` are separate clients + and need their own override, especially when changing their base URL. + """ + if not isinstance(client, (OpenAI, AsyncOpenAI)): + raise TypeError("client must be an OpenAI or AsyncOpenAI instance") + if provider is not None and not ( + isinstance(provider, str) and _PROVIDER_NAME.fullmatch(provider) + ): + raise ValueError( + "provider must be a lowercase identifier of 1-64 characters " + "(letters, digits, '.', '_' or '-', starting with a letter), or None" + ) + setattr(client, _PROVIDER_ATTRIBUTE, provider) + + +def _get_client_provider(client: _Client) -> Optional[str]: + provider = getattr(client, _PROVIDER_ATTRIBUTE, None) + return provider if isinstance(provider, str) else None diff --git a/python/frameworks/openai/traceai_openai/_providers.py b/python/frameworks/openai/traceai_openai/_providers.py index 45bd0d150..6edf3979f 100644 --- a/python/frameworks/openai/traceai_openai/_providers.py +++ b/python/frameworks/openai/traceai_openai/_providers.py @@ -15,6 +15,7 @@ # Exact API hosts. _HOSTS = { "api.openai.com": FiLLMProviderValues.OPENAI.value, + "ollama.com": "ollama", "generativelanguage.googleapis.com": FiLLMProviderValues.GOOGLE.value, "api.anthropic.com": FiLLMProviderValues.ANTHROPIC.value, "api.mistral.ai": FiLLMProviderValues.MISTRALAI.value, diff --git a/python/frameworks/openai/traceai_openai/_request.py b/python/frameworks/openai/traceai_openai/_request.py index 61f6a9aab..b90fe9715 100644 --- a/python/frameworks/openai/traceai_openai/_request.py +++ b/python/frameworks/openai/traceai_openai/_request.py @@ -16,6 +16,7 @@ from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY from opentelemetry.trace import INVALID_SPAN from opentelemetry.util.types import AttributeValue +from traceai_openai._client_provider import _get_client_provider from traceai_openai._providers import provider_from_host from traceai_openai._request_attributes_extractor import _RequestAttributesExtractor from traceai_openai._response_accumulator import ( @@ -140,12 +141,12 @@ def _get_span_kind(self, cast_to: type) -> str: def _get_attributes_from_instance( self, instance: Any ) -> Iterator[Tuple[str, AttributeValue]]: - # The provider is the vendor behind the client's base_url, not the SDK: - # known OpenAI-compatible hosts get their own label, anything else "openai". + # Explicit client attribution wins; never guess the vendor of a local server. base_url = getattr(instance, "base_url", None) - yield SpanAttributes.GEN_AI_PROVIDER_NAME, provider_from_host( - getattr(base_url, "host", None) - ) + provider = _get_client_provider(instance) + if provider is None: + provider = provider_from_host(getattr(base_url, "host", None)) + yield SpanAttributes.GEN_AI_PROVIDER_NAME, provider def _get_attributes_from_request( self, From f366a91f424b90e68fbd05782b59c3d7d01ad3e4 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 19:15:05 -0700 Subject: [PATCH 2/2] fix(vllm): complete shared client migration without duplicate spans Preserve the standalone OpenAI 1.0 entry point, yield to the shared instrumentor in either setup order, document the chat-only legacy boundary, and execute modern SDK README examples. Add offline floor/current migration and standalone legacy CI contracts. Co-Authored-By: Claude Code --- .github/workflows/ollama-provider-tests.yml | 46 +++- python/frameworks/openai/README.md | 23 +- .../openai/tests/test_vllm_migration.py | 204 ++++++++++++++++++ python/frameworks/vllm/README.md | 180 ++++++++-------- .../vllm/tests/test_instrumentor.py | 12 ++ .../frameworks/vllm/tests/test_legacy_sdk.py | 115 ++++++++++ .../frameworks/vllm/traceai_vllm/__init__.py | 46 ++-- .../frameworks/vllm/traceai_vllm/_wrappers.py | 18 +- 8 files changed, 500 insertions(+), 144 deletions(-) create mode 100644 python/frameworks/openai/tests/test_vllm_migration.py create mode 100644 python/frameworks/vllm/tests/test_legacy_sdk.py diff --git a/.github/workflows/ollama-provider-tests.yml b/.github/workflows/ollama-provider-tests.yml index f877b4ddd..348445463 100644 --- a/.github/workflows/ollama-provider-tests.yml +++ b/.github/workflows/ollama-provider-tests.yml @@ -1,10 +1,11 @@ -name: Ollama provider tests +name: OpenAI provider migrations on: pull_request: branches: [main, dev] paths: - 'python/frameworks/openai/**' + - 'python/frameworks/vllm/**' - 'python/fi_instrumentation/**' - 'python/pyproject.toml' - '.github/workflows/ollama-provider-tests.yml' @@ -12,6 +13,7 @@ on: branches: [dev] paths: - 'python/frameworks/openai/**' + - 'python/frameworks/vllm/**' - 'python/fi_instrumentation/**' - 'python/pyproject.toml' - '.github/workflows/ollama-provider-tests.yml' @@ -22,7 +24,7 @@ permissions: jobs: provider: - name: Ollama provider (OpenAI ${{ matrix.openai }}) + name: Ollama and vLLM (OpenAI ${{ matrix.openai }}) runs-on: ubuntu-24.04 timeout-minutes: 10 strategy: @@ -40,13 +42,43 @@ jobs: env: OPENAI_VERSION: ${{ matrix.openai }} run: >- - python -m pip install -e python -e python/frameworks/openai - "openai==$OPENAI_VERSION" 'pytest==8.4.2' 'pytest-socket==0.7.0' + python -m pip install -e python -e python/frameworks/openai -e python/frameworks/vllm + "openai==$OPENAI_VERSION" 'pytest==8.4.2' 'pytest-asyncio==0.24.0' 'pytest-socket==0.7.0' - name: Test attribution over localhost OTLP env: FI_BASE_URL: http://127.0.0.1:9 FI_GRPC_URL: http://127.0.0.1:9 + run: | + sudo unshare --net -- bash -e -c ' + ip link set lo up + "$1" -m pytest "$2/python/frameworks/openai/tests/test_ollama_provider.py" \ + "$2/python/frameworks/openai/tests/test_vllm_migration.py" \ + -q -o addopts="" -p no:cacheprovider --disable-socket \ + --allow-hosts=127.0.0.1 --allow-unix-socket + "$1" -m pytest "$2/python/frameworks/vllm/tests" \ + -q -o addopts="" -p no:cacheprovider --disable-socket --allow-unix-socket + ' _ "$(command -v python)" "$PWD" + + legacy: + name: vLLM legacy (OpenAI 1.0.0) + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.10' + - name: Install legacy package without the shared instrumentor run: >- - python -m pytest python/frameworks/openai/tests/test_ollama_provider.py - -q -o addopts='' -p no:cacheprovider --disable-socket - --allow-hosts=127.0.0.1 --allow-unix-socket + python -m pip install -e python -e python/frameworks/vllm + 'openai==1.0.0' 'httpx==0.27.2' 'pytest==8.4.2' + 'pytest-asyncio==0.24.0' 'pytest-socket==0.7.0' + - name: Test legacy SDK with outbound network disabled + run: | + sudo unshare --net -- bash -e -c ' + ip link set lo up + "$1" -m pytest "$2/python/frameworks/vllm/tests" \ + -q -o addopts="" -p no:cacheprovider --disable-socket --allow-unix-socket + ' _ "$(command -v python)" "$PWD" diff --git a/python/frameworks/openai/README.md b/python/frameworks/openai/README.md index 9969ece2f..fb8e0eff3 100644 --- a/python/frameworks/openai/README.md +++ b/python/frameworks/openai/README.md @@ -87,18 +87,17 @@ OpenAIInstrumentor().instrument(tracer_provider=trace_provider) Set up your OpenAI client with built-in observability. ```python -import openai - -openai.api_key = os.environ["OPENAI_API_KEY"] - -response = openai.ChatCompletion.create( - model="gpt-4-0", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Can you tell me a joke?"} - ] -) +from openai import OpenAI -print(response.choices[0].message['content'].strip()) +# OpenAI reads OPENAI_API_KEY from your environment. +with OpenAI() as client: + response = client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Can you tell me a joke?"}, + ], + ) + print((response.choices[0].message.content or "").strip()) ``` diff --git a/python/frameworks/openai/tests/test_vllm_migration.py b/python/frameworks/openai/tests/test_vllm_migration.py new file mode 100644 index 000000000..163ebcdef --- /dev/null +++ b/python/frameworks/openai/tests/test_vllm_migration.py @@ -0,0 +1,204 @@ +"""Real SDK migration contracts; synthetic HTTP and localhost OTLP only.""" + +import asyncio +import re +from pathlib import Path + +import openai +import pytest +import requests +from fi_instrumentation import TraceConfig +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_openai import OpenAIInstrumentor, set_client_provider + +from .test_ollama_provider import ( + FAKE_KEY, + MODEL, + PRIVATE_INPUT, + PRIVATE_OUTPUT, + assert_response, + attributes, + call, + collector, # noqa: F401 -- shared localhost receiver fixture + handler_for, + httpx, + make_client, + spans_on_wire, +) + +VLLMInstrumentor = pytest.importorskip("traceai_vllm").VLLMInstrumentor +BASE_URL = "http://localhost:8000/v1" + + +@pytest.fixture +def telemetry(collector): # noqa: F811 -- pytest resolves the imported fixture + port, bodies = collector + bodies.clear() + session = requests.Session() + session.trust_env = False + provider = TracerProvider() + provider.add_span_processor( + SimpleSpanProcessor( + OTLPSpanExporter( + endpoint=f"http://127.0.0.1:{port}/v1/traces", session=session + ) + ) + ) + yield provider, bodies + for instrumentor in (VLLMInstrumentor(), OpenAIInstrumentor()): + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + provider.shutdown() + session.close() + + +def invoke(operation, use_async): + async def run_async(): + async with make_client(BASE_URL, operation, True) as client: + set_client_provider(client, provider="vllm") + result = await call(client, operation) + if operation.endswith("-stream"): + assert [chunk async for chunk in result] + else: + assert_response(result, operation) + + def run_sync(): + with make_client(BASE_URL, operation) as client: + set_client_provider(client, provider="vllm") + result = call(client, operation) + if operation.endswith("-stream"): + assert list(result) + else: + assert_response(result, operation) + + if operation.startswith("error"): + with pytest.raises(openai.AuthenticationError): + asyncio.run(run_async()) if use_async else run_sync() + else: + asyncio.run(run_async()) if use_async else run_sync() + + +@pytest.mark.parametrize("first", ["shared", "legacy"]) +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("private", [False, True]) +@pytest.mark.parametrize( + "operation", + [ + "chat", + "chat-stream", + "embeddings", + "responses", + "responses-stream", + "error-chat", + "error-embeddings", + "error-responses", + ], +) +def test_shared_owns_one_span(telemetry, first, use_async, private, operation): + provider, bodies = telemetry + shared, legacy = OpenAIInstrumentor(), VLLMInstrumentor() + order = [shared, legacy] if first == "shared" else [legacy, shared] + # Different providers/configs prove the shared route owns privacy and export. + unused_provider = TracerProvider() + legacy_exporter = InMemorySpanExporter() + unused_provider.add_span_processor(SimpleSpanProcessor(legacy_exporter)) + try: + for inst in order: + inst.instrument( + tracer_provider=provider if inst is shared else unused_provider, + config=TraceConfig( + hide_inputs=private if inst is shared else False, + hide_outputs=private if inst is shared else False, + hide_embedding_vectors=private, + hide_llm_invocation_parameters=private, + hide_error_details=private, + ), + ) + invoke(operation, use_async) + spans = spans_on_wire(bodies) + assert len(spans) == 1 + assert not legacy_exporter.get_finished_spans() + attrs = attributes(spans[0]) + assert attrs["gen_ai.provider.name"] == "vllm" + assert attrs["gen_ai.request.model"] == MODEL + assert spans[0].status.code == (2 if operation.startswith("error") else 1) + payload = str(spans) + assert FAKE_KEY not in payload + if private: + assert PRIVATE_INPUT not in payload + assert PRIVATE_OUTPUT not in payload + finally: + unused_provider.shutdown() + + +@pytest.mark.parametrize("first", ["shared", "legacy"]) +@pytest.mark.parametrize("remove", ["shared", "legacy"]) +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("operation", ["chat", "chat-stream", "error-chat"]) +def test_independent_teardown_and_reenable( + telemetry, first, remove, use_async, operation +): + provider, bodies = telemetry + instruments = {"shared": OpenAIInstrumentor(), "legacy": VLLMInstrumentor()} + order = [first, "legacy" if first == "shared" else "shared"] + for name in order: + instruments[name].instrument(tracer_provider=provider) + invoke(operation, use_async) + assert len(spans_on_wire(bodies)) == 1 + bodies.clear() + instruments[remove].uninstrument() + invoke(operation, use_async) + spans = spans_on_wire(bodies) + assert len(spans) == 1 + assert attributes(spans[0])["gen_ai.provider.name"] == "vllm" + bodies.clear() + instruments[remove].instrument(tracer_provider=provider) + invoke(operation, use_async) + assert len(spans_on_wire(bodies)) == 1 + for name in reversed(order): + instruments[name].uninstrument() + bodies.clear() + invoke(operation, use_async) + assert spans_on_wire(bodies) == [] + + +@pytest.mark.parametrize("package", ["openai", "vllm"]) +def test_readme_quickstart(telemetry, monkeypatch, package): + import fi_instrumentation + + provider, bodies = telemetry + original_init = openai.OpenAI.__init__ + + def mock_transport(self, *args, **kwargs): + kwargs["http_client"] = httpx.Client( + transport=httpx.MockTransport(handler_for("chat")), trust_env=False + ) + original_init(self, *args, **kwargs) + + monkeypatch.setattr(openai.OpenAI, "__init__", mock_transport) + monkeypatch.setattr(fi_instrumentation, "register", lambda **kwargs: provider) + monkeypatch.setenv("OPENAI_API_KEY", FAKE_KEY) + monkeypatch.setenv("VLLM_API_KEY", FAKE_KEY) + monkeypatch.setenv("VLLM_MODEL", MODEL) + text = (Path(__file__).parents[2] / package / "README.md").read_text() + if package == "openai": + text = text.split("## Quickstart", 1)[1] + snippets = re.findall(r"```python\n(.*?)\n```", text, re.S) + namespace = {} + for snippet in snippets[:3] if package == "openai" else snippets[:1]: + exec(compile(snippet, f"{package}-README", "exec"), namespace) + spans = spans_on_wire(bodies) + assert len(spans) == 1 + assert attributes(spans[0])["gen_ai.provider.name"] == ( + "openai" if package == "openai" else "vllm" + ) + assert FAKE_KEY not in str(spans) + + +def test_legacy_warns_without_removing_entrypoint(telemetry): + provider, _ = telemetry + with pytest.warns(DeprecationWarning, match="OpenAIInstrumentor"): + VLLMInstrumentor().instrument(tracer_provider=provider) diff --git a/python/frameworks/vllm/README.md b/python/frameworks/vllm/README.md index 1b910a577..445f1a878 100644 --- a/python/frameworks/vllm/README.md +++ b/python/frameworks/vllm/README.md @@ -1,110 +1,108 @@ -# TraceAI vLLM Instrumentation +# vLLM client tracing -OpenTelemetry instrumentation for vLLM, enabling comprehensive observability for local LLM inference. +Trace requests from the OpenAI Python client to a vLLM server with the shared +`traceAI-openai` integration. For inference-engine and server spans, use +[vLLM's native OpenTelemetry setup](https://docs.vllm.ai/en/latest/examples/observability/opentelemetry/) +separately; this client integration does not instrument the server. -## Installation +## Install and configure + +Use Python 3.10–3.13 and OpenAI >= 1.69.0 for the shared route. Install a +`traceAI-openai` release that exports `set_client_provider`: ```bash -pip install traceai-vllm +pip install traceAI-openai ``` -## Quick Start +Set `FI_API_KEY` and `FI_SECRET_KEY` to your Future AGI credentials. Set +`VLLM_MODEL` to the served model name and `VLLM_API_KEY` to your server's API key +(or a non-empty placeholder only if your server has authentication disabled). +`VLLM_BASE_URL` defaults to `http://localhost:8000/v1` in this example. ```python +import os + from fi_instrumentation import register from fi_instrumentation.fi_types import ProjectType -from traceai_vllm import VLLMInstrumentor +from openai import OpenAI +from traceai_openai import OpenAIInstrumentor, set_client_provider -# Setup TraceAI trace_provider = register( project_type=ProjectType.OBSERVE, - project_name="my-vllm-app", -) - -# Instrument vLLM (specify your server URLs) -VLLMInstrumentor( - vllm_base_urls=["localhost:8000"] # Your vLLM server(s) -).instrument(tracer_provider=trace_provider) - -# Now use OpenAI client with vLLM -from openai import OpenAI - -client = OpenAI( - api_key="token", # vLLM doesn't require a real API key - base_url="http://localhost:8000/v1", + project_name="vllm-app", ) - -response = client.chat.completions.create( - model="meta-llama/Llama-2-7b-chat-hf", - messages=[{"role": "user", "content": "Hello!"}] -) -``` - -## Configuration - -### Custom vLLM Server URLs - -You can specify multiple vLLM server URLs to instrument: - -```python -VLLMInstrumentor( - vllm_base_urls=[ - "localhost:8000", - "production-vllm.internal:8000", - "staging-vllm.internal:8080", - ] -).instrument(tracer_provider=trace_provider) +OpenAIInstrumentor().instrument(tracer_provider=trace_provider) + +try: + with OpenAI( + base_url=os.environ.get("VLLM_BASE_URL", "http://localhost:8000/v1"), + api_key=os.environ["VLLM_API_KEY"], + ) as client: + set_client_provider(client, provider="vllm") + response = client.chat.completions.create( + model=os.environ["VLLM_MODEL"], + messages=[{"role": "user", "content": "Explain continuous batching briefly."}], + ) + print(response.choices[0].message.content or "") +finally: + trace_provider.shutdown() ``` -## Features - -- Automatic tracing of vLLM API calls -- Support for both synchronous and asynchronous operations -- Streaming response support -- Token usage tracking -- Request/response attribute capture -- Support for multiple vLLM server endpoints -- OpenTelemetry semantic conventions for GenAI - -## Captured Attributes - -- `gen_ai.request.model` - Model name -- `gen_ai.request.max_tokens` - Maximum tokens -- `gen_ai.request.temperature` - Temperature setting -- `gen_ai.usage.input_tokens` - Input token count -- `gen_ai.usage.output_tokens` - Output token count -- `gen_ai.prompt.{n}.role` - Message roles -- `gen_ai.prompt.{n}.content` - Message contents -- `gen_ai.completion.{n}.content` - Response content - -## vLLM-Specific Parameters - -The instrumentation also captures vLLM-specific parameters when provided: - -- `best_of` - Number of output sequences to generate -- `use_beam_search` - Whether to use beam search - -## Requirements - -- Python >= 3.9 -- openai >= 1.0.0 -- fi-instrumentation >= 0.1.0 -- opentelemetry-api >= 1.0.0 -- opentelemetry-sdk >= 1.0.0 - -## Running vLLM Server - -To use this instrumentation, you need a running vLLM server: - -```bash -# Install vLLM -pip install vllm - -# Start the server -python -m vllm.entrypoints.openai.api_server \ - --model meta-llama/Llama-2-7b-chat-hf \ - --port 8000 -``` +Open the `vllm-app` project in Future AGI to inspect the client request span. +The explicit provider setting labels it `vllm`; a localhost address alone does +not identify a provider. Set it on each `OpenAI` or `AsyncOpenAI` client before +requests, including clients created by `copy()` or `with_options()`. It changes +trace metadata only and is not sent to the server. + +The shared instrumentor captures sync/async chat, consumed streams, embeddings, +Responses and request failures. Use only APIs supported by your deployed vLLM +version and model. Embeddings require an embedding-capable model. Streaming +spans finish when the stream finishes. Configure `TraceConfig` on the shared +instrumentor to control input, output, embedding and error-detail capture. + +## Migrate from `VLLMInstrumentor` + +1. Upgrade OpenAI to >= 1.69.0 and install the shared package above. +2. Replace the `VLLMInstrumentor(...).instrument(...)` call with + `OpenAIInstrumentor().instrument(...)`, preserving your tracer provider and + privacy configuration. +3. Replace `vllm_base_urls` configuration with + `set_client_provider(client, provider="vllm")` on each vLLM client. +4. Remove the legacy instrumentor from your application. You can then uninstall + `traceAI-vllm` if no other application in the environment uses it. + +During migration, if both instrumentors are enabled, the shared instrumentor +owns client spans and its tracer provider and privacy configuration apply. +The legacy wrapper does not emit a second span. This works in either setup +order; disabling the shared instrumentor restores legacy chat tracing if the +legacy instrumentor is still enabled. Legacy URL settings do not configure the +shared instrumentor: keep the explicit per-client provider setting. + +## Legacy compatibility + +`traceAI-vllm` retains its public `VLLMInstrumentor` entry point and its +OpenAI >= 1.0.0 dependency floor for existing applications. It emits a +`DeprecationWarning` when enabled; the package has not been removed. Applications +that must stay on OpenAI 1.0–1.68 can continue using their existing setup. + +The legacy instrumentor is **chat-completions only**, including sync/async chat +and consumed text streams for configured URLs (`localhost:8000` by default). +It does not trace embeddings, Responses, the native vLLM engine, or server-side +scheduling. Migrate to the shared route for the additional client APIs; enable +native server telemetry separately when you need server spans. Do not interpret +a client span as an inference-engine span or assume automatic correlation with +native server spans. + +## Troubleshooting + +- **`set_client_provider` cannot be imported:** your installed `traceAI-openai` + release predates this API. Upgrade to a release containing it before switching + instrumentors; do not remove a working legacy setup first. +- **Provider is `openai`:** set the override on the exact client making the + request, including any copied client. +- **No embedding or Responses span with the legacy package:** those APIs are + outside its chat-only scope. Use the shared route and confirm that your server + supports the requested API. ## License diff --git a/python/frameworks/vllm/tests/test_instrumentor.py b/python/frameworks/vllm/tests/test_instrumentor.py index fdeb0f0e4..b161ff1ab 100644 --- a/python/frameworks/vllm/tests/test_instrumentor.py +++ b/python/frameworks/vllm/tests/test_instrumentor.py @@ -7,6 +7,18 @@ from traceai_vllm import VLLMInstrumentor, __version__ +@pytest.fixture(autouse=True) +def isolate_mock_instrumentor(monkeypatch): + """Do not leak mocked singleton state or methods into real SDK contracts.""" + from openai.resources.chat import AsyncCompletions, Completions + + monkeypatch.setattr(VLLMInstrumentor, "_instance", None) + # Register the real methods for restoration even when a test replaces them + # through _uninstrument() while mocking the import machinery. + monkeypatch.setattr(Completions, "create", Completions.create) + monkeypatch.setattr(AsyncCompletions, "create", AsyncCompletions.create) + + class TestVLLMInstrumentor: """Tests for VLLMInstrumentor class.""" diff --git a/python/frameworks/vllm/tests/test_legacy_sdk.py b/python/frameworks/vllm/tests/test_legacy_sdk.py new file mode 100644 index 000000000..91df79ab5 --- /dev/null +++ b/python/frameworks/vllm/tests/test_legacy_sdk.py @@ -0,0 +1,115 @@ +"""Keep the standalone legacy entry point usable at OpenAI 1.0 and current.""" + +import asyncio +import json +import sys + +import openai +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_vllm import VLLMInstrumentor + +if int(openai.__version__.split(".")[0]) >= 3: + import httpx2 as httpx +else: + import httpx + + +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("operation", ["chat", "stream", "error"]) +def test_real_sdk_without_shared_package(use_async, operation): + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = VLLMInstrumentor() + assert instrumentor.instrumentation_dependencies() == ("openai >= 1.0.0",) + shared_was_loaded = "traceai_openai" in sys.modules + + def handle(request): + if operation == "error": + return httpx.Response(401, json={"error": {"message": "Synthetic error"}}) + response = { + "id": "c1", + "object": "chat.completion", + "created": 1, + "model": "test-vllm-model", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + "content": "Synthetic output", + }, + } + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + } + if operation == "stream": + response["object"] = "chat.completion.chunk" + response["choices"][0]["delta"] = response["choices"][0].pop("message") + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=f"data: {json.dumps(response)}\n\ndata: [DONE]\n\n", + ) + return httpx.Response(200, json=response) + + options = { + "base_url": "http://localhost:8000/v1", + "api_key": "synthetic-not-a-real-key", + "max_retries": 0, + } + params = { + "model": "test-vllm-model", + "messages": [{"role": "user", "content": "Hello"}], + "stream": operation == "stream", + } + + async def run_async(): + async with openai.AsyncOpenAI( + **options, + http_client=httpx.AsyncClient( + transport=httpx.MockTransport(handle), trust_env=False + ), + ) as client: + result = await client.chat.completions.create(**params) + if operation == "stream": + assert [chunk async for chunk in result] + else: + assert result.choices[0].message.content == "Synthetic output" + + def run_sync(): + with openai.OpenAI( + **options, + http_client=httpx.Client( + transport=httpx.MockTransport(handle), trust_env=False + ), + ) as client: + result = client.chat.completions.create(**params) + if operation == "stream": + assert list(result) + else: + assert result.choices[0].message.content == "Synthetic output" + + try: + with pytest.warns(DeprecationWarning, match="OpenAIInstrumentor"): + instrumentor.instrument(tracer_provider=provider) + if operation == "error": + with pytest.raises(openai.AuthenticationError): + asyncio.run(run_async()) if use_async else run_sync() + else: + asyncio.run(run_async()) if use_async else run_sync() + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].attributes["gen_ai.provider.name"] == "vllm" + assert spans[0].attributes["gen_ai.request.model"] == "test-vllm-model" + assert spans[0].status.is_ok == (operation != "error") + assert "synthetic-not-a-real-key" not in str(spans[0].attributes) + # Instrumentation must not load a package incompatible with the old SDK. + assert ("traceai_openai" in sys.modules) == shared_was_loaded + finally: + instrumentor.uninstrument() + provider.shutdown() diff --git a/python/frameworks/vllm/traceai_vllm/__init__.py b/python/frameworks/vllm/traceai_vllm/__init__.py index 6838992a7..94413cd86 100644 --- a/python/frameworks/vllm/traceai_vllm/__init__.py +++ b/python/frameworks/vllm/traceai_vllm/__init__.py @@ -1,38 +1,13 @@ -"""TraceAI instrumentation for vLLM. +"""Legacy chat-only vLLM client instrumentation. -This module provides OpenTelemetry instrumentation for vLLM, -enabling comprehensive observability for vLLM API calls. -vLLM uses an OpenAI-compatible API. - -Example usage: - - from fi_instrumentation import register - from fi_instrumentation.fi_types import ProjectType - from traceai_vllm import VLLMInstrumentor - - # Setup TraceAI - trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="vllm-app", - ) - - # Instrument vLLM - VLLMInstrumentor().instrument(tracer_provider=trace_provider) - - # Now use OpenAI client with vLLM - from openai import OpenAI - - client = OpenAI( - api_key="token", # vLLM doesn't require real key - base_url="http://localhost:8000/v1", # Your vLLM server - ) - response = client.chat.completions.create( - model="meta-llama/Llama-2-7b-chat-hf", - messages=[{"role": "user", "content": "Hello!"}] - ) +VLLMInstrumentor remains available for OpenAI >= 1.0.0. New applications should +use traceai_openai.OpenAIInstrumentor and set_client_provider(client, +provider="vllm") with OpenAI >= 1.69.0. See the package README for migration. +This package does not instrument the vLLM server or inference engine. """ import logging +import warnings from importlib import import_module from typing import Any, Collection @@ -82,6 +57,15 @@ def instrumentation_dependencies(self) -> Collection[str]: return _instruments def _instrument(self, **kwargs: Any) -> None: + warnings.warn( + "VLLMInstrumentor is deprecated. Use traceai_openai.OpenAIInstrumentor " + "and set_client_provider(client, provider='vllm') instead " + "(requires OpenAI >= 1.69.0). When both instrumentors are enabled, " + "OpenAIInstrumentor owns tracing and its tracer_provider and " + "TraceConfig apply.", + DeprecationWarning, + stacklevel=3, + ) if not (tracer_provider := kwargs.get("tracer_provider")): tracer_provider = trace_api.get_tracer_provider() if not (config := kwargs.get("config")): diff --git a/python/frameworks/vllm/traceai_vllm/_wrappers.py b/python/frameworks/vllm/traceai_vllm/_wrappers.py index 299189f7d..7fe061aee 100644 --- a/python/frameworks/vllm/traceai_vllm/_wrappers.py +++ b/python/frameworks/vllm/traceai_vllm/_wrappers.py @@ -1,6 +1,7 @@ """Wrapper classes for vLLM instrumentation.""" import logging +import sys from abc import ABC from contextlib import contextmanager from typing import Any, Dict, Iterator, List, Tuple @@ -15,6 +16,18 @@ logger.addHandler(logging.NullHandler()) +def _shared_instrumentor_enabled() -> bool: + """Defer to the shared request-level instrumentor while it is enabled. + + Inspect live state, not installation order. Do not import the optional + package: legacy users can keep OpenAI 1.0, below the shared package's floor. + This mirrors the Fireworks compatibility route. + """ + module = sys.modules.get("traceai_openai") + instrumentor = getattr(module, "OpenAIInstrumentor", None) + return bool(instrumentor and instrumentor().is_instrumented_by_opentelemetry) + + class _WithTracer(ABC): """Base class for wrappers that need a tracer.""" @@ -73,7 +86,6 @@ def _get_input_attributes( attributes: Dict[str, AttributeValue] = { SpanAttributes.GEN_AI_SPAN_KIND: FiSpanKindValues.LLM.value, SpanAttributes.GEN_AI_PROVIDER_NAME: "vllm", - SpanAttributes.GEN_AI_PROVIDER_NAME: "vllm", SpanAttributes.GEN_AI_REQUEST_MODEL: model, } @@ -214,7 +226,7 @@ def __call__( kwargs: Dict[str, Any], ) -> Any: # Check if this is a vLLM client - if not self._is_vllm_client(instance): + if _shared_instrumentor_enabled() or not self._is_vllm_client(instance): return wrapped(*args, **kwargs) # Extract parameters @@ -285,7 +297,7 @@ async def __call__( kwargs: Dict[str, Any], ) -> Any: # Check if this is a vLLM client - if not self._is_vllm_client(instance): + if _shared_instrumentor_enabled() or not self._is_vllm_client(instance): return await wrapped(*args, **kwargs) # Extract parameters