From 07fe41dd7a7ea936ca65007435309f8ca78d1ee6 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 08:05:45 -0700 Subject: [PATCH 1/2] fix(openai): attribute Novita chat requests to their provider Keep the shared OpenAI instrumentor and exact-host fallback behavior. Verify sync/async success, streams, usage and errors over local OTLP HTTP. Co-Authored-By: Claude Code --- .../openai/tests/test_novita_provider.py | 195 ++++++++++++++++++ .../openai/traceai_openai/_providers.py | 1 + 2 files changed, 196 insertions(+) create mode 100644 python/frameworks/openai/tests/test_novita_provider.py diff --git a/python/frameworks/openai/tests/test_novita_provider.py b/python/frameworks/openai/tests/test_novita_provider.py new file mode 100644 index 000000000..834382fd1 --- /dev/null +++ b/python/frameworks/openai/tests/test_novita_provider.py @@ -0,0 +1,195 @@ +"""Novita uses the shared OpenAI instrumentor, not a separate SDK wrapper. + +Vendor traffic is synthetic. Traces cross a localhost OTLP/HTTP protobuf wire +before assertions, covering sync/async success, streams, errors and look-alikes. +""" + +import asyncio +import json +from http.server import BaseHTTPRequestHandler, HTTPServer +from queue import Queue +from threading import Thread + +import httpx +import openai +import pytest +import requests +from fi_instrumentation import TraceConfig +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.proto.trace.v1.trace_pb2 import Status +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor + +from traceai_openai import OpenAIInstrumentor + +from .test_openai_compatible_spans import ( + ANSWER, + REQUEST_MODEL, + RESPONSE_MODEL, + async_client, + chat_completion, + chat_stream, + client, + unauthorized, +) + + +@pytest.fixture(scope="module") +def otlp_receiver(): + received = Queue() + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + received.put((self.path, self.headers["Content-Type"], body)) + self.send_response(200) + self.send_header("Content-Type", "application/x-protobuf") + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *_args): + pass + + server = HTTPServer(("127.0.0.1", 0), Handler) + worker = Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + yield f"http://127.0.0.1:{server.server_port}/v1/traces", received + finally: + server.shutdown() + worker.join(timeout=5) + server.server_close() + + +@pytest.fixture +def wire_trace(otlp_receiver): + endpoint, received = otlp_receiver + assert received.empty() + session = requests.Session() + session.trust_env = False + exporter = OTLPSpanExporter(endpoint=endpoint, session=session, timeout=5) + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = OpenAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig()) + try: + yield received + finally: + instrumentor.uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize("base_url,expected_provider", [ + ("https://api.novita.ai/openai/v1", "novita"), + ("https://API.NOVITA.AI./openai/v1", "novita"), + ("https://api.novita.ai:443/openai/v1", "novita"), + ("https://api.novita.ai.customer.example/openai/v1", "openai"), + ("https://notapi.novita.ai/openai/v1", "openai"), + ("https://proxy.api.novita.ai/openai/v1", "openai"), + ("https://api.novita-ai.example/openai/v1", "openai"), + ("https://llm-proxy.customer.example/openai/v1", "openai"), + ("https://api.openai.com/openai/v1", "openai"), +]) +@pytest.mark.parametrize("outcome", ["success", "stream", "stream-usage", "error"]) +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +def test_novita_provider_over_otlp(wire_trace, base_url, expected_provider, outcome, use_async): + calls = [] + streaming = outcome.startswith("stream") + + def handler(request): + assert request.url.path == "/openai/v1/chat/completions" + body = json.loads(request.content) + assert body["model"] == REQUEST_MODEL + assert body.get("stream", False) == streaming + calls.append(request.url.host) + if outcome == "error": + return unauthorized(request) + if streaming: + response = chat_stream() + if outcome == "stream-usage": + assert body["stream_options"] == {"include_usage": True} + usage = { + "id": "c", "object": "chat.completion.chunk", "created": 1, + "model": RESPONSE_MODEL, "choices": [], + "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7}, + } + content = response.content.replace( + b"data: [DONE]", f"data: {json.dumps(usage)}\n\ndata: [DONE]".encode(), + ) + return httpx.Response(200, headers=response.headers, content=content) + return response + return httpx.Response(200, json=chat_completion()) + + kwargs = { + "model": REQUEST_MODEL, + "messages": [{"role": "user", "content": "Capital of Japan?"}], + "max_tokens": 128, + "stream": streaming, + } + if outcome == "stream-usage": + kwargs["stream_options"] = {"include_usage": True} + + async def run_async(): + async with async_client(handler, base_url=base_url) as sdk: + response = await sdk.chat.completions.create(**kwargs) + if streaming: + text = [] + async for chunk in response: + text.extend(choice.delta.content or "" for choice in chunk.choices) + assert "".join(text) == ANSWER + else: + assert response.choices[0].message.content == ANSWER + + def run_sync(): + with client(handler, base_url=base_url) as sdk: + response = sdk.chat.completions.create(**kwargs) + if streaming: + text = "".join( + choice.delta.content or "" for chunk in response for choice in chunk.choices + ) + assert text == ANSWER + else: + assert response.choices[0].message.content == ANSWER + + def run(): + if use_async: + asyncio.run(run_async()) + else: + run_sync() + + if outcome == "error": + with pytest.raises(openai.AuthenticationError): + run() + else: + run() + + assert calls == [httpx.URL(base_url).host] + path, content_type, payload = wire_trace.get(timeout=5) + assert path == "/v1/traces" + assert content_type == "application/x-protobuf" + message = ExportTraceServiceRequest.FromString(payload) + spans = [span for resource in message.resource_spans + for scope in resource.scope_spans for span in scope.spans] + assert len(spans) == 1 + assert wire_trace.empty(), "The shared instrumentor must not duplicate spans" + span = spans[0] + attrs = {item.key: getattr(item.value, item.value.WhichOneof("value")) + for item in span.attributes} + assert attrs["gen_ai.provider.name"] == expected_provider + assert attrs["gen_ai.request.model"] == ( + RESPONSE_MODEL if outcome == "success" else REQUEST_MODEL + ) + assert attrs["gen_ai.span.kind"] == "LLM" + assert span.status.code == ( + Status.STATUS_CODE_ERROR if outcome == "error" else Status.STATUS_CODE_OK + ) + if outcome in ("success", "stream-usage"): + assert attrs["gen_ai.usage.input_tokens"] == 5 + assert attrs["gen_ai.usage.output_tokens"] == 2 + assert attrs["gen_ai.usage.total_tokens"] == 7 + else: + assert not any(key.startswith("gen_ai.usage.") for key in attrs) + if streaming: + assert attrs["output.value"] == ANSWER + assert b"test-vendor-key" not in payload diff --git a/python/frameworks/openai/traceai_openai/_providers.py b/python/frameworks/openai/traceai_openai/_providers.py index 45bd0d150..7ac12e7d0 100644 --- a/python/frameworks/openai/traceai_openai/_providers.py +++ b/python/frameworks/openai/traceai_openai/_providers.py @@ -25,6 +25,7 @@ "dashscope.aliyuncs.com": "dashscope", "dashscope-intl.aliyuncs.com": "dashscope", "dashscope-us.aliyuncs.com": "dashscope", + "api.novita.ai": "novita", "api.perplexity.ai": "perplexity", "api.orcarouter.ai": "orcarouter", "api.anannas.ai": "anannas", From 454fc2da277ecd675e35136118cd208247d3a1ee Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 9 Nov 2026 14:47:30 -0800 Subject: [PATCH 2/2] test(openai): qualify Novita host boundaries and private wire paths Keep Novita adjacent to Mistral in the explicit host map and cover port/scheme normalization, look-alike fallbacks, and hidden content. Co-Authored-By: Claude Code --- .../openai/tests/test_novita_provider.py | 58 +++++++++++++++++-- .../openai/traceai_openai/_providers.py | 2 +- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/python/frameworks/openai/tests/test_novita_provider.py b/python/frameworks/openai/tests/test_novita_provider.py index 834382fd1..e1339b4f6 100644 --- a/python/frameworks/openai/tests/test_novita_provider.py +++ b/python/frameworks/openai/tests/test_novita_provider.py @@ -6,6 +6,7 @@ import asyncio import json +import os from http.server import BaseHTTPRequestHandler, HTTPServer from queue import Queue from threading import Thread @@ -22,6 +23,7 @@ from opentelemetry.sdk.trace.export import SimpleSpanProcessor from traceai_openai import OpenAIInstrumentor +from traceai_openai._providers import provider_from_host from .test_openai_compatible_spans import ( ANSWER, @@ -35,6 +37,32 @@ ) +PROMPT = "synthetic-novita-private-prompt" + + +@pytest.mark.parametrize("host,expected", [ + ("api.novita.ai", "novita"), + ("API.NOVITA.AI.", "novita"), + ("novita.ai", "openai"), + ("proxy.api.novita.ai", "openai"), + ("api.novita.ai.evil.example", "openai"), + ("notapi.novita.ai", "openai"), + ("api.novita-ai.example", "openai"), + # The resolver accepts a hostname, not a URL or authority. The SDK extracts + # the host first; schemes and ports on a genuine Novita URL do not change it. + ("https://api.novita.ai", "openai"), + ("http://api.novita.ai", "openai"), + ("api.novita.ai:443", "openai"), + ("api.novita.ai:8443", "openai"), + ("api.novita.ai@evil.example", "openai"), + (None, "openai"), + ("", "openai"), + (443, "openai"), +]) +def test_novita_host_boundaries(host, expected): + assert provider_from_host(host) == expected + + @pytest.fixture(scope="module") def otlp_receiver(): received = Queue() @@ -63,7 +91,11 @@ def log_message(self, *_args): @pytest.fixture -def wire_trace(otlp_receiver): +def wire_trace(otlp_receiver, monkeypatch, hidden): + # Keep credentials, telemetry destinations and privacy flags synthetic. + for name in tuple(os.environ): + if name.startswith(("FI_", "OTEL_", "OPENAI_")): + monkeypatch.delenv(name) endpoint, received = otlp_receiver assert received.empty() session = requests.Session() @@ -72,7 +104,10 @@ def wire_trace(otlp_receiver): provider = TracerProvider() provider.add_span_processor(SimpleSpanProcessor(exporter)) instrumentor = OpenAIInstrumentor() - instrumentor.instrument(tracer_provider=provider, config=TraceConfig()) + config = TraceConfig( + hide_inputs=True, hide_outputs=True, hide_llm_invocation_parameters=True, + ) if hidden else TraceConfig() + instrumentor.instrument(tracer_provider=provider, config=config) try: yield received finally: @@ -84,6 +119,12 @@ def wire_trace(otlp_receiver): ("https://api.novita.ai/openai/v1", "novita"), ("https://API.NOVITA.AI./openai/v1", "novita"), ("https://api.novita.ai:443/openai/v1", "novita"), + # Match existing host-only mappings: a port or HTTP scheme is not a new host. + ("https://api.novita.ai:8443/openai/v1", "novita"), + ("http://api.novita.ai:8080/openai/v1", "novita"), + ("https://api.novita.ai.evil.example:8443/openai/v1", "openai"), + ("http://proxy.api.novita.ai:8080/openai/v1", "openai"), + ("https://api.novita.ai@evil.example/openai/v1", "openai"), ("https://api.novita.ai.customer.example/openai/v1", "openai"), ("https://notapi.novita.ai/openai/v1", "openai"), ("https://proxy.api.novita.ai/openai/v1", "openai"), @@ -93,7 +134,8 @@ def wire_trace(otlp_receiver): ]) @pytest.mark.parametrize("outcome", ["success", "stream", "stream-usage", "error"]) @pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) -def test_novita_provider_over_otlp(wire_trace, base_url, expected_provider, outcome, use_async): +@pytest.mark.parametrize("hidden", [False, True], ids=["default", "hidden"]) +def test_novita_provider_over_otlp(wire_trace, base_url, expected_provider, outcome, use_async, hidden): calls = [] streaming = outcome.startswith("stream") @@ -123,7 +165,7 @@ def handler(request): kwargs = { "model": REQUEST_MODEL, - "messages": [{"role": "user", "content": "Capital of Japan?"}], + "messages": [{"role": "user", "content": PROMPT}], "max_tokens": 128, "stream": streaming, } @@ -177,6 +219,8 @@ def run(): attrs = {item.key: getattr(item.value, item.value.WhichOneof("value")) for item in span.attributes} assert attrs["gen_ai.provider.name"] == expected_provider + # This instrumentor emits the modern provider field, not the legacy system. + assert "gen_ai.system" not in attrs assert attrs["gen_ai.request.model"] == ( RESPONSE_MODEL if outcome == "success" else REQUEST_MODEL ) @@ -190,6 +234,10 @@ def run(): assert attrs["gen_ai.usage.total_tokens"] == 7 else: assert not any(key.startswith("gen_ai.usage.") for key in attrs) - if streaming: + if hidden: + assert PROMPT.encode() not in payload + assert ANSWER.encode() not in payload + assert "gen_ai.request.parameters" not in attrs + elif streaming: assert attrs["output.value"] == ANSWER assert b"test-vendor-key" not in payload diff --git a/python/frameworks/openai/traceai_openai/_providers.py b/python/frameworks/openai/traceai_openai/_providers.py index b54a52148..f97caad5d 100644 --- a/python/frameworks/openai/traceai_openai/_providers.py +++ b/python/frameworks/openai/traceai_openai/_providers.py @@ -21,6 +21,7 @@ "generativelanguage.googleapis.com": FiLLMProviderValues.GOOGLE.value, "api.anthropic.com": FiLLMProviderValues.ANTHROPIC.value, "api.mistral.ai": FiLLMProviderValues.MISTRALAI.value, + "api.novita.ai": "novita", "api.deepseek.com": FiLLMProviderValues.DEEPSEEK.value, "api.x.ai": FiLLMProviderValues.XAI.value, "api.z.ai": "zai", @@ -30,7 +31,6 @@ "dashscope.aliyuncs.com": "dashscope", "dashscope-intl.aliyuncs.com": "dashscope", "dashscope-us.aliyuncs.com": "dashscope", - "api.novita.ai": "novita", "api.perplexity.ai": "perplexity", "api.parallel.ai": "parallel", "api.orcarouter.ai": "orcarouter",