Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
243 changes: 243 additions & 0 deletions python/frameworks/openai/tests/test_novita_provider.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,243 @@
"""Novita uses the shared OpenAI instrumentor, not a separate SDK wrapper.

Vendor traffic is synthetic. Traces cross a localhost OTLP/HTTP protobuf wire
before assertions, covering sync/async success, streams, errors and look-alikes.
"""

import asyncio
import json
import os
from http.server import BaseHTTPRequestHandler, HTTPServer
from queue import Queue
from threading import Thread

import httpx
import openai
import pytest
import requests
from fi_instrumentation import TraceConfig
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
from opentelemetry.proto.trace.v1.trace_pb2 import Status
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor

from traceai_openai import OpenAIInstrumentor
from traceai_openai._providers import provider_from_host

from .test_openai_compatible_spans import (
ANSWER,
REQUEST_MODEL,
RESPONSE_MODEL,
async_client,
chat_completion,
chat_stream,
client,
unauthorized,
)


PROMPT = "synthetic-novita-private-prompt"


@pytest.mark.parametrize("host,expected", [
("api.novita.ai", "novita"),
("API.NOVITA.AI.", "novita"),
("novita.ai", "openai"),
("proxy.api.novita.ai", "openai"),
("api.novita.ai.evil.example", "openai"),
("notapi.novita.ai", "openai"),
("api.novita-ai.example", "openai"),
# The resolver accepts a hostname, not a URL or authority. The SDK extracts
# the host first; schemes and ports on a genuine Novita URL do not change it.
("https://api.novita.ai", "openai"),
("http://api.novita.ai", "openai"),
("api.novita.ai:443", "openai"),
("api.novita.ai:8443", "openai"),
("api.novita.ai@evil.example", "openai"),
(None, "openai"),
("", "openai"),
(443, "openai"),
])
def test_novita_host_boundaries(host, expected):
assert provider_from_host(host) == expected


@pytest.fixture(scope="module")
def otlp_receiver():
received = Queue()

class Handler(BaseHTTPRequestHandler):
def do_POST(self):
body = self.rfile.read(int(self.headers["Content-Length"]))
received.put((self.path, self.headers["Content-Type"], body))
self.send_response(200)
self.send_header("Content-Type", "application/x-protobuf")
self.send_header("Content-Length", "0")
self.end_headers()

def log_message(self, *_args):
pass

server = HTTPServer(("127.0.0.1", 0), Handler)
worker = Thread(target=server.serve_forever, daemon=True)
worker.start()
try:
yield f"http://127.0.0.1:{server.server_port}/v1/traces", received
finally:
server.shutdown()
worker.join(timeout=5)
server.server_close()


@pytest.fixture
def wire_trace(otlp_receiver, monkeypatch, hidden):
# Keep credentials, telemetry destinations and privacy flags synthetic.
for name in tuple(os.environ):
if name.startswith(("FI_", "OTEL_", "OPENAI_")):
monkeypatch.delenv(name)
endpoint, received = otlp_receiver
assert received.empty()
session = requests.Session()
session.trust_env = False
exporter = OTLPSpanExporter(endpoint=endpoint, session=session, timeout=5)
provider = TracerProvider()
provider.add_span_processor(SimpleSpanProcessor(exporter))
instrumentor = OpenAIInstrumentor()
config = TraceConfig(
hide_inputs=True, hide_outputs=True, hide_llm_invocation_parameters=True,
) if hidden else TraceConfig()
instrumentor.instrument(tracer_provider=provider, config=config)
try:
yield received
finally:
instrumentor.uninstrument()
provider.shutdown()


@pytest.mark.parametrize("base_url,expected_provider", [
("https://api.novita.ai/openai/v1", "novita"),
("https://API.NOVITA.AI./openai/v1", "novita"),
("https://api.novita.ai:443/openai/v1", "novita"),
# Match existing host-only mappings: a port or HTTP scheme is not a new host.
("https://api.novita.ai:8443/openai/v1", "novita"),
("http://api.novita.ai:8080/openai/v1", "novita"),
("https://api.novita.ai.evil.example:8443/openai/v1", "openai"),
("http://proxy.api.novita.ai:8080/openai/v1", "openai"),
("https://api.novita.ai@evil.example/openai/v1", "openai"),
("https://api.novita.ai.customer.example/openai/v1", "openai"),
("https://notapi.novita.ai/openai/v1", "openai"),
("https://proxy.api.novita.ai/openai/v1", "openai"),
("https://api.novita-ai.example/openai/v1", "openai"),
("https://llm-proxy.customer.example/openai/v1", "openai"),
("https://api.openai.com/openai/v1", "openai"),
])
@pytest.mark.parametrize("outcome", ["success", "stream", "stream-usage", "error"])
@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"])
@pytest.mark.parametrize("hidden", [False, True], ids=["default", "hidden"])
def test_novita_provider_over_otlp(wire_trace, base_url, expected_provider, outcome, use_async, hidden):
calls = []
streaming = outcome.startswith("stream")

def handler(request):
assert request.url.path == "/openai/v1/chat/completions"
body = json.loads(request.content)
assert body["model"] == REQUEST_MODEL
assert body.get("stream", False) == streaming
calls.append(request.url.host)
if outcome == "error":
return unauthorized(request)
if streaming:
response = chat_stream()
if outcome == "stream-usage":
assert body["stream_options"] == {"include_usage": True}
usage = {
"id": "c", "object": "chat.completion.chunk", "created": 1,
"model": RESPONSE_MODEL, "choices": [],
"usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
}
content = response.content.replace(
b"data: [DONE]", f"data: {json.dumps(usage)}\n\ndata: [DONE]".encode(),
)
return httpx.Response(200, headers=response.headers, content=content)
return response
return httpx.Response(200, json=chat_completion())

kwargs = {
"model": REQUEST_MODEL,
"messages": [{"role": "user", "content": PROMPT}],
"max_tokens": 128,
"stream": streaming,
}
if outcome == "stream-usage":
kwargs["stream_options"] = {"include_usage": True}

async def run_async():
async with async_client(handler, base_url=base_url) as sdk:
response = await sdk.chat.completions.create(**kwargs)
if streaming:
text = []
async for chunk in response:
text.extend(choice.delta.content or "" for choice in chunk.choices)
assert "".join(text) == ANSWER
else:
assert response.choices[0].message.content == ANSWER

def run_sync():
with client(handler, base_url=base_url) as sdk:
response = sdk.chat.completions.create(**kwargs)
if streaming:
text = "".join(
choice.delta.content or "" for chunk in response for choice in chunk.choices
)
assert text == ANSWER
else:
assert response.choices[0].message.content == ANSWER

def run():
if use_async:
asyncio.run(run_async())
else:
run_sync()

if outcome == "error":
with pytest.raises(openai.AuthenticationError):
run()
else:
run()

assert calls == [httpx.URL(base_url).host]
path, content_type, payload = wire_trace.get(timeout=5)
assert path == "/v1/traces"
assert content_type == "application/x-protobuf"
message = ExportTraceServiceRequest.FromString(payload)
spans = [span for resource in message.resource_spans
for scope in resource.scope_spans for span in scope.spans]
assert len(spans) == 1
assert wire_trace.empty(), "The shared instrumentor must not duplicate spans"
span = spans[0]
attrs = {item.key: getattr(item.value, item.value.WhichOneof("value"))
for item in span.attributes}
assert attrs["gen_ai.provider.name"] == expected_provider
# This instrumentor emits the modern provider field, not the legacy system.
assert "gen_ai.system" not in attrs
assert attrs["gen_ai.request.model"] == (
RESPONSE_MODEL if outcome == "success" else REQUEST_MODEL
)
assert attrs["gen_ai.span.kind"] == "LLM"
assert span.status.code == (
Status.STATUS_CODE_ERROR if outcome == "error" else Status.STATUS_CODE_OK
)
if outcome in ("success", "stream-usage"):
assert attrs["gen_ai.usage.input_tokens"] == 5
assert attrs["gen_ai.usage.output_tokens"] == 2
assert attrs["gen_ai.usage.total_tokens"] == 7
else:
assert not any(key.startswith("gen_ai.usage.") for key in attrs)
if hidden:
assert PROMPT.encode() not in payload
assert ANSWER.encode() not in payload
assert "gen_ai.request.parameters" not in attrs
elif streaming:
assert attrs["output.value"] == ANSWER
assert b"test-vendor-key" not in payload
1 change: 1 addition & 0 deletions python/frameworks/openai/traceai_openai/_providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
"generativelanguage.googleapis.com": FiLLMProviderValues.GOOGLE.value,
"api.anthropic.com": FiLLMProviderValues.ANTHROPIC.value,
"api.mistral.ai": FiLLMProviderValues.MISTRALAI.value,
"api.novita.ai": "novita",
"api.deepseek.com": FiLLMProviderValues.DEEPSEEK.value,
"api.x.ai": FiLLMProviderValues.XAI.value,
"api.z.ai": "zai",
Expand Down
Loading