diff --git a/.github/workflows/fireworks-python-tests.yml b/.github/workflows/fireworks-python-tests.yml new file mode 100644 index 000000000..36100f459 --- /dev/null +++ b/.github/workflows/fireworks-python-tests.yml @@ -0,0 +1,66 @@ +name: Fireworks Python compatibility + +on: + pull_request: + paths: + - 'python/frameworks/fireworks/**' + - 'python/frameworks/openai/**' + - 'python/fi_instrumentation/**' + - 'python/tests/harness/**' + - '.github/workflows/fireworks-python-tests.yml' + push: + branches: [dev] + paths: + - 'python/frameworks/fireworks/**' + - 'python/frameworks/openai/**' + - 'python/fi_instrumentation/**' + - 'python/tests/harness/**' + - '.github/workflows/fireworks-python-tests.yml' + +permissions: + contents: read + +jobs: + contract: + name: Fireworks (openai ${{ matrix.sdk }}) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - sdk: '1.0.0' + python: '3.10' + - sdk: '1.69.0' + python: '3.10' + - sdk: '3.42.0' + python: '3.12' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ matrix.python }} + - name: Install source integrations and the selected SDK + env: + SDK: ${{ matrix.sdk }} + run: | + python -m venv .venv + .venv/bin/pip install -e python -e python/frameworks/fireworks "openai==$SDK" 'httpx<0.28' pytest pytest-asyncio + if [ "$SDK" != '1.0.0' ]; then + .venv/bin/pip install -e python/frameworks/openai + fi + - name: Run with outbound network disabled + env: + SDK: ${{ matrix.sdk }} + run: | + sudo unshare --net -- bash -e -c ' + ip link set lo up + "$1" -I -m pytest "$2/python/frameworks/fireworks/tests" -q -o addopts="" + if [ "$3" = "1.0.0" ]; then + "$1" -I -m pytest "$2/python/frameworks/openai/tests/test_fireworks_route.py" -q -o addopts="" + else + "$1" -I -m pytest "$2/python/frameworks/openai/tests" -q -o addopts="" + fi + ' _ "$PWD/.venv/bin/python" "$PWD" "$SDK" diff --git a/python/frameworks/fireworks/README.md b/python/frameworks/fireworks/README.md index ac99fe13a..80a5e57d5 100644 --- a/python/frameworks/fireworks/README.md +++ b/python/frameworks/fireworks/README.md @@ -1,71 +1,82 @@ -# TraceAI Fireworks AI Instrumentation +# Trace Fireworks with the OpenAI integration -OpenTelemetry instrumentation for Fireworks AI, enabling comprehensive observability for LLM API calls. +Use `traceAI-openai` to trace Fireworks' OpenAI-compatible Python API. You do not need a separate Fireworks instrumentor for new applications. -## Installation +## Install + +Use Python 3.10–3.13 and OpenAI SDK 1.69.0 or later. ```bash -pip install traceai-fireworks +pip install --upgrade traceAI-openai 'fi-instrumentation-otel>=1.2.2' openai ``` -## Quick Start +Set `FIREWORKS_API_KEY`, `FIREWORKS_MODEL`, `FI_API_KEY` and `FI_SECRET_KEY` in your application environment. Use a Fireworks model ID available to your account. + +## Trace a request ```python +import os + from fi_instrumentation import register from fi_instrumentation.fi_types import ProjectType -from traceai_fireworks import FireworksInstrumentor +from openai import OpenAI +from traceai_openai import OpenAIInstrumentor -# Setup TraceAI -trace_provider = register( +provider = register( project_type=ProjectType.OBSERVE, - project_name="my-fireworks-app", + project_name="fireworks-app", ) +OpenAIInstrumentor().instrument(tracer_provider=provider) + +try: + with OpenAI( + api_key=os.environ["FIREWORKS_API_KEY"], + base_url="https://api.fireworks.ai/inference/v1", + ) as client: + response = client.chat.completions.create( + model=os.environ["FIREWORKS_MODEL"], + messages=[{"role": "user", "content": "Suggest two names for a weather app."}], + ) + print(response.choices[0].message.content) +finally: + provider.shutdown() +``` -# Instrument Fireworks -FireworksInstrumentor().instrument(tracer_provider=trace_provider) +In a long-running application, shut down the provider only when the application stops. Use `AsyncOpenAI` for asynchronous requests; the same instrumentor handles both clients and streaming responses. -# Now use OpenAI client with Fireworks -from openai import OpenAI +The span includes the Fireworks provider, model, elapsed time, messages, tool-call arguments and token usage when returned by the API. Tool-call arguments belong to the model span; this does not trace the execution of your tool functions. This route covers the OpenAI-compatible API, not the separate `fireworks-ai` SDK or its training and management APIs. -client = OpenAI( - api_key="your-fireworks-api-key", - base_url="https://api.fireworks.ai/inference/v1", -) +## Protect request and response content + +Tracing captures content by default. Pass a `TraceConfig` when enabling the instrumentor to hide sensitive content while retaining model and usage metadata: -response = client.chat.completions.create( - model="accounts/fireworks/models/llama-v3p1-8b-instruct", - messages=[{"role": "user", "content": "Hello!"}] +```python +from fi_instrumentation import TraceConfig + +OpenAIInstrumentor().instrument( + tracer_provider=provider, + config=TraceConfig( + hide_inputs=True, + hide_outputs=True, + hide_llm_invocation_parameters=True, + hide_error_details=True, + ), ) ``` -## Features - -- Automatic tracing of Fireworks AI API calls -- Support for both synchronous and asynchronous operations -- Streaming response support -- Token usage tracking -- Request/response attribute capture -- OpenTelemetry semantic conventions for GenAI +Use this call **instead of** the default instrumentation call above, not after it. These settings change telemetry, not the request sent to Fireworks. -## Captured Attributes +## Migrate from `traceAI-fireworks` -- `gen_ai.request.model` - Model name -- `gen_ai.request.max_tokens` - Maximum tokens -- `gen_ai.request.temperature` - Temperature setting -- `gen_ai.usage.input_tokens` - Input token count -- `gen_ai.usage.output_tokens` - Output token count -- `gen_ai.prompt.{n}.role` - Message roles -- `gen_ai.prompt.{n}.content` - Message contents -- `gen_ai.completion.{n}.content` - Response content +`FireworksInstrumentor` is deprecated but its import, `instrument()` and `uninstrument()` entry points remain available. Existing applications using the older OpenAI SDK floor of 1.0.0 can continue using it while upgrading. -## Requirements +1. Upgrade to OpenAI SDK 1.69.0 or later and install `traceAI-openai`. +2. Replace the `FireworksInstrumentor` import and setup with `OpenAIInstrumentor` as above. Move your existing `tracer_provider` and `TraceConfig` arguments to that call. +3. Remove the old setup and uninstall `traceAI-fireworks` after checking that nothing else in your application imports it. -- Python >= 3.9 -- openai >= 1.0.0 -- fi-instrumentation >= 0.1.0 -- opentelemetry-api >= 1.0.0 -- opentelemetry-sdk >= 1.0.0 +If both updated instrumentors are enabled during migration, `OpenAIInstrumentor` owns the request span. Its provider and privacy configuration apply; the deprecated instrumentor does not add a second span. Uninstrumenting either route does not disable the other route. -## License +## References -Apache-2.0 +- [Fireworks OpenAI compatibility](https://docs.fireworks.ai/tools-sdks/openai-compatibility) +- [traceAI OpenAI integration](https://github.com/future-agi/traceAI/tree/dev/python/frameworks/openai) diff --git a/python/frameworks/fireworks/traceai_fireworks/__init__.py b/python/frameworks/fireworks/traceai_fireworks/__init__.py index 40b725cba..86bb446e0 100644 --- a/python/frameworks/fireworks/traceai_fireworks/__init__.py +++ b/python/frameworks/fireworks/traceai_fireworks/__init__.py @@ -33,6 +33,7 @@ """ import logging +import warnings from importlib import import_module from typing import Any, Collection @@ -71,6 +72,14 @@ def instrumentation_dependencies(self) -> Collection[str]: return _instruments def _instrument(self, **kwargs: Any) -> None: + warnings.warn( + "FireworksInstrumentor is deprecated. Use traceai_openai.OpenAIInstrumentor " + "with base_url='https://api.fireworks.ai/inference/v1' instead. " + "When both are enabled, OpenAIInstrumentor owns tracing and its " + "tracer_provider and TraceConfig apply.", + DeprecationWarning, + stacklevel=3, + ) if not (tracer_provider := kwargs.get("tracer_provider")): tracer_provider = trace_api.get_tracer_provider() if not (config := kwargs.get("config")): diff --git a/python/frameworks/fireworks/traceai_fireworks/_wrappers.py b/python/frameworks/fireworks/traceai_fireworks/_wrappers.py index 76cf3dc26..ff3546a6b 100644 --- a/python/frameworks/fireworks/traceai_fireworks/_wrappers.py +++ b/python/frameworks/fireworks/traceai_fireworks/_wrappers.py @@ -1,9 +1,11 @@ """Wrapper classes for Fireworks AI instrumentation.""" import logging +import sys from abc import ABC from contextlib import contextmanager from typing import Any, Dict, Iterator, Tuple +from urllib.parse import urlsplit from fi_instrumentation import safe_json_dumps from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes, MessageAttributes @@ -52,13 +54,25 @@ def _is_fireworks_client(instance: Any) -> bool: if hasattr(instance, "_client"): client = instance._client if hasattr(client, "base_url"): - base_url = str(client.base_url) - return FIREWORKS_BASE_URL in base_url + host = urlsplit(str(client.base_url)).hostname + return bool(host and host.lower().rstrip(".") == FIREWORKS_BASE_URL) except Exception: pass return False +def _shared_instrumentor_enabled() -> bool: + """Let the request-level instrumentor own spans when both routes are enabled. + + Do not import the optional package here: legacy users can keep openai 1.0, + which predates the shared instrumentor's supported SDK floor. Inspect live + state on each call so either installation/uninstallation order works. + """ + module = sys.modules.get("traceai_openai") + instrumentor = getattr(module, "OpenAIInstrumentor", None) + return bool(instrumentor and instrumentor().is_instrumented_by_opentelemetry) + + def _get_input_attributes( model: str, messages: list, @@ -68,7 +82,6 @@ def _get_input_attributes( attributes: Dict[str, AttributeValue] = { SpanAttributes.GEN_AI_SPAN_KIND: FiSpanKindValues.LLM.value, SpanAttributes.GEN_AI_PROVIDER_NAME: "fireworks", - SpanAttributes.GEN_AI_PROVIDER_NAME: "fireworks", SpanAttributes.GEN_AI_REQUEST_MODEL: model, } @@ -175,12 +188,19 @@ def _get_stream_output_attributes( if hasattr(last_chunk, "usage") and last_chunk.usage: usage = last_chunk.usage - if hasattr(usage, "prompt_tokens"): - attributes[SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS] = usage.prompt_tokens - if hasattr(usage, "completion_tokens"): - attributes[SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] = usage.completion_tokens - if hasattr(usage, "total_tokens"): - attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] = usage.total_tokens + # OpenAI 1.0 exposes Fireworks' extra streaming usage as a dict. + for field, attribute in ( + ("prompt_tokens", SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS), + ("completion_tokens", SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS), + ("total_tokens", SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS), + ): + value = ( + usage.get(field) + if isinstance(usage, dict) + else getattr(usage, field, None) + ) + if value is not None: + attributes[attribute] = value # Add raw output try: @@ -203,8 +223,8 @@ def __call__( args: Tuple[Any, ...], kwargs: Dict[str, Any], ) -> Any: - # Check if this is a Fireworks client - if not _is_fireworks_client(instance): + # One request must have one owner, regardless of setup order. + if _shared_instrumentor_enabled() or not _is_fireworks_client(instance): return wrapped(*args, **kwargs) # Extract parameters @@ -274,8 +294,8 @@ async def __call__( args: Tuple[Any, ...], kwargs: Dict[str, Any], ) -> Any: - # Check if this is a Fireworks client - if not _is_fireworks_client(instance): + # One request must have one owner, regardless of setup order. + if _shared_instrumentor_enabled() or not _is_fireworks_client(instance): return await wrapped(*args, **kwargs) # Extract parameters diff --git a/python/frameworks/openai/tests/test_fireworks_route.py b/python/frameworks/openai/tests/test_fireworks_route.py new file mode 100644 index 000000000..d5698b981 --- /dev/null +++ b/python/frameworks/openai/tests/test_fireworks_route.py @@ -0,0 +1,370 @@ +"""Fireworks' OpenAI-compatible route, using fake HTTP and real OTLP/HTTP. + +Run with both source packages installed. No Fireworks or Future AGI endpoint is +contacted; the only socket destination is the loopback OTLP receiver. +""" + +import asyncio +import json +import sys +from pathlib import Path + +import httpx +import openai +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from packaging.version import Version + +# Generic OpenAI-only jobs need not install the deprecated optional package; +# the dedicated Fireworks CI matrix installs both and must run this contract. +FireworksInstrumentor = pytest.importorskip("traceai_fireworks").FireworksInstrumentor + +# Reuse the repository's OTLP receiver, rather than asserting only Python spans. +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) +from harness import Receiver, _flatten_attributes + +MODERN_SDK = Version(openai.__version__) >= Version("1.69.0") +if MODERN_SDK: + from traceai_openai import OpenAIInstrumentor + +BASE_URL = "https://api.fireworks.ai/inference/v1" +MODEL = "accounts/fireworks/models/test-model" +INPUT = "PRIVATE_INPUT_SENTINEL" +OUTPUT = "PRIVATE_OUTPUT_SENTINEL" +ARGUMENT = "PRIVATE_TOOL_ARGUMENT_SENTINEL" +ERROR = "PRIVATE_ERROR_SENTINEL" + + +@pytest.fixture(scope="module") +def receiver(): + with Receiver() as receiver: + yield receiver + + +def response(outcome): + usage = {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7} + tool = { + "id": "call_1", + "type": "function", + "function": {"name": "lookup", "arguments": json.dumps({"query": ARGUMENT})}, + } + message = {"role": "assistant", "content": OUTPUT} + if outcome == "tool": + message = {"role": "assistant", "content": None, "tool_calls": [tool]} + if outcome == "error": + return httpx.Response( + 401, json={"error": {"message": ERROR, "type": "authentication_error"}} + ) + if outcome in ("stream", "tool_stream"): + deltas = ( + [{"role": "assistant", "content": OUTPUT[:10]}, {"content": OUTPUT[10:]}] + if outcome == "stream" + else [ + { + "role": "assistant", + "tool_calls": [ + { + **tool, + "index": 0, + "function": {"name": "lookup", "arguments": '{"query": "'}, + } + ], + }, + { + "tool_calls": [ + {"index": 0, "function": {"arguments": ARGUMENT + '"}'}} + ] + }, + ] + ) + chunks = [ + { + "id": "c", + "object": "chat.completion.chunk", + "created": 1, + "model": MODEL, + "choices": [{"index": 0, "delta": delta, "finish_reason": None}], + } + for delta in deltas + ] + chunks.append( + { + "id": "c", + "object": "chat.completion.chunk", + "created": 1, + "model": MODEL, + "choices": [], + "usage": usage, + } + ) + body = ( + "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + + "data: [DONE]\n\n" + ) + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, content=body.encode() + ) + return httpx.Response( + 200, + json={ + "id": "c", + "object": "chat.completion", + "created": 1, + "model": MODEL, + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls" if outcome == "tool" else "stop", + "message": message, + } + ], + "usage": usage, + }, + ) + + +@pytest.fixture +def setup_route(receiver): + providers = [] + + def setup(route, private=False): + if route != "legacy" and not MODERN_SDK: + pytest.skip("The shared OpenAI instrumentor requires openai >= 1.69.0") + receiver.clear() + provider = TracerProvider() + provider.add_span_processor( + SimpleSpanProcessor( + OTLPSpanExporter(endpoint=receiver.endpoint, headers={}, timeout=5) + ) + ) + providers.append(provider) + config = TraceConfig( + hide_inputs=private, + hide_outputs=private, + hide_llm_invocation_parameters=private, + hide_error_details=private, + ) + instruments = { + "legacy": [FireworksInstrumentor()], + } + if MODERN_SDK: + instruments.update( + { + "shared": [OpenAIInstrumentor()], + "legacy_first": [FireworksInstrumentor(), OpenAIInstrumentor()], + "shared_first": [OpenAIInstrumentor(), FireworksInstrumentor()], + } + ) + for instrumentor in instruments[route]: + if isinstance(instrumentor, FireworksInstrumentor): + with pytest.warns(DeprecationWarning, match="OpenAIInstrumentor"): + instrumentor.instrument(tracer_provider=provider, config=config) + else: + instrumentor.instrument(tracer_provider=provider, config=config) + return provider + + yield setup + for instrumentor in ( + [FireworksInstrumentor(), OpenAIInstrumentor()] + if MODERN_SDK + else [FireworksInstrumentor()] + ): + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + for provider in providers: + provider.shutdown() + + +def invoke(outcome="success", use_async=False, base_url=BASE_URL): + requests = [] + + def handle(request): + requests.append(request) + return response(outcome) + + params = { + "model": MODEL, + "messages": [{"role": "user", "content": INPUT}], + "stream": outcome in ("stream", "tool_stream"), + } + if outcome in ("tool", "tool_stream"): + params["tools"] = [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + }, + }, + } + ] + + async def run_async(): + async with openai.AsyncOpenAI( + base_url=base_url, + api_key="synthetic-not-a-real-key", + max_retries=0, + http_client=httpx.AsyncClient( + transport=httpx.MockTransport(handle), trust_env=False + ), + ) as client: + result = await client.chat.completions.create(**params) + if params["stream"]: + return [chunk async for chunk in result] + return result + + def run_sync(): + with openai.OpenAI( + base_url=base_url, + api_key="synthetic-not-a-real-key", + max_retries=0, + http_client=httpx.Client( + transport=httpx.MockTransport(handle), trust_env=False + ), + ) as client: + result = client.chat.completions.create(**params) + return list(result) if params["stream"] else result + + if outcome == "error": + with pytest.raises(openai.AuthenticationError, match=ERROR): + asyncio.run(run_async()) if use_async else run_sync() + result = None + else: + result = asyncio.run(run_async()) if use_async else run_sync() + if outcome == "success": + assert result.choices[0].message.content == OUTPUT + elif outcome == "stream": + assert ( + "".join( + chunk.choices[0].delta.content or "" + for chunk in result + if chunk.choices + ) + == OUTPUT + ) + elif outcome == "tool": + assert json.loads( + result.choices[0].message.tool_calls[0].function.arguments + ) == {"query": ARGUMENT} + assert len(requests) == 1 + assert requests[0].url.path == "/inference/v1/chat/completions" + return result + + +@pytest.mark.parametrize("route", ["shared", "legacy", "legacy_first", "shared_first"]) +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize( + "outcome", ["success", "stream", "tool", "tool_stream", "error"] +) +@pytest.mark.parametrize("private", [False, True]) +def test_one_fireworks_span_on_the_wire( + setup_route, receiver, route, use_async, outcome, private +): + if outcome in ("tool", "tool_stream") and Version(openai.__version__) < Version( + "1.1.0" + ): + pytest.skip("OpenAI 1.0 does not accept the tools parameter") + setup_route(route, private) + invoke(outcome, use_async) + spans = receiver.spans() + assert len(spans) == 1, [span["name"] for span in spans] + span = spans[0] + attrs = _flatten_attributes(span["attributes"]) + assert attrs["gen_ai.provider.name"] == "fireworks" + assert attrs["gen_ai.request.model"] == MODEL + assert attrs["gen_ai.span.kind"] == "LLM" + assert "parentSpanId" not in span + assert span["status"]["code"] == ( + "STATUS_CODE_ERROR" if outcome == "error" else "STATUS_CODE_OK" + ) + wire = json.dumps(spans) + assert "synthetic-not-a-real-key" not in wire + if private: + for secret in (INPUT, OUTPUT, ARGUMENT, ERROR): + assert secret not in wire + else: + assert INPUT in wire + assert ( + ERROR + if outcome == "error" + else ARGUMENT + if outcome in ("tool", "tool_stream") + else OUTPUT + ) in wire + if not private and route != "legacy" and outcome in ("tool", "tool_stream"): + prefix = "gen_ai.output.messages.0.message.tool_calls.0.tool_call.function." + assert attrs[prefix + "name"] == "lookup" + assert json.loads(attrs[prefix + "arguments"]) == {"query": ARGUMENT} + assert ARGUMENT in attrs["output.value"] + if outcome != "error": + assert int(attrs["gen_ai.usage.input_tokens"]) == 5 + assert int(attrs["gen_ai.usage.output_tokens"]) == 2 + assert int(attrs["gen_ai.usage.total_tokens"]) == 7 + + +@pytest.mark.parametrize( + "host,provider", + [ + ("API.FIREWORKS.AI.", "fireworks"), + ("api.fireworks.ai.customer.example", "openai"), + ("proxy.api.fireworks.ai", "openai"), + ("notapi.fireworks.ai", "openai"), + ("api.openai.com", "openai"), + ], +) +@pytest.mark.parametrize("route", ["shared", "legacy_first", "shared_first"]) +def test_exact_host_attribution(setup_route, receiver, host, provider, route): + setup_route(route) + invoke(base_url=f"https://{host}/inference/v1") + spans = receiver.spans() + assert len(spans) == 1 + assert ( + _flatten_attributes(spans[0]["attributes"])["gen_ai.provider.name"] == provider + ) + + +@pytest.mark.parametrize("first", ["legacy_first", "shared_first"]) +def test_uninstrument_lifecycles_are_independent(setup_route, receiver, first): + setup_route(first) + OpenAIInstrumentor().uninstrument() + invoke() + assert len(receiver.spans()) == 1 + assert receiver.spans()[0]["name"] == "Fireworks Chat Completions" + receiver.clear() + FireworksInstrumentor().uninstrument() + invoke() + assert receiver.spans() == [] + + +@pytest.mark.parametrize("route", ["legacy_first", "shared_first"]) +def test_removing_legacy_does_not_disable_shared(setup_route, receiver, route): + setup_route(route) + FireworksInstrumentor().uninstrument() + invoke() + spans = receiver.spans() + assert len(spans) == 1 + assert spans[0]["name"] == "ChatCompletion" + assert ( + _flatten_attributes(spans[0]["attributes"])["gen_ai.provider.name"] + == "fireworks" + ) + + +@pytest.mark.parametrize( + "host,expected", + [ + ("API.FIREWORKS.AI.", 1), + ("api.fireworks.ai.customer.example", 0), + ("proxy.api.fireworks.ai", 0), + ("notapi.fireworks.ai", 0), + ], +) +def test_legacy_exact_host(setup_route, receiver, host, expected): + setup_route("legacy") + invoke(base_url=f"https://{host}/inference/v1") + assert len(receiver.spans()) == expected diff --git a/python/frameworks/openai/tests/test_streamed_tool_arguments.py b/python/frameworks/openai/tests/test_streamed_tool_arguments.py new file mode 100644 index 000000000..9b0cba5a7 --- /dev/null +++ b/python/frameworks/openai/tests/test_streamed_tool_arguments.py @@ -0,0 +1,112 @@ +"""Streamed tool arguments use the same indexed message schema as full replies.""" + +import json + +import openai +import pytest +from openai.types.chat import ChatCompletionChunk +from traceai_openai._response_accumulator import _ChatCompletionAccumulator +from traceai_openai._response_attributes_extractor import _ResponseAttributesExtractor + + +@pytest.mark.parametrize( + "usage", [None, {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}] +) +def test_interleaved_tools_and_choices_keep_complete_arguments(usage): + accumulator = _ChatCompletionAccumulator( + request_parameters={"model": "requested-model"}, + chat_completion_type=openai.types.chat.ChatCompletion, + response_attributes_extractor=_ResponseAttributesExtractor(openai=openai), + ) + + def feed(choices): + accumulator.process_chunk( + ChatCompletionChunk( + id="c", + created=1, + model="served-model", + object="chat.completion.chunk", + choices=choices, + usage=usage, + ) + ) + + feed( + [ + { + "index": choice, + "delta": { + "role": "assistant", + "tool_calls": [ + { + "index": tool, + "id": f"call_{choice}_{tool}", + "type": "function", + "function": { + "name": "lookup_", + "arguments": '{"query": "', + }, + } + for tool in (1, 0) + ], + }, + } + for choice in (1, 0) + ] + ) + # Reading a partial result must not freeze the cache before later chunks. + dict(accumulator.get_attributes()) + feed( + [ + { + "index": choice, + "delta": { + "tool_calls": [ + { + "index": tool, + "function": { + "name": f"{choice}_{tool}", + "arguments": f"value-{choice}-{tool}" + '"}', + }, + } + for tool in (0, 1) + ] + }, + } + for choice in (0, 1) + ] + ) + attrs = dict(accumulator.get_extra_attributes()) + for choice in (0, 1): + for tool in (0, 1): + prefix = ( + f"gen_ai.output.messages.{choice}.message.tool_calls.{tool}.tool_call." + ) + assert attrs[prefix + "id"] == f"call_{choice}_{tool}" + assert attrs[prefix + "function.name"] == f"lookup_{choice}_{tool}" + assert json.loads(attrs[prefix + "function.arguments"]) == { + "query": f"value-{choice}-{tool}" + } + output = dict(accumulator.get_attributes()) + assert output["output.mime_type"] == "application/json" + assert len(json.loads(output["output.value"])) == 2 + # Preserve the existing requested-model policy for streamed calls. + assert "gen_ai.request.model" not in attrs + + +def test_text_stream_with_explicit_null_usage_still_has_output(): + accumulator = _ChatCompletionAccumulator( + request_parameters={}, + chat_completion_type=openai.types.chat.ChatCompletion, + ) + accumulator.process_chunk( + ChatCompletionChunk( + id="c", + created=1, + model="test-model", + object="chat.completion.chunk", + choices=[{"index": 0, "delta": {"role": "assistant", "content": "Hello"}}], + usage=None, + ) + ) + assert dict(accumulator.get_attributes()) == {"output.value": "Hello"} diff --git a/python/frameworks/openai/tests/test_streamed_tool_boundaries.py b/python/frameworks/openai/tests/test_streamed_tool_boundaries.py new file mode 100644 index 000000000..4cf96a7ed --- /dev/null +++ b/python/frameworks/openai/tests/test_streamed_tool_boundaries.py @@ -0,0 +1,165 @@ +"""Real-SDK regressions for fragmented names and OTel attribute eviction.""" + +import asyncio +import json + +import openai +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.sdk.trace import SpanLimits, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_openai import OpenAIInstrumentor + +if int(openai.__version__.split(".")[0]) >= 3: + import httpx2 as httpx +else: + import httpx + + +def stream_body(tool_count, legacy): + def delta(name, arguments, first=False): + function = {"name": name, "arguments": arguments} + if legacy: + return {"function_call": function} + return { + "tool_calls": [ + { + "index": i, + **({"id": f"call_{i}", "type": "function"} if first else {}), + "function": function, + } + for i in range(tool_count) + ] + } + + events = [ + {"role": "assistant", **delta("get_", '{"city":"', first=True)}, + delta("weather", 'Paris"}'), + ] + chunks = [ + { + "id": "c", + "object": "chat.completion.chunk", + "created": 1, + "model": "served-model", + "choices": [{"index": 0, "delta": event, "finish_reason": None}], + } + for event in events + ] + chunks.append( + { + "id": "c", + "object": "chat.completion.chunk", + "created": 1, + "model": "served-model", + "choices": [], + "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7}, + } + ) + return ( + "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + + "data: [DONE]\n\n" + ) + + +def collect(*, tool_count=1, legacy=False, private=False, use_async=False, limit=128): + exporter = InMemorySpanExporter() + provider = TracerProvider(span_limits=SpanLimits(max_attributes=limit)) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = OpenAIInstrumentor() + instrumentor.instrument( + tracer_provider=provider, config=TraceConfig(hide_outputs=private) + ) + + def handle(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=stream_body(tool_count, legacy), + ) + + options = { + "api_key": "synthetic-not-real", + "base_url": "https://api.fireworks.ai/inference/v1", + "max_retries": 0, + } + params = { + "model": "requested-model", + "messages": [{"role": "user", "content": "Use weather tool"}], + "stream": True, + } + + async def run_async(): + async with openai.AsyncOpenAI( + **options, + http_client=httpx.AsyncClient( + transport=httpx.MockTransport(handle), trust_env=False + ), + ) as client: + stream = await client.chat.completions.create(**params) + assert len([chunk async for chunk in stream]) == 3 + + try: + if use_async: + asyncio.run(run_async()) + else: + with openai.OpenAI( + **options, + http_client=httpx.Client( + transport=httpx.MockTransport(handle), trust_env=False + ), + ) as client: + assert len(list(client.chat.completions.create(**params))) == 3 + provider.force_flush() + spans = exporter.get_finished_spans() + assert len(spans) == 1 + return dict(spans[0].attributes) + finally: + instrumentor.uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("tool_count", [1, 50]) +@pytest.mark.parametrize("private", [False, True]) +@pytest.mark.parametrize("limit", [16, 128]) +def test_many_tools_keep_primary_attributes(use_async, tool_count, private, limit): + attrs = collect( + tool_count=tool_count, private=private, use_async=use_async, limit=limit + ) + expected = { + "gen_ai.provider.name": "fireworks", + "gen_ai.request.model": "requested-model", + "gen_ai.span.kind": "LLM", + "gen_ai.usage.input_tokens": 5, + "gen_ai.usage.output_tokens": 2, + "gen_ai.usage.total_tokens": 7, + } + assert {key: attrs.get(key) for key in expected} == expected + if private: + assert "Paris" not in json.dumps(attrs) + assert "get_weather" not in json.dumps(attrs) + else: + messages = json.loads(attrs["output.value"]) + assert len(messages[0]["tool_calls"]) == tool_count + assert messages[0]["tool_calls"][-1]["function"] == { + "name": "get_weather", + "arguments": '{"city":"Paris"}', + } + + +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("legacy", [False, True]) +def test_fragmented_function_names_are_concatenated(use_async, legacy): + attrs = collect(use_async=use_async, legacy=legacy) + prefix = "gen_ai.output.messages.0.message." + if legacy: + assert attrs[prefix + "function_call_name"] == "get_weather" + else: + assert attrs[prefix + "tool_calls.0.tool_call.function.name"] == "get_weather" + message = json.loads(attrs["output.value"])[0] + function = ( + message["function_call"] if legacy else message["tool_calls"][0]["function"] + ) + assert function == {"name": "get_weather", "arguments": '{"city":"Paris"}'} diff --git a/python/frameworks/openai/traceai_openai/_providers.py b/python/frameworks/openai/traceai_openai/_providers.py index 45bd0d150..21c5a2345 100644 --- a/python/frameworks/openai/traceai_openai/_providers.py +++ b/python/frameworks/openai/traceai_openai/_providers.py @@ -21,6 +21,7 @@ "api.deepseek.com": FiLLMProviderValues.DEEPSEEK.value, "api.x.ai": FiLLMProviderValues.XAI.value, "api.together.ai": "together", + "api.fireworks.ai": "fireworks", "inference.baseten.co": "baseten", "dashscope.aliyuncs.com": "dashscope", "dashscope-intl.aliyuncs.com": "dashscope", diff --git a/python/frameworks/openai/traceai_openai/_request.py b/python/frameworks/openai/traceai_openai/_request.py index 61f6a9aab..5e36234ff 100644 --- a/python/frameworks/openai/traceai_openai/_request.py +++ b/python/frameworks/openai/traceai_openai/_request.py @@ -57,11 +57,12 @@ def _start_as_current_span( context_attributes: Iterable[Tuple[str, AttributeValue]], extra_attributes: Iterable[Tuple[str, AttributeValue]], ) -> Iterator[_WithSpan]: - # Because OTEL has a default limit of 128 attributes, we split our attributes into - # two tiers, where the addition of "extra_attributes" is deferred until the end - # and only after the "attributes" are added. + # Keep primary attributes so finalization can restore them after optional + # details: OTel's bounded attribute map evicts the oldest entries. + initial_attributes = {} try: - span = self._tracer.start_span(name=span_name, attributes=dict(attributes)) + initial_attributes = dict(attributes) + span = self._tracer.start_span(name=span_name, attributes=initial_attributes) except Exception: logger.exception("Failed to start span") span = INVALID_SPAN @@ -73,6 +74,7 @@ def _start_as_current_span( ) as span: yield _WithSpan( span=span, + initial_attributes=initial_attributes, context_attributes=dict(context_attributes), extra_attributes=dict(extra_attributes), ) diff --git a/python/frameworks/openai/traceai_openai/_response_accumulator.py b/python/frameworks/openai/traceai_openai/_response_accumulator.py index 744e05d5e..17417d213 100644 --- a/python/frameworks/openai/traceai_openai/_response_accumulator.py +++ b/python/frameworks/openai/traceai_openai/_response_accumulator.py @@ -143,12 +143,14 @@ def __init__( message=_ValuesAccumulator( content=_StringAccumulator(), function_call=_ValuesAccumulator( - arguments=_StringAccumulator() + name=_StringAccumulator(), + arguments=_StringAccumulator(), ), tool_calls=_IndexedAccumulator( lambda: _ValuesAccumulator( function=_ValuesAccumulator( - arguments=_StringAccumulator() + name=_StringAccumulator(), + arguments=_StringAccumulator(), ), ) ), @@ -161,6 +163,19 @@ def process_chunk(self, chunk: Any) -> None: raw_chunk = _to_dict(chunk) self._raw_data.append(raw_chunk) self._is_null = False + self._cached_result = None + + # Streaming choices carry message deltas. Feed the existing indexed + # accumulators so fragmented tool arguments and multiple choices survive + # serialization, just as non-streamed assistant messages do. Keep the + # requested-model convention for streams; only aggregate message fields. + self._values += { + "choices": [ + {"index": choice["index"], "message": choice.get("delta") or {}} + for choice in raw_chunk.get("choices", []) + if choice.get("index") is not None + ] + } if "choices" in raw_chunk: choices = raw_chunk["choices"] @@ -183,7 +198,7 @@ def get_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: if self._raw_data: last_chunk = self._raw_data[-1] - if "usage" in last_chunk: + if isinstance(last_chunk.get("usage"), Mapping): usage = last_chunk["usage"] total_tokens = usage.get("total_tokens") prompt_tokens = usage.get("prompt_tokens") @@ -196,7 +211,17 @@ def get_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: if completion_tokens is not None: yield SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens - # Only yield the assistant's response content, not the raw API response + # A tool-only completion has no text, but still has useful output. + if not output_value and (result := self._result()): + messages = [ + choice["message"] + for choice in result.get("choices", []) + if choice.get("message", {}).get("tool_calls") + or choice.get("message", {}).get("function_call") + ] + if messages: + yield SpanAttributes.OUTPUT_MIME_TYPE, FiMimeTypeValues.JSON.value + output_value = safe_json_dumps(messages) yield SpanAttributes.OUTPUT_VALUE, output_value def get_extra_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: diff --git a/python/frameworks/openai/traceai_openai/_with_span.py b/python/frameworks/openai/traceai_openai/_with_span.py index b2bc6fb2a..5f5419ad0 100644 --- a/python/frameworks/openai/traceai_openai/_with_span.py +++ b/python/frameworks/openai/traceai_openai/_with_span.py @@ -1,16 +1,28 @@ import logging from typing import Any, Optional +from fi_instrumentation.fi_types import SpanAttributes from opentelemetry import trace as trace_api from opentelemetry.util.types import Attributes logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) +# Non-streamed responses expose usage as extra attributes rather than primary +# attributes. Keep these alongside the primary request/response data. +_USAGE_ATTRIBUTES = frozenset( + ( + SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, + SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, + SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS, + ) +) + class _WithSpan: __slots__ = ( "_span", + "_primary_attributes", "_context_attributes", "_extra_attributes", "_is_finished", @@ -21,8 +33,10 @@ def __init__( span: trace_api.Span, context_attributes: Attributes = None, extra_attributes: Attributes = None, + initial_attributes: Attributes = None, ) -> None: self._span = span + self._primary_attributes = dict(initial_attributes or {}) self._context_attributes = context_attributes self._extra_attributes = extra_attributes try: @@ -59,6 +73,7 @@ def finish_tracing( ) -> None: if self._is_finished: return + primary_attributes = {**self._primary_attributes, **(attributes or {})} for mapping in ( attributes, self._context_attributes, @@ -70,10 +85,15 @@ def finish_tracing( for key, value in mapping.items(): if value is None: continue - try: - self._span.set_attribute(key, value) - except Exception: - logger.exception("Failed to set attribute on span") + if key in primary_attributes or key in _USAGE_ATTRIBUTES: + # Preserve response-over-request precedence (e.g. served model). + primary_attributes[key] = value + self.set_attribute(key, value) + # Restore primary data last, through the same privacy-masking span API. + # Indexed tool/message details may exceed even a custom attribute limit. + for key, value in primary_attributes.items(): + if value is not None: + self.set_attribute(key, value) if status is not None: try: self._span.set_status(status=status) @@ -91,5 +111,7 @@ def set_attribute(self, key: str, value: Any) -> None: return try: self._span.set_attribute(key, value) + if key in self._primary_attributes and value is not None: + self._primary_attributes[key] = value except Exception: logger.exception(f"Failed to set attribute {key} on span")