Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions .github/workflows/parallel-python-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ jobs:
distribution: wheel
- python: '3.13'
parallel: '1.3.7'
openai: '3.42.0'
openai: '3.45.0'
distribution: sdist
env:
FI_BASE_URL: http://127.0.0.1:9
Expand Down Expand Up @@ -66,7 +66,13 @@ jobs:
packages=(python/dist/*.tar.gz python/frameworks/{parallel,openai}/dist/*.tar.gz)
fi
.parallel-test-venv/bin/pip install "${packages[@]}" \
"parallel-web==$PARALLEL_VERSION" "openai==$OPENAI_VERSION" pytest pytest-asyncio
"parallel-web==$PARALLEL_VERSION" "openai==$OPENAI_VERSION" \
./python/frameworks/{deepseek,fireworks,vllm} pytest pytest-asyncio
- name: Prevent source-tree imports from shadowing installed distributions
shell: bash
run: |
mv python/frameworks/openai/traceai_openai "$RUNNER_TEMP/openai-source"
mv python/frameworks/parallel/traceai_parallel "$RUNNER_TEMP/parallel-source"
- name: Test with only loopback networking
shell: bash
run: |
Expand Down
36 changes: 29 additions & 7 deletions python/examples/perplexity/tests/test_perplexity_recipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,19 @@
== Path(CORE_DISTRIBUTION.locate_file("fi_instrumentation/__init__.py")).resolve()
)

# The recipe CI deliberately retains the old published SDK as a compatibility
# control. Only that exact installed distribution exports echoed instructions;
# current source must not reintroduce them to satisfy legacy expectations.
try:
OPENAI_DISTRIBUTION = distribution("traceAI-openai")
except PackageNotFoundError:
LEGACY_OUTPUT = False
else:
LEGACY_OUTPUT = OPENAI_DISTRIBUTION.version == "0.1.10" and (
Path(traceai_openai.__file__).resolve()
== Path(OPENAI_DISTRIBUTION.locate_file("traceai_openai/__init__.py")).resolve()
)

TEST_DIR = Path(__file__).resolve().parent
RECIPE_DIR = TEST_DIR.parent
VENDOR_KEY = "placeholder-perplexity-key"
Expand Down Expand Up @@ -254,7 +267,10 @@ def test_responses_at_documented_host(tracing):
assert raw["usage"]["input_tokens_details"][key] == value
assert raw["usage"]["output_tokens_details"] == fake.USAGE["output_tokens_details"]
assert raw["usage"]["tool_calls_details"] == fake.USAGE["tool_calls_details"]
assert fake.INSTRUCTIONS_MARKER in raw["instructions"]
if LEGACY_OUTPUT:
assert fake.INSTRUCTIONS_MARKER in raw["instructions"]
else:
assert "instructions" not in raw
assert fake.SEARCH_URL in attrs["output.value"]
assert fake.SEARCH_SNIPPET in attrs["output.value"]

Expand Down Expand Up @@ -349,9 +365,11 @@ def handler(request):
assert _call(handler, prompt=marker) == fake.ANSWER
span, _ = _assert_export(receiver, provider, project, hide_inputs=hidden)
assert (marker in json.dumps(span)) is (not hidden)
# Output echoes remain visible even when the input itself is hidden.
for output_marker in (fake.QUERY_MARKER, fake.INSTRUCTIONS_MARKER):
assert output_marker in _attributes(span)["output.value"]
# Search queries are response-owned output; instructions are input.
output = _attributes(span)["output.value"]
assert fake.QUERY_MARKER in output
assert (fake.INSTRUCTIONS_MARKER in output) is LEGACY_OUTPUT
assert fake.ANSWER in output


@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"])
Expand All @@ -363,8 +381,11 @@ def test_hide_outputs_with_control(tracing, monkeypatch, stream):
span, attrs = _assert_export(receiver, provider, project)
_assert_usage(attrs)
assert (fake.ANSWER in json.dumps(span)) is (not hidden)
for marker in (fake.SEARCH_URL, fake.SEARCH_SNIPPET, fake.QUERY_MARKER, fake.INSTRUCTIONS_MARKER):
for marker in (fake.SEARCH_URL, fake.SEARCH_SNIPPET, fake.QUERY_MARKER):
assert (marker in json.dumps(span)) is (not hidden and not stream)
assert (fake.INSTRUCTIONS_MARKER in json.dumps(span)) is (
LEGACY_OUTPUT and not hidden and not stream
)


@pytest.mark.parametrize("hidden", [False, True], ids=["visible", "hidden"])
Expand All @@ -381,8 +402,9 @@ def test_empty_text_stream_raw_fallback(tracing, monkeypatch, hidden):
list(client.responses.create(model=fake.REQUEST_MODEL, input="empty stream", stream=True))
span, attrs = _assert_export(receiver, provider, project)
_assert_usage(attrs)
for marker in (fake.SEARCH_URL, fake.QUERY_MARKER, fake.INSTRUCTIONS_MARKER):
for marker in (fake.SEARCH_URL, fake.QUERY_MARKER):
assert (marker in json.dumps(span)) is (not hidden)
assert (fake.INSTRUCTIONS_MARKER in json.dumps(span)) is (LEGACY_OUTPUT and not hidden)


def test_cache_details_require_output_details(tracing):
Expand Down Expand Up @@ -457,7 +479,7 @@ def test_subprocess_app(tmp_path, stream, cli_model):
assert (fake.SEARCH_URL in json.dumps(attrs)) is (not stream)
assert (fake.SEARCH_SNIPPET in json.dumps(attrs)) is (not stream)
assert (fake.QUERY_MARKER in json.dumps(attrs)) is (not stream)
assert (fake.INSTRUCTIONS_MARKER in json.dumps(attrs)) is (not stream)
assert (fake.INSTRUCTIONS_MARKER in json.dumps(attrs)) is (LEGACY_OUTPUT and not stream)
[export] = receiver.requests()
assert export["path"] == "/tracer/v1/traces"
assert export["headers"]["x-api-key"] == FI_KEY
Expand Down
42 changes: 42 additions & 0 deletions python/frameworks/openai/tests/test_core_dependency_privacy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
"""Synthetic fixture to verify declared core compatibility and hidden error details."""
import asyncio
import httpx
import openai
import pytest
from fi_instrumentation import TraceConfig, register
from fi_instrumentation.fi_types import ProjectType
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from traceai_openai import OpenAIInstrumentor

MESSAGE = 'synthetic-private-upstream-error-483129'

@pytest.mark.parametrize('async_', [False, True])
def test_hidden_failed_response(async_, monkeypatch):
monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', 'true')
exporter = InMemorySpanExporter()
provider = register(project_name='core-floor-probe', project_type=ProjectType.OBSERVE,
span_exporter=exporter, batch=False, verbose=False)
instrumentor = OpenAIInstrumentor()
instrumentor.instrument(tracer_provider=provider, config=TraceConfig(hide_inputs=True))
body = {'id': 'resp_probe', 'object': 'response', 'created_at': 1, 'model': 'synthetic',
'status': 'failed', 'output': [], 'error': {'code': 'server_error', 'message': MESSAGE},
'usage': {'input_tokens': 1, 'output_tokens': 0, 'total_tokens': 1}}
def handler(req): return httpx.Response(200, json=body)
def make_client():
http = (httpx.AsyncClient if async_ else httpx.Client)(transport=httpx.MockTransport(handler), trust_env=False)
return (openai.AsyncOpenAI if async_ else openai.OpenAI)(api_key='synthetic-key', base_url='https://api.parallel.ai/v1', max_retries=0, http_client=http)
try:
if async_:
async def run():
async with make_client() as client:
assert (await client.responses.create(model='synthetic', input='private input')).status == 'failed'
asyncio.run(run())
else:
with make_client() as client:
assert client.responses.create(model='synthetic', input='private input').status == 'failed'
spans = exporter.get_finished_spans()
assert len(spans) == 1
assert MESSAGE not in spans[0].to_json()
finally:
instrumentor.uninstrument()
provider.shutdown()
8 changes: 4 additions & 4 deletions python/frameworks/openai/tests/test_deepseek_raw_response.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,9 +134,9 @@ async def run():
assert b"PRIVATE_STREAM_ERROR" not in spans[0].SerializeToString()


@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("asynchronous,close_method", [(False, "close"), (True, "close"), (True, "aclose")])
@pytest.mark.parametrize("parse_first", [False, True])
def test_raw_stream_close_before_exhaustion(wire, asynchronous, parse_first):
def test_raw_stream_close_before_exhaustion(wire, asynchronous, parse_first, close_method):
spans = wire("shared")
params = dict(
model="deepseek-reasoner",
Expand All @@ -155,8 +155,8 @@ async def run():
if parse_first:
stream = raw.parse()
await stream.__anext__()
await raw.aclose()
await raw.aclose()
await getattr(raw, close_method)()
await getattr(raw, close_method)()

if asynchronous:
asyncio.run(run())
Expand Down
3 changes: 2 additions & 1 deletion python/frameworks/openai/tests/test_deepseek_route.py
Original file line number Diff line number Diff line change
Expand Up @@ -555,7 +555,8 @@ def respond(request):
if streaming:
assert a["output.value"] == ANSWER
assert a["gen_ai.output.messages.0.message.content"] == ANSWER
assert spans[0].status.code == (2 if event_type == "response.failed" else 1)
# A terminal incomplete response is not a successful generation either.
assert spans[0].status.code == (1 if event_type == "response.completed" else 2)
assert b"PRIVATE_VENDOR_ERROR" not in spans[0].SerializeToString()


Expand Down
Loading
Loading