From 92241331623a0adb0f54beefa7bbf2ed16f32d03 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 19:28:04 -0700 Subject: [PATCH] fix(parallel): cover Task workflows and Responses attribution Keep Search/Extract instrumentation unchanged, document application-level Task timing, and identify Parallel Responses in the shared OpenAI instrumentor. Co-Authored-By: Claude Code --- .github/workflows/parallel-python-tests.yml | 78 ++++++ .../openai/tests/test_parallel_provider.py | 257 ++++++++++++++++++ .../openai/traceai_openai/_providers.py | 1 + python/frameworks/parallel/README.md | 119 +++++++- .../parallel/examples/task_workflow.py | 83 ++++++ .../parallel/tests/test_task_workflow.py | 207 ++++++++++++++ 6 files changed, 731 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/parallel-python-tests.yml create mode 100644 python/frameworks/openai/tests/test_parallel_provider.py create mode 100644 python/frameworks/parallel/examples/task_workflow.py create mode 100644 python/frameworks/parallel/tests/test_task_workflow.py diff --git a/.github/workflows/parallel-python-tests.yml b/.github/workflows/parallel-python-tests.yml new file mode 100644 index 000000000..553945dd8 --- /dev/null +++ b/.github/workflows/parallel-python-tests.yml @@ -0,0 +1,78 @@ +name: Parallel Python contracts + +on: + pull_request: + branches: [main, dev] + paths: + - 'python/frameworks/parallel/**' + - 'python/frameworks/openai/**' + - 'python/fi_instrumentation/**' + - 'python/tests/harness/**' + - '.github/workflows/parallel-python-tests.yml' + push: + branches: [dev] + paths: + - 'python/frameworks/parallel/**' + - 'python/frameworks/openai/**' + - 'python/fi_instrumentation/**' + - 'python/tests/harness/**' + - '.github/workflows/parallel-python-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + contract: + name: Parallel (${{ matrix.parallel }}, OpenAI ${{ matrix.openai }}, ${{ matrix.distribution }}) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - python: '3.10' + parallel: '1.0.1' + openai: '1.69.0' + distribution: wheel + - python: '3.13' + parallel: '1.3.7' + openai: '3.42.0' + distribution: sdist + env: + FI_BASE_URL: http://127.0.0.1:9 + FI_GRPC_URL: http://127.0.0.1:9 + PARALLEL_VERSION: ${{ matrix.parallel }} + OPENAI_VERSION: ${{ matrix.openai }} + DISTRIBUTION: ${{ matrix.distribution }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ matrix.python }} + - name: Build and install distributions + shell: bash + run: | + python -m venv .parallel-test-venv + .parallel-test-venv/bin/pip install build + .parallel-test-venv/bin/python -m build "--$DISTRIBUTION" python/frameworks/parallel + .parallel-test-venv/bin/python -m build "--$DISTRIBUTION" python/frameworks/openai + if [ "$DISTRIBUTION" = wheel ]; then + packages=(python/frameworks/{parallel,openai}/dist/*.whl) + else + packages=(python/frameworks/{parallel,openai}/dist/*.tar.gz) + fi + .parallel-test-venv/bin/pip install "${packages[@]}" \ + "parallel-web==$PARALLEL_VERSION" "openai==$OPENAI_VERSION" pytest pytest-asyncio + - name: Test with only loopback networking + shell: bash + run: | + sudo --preserve-env=FI_BASE_URL,FI_GRPC_URL unshare --net bash -c ' + set -e + ip link set lo up + "$1" -I -m pytest \ + python/frameworks/parallel/tests python/frameworks/openai/tests \ + -q -p no:cacheprovider + ' _ "$PWD/.parallel-test-venv/bin/python" diff --git a/python/frameworks/openai/tests/test_parallel_provider.py b/python/frameworks/openai/tests/test_parallel_provider.py new file mode 100644 index 000000000..d0a234cc4 --- /dev/null +++ b/python/frameworks/openai/tests/test_parallel_provider.py @@ -0,0 +1,257 @@ +"""Parallel Responses uses the shared OpenAI instrumentor, not a second patch. + +Upstream route: https://docs.parallel.ai/responses-api/responses-quickstart +SDK calls use MockTransport. Only the loopback OTLP receiver opens a socket. +JSON is a test exporter using the upstream encoder, not a new product transport. +""" + +import asyncio +import base64 +import gzip +import json +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread + +import httpx +import openai +import pytest +import requests +from fi_instrumentation import TraceConfig +from google.protobuf.json_format import MessageToDict +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.exporter.otlp.proto.http import Compression +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExporter, SpanExportResult + +from traceai_openai import OpenAIInstrumentor +from traceai_openai._providers import provider_from_host + +from .test_openai_compatible_spans import async_client, client, responses_body, unauthorized + +PROMPT = "synthetic-parallel-private-question" +ANSWER = "synthetic-parallel-private-answer" +BASE_URL = "https://api.parallel.ai/v1" + + +def otlp_json(message): + payload = MessageToDict(message, use_integers_for_enums=True) + + def visit(value): + if isinstance(value, dict): + for key, child in value.items(): + if key in {"traceId", "spanId", "parentSpanId"}: + value[key] = base64.b64decode(child).hex() + else: + visit(child) + elif isinstance(value, list): + for child in value: + visit(child) + + visit(payload) + return payload + + +class JsonExporter(SpanExporter): + def __init__(self, endpoint, session, compress): + self.endpoint, self.session, self.compress = endpoint, session, compress + + def export(self, spans): + body = json.dumps(otlp_json(encode_spans(spans))).encode() + headers = {"Content-Type": "application/json"} + if self.compress: + body = gzip.compress(body) + headers["Content-Encoding"] = "gzip" + response = self.session.post(self.endpoint, data=body, headers=headers, timeout=3) + response.raise_for_status() + return SpanExportResult.SUCCESS + + def shutdown(self): + pass + + +@pytest.fixture +def wire_tracer(): + payloads = [] + + class Receiver(BaseHTTPRequestHandler): + def do_POST(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + if self.headers.get("Content-Encoding") == "gzip": + body = gzip.decompress(body) + payload = (json.loads(body) if self.headers["Content-Type"] == "application/json" + else otlp_json(ExportTraceServiceRequest.FromString(body))) + payloads.append((self.path, self.headers, payload)) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *_): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Receiver) + thread = Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) + thread.start() + provider = TracerProvider() + session = requests.Session() + session.trust_env = False + instrumentor = OpenAIInstrumentor() + + def configure(encoding, compress, hidden): + endpoint = f"http://127.0.0.1:{server.server_port}/v1/traces" + exporter = (JsonExporter(endpoint, session, compress) if encoding == "json" else + OTLPSpanExporter(endpoint=endpoint, session=session, timeout=3, + compression=Compression.Gzip if compress else Compression.NoCompression)) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor.instrument(tracer_provider=provider, config=TraceConfig( + hide_inputs=hidden, hide_outputs=hidden, hide_llm_invocation_parameters=hidden, + )) + return payloads + + try: + yield configure + finally: + instrumentor.uninstrument() + provider.shutdown() + session.close() + server.shutdown() + server.server_close() + thread.join(timeout=3) + + +def stream_response(): + completed = responses_body(text=ANSWER, model="parallel") + events = [ + {"type": "response.created", "sequence_number": 0, + "response": {**completed, "status": "in_progress", "output": []}}, + {"type": "response.output_text.delta", "sequence_number": 1, "item_id": "msg_1", + "output_index": 0, "content_index": 0, "delta": ANSWER, "logprobs": []}, + {"type": "response.completed", "sequence_number": 2, "response": completed}, + ] + body = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=body.encode()) + + +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +@pytest.mark.parametrize("compress", [False, True], ids=["plain", "gzip"]) +@pytest.mark.parametrize("hidden", [False, True], ids=["content", "masked"]) +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("outcome", ["success", "stream", "error"]) +def test_parallel_responses_wire(wire_tracer, encoding, compress, hidden, use_async, outcome): + payloads = wire_tracer(encoding, compress, hidden) + seen = [] + + def handler(request): + seen.append(request) + assert request.url.path == "/v1/responses" + assert json.loads(request.content)["input"] == PROMPT + if outcome == "error": + return unauthorized(request) + if outcome == "stream": + return stream_response() + return httpx.Response(200, json=responses_body(text=ANSWER, model="parallel")) + + kwargs = dict(model="parallel", input=PROMPT, stream=outcome == "stream") + + async def run_async(): + async with async_client(handler, base_url=BASE_URL) as sdk: + response = await sdk.responses.create(**kwargs) + if outcome == "stream": + assert "".join([event.delta async for event in response + if event.type == "response.output_text.delta"]) == ANSWER + else: + assert response.output_text == ANSWER + + def run_sync(): + with client(handler, base_url=BASE_URL) as sdk: + response = sdk.responses.create(**kwargs) + if outcome == "stream": + assert "".join(event.delta for event in response + if event.type == "response.output_text.delta") == ANSWER + else: + assert response.output_text == ANSWER + + def run(): + asyncio.run(run_async()) if use_async else run_sync() + + if outcome == "error": + with pytest.raises(openai.AuthenticationError): + run() + else: + run() + assert len(seen) == len(payloads) == 1 + path, headers, payload = payloads[0] + assert path == "/v1/traces" + assert headers.get("Content-Encoding") == ("gzip" if compress else None) + assert headers["Content-Type"] == ("application/json" if encoding == "json" else "application/x-protobuf") + spans = [span for resource in payload["resourceSpans"] + for scope in resource["scopeSpans"] for span in scope["spans"]] + assert len(spans) == 1 + span = spans[0] + attrs = {a["key"]: a["value"] for a in span["attributes"]} + assert attrs["gen_ai.provider.name"] == {"stringValue": "parallel"} + assert attrs["gen_ai.request.model"] == {"stringValue": "parallel"} + assert attrs["gen_ai.span.kind"] == {"stringValue": "LLM"} + assert span["status"]["code"] == (2 if outcome == "error" else 1) + if outcome != "error": + assert attrs["gen_ai.usage.input_tokens"] == {"intValue": "5"} + assert attrs["gen_ai.usage.output_tokens"] == {"intValue": "2"} + wire = json.dumps(payload) + assert "test-vendor-key" not in wire + assert (PROMPT in wire) == (not hidden) + if hidden: + assert ANSWER not in wire + + +@pytest.mark.parametrize("host,expected", [ + ("api.parallel.ai", "parallel"), ("API.PARALLEL.AI.", "parallel"), + ("api.parallel.ai.customer.example", "openai"), ("proxy.api.parallel.ai", "openai"), + ("notapi.parallel.ai", "openai"), ("parallel.ai", "openai"), + ("localhost", "openai"), ("api.openai.com", "openai"), (None, "openai"), +]) +def test_parallel_exact_host(host, expected): + assert provider_from_host(host) == expected + + +def test_parallel_readme_responses_snippet(): + import os + import subprocess + import sys + from pathlib import Path + + sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) + from harness import Receiver, _flatten_attributes + + readme = Path(__file__).resolve().parents[2] / "parallel" / "README.md" + snippet = readme.read_text().split("## Trace the Responses API")[1].split("```python\n")[1].split("```")[0] + # Inject only an HTTP transport, then execute the README verbatim. The real + # SDK, instrumentor, register() and OTLP exporter all run in the subprocess. + bootstrap = ''' +import httpx +import openai +_original_init = openai.OpenAI.__init__ +def handler(request): + assert request.url.host == "api.parallel.ai" + assert request.url.path == "/v1/responses" + return httpx.Response(200, json=RESPONSE_BODY) +def init(self, *args, **kwargs): + kwargs.update(http_client=httpx.Client(transport=httpx.MockTransport(handler), trust_env=False), max_retries=0) + _original_init(self, *args, **kwargs) +openai.OpenAI.__init__ = init +'''.replace("RESPONSE_BODY", repr(responses_body(text=ANSWER, model="parallel"))) + with Receiver() as receiver: + env = {k: v for k, v in os.environ.items() if not k.startswith(("FI_", "PARALLEL_", "OPENAI_", "OTEL_"))} + env.update(FI_BASE_URL=receiver.origin, FI_API_KEY="synthetic-fi-key", + FI_SECRET_KEY="synthetic-fi-secret", PARALLEL_API_KEY="synthetic-parallel-key") + result = subprocess.run([sys.executable, "-I", "-c", bootstrap + "\n" + snippet], + env=env, capture_output=True, timeout=30) + assert result.returncode == 0, result.stderr.decode() + spans = receiver.spans() + assert len(spans) == 1 + attrs = _flatten_attributes(spans[0]["attributes"]) + assert attrs["gen_ai.provider.name"] == "parallel" + assert attrs["gen_ai.request.model"] == "parallel" + assert ANSWER in json.dumps(spans) + assert "synthetic-parallel-key" not in json.dumps(spans) + assert all(r["path"] == "/tracer/v1/traces" for r in receiver.requests()) diff --git a/python/frameworks/openai/traceai_openai/_providers.py b/python/frameworks/openai/traceai_openai/_providers.py index 41b175fdd..e9a324f38 100644 --- a/python/frameworks/openai/traceai_openai/_providers.py +++ b/python/frameworks/openai/traceai_openai/_providers.py @@ -27,6 +27,7 @@ "dashscope-intl.aliyuncs.com": "dashscope", "dashscope-us.aliyuncs.com": "dashscope", "api.perplexity.ai": "perplexity", + "api.parallel.ai": "parallel", "api.orcarouter.ai": "orcarouter", "api.anannas.ai": "anannas", "anannas.ai": "anannas", diff --git a/python/frameworks/parallel/README.md b/python/frameworks/parallel/README.md index 68e033e37..ab85d79d7 100644 --- a/python/frameworks/parallel/README.md +++ b/python/frameworks/parallel/README.md @@ -4,9 +4,11 @@ OpenTelemetry instrumentation for the Search and Extract methods of the [`parallel-web`](https://pypi.org/project/parallel-web/) client (Parallel Web Systems). -It traces `search` and `extract` only. The Task API (`task_run`, -`task_group`), Monitor, FindAll and the rest of `client.beta` are different -products and produce no spans from this package. +Automatic instrumentation covers `search` and `extract`. For Task submission +and result waiting, use the [application-span recipe](#trace-a-task-workflow) +below. For the OpenAI-compatible Responses API, use the +[shared OpenAI instrumentor](#trace-the-responses-api). TaskGroup, Monitor, +FindAll and the rest of `client.beta` are not automatically instrumented. ## Installation @@ -15,8 +17,7 @@ pip install traceAI-parallel ``` It accepts `parallel-web>=1.0.1,<2` and `fi-instrumentation-otel>=1.1.0`. -The suite runs with `parallel-web` 1.0.1 and 1.3.5 on Python 3.10, 3.11, 3.12 -and 3.13. +Python 3.10–3.13 is supported. ## Usage @@ -208,13 +209,103 @@ skips the span. - Do not combine this package with another instrumentor that wraps the same `parallel-web` methods, or calls will produce duplicate spans. -## Live vendor verification (2026-10-17) +## Trace a Task workflow -Run against the real Parallel API with `parallel-web` 1.3.5 and -`traceAI-parallel` 0.1.0 on Python 3.12: `search` and `extract` (sync and -async) produced `RETRIEVER` and `TOOL` spans with the documented attributes, -and a call with a bad API key produced an `ERROR` span (`AuthenticationError`, -HTTP 401) whose stacktrace holds the exception line only. With -`FI_HIDE_INPUTS=true` and `FI_HIDE_OUTPUTS=true` the queries were absent from -every stored column. Not run live: `with_raw_response` and -`with_streaming_response` (package tests only). +Set `PARALLEL_API_KEY`, `FI_API_KEY` and `FI_SECRET_KEY` in your environment. +Use an application span around both `task_run.create()` and the blocking +`task_run.result()` call. This shows the time your application spends submitting +and waiting, rather than treating submission latency as the completed task's +duration. You do not need `ParallelInstrumentor` for this recipe. + +```python +from fi_instrumentation import FITracer, register +from fi_instrumentation.fi_types import ProjectType +from opentelemetry.trace import Status, StatusCode +from parallel import Parallel +from parallel.types import TaskRunResult + +def research(client: Parallel, tracer: FITracer, question: str) -> TaskRunResult: + with tracer.start_as_current_span( + "parallel.task", attributes={"gen_ai.span.kind": "CHAIN", "gen_ai.provider.name": "parallel"}, + record_exception=False, set_status_on_exception=False, + ) as span: + try: + run = client.task_run.create(input=question, processor="base") + span.set_attribute("parallel.run_id", run.run_id) + span.set_attribute("parallel.processor", "base") + result = client.task_run.result(run.run_id, api_timeout=600, timeout=610) + except Exception: + span.set_status(Status(StatusCode.ERROR, "Task request failed")) + raise + span.set_attribute("parallel.task.status", result.run.status) + span.set_status(Status(StatusCode.OK) if result.run.status == "completed" + else Status(StatusCode.ERROR, "Task did not complete")) + return result + + +provider = register(project_name="parallel-tasks", project_type=ProjectType.OBSERVE) +tracer = FITracer(provider.get_tracer("parallel-workflow")) +try: + with Parallel() as client: + result = research(client, tracer, "What is OpenTelemetry and who maintains it?") + print(result.output.content) +finally: + provider.force_flush() + provider.shutdown() +``` + +Open the `parallel-tasks` project in Future AGI to see the `parallel.task` +chain span with its run ID, processor and terminal status. Task inputs, outputs +and raw exception text are not attached. SDK errors are re-raised unchanged; +only a generic error status is recorded. The `result()` call waits on the +server for up to 600 seconds, with a 610-second HTTP timeout. A wait timeout +is not a remote task failure: use the run ID to retrieve its status or resume +waiting. The span does not expose Parallel's internal research steps. + +The complete [sync and async example](examples/task_workflow.py) also handles +`asyncio.CancelledError`. Cancelling your local wait ends the span as an error +but does not cancel the remote task. Run it with `--async` to use `AsyncParallel`. + +## Trace the Responses API + +Install `traceAI-openai` and use Parallel's +[OpenAI-compatible endpoint](https://docs.parallel.ai/responses-api/responses-quickstart). +Do not add a separate Parallel SDK patch for this route. + +```bash +pip install traceAI-openai +``` + +```python +import os + +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from openai import OpenAI +from traceai_openai import OpenAIInstrumentor + +provider = register(project_name="parallel-responses", project_type=ProjectType.OBSERVE) +OpenAIInstrumentor().instrument(tracer_provider=provider) +try: + with OpenAI( + api_key=os.environ["PARALLEL_API_KEY"], + base_url="https://api.parallel.ai/v1", + timeout=120, + ) as client: + response = client.responses.create( + model="parallel", + input="What is OpenTelemetry and who maintains it?", + reasoning={"effort": "low"}, + ) + print(response.output_text) +finally: + provider.force_flush() + provider.shutdown() +``` + +This produces one LLM span per Responses call, attributed to `parallel` at +`api.parallel.ai`. Sync, async and streamed Responses calls use the same +instrumentor. The model and token usage come from the request or response; +no Task processor is treated as a model and no Task token usage is invented. +Use `FI_HIDE_INPUTS=true` and `FI_HIDE_OUTPUTS=true` before starting your app +if Responses prompts and answers should not be exported. diff --git a/python/frameworks/parallel/examples/task_workflow.py b/python/frameworks/parallel/examples/task_workflow.py new file mode 100644 index 000000000..a3c32c20f --- /dev/null +++ b/python/frameworks/parallel/examples/task_workflow.py @@ -0,0 +1,83 @@ +"""Application spans cover Task submission AND waiting, not server internals. + +Set PARALLEL_API_KEY, FI_API_KEY and FI_SECRET_KEY before running. +Use --async for the AsyncParallel example. Task content and raw error text +are deliberately not attached to these metadata-only workflow spans. +""" + +import asyncio +import sys + +from fi_instrumentation import FITracer, register +from fi_instrumentation.fi_types import ProjectType +from opentelemetry.trace import Status, StatusCode +from parallel import AsyncParallel, Parallel +from parallel.types import TaskRunResult + + +def research(client: Parallel, tracer: FITracer, question: str) -> TaskRunResult: + with tracer.start_as_current_span( + "parallel.task", attributes={"gen_ai.span.kind": "CHAIN", "gen_ai.provider.name": "parallel"}, + record_exception=False, set_status_on_exception=False, + ) as span: + try: + run = client.task_run.create(input=question, processor="base") + span.set_attribute("parallel.run_id", run.run_id) + span.set_attribute("parallel.processor", "base") + result = client.task_run.result(run.run_id, api_timeout=600, timeout=610) + except Exception: + span.set_status(Status(StatusCode.ERROR, "Task request failed")) + raise + span.set_attribute("parallel.task.status", result.run.status) + span.set_status(Status(StatusCode.OK) if result.run.status == "completed" + else Status(StatusCode.ERROR, "Task did not complete")) + return result + + +async def research_async(client: AsyncParallel, tracer: FITracer, question: str) -> TaskRunResult: + with tracer.start_as_current_span( + "parallel.task", attributes={"gen_ai.span.kind": "CHAIN", "gen_ai.provider.name": "parallel"}, + record_exception=False, set_status_on_exception=False, + ) as span: + try: + run = await client.task_run.create(input=question, processor="base") + span.set_attribute("parallel.run_id", run.run_id) + span.set_attribute("parallel.processor", "base") + result = await client.task_run.result(run.run_id, api_timeout=600, timeout=610) + except asyncio.CancelledError: + span.set_attribute("parallel.cancelled", True) + span.set_status(Status(StatusCode.ERROR, "Task wait cancelled")) + raise + except Exception: + span.set_status(Status(StatusCode.ERROR, "Task request failed")) + raise + span.set_attribute("parallel.task.status", result.run.status) + span.set_status(Status(StatusCode.OK) if result.run.status == "completed" + else Status(StatusCode.ERROR, "Task did not complete")) + return result + + +def main() -> None: + provider = register(project_name="parallel-tasks", project_type=ProjectType.OBSERVE) + tracer = FITracer(provider.get_tracer("parallel-workflow")) + question = "What is OpenTelemetry and who maintains it?" + + async def run_async() -> None: + async with AsyncParallel() as client: + result = await research_async(client, tracer, question) + print(result.output.content) + + try: + if "--async" in sys.argv: + asyncio.run(run_async()) + else: + with Parallel() as client: + result = research(client, tracer, question) + print(result.output.content) + finally: + provider.force_flush() + provider.shutdown() + + +if __name__ == "__main__": + main() diff --git a/python/frameworks/parallel/tests/test_task_workflow.py b/python/frameworks/parallel/tests/test_task_workflow.py new file mode 100644 index 000000000..52d40bdb4 --- /dev/null +++ b/python/frameworks/parallel/tests/test_task_workflow.py @@ -0,0 +1,207 @@ +"""Exercise the documented application span with real sync/async Parallel SDKs.""" + +import asyncio +import importlib.util +import json +import os +import subprocess +import sys +from pathlib import Path + +import httpx +import pytest +from fi_instrumentation import FITracer, using_session +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from parallel import AsyncParallel, AuthenticationError, Parallel +from traceai_parallel import ParallelInstrumentor + +EXAMPLE = Path(__file__).resolve().parents[1] / "examples" / "task_workflow.py" +spec = importlib.util.spec_from_file_location("parallel_task_example", EXAMPLE) +example = importlib.util.module_from_spec(spec) +spec.loader.exec_module(example) +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) +from harness import Receiver, _flatten_attributes # noqa: E402 + +KEY = "synthetic-parallel-key" +QUESTION = "synthetic-private-task-question" +ANSWER = "synthetic-private-task-answer" + + +def run_body(status="completed"): + return {"run_id": "run_1", "interaction_id": "interaction_1", "processor": "base", + "status": status, "is_active": status == "queued"} + + +def result_body(status="completed"): + return {"run": run_body(status), "output": {"type": "text", "content": ANSWER, "basis": []}} + + +@pytest.fixture +def traced(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = ParallelInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + yield FITracer(provider.get_tracer("task-example")), exporter + finally: + instrumentor.uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("outcome", ["completed", "failed", "cancelled", "create-error", "result-error"]) +def test_workflow_waits_for_result_and_preserves_errors(traced, use_async, outcome): + tracer, exporter = traced + paths = [] + + def handler(request): + paths.append(request.url.path) + # Submission must not end the workflow span. Waiting stays inside it. + assert exporter.get_finished_spans() == () + assert request.headers["x-api-key"] == KEY + if (outcome == "create-error" and len(paths) == 1) or ( + outcome == "result-error" and len(paths) == 2 + ): + return httpx.Response(401, json={"error": {"message": KEY + QUESTION + ANSWER}}) + if len(paths) == 1: + assert json.loads(request.content)["input"] == QUESTION + return httpx.Response(200, json=run_body("queued")) + assert request.url.params["timeout"] == "600" + return httpx.Response(200, json=result_body(outcome)) + + async def run_async(): + async with AsyncParallel(api_key=KEY, max_retries=0, http_client=httpx.AsyncClient( + transport=httpx.MockTransport(handler), trust_env=False, + )) as client: + return await example.research_async(client, tracer, QUESTION) + + def run_sync(): + with Parallel(api_key=KEY, max_retries=0, http_client=httpx.Client( + transport=httpx.MockTransport(handler), trust_env=False, + )) as client: + return example.research(client, tracer, QUESTION) + + with using_session("task-session"), tracer.start_as_current_span("application") as parent: + if outcome.endswith("error"): + with pytest.raises(AuthenticationError) as caught: + asyncio.run(run_async()) if use_async else run_sync() + assert KEY in str(caught.value) # Original SDK exception is unchanged. + else: + result = asyncio.run(run_async()) if use_async else run_sync() + assert result.output.content == ANSWER + + spans = exporter.get_finished_spans() + assert len(spans) == 2 # No submission or polling monkeypatch spans. + span = spans[0] + assert span.name == "parallel.task" + assert span.parent.span_id == parent.get_span_context().span_id + assert span.attributes["session.id"] == "task-session" + assert span.attributes["gen_ai.span.kind"] == "CHAIN" + assert span.attributes["gen_ai.provider.name"] == "parallel" + assert span.status.status_code is (StatusCode.OK if outcome == "completed" else StatusCode.ERROR) + if outcome != "create-error": + assert span.attributes["parallel.run_id"] == "run_1" + if not outcome.endswith("error"): + assert span.attributes["parallel.task.status"] == outcome + assert paths == (["/v1/tasks/runs"] if outcome == "create-error" else + ["/v1/tasks/runs", "/v1/tasks/runs/run_1/result"]) + assert span.events == () + wire = span.to_json() + for secret in (KEY, QUESTION, ANSWER): + assert secret not in wire + + +def test_cancelling_the_wait_ends_one_span_without_cancelling_remote_task(traced): + tracer, exporter = traced + paths = [] + + async def run(): + waiting = asyncio.Event() + + async def handler(request): + paths.append(request.url.path) + if len(paths) == 1: + return httpx.Response(200, json=run_body("queued")) + waiting.set() + await asyncio.Event().wait() + + async with AsyncParallel(api_key=KEY, max_retries=0, http_client=httpx.AsyncClient( + transport=httpx.MockTransport(handler), trust_env=False, + )) as client: + task = asyncio.create_task(example.research_async(client, tracer, QUESTION)) + await asyncio.wait_for(waiting.wait(), timeout=3) + assert exporter.get_finished_spans() == () + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + asyncio.run(run()) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].attributes["parallel.cancelled"] is True + assert spans[0].status.status_code is StatusCode.ERROR + assert spans[0].status.description == "Task wait cancelled" + assert spans[0].events == () + assert paths == ["/v1/tasks/runs", "/v1/tasks/runs/run_1/result"] + + +@pytest.mark.parametrize("mode", ["sync", "async", "readme"]) +def test_example_cli_with_loopback_sdk_and_real_exporter(mode): + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + from threading import Thread + + class API(BaseHTTPRequestHandler): + def do_POST(self): + self.rfile.read(int(self.headers["Content-Length"])) + self.respond(run_body("queued")) + + def do_GET(self): + self.respond(result_body()) + + def respond(self, value): + body = json.dumps(value).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), API) + thread = Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) + thread.start() + try: + with Receiver() as receiver: + env = {k: v for k, v in os.environ.items() if not k.startswith(("FI_", "PARALLEL_", "OTEL_"))} + env.update(FI_BASE_URL=receiver.origin, FI_API_KEY="synthetic-fi-key", + FI_SECRET_KEY="synthetic-fi-secret", PARALLEL_API_KEY=KEY, + PARALLEL_BASE_URL=f"http://127.0.0.1:{server.server_port}") + if mode == "readme": + section = (EXAMPLE.parents[1] / "README.md").read_text().split("## Trace a Task workflow")[1] + snippet = section.split("```python\n")[1].split("```")[0] + command = [sys.executable, "-I", "-c", snippet] + else: + command = [sys.executable, "-I", str(EXAMPLE)] + (["--async"] if mode == "async" else []) + result = subprocess.run(command, env=env, capture_output=True, timeout=30) + assert result.returncode == 0, result.stderr.decode() + spans = receiver.spans() + assert len(spans) == 1 + assert spans[0]["name"] == "parallel.task" + attrs = _flatten_attributes(spans[0]["attributes"]) + assert attrs["parallel.run_id"] == "run_1" + assert attrs["parallel.task.status"] == "completed" + assert attrs["gen_ai.span.kind"] == "CHAIN" + assert all(r["path"] == "/tracer/v1/traces" for r in receiver.requests()) + assert KEY not in json.dumps(spans) + assert ANSWER not in json.dumps(spans) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=3)