diff --git a/.github/workflows/python-voice-attachments.yml b/.github/workflows/python-voice-attachments.yml new file mode 100644 index 000000000..967921ed3 --- /dev/null +++ b/.github/workflows/python-voice-attachments.yml @@ -0,0 +1,57 @@ +name: Python Voice Attachments + +on: + pull_request: + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/livekit/**' + - 'python/frameworks/pipecat/**' + - 'python/voice-tests/**' + - 'python/traceai/**' + - 'python/tests/attachments/**' + - '.github/workflows/python-voice-attachments.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + components: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + # Pipecat 1.15.0 (the qualified recorder surface) requires Python >=3.11. + python: ['3.11', '3.12'] + adapter: + - {name: pipecat, upstream: pipecat-ai, version: '1.15.0'} + - {name: livekit, upstream: livekit-agents, version: '1.8.10'} + - {name: livekit, upstream: livekit-agents, version: '1.8.11'} + include: + - python: '3.10' + adapter: {name: livekit, upstream: livekit-agents, version: '1.8.10'} + - python: '3.10' + adapter: {name: livekit, upstream: livekit-agents, version: '1.8.11'} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install exact upstream with this checkout + run: | + python -m pip install -e python -e python/frameworks/${{ matrix.adapter.name }} \ + '${{ matrix.adapter.upstream }}==${{ matrix.adapter.version }}' \ + opentelemetry-exporter-otlp-proto-grpc pytest pytest-asyncio jsonschema + python -m pip check + python -m pip freeze > voice-versions.txt + - name: Run voice components and negative controls + run: python -m pytest python/voice-tests/test_${{ matrix.adapter.name }}_attachments.py -q --tb=short + - name: Run existing LiveKit regression tests + if: matrix.adapter.name == 'livekit' + run: python -m pytest python/frameworks/livekit/tests -q --tb=short + - uses: actions/upload-artifact@v4 + if: always() + with: + name: voice-versions-${{ matrix.python }}-${{ matrix.adapter.name }}-${{ matrix.adapter.version }} + path: voice-versions.txt diff --git a/python/frameworks/livekit/README.md b/python/frameworks/livekit/README.md index ea3feb2b5..5c7b173bc 100644 --- a/python/frameworks/livekit/README.md +++ b/python/frameworks/livekit/README.md @@ -47,6 +47,39 @@ drops the Future AGI exporter. LiveKit installs its privacy filter once per prov ignored. To keep conversation content out of your traces, set `LIVEKIT_TELEMETRY_ALLOW_PII=false` before calling `enable_*` (livekit-agents 1.8 and later). +### Grouping a call into a session + +After enabling attribute mapping, use `using_session` inside each job entrypoint. Wrap the call's +`AgentSession` startup and work, not just tracing setup at worker import time: + +```python +from fi_instrumentation import using_session +from livekit.agents import Agent, AgentSession + +# Inside entrypoint(ctx), after register() and enable_http_attribute_mapping(): +with using_session(session_id=ctx.room.name): + await ctx.connect() + session = AgentSession() + await session.start(agent=Agent(instructions="Answer briefly."), room=ctx.room) + # Run your conversation here. +``` + +The native spans capture `session.id` when they start, so concurrent calls can use different IDs +on the same provider. An explicit `session.id` takes precedence; an existing `gen_ai.conversation.id` +(including LiveKit's own conversation ID) is preserved. Grouping does not enable attachments or +recording. With no session context, no `session.id` is added. + +On livekit-agents 1.8, context-derived IDs follow `LIVEKIT_TELEMETRY_ALLOW_PII` and project +redaction. Set your privacy choice before enabling mapping; changing it afterward does not +replace LiveKit's installed filter. The IDs are mapped only on the Future AGI export copy, so +they cannot bypass LiveKit Cloud redaction. + +A child can inherit from its in-process parent span even after leaving the context block. A +bare `SpanContext` or `NonRecordingSpan` carries only tracing IDs, not session metadata: watchdog +diagnostics such as `event_loop_blocked` may lack a session when emitted with only those IDs. Likewise, +a span started before entering the block is not retroactively grouped. Pass the context through +your own asynchronous work instead of assigning one session globally to the worker. + ### Flushing at the end of a job On Linux a LiveKit job process ends without running `atexit`, and LiveKit flushes only its own processors, so the spans @@ -57,8 +90,9 @@ them twice over: trace, `agent_session` included, as soon as the call ends. - **When the process exits**, just before it stops: this sends what LiveKit ends after the shutdown callbacks, its own `job_shutdown` span, and the job's root `job_entrypoint` span when tracing is set up before the job starts. With - set-up inside the entrypoint, LiveKit starts `job_entrypoint` before tracing is on, so it is never recorded and the - job trace arrives rooted at `job_shutdown`. + set-up inside the entrypoint, LiveKit starts `job_entrypoint` before tracing is on, so it is never recorded. The call + then arrives as separate traces: the session trace rooted at `agent_session`, and `job_shutdown` as a root of its own + (see "How Future AGI Voice finds a LiveKit call"). Each waits at most 3 seconds for the collector, well inside the 10 seconds LiveKit gives a job process to exit, so its own clean-up still runs when the collector is slow. @@ -188,3 +222,36 @@ The pinned development lock contains LiveKit Agents 1.8.6, API 1.2.1, protocol 1 Check the `examples/` directory for agent examples (validate recording SDK compatibility before deployment): * `simple_agent.py`: A basic voice assistant agent. * `egress_agent.py`: An agent with opt-in Egress metadata capture, an explicit session envelope and bounded stop/close cleanup. The modern `LiveKitAPI().egress` request shape and native-session linkage require the deployment's SDK/mode validation. + +## Call recordings as trace attachments (opt-in) + +Attachments are off by default. Both paths below need the attachment-enabled `fi-instrumentation-otel` and `livekit-agents` 1.8.10 or 1.8.11. Neither starts, stops or enables recording. + +**Local session recording.** If your session already records audio (`record=True`), bind it after `await session.start(...)`: + +```python +binding = traceai.bind_livekit_session(session, ctx, owned_recording_files=True) +``` + +When the session ends, the SDK's `audio.ogg` is uploaded as one `mixed` file. This happens before LiveKit deletes its temporary directory, with a deadline of up to 30 s. Hard links, symlinks, foreign owners and replaced files are refused. LiveKit redaction, `LIVEKIT_TELEMETRY_ALLOW_PII` and FI hide flags all deny. The session span is marked `fi.span.kind=CONVERSATION` only if it has no parent, which depends on where tracing is set up (below). + +**How Future AGI Voice finds a LiveKit call.** The SDK never invents a conversation root: it marks the session span only when that span is already the root of its trace. Which span is the root depends on where tracing is set up (checked with a real livekit-agents 1.8.10 and 1.8.11 worker): + +- **Tracing set up inside the entrypoint** (the Quick Start above). LiveKit starts `job_entrypoint` before tracing is on, so it is not recorded and `agent_session` is the root of the call's trace (it carries `lk.job_id`). An admitted `bind_livekit_session(..., owned_recording_files=True)` marks `agent_session` as `CONVERSATION`, the recording is attached to it, and the call is listed in Voice. Without an admitted binding (attachments off, a denial, or `owned_recording_files=False`) nothing is marked, and the call is not listed. `job_shutdown` (with the session-end upload span) and `room_connect` arrive as separate root traces. +- **Tracing set up before the job starts** (for example in `setup_fnc`, or at worker start-up). LiveKit's `job_entrypoint` span is the trace root and carries `lk.job_id`; `agent_session` and `job_shutdown` are its children, so the binding does not mark `agent_session`. When attachments are enabled on the Future AGI server, ingest treats an untyped, parentless `job_entrypoint` span with a non-empty `lk.job_id` as the call's conversation root, so the call is listed in Voice and its recording is found on the child `agent_session` span. +- Traces ingested before attachments were enabled on the server are not re-classified. +- Do not set `fi.span.kind` on `job_entrypoint` or `agent_session` yourself to force a root; that changes how the whole trace is classified. + +**Completed Egress.** Pass a completed audio-only Egress that your app is authorized to use: + +```python +handle = await traceai.import_livekit_completed_egress( + egress_info, session_context=session_span, egress_id=egress_info.egress_id, + room_name=room, connection_id=CONNECTION_ID, registry=registry, existing_source_authorized=True) +``` + +- `egress_info` is an object you already have. Alternatively, pass `egress_client=lkapi.egress`, which makes one read by that ID. +- `connection_id` and `registry` identify the output that Future AGI has registered for your LiveKit connection. +- Only `EGRESS_COMPLETE` jobs from audio-only room or track composites with exactly one file are accepted. Anything else returns `source_unavailable`. +- No location, filename, URL or credential is ever sent. +- `handle.reason == "queued"` means the server accepted the import. It does not mean the recording is playable yet. diff --git a/python/frameworks/livekit/tests/test_integration.py b/python/frameworks/livekit/tests/test_integration.py index 74e469924..59349226d 100644 --- a/python/frameworks/livekit/tests/test_integration.py +++ b/python/frameworks/livekit/tests/test_integration.py @@ -86,7 +86,10 @@ def test_enabling_http_mapping_keeps_the_exporter_alongside_livekits_privacy_fil assert enable_http_attribute_mapping() is True processors = _processors(provider) - assert len(processors) == 2 + from traceai_livekit._session_context import _SessionContextProcessor + + assert len(processors) == 3 + assert isinstance(processors[2], _SessionContextProcessor) assert isinstance(processors[0], pii._PIIFilteringSpanProcessor) assert isinstance(processors[1], BatchSpanProcessor) assert isinstance(processors[1].span_exporter, MappedHTTPSpanExporter) @@ -406,6 +409,59 @@ def test_the_allow_pii_opt_out_keeps_content_out_of_the_export( assert [k for k in attributes if k.startswith("lk.pii.")] == [] +@pytest.mark.parametrize("allow_pii", [True, False]) +def test_overlapping_native_sessions_reach_collector_with_isolated_grouping( + monkeypatch: pytest.MonkeyPatch, collector: _Collector, make_provider: Any, allow_pii: bool, +) -> None: + from fi_instrumentation import using_session + from livekit.agents.telemetry import tracer + + monkeypatch.setenv("LIVEKIT_TELEMETRY_ALLOW_PII", str(allow_pii).lower()) + provider = make_provider() + enable_http_attribute_mapping() + roots = {} + + async def run(): + entered = [asyncio.Event(), asyncio.Event()] + + async def call(index): + with using_session("native-call-" + str(index)): + async with AgentSession(llm=_FakeLLM()) as session: + await session.start(Agent(instructions="Synthetic session isolation test.")) + roots[session._session_span.get_span_context().trace_id] = "native-call-" + str(index) + entered[index].set() + await entered[1 - index].wait() + await session.run(user_input=USER_TEXT) + + await asyncio.gather(call(0), call(1)) + tracer.start_span("outside-session").end() + + asyncio.run(run()) + provider.force_flush() + assert len(roots) == 2 + names = {trace_id: set() for trace_id in roots} + outside = [] + for body in collector.bodies: + request = ExportTraceServiceRequest.FromString(body) + for resource in request.resource_spans: + for scope in resource.scope_spans: + for span in scope.spans: + attrs = {a.key: _any_value(a.value) for a in span.attributes} + trace_id = int.from_bytes(span.trace_id, "big") + if trace_id in roots: + names[trace_id].add(span.name) + if allow_pii: + assert attrs[SpanAttributes.SESSION_ID] == roots[trace_id] + else: + assert SpanAttributes.SESSION_ID not in attrs + elif span.name == "outside-session": + outside.append(attrs) + for seen in names.values(): + assert {"agent_session", "agent_turn", "llm_node", "session_close"} <= seen + assert len(outside) == 1 + assert SpanAttributes.SESSION_ID not in outside[0] + + # Recording rows use real LiveKit telemetry, but never start an RPC/collector. @pytest.mark.parametrize('allow_pii', [True, False]) def test_recording_metadata_respects_livekit_pii_opt_out_and_cloud_redaction( diff --git a/python/frameworks/livekit/tests/test_session_context.py b/python/frameworks/livekit/tests/test_session_context.py new file mode 100644 index 000000000..0196f8fdb --- /dev/null +++ b/python/frameworks/livekit/tests/test_session_context.py @@ -0,0 +1,235 @@ +"""Call-local native LiveKit grouping, independent of attachment capture.""" + +import asyncio + +import pytest +from fi_instrumentation import using_session +from fi_instrumentation.fi_types import SpanAttributes +from fi_instrumentation.otel import GRPCSpanExporter, HTTPSpanExporter, Transport +from opentelemetry import context, trace +from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_livekit import enable_grpc_attribute_mapping, enable_http_attribute_mapping +from traceai_livekit._session_context import _CONVERSATION, _SESSION, _SessionContextProcessor + +SID = SpanAttributes.SESSION_ID +CID = SpanAttributes.GEN_AI_CONVERSATION_ID + + +@pytest.fixture +def mapped_spans(monkeypatch): + spans = [] + + def export(self, batch): + spans.extend(batch) + return SpanExportResult.SUCCESS + + # Exercise mapping, privacy and batch execution; replace only the HTTP send. + monkeypatch.setattr(HTTPSpanExporter, "export", export) + monkeypatch.setattr(GRPCSpanExporter, "export", export) + return spans + + +@pytest.mark.parametrize("transport,enable", [ + (Transport.HTTP, enable_http_attribute_mapping), + (Transport.GRPC, enable_grpc_attribute_mapping), +]) +def test_capture_at_start_not_setup_or_batch_export(make_provider, mapped_spans, transport, enable): + with using_session("setup-only"): + provider = make_provider(transport) + assert enable() + # Context grouping does not depend on (or turn on) attachment capture. + manager = getattr(provider, "attachment_manager", None) + assert manager is None or manager.status().state == "disabled" + tracer = provider.get_tracer("livekit-agents") + with using_session("call"): + span = tracer.start_span("agent_session") + span.end() + tracer.start_span("outside").end() + with using_session("export-only"): + provider.force_flush() + by_name = {s.name: s.attributes for s in mapped_spans} + assert by_name["agent_session"][SID] == "call" + assert by_name["agent_session"][CID] == "call" + assert SID not in by_name["outside"] + assert CID not in by_name["outside"] + + +def test_installation_is_idempotent_and_native_scope_only(make_provider, mapped_spans): + provider = make_provider() + enable_http_attribute_mapping() + enable_http_attribute_mapping() + assert sum(isinstance(p, _SessionContextProcessor) for p in + provider._active_span_processor._span_processors) == 1 + with using_session("native-only"): + provider.get_tracer("application").start_span("unrelated").end() + provider.force_flush() + assert SID not in mapped_spans[0].attributes + assert _SESSION not in mapped_spans[0].attributes + + +@pytest.mark.parametrize("separate_providers", [False, True]) +def test_overlapping_calls_and_no_context_are_isolated( + make_provider, mapped_spans, separate_providers, +): + first = make_provider() + enable_http_attribute_mapping() + second = make_provider() if separate_providers else first + if separate_providers: + enable_http_attribute_mapping() + tracers = [p.get_tracer("livekit-agents") for p in (first, second)] + + async def run(): + entered = [asyncio.Event(), asyncio.Event()] + + async def call(index): + with using_session("call-" + str(index)): + with tracers[index].start_as_current_span("root-" + str(index)): + entered[index].set() + await entered[1 - index].wait() + await asyncio.sleep(0) + tracers[index].start_span("child-" + str(index)).end() + + async def outside(): + await asyncio.gather(*(event.wait() for event in entered)) + tracers[0].start_span("outside").end() + + await asyncio.gather(call(0), call(1), outside()) + + asyncio.run(run()) + first.force_flush() + second.force_flush() + by_name = {s.name: s for s in mapped_spans} + for index in (0, 1): + root, child = (by_name[prefix + str(index)] for prefix in ("root-", "child-")) + assert root.attributes[SID] == child.attributes[SID] == "call-" + str(index) + assert root.attributes[CID] == child.attributes[CID] == "call-" + str(index) + assert child.parent.span_id == root.context.span_id + assert SID not in by_name["outside"].attributes + assert by_name["root-0"].context.trace_id != by_name["root-1"].context.trace_id + + +def test_explicit_parent_context_and_late_children(make_provider, mapped_spans): + provider = make_provider() + enable_http_attribute_mapping() + tracer = provider.get_tracer("livekit-agents") + with using_session("call"): + root = tracer.start_span("agent_session") + saved = context.get_current() + # LiveKit starts shutdown work outside the entrypoint's Python context. + with trace.use_span(root, end_on_exit=False): + tracer.start_span("session_close").end() + with using_session("unrelated"): + tracer.start_span("explicit-context", context=saved).end() + tracer.start_span("isolated-root", context=context.Context()).end() + # A bare span id carries no session metadata; do not look it up globally. + with trace.use_span(trace.NonRecordingSpan(root.context), end_on_exit=False): + tracer.start_span("id-only-parent").end() + root.end() + provider.force_flush() + by_name = {s.name: s.attributes for s in mapped_spans} + for name in ("agent_session", "session_close", "explicit-context"): + assert by_name[name][SID] == "call" + for name in ("isolated-root", "id-only-parent"): + assert SID not in by_name[name] + + +@pytest.mark.parametrize("attributes,expected_conversation", [ + ({SID: "explicit"}, "explicit"), + ({SID: "explicit", CID: "explicit-conversation"}, "explicit-conversation"), +]) +def test_explicit_ids_win_over_context_and_room( + make_provider, mapped_spans, attributes, expected_conversation, +): + provider = make_provider() + enable_http_attribute_mapping() + original = dict(attributes, **{"lk.pii.room_name": "room"}) + with using_session("ambient"): + provider.get_tracer("livekit-agents").start_span("agent_session", attributes=original).end() + provider.force_flush() + assert mapped_spans[0].attributes[SID] == "explicit" + assert mapped_spans[0].attributes[CID] == expected_conversation + assert original == dict(attributes, **{"lk.pii.room_name": "room"}) + + +@pytest.mark.parametrize("allow_pii", [True, False]) +def test_privacy_filter_and_cloud_gate_do_not_receive_unqualified_copies( + monkeypatch, make_provider, mapped_spans, allow_pii, +): + from livekit.agents.telemetry.traces import _GatedSpanExporter + + monkeypatch.setenv("LIVEKIT_TELEMETRY_ALLOW_PII", str(allow_pii).lower()) + provider = make_provider(batch=False) + enable_http_attribute_mapping() + raw = InMemorySpanExporter() + cloud = InMemorySpanExporter() + gate = _GatedSpanExporter(cloud, export_jobs={}) + gate.open_job("job") + provider.add_span_processor(SimpleSpanProcessor(raw)) + provider.add_span_processor(SimpleSpanProcessor(gate)) + with using_session("private-call"): + provider.get_tracer("livekit-agents").start_span("agent_session", attributes={"job_id": "job"}).end() + gate.job_registered("job", traces_enabled=True, redacted=True) + provider.force_flush() + for span in (*raw.get_finished_spans(), *cloud.get_finished_spans()): + assert SID not in span.attributes + assert CID not in span.attributes + cloud_attributes = cloud.get_finished_spans()[0].attributes + assert _SESSION not in cloud_attributes + assert _CONVERSATION not in cloud_attributes + if allow_pii: + assert mapped_spans[0].attributes[SID] == "private-call" + assert raw.get_finished_spans()[0].attributes[_SESSION] == "private-call" + else: + assert SID not in mapped_spans[0].attributes + assert CID not in mapped_spans[0].attributes + assert _SESSION not in raw.get_finished_spans()[0].attributes + + +def test_project_redaction_wins_over_provider_permission(monkeypatch, make_provider, mapped_spans): + from livekit.agents.types import ATTRIBUTE_REDACTION_ENABLED + + monkeypatch.setenv("LIVEKIT_TELEMETRY_ALLOW_PII", "true") + provider = make_provider() + enable_http_attribute_mapping() + with using_session("project-withheld"): + span = provider.get_tracer("livekit-agents").start_span( + "agent_session", attributes={ATTRIBUTE_REDACTION_ENABLED: True}, + ) + # End off the call context; the span's own project stamp still wins. + span.end() + provider.force_flush() + assert SID not in mapped_spans[0].attributes + assert CID not in mapped_spans[0].attributes + assert not any("pii" in key.split(".") for key in mapped_spans[0].attributes) + + +def test_nested_context_restores_outer_session(make_provider, mapped_spans): + provider = make_provider() + enable_http_attribute_mapping() + tracer = provider.get_tracer("livekit-agents") + with using_session("outer"): + with tracer.start_as_current_span("outer-root"): + with using_session("inner"): + tracer.start_span("inner-child").end() + tracer.start_span("outer-child").end() + tracer.start_span("outside").end() + provider.force_flush() + by_name = {s.name: s.attributes for s in mapped_spans} + assert by_name["outer-root"][SID] == by_name["outer-child"][SID] == "outer" + assert by_name["inner-child"][SID] == "inner" + assert SID not in by_name["outside"] + + +def test_first_privacy_denial_wins_over_later_enable(monkeypatch, make_provider, mapped_spans): + monkeypatch.setenv("LIVEKIT_TELEMETRY_ALLOW_PII", "false") + provider = make_provider() + enable_http_attribute_mapping() + monkeypatch.setenv("LIVEKIT_TELEMETRY_ALLOW_PII", "true") + enable_http_attribute_mapping() + with using_session("withheld"): + provider.get_tracer("livekit-agents").start_span("agent_session").end() + provider.force_flush() + assert SID not in mapped_spans[0].attributes + assert CID not in mapped_spans[0].attributes diff --git a/python/frameworks/livekit/traceai_livekit/__init__.py b/python/frameworks/livekit/traceai_livekit/__init__.py index ce846a40d..c39c0c497 100644 --- a/python/frameworks/livekit/traceai_livekit/__init__.py +++ b/python/frameworks/livekit/traceai_livekit/__init__.py @@ -19,7 +19,16 @@ flush_on_job_shutdown, ) -from .recording import record_egress, wrap_egress_client +from .recording import LiveKitEgressResolver, import_completed_egress, record_egress, wrap_egress_client + + +def bind_session(session, job_context, *, manager, owned_recording_files=False, + finalization_timeout=30, capture_policy=None): + """Opt-in existing recording import; requires the attachment-enabled core.""" + from .attachments import bind_session as bind + return bind(session, job_context, manager=manager, owned_recording_files=owned_recording_files, + finalization_timeout=finalization_timeout, capture_policy=capture_policy) + __version__ = "0.2.1" # x-release-please-version @@ -30,6 +39,9 @@ "create_mapped_http_exporter", "create_mapped_grpc_exporter", "flush_on_job_shutdown", + "bind_session", + "import_completed_egress", + "LiveKitEgressResolver", "record_egress", "wrap_egress_client", # Exporters diff --git a/python/frameworks/livekit/traceai_livekit/_session_context.py b/python/frameworks/livekit/traceai_livekit/_session_context.py new file mode 100644 index 000000000..ea79b0405 --- /dev/null +++ b/python/frameworks/livekit/traceai_livekit/_session_context.py @@ -0,0 +1,54 @@ +"""Bridge call-local FI session context into LiveKit's native SDK spans. + +Keep identifiers under LiveKit's PII namespace until our exporter maps a copy. +There is no provider/process-wide session value or trace-id registry: a late child +can inherit only from its actual in-process parent, not from another call. +""" + +from typing import Optional + +from opentelemetry import context, trace +from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor, TracerProvider + +from fi_instrumentation.fi_types import SpanAttributes + +_SESSION = "lk.pii.traceai.session_id" +_CONVERSATION = "lk.pii.traceai.conversation_id" + + +class _SessionContextProcessor(SpanProcessor): + def on_start(self, span: Span, parent_context: Optional[context.Context] = None) -> None: + scope = span.instrumentation_scope + if scope is None or scope.name != "livekit-agents": + return + + attributes = span.attributes or {} + parent = trace.get_current_span(parent_context) + inherited = getattr(parent, "attributes", None) or {} + # An explicit OTel context is authoritative, including an empty context. + # Read on creation, never on export (which may run in a batch thread). + session = attributes.get(SpanAttributes.SESSION_ID) + if session is None: + session = context.get_value(SpanAttributes.SESSION_ID, parent_context) + if session is None: + session = inherited.get(_SESSION, inherited.get(SpanAttributes.SESSION_ID)) + if session is None: + return + + # Do not copy an explicit unqualified attribute: the application owns it. + # Its presence also prevents an ambient session from supplying an alias. + if SpanAttributes.SESSION_ID not in attributes and _SESSION not in attributes: + span.set_attribute(_SESSION, session) + if SpanAttributes.GEN_AI_CONVERSATION_ID not in attributes and _CONVERSATION not in attributes: + span.set_attribute(_CONVERSATION, session) + + def on_end(self, span: ReadableSpan) -> None: + pass + + +def _install_session_context(provider: TracerProvider) -> None: + """Install once per provider without retaining any call's context.""" + active = getattr(provider, "_active_span_processor", None) + processors = getattr(active, "_span_processors", (active,)) + if not any(isinstance(p, _SessionContextProcessor) for p in processors): + provider.add_span_processor(_SessionContextProcessor()) diff --git a/python/frameworks/livekit/traceai_livekit/attachments.py b/python/frameworks/livekit/traceai_livekit/attachments.py new file mode 100644 index 000000000..e2856c098 --- /dev/null +++ b/python/frameworks/livekit/traceai_livekit/attachments.py @@ -0,0 +1,314 @@ +"""Opt-in import of a bound LiveKit session's existing local recording. + +This adapter never enables recording. It intentionally supports only the exact +Agents versions exercised by the component suite: 1.8.10 and 1.8.11. The private +SDK lifecycle seam is confined here and fails closed on unknown versions. +""" + +import asyncio +import inspect +import math +import os +import stat +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path + +from fi_instrumentation.attachments import ( + CapturePolicy, CorrelationHandle, RecordingFileScope, +) + +_SUPPORTED = frozenset(("1.8.10", "1.8.11")) +_BINDING = "_traceai_recording_binding" + + +def _admission(manager, capture, correlation, *, binding_time=False): + """The core's public preflight decides; this adapter adds no weaker lattice. + + Nothing is retained: the one-byte ticket is released at once. At bind time + a still-running bootstrap is not a denial (finalization re-checks). + """ + if correlation is None: + return "correlation_unavailable" + try: + ticket = manager.preflight(correlation, mime_type="audio/ogg", direction="mixed", max_bytes=1, + operation="import_existing", source_kind="OwnedRecordingFile", + purpose="conversation_recording", capture_policy=capture) + reason = ticket.reason + ticket.release() + except Exception: + return "source_unavailable" + if reason == "queued" or (binding_time and reason == "bootstrap_pending"): + return None + return reason + + +def mark_conversation_root(span) -> bool: + """Mark ``span`` as the conversation root ONLY when it is a true root. + + A true root is a live SDK span with no parent at all (local or remote). + """ + try: + from opentelemetry.sdk.trace import Span as SdkSpan + if not isinstance(span, SdkSpan) or span.parent is not None or not span.is_recording(): + return False + span.set_attribute("fi.span.kind", "CONVERSATION") + span.set_attribute("gen_ai.span.kind", "CONVERSATION") + return True + except Exception: + return False + + +def _native_denial(session, job, processors): + # Native integration restrictions precede even recording-path inspection. + # Text redaction never sanitizes binary recordings. + from livekit.agents.telemetry.utils import allow_pii_from_env + if allow_pii_from_env() is False or any(p._allow_pii is not True for p in processors): + return "policy_denied" + options = session.options.recording_options + if (options.get("redaction", False) is not False + or getattr(job, "_redaction_enabled", True) is not False): + return "redaction_unavailable" + if options.get("audio") is not True: + return "policy_denied" + return None + + +class SessionRecordingBinding: + """A content-free status and awaitable, at-most-once finalizer. + + ``validating`` means the bounded file reached quarantine, NOT playable. + SessionReport files larger than the core's limit remain ``too_large``. + ``close()`` only detaches this adapter; it never stops the application's + session, shuts down the shared manager or enables/deletes a recording. + """ + + def __init__(self, session, job_context, manager, *, enabled, timeout, capture): + self._session = session + self._job = job_context + self._manager = manager + self._capture = capture + self._enabled = enabled + self._timeout = timeout + self._correlation = None + self._privacy_processors = () + self._root = None + self._original = None + self._wrapper = None + self._lock = asyncio.Lock() + self._finished = False + self._closed = False + self._scope = None + self._handle = None + self.root_marked = False + self.status = "session_bound" + + @property + def handle(self): + return self._handle + + async def finalize(self): + """Await delivery before SDK tempdir cleanup, with an explicit deadline.""" + async with self._lock: + if self._finished or self._closed: + return self.status + self._finished = True + if not self._enabled: + self.status = "disabled" + self.close() + return self.status + reason = _admission(self._manager, self._capture, self._correlation) + if reason: + self.status = reason + self.close() + return self.status + try: + # All file work is in this finalization callback, never in an + # inference/audio callback. A bounded source is handed to the + # manager; no caller stream is read or advanced. + await asyncio.wait_for(self._capture_report(), timeout=self._timeout) + except asyncio.TimeoutError: + self.status = "incomplete" + except asyncio.CancelledError: + self.status = "incomplete" + raise + except Exception: + # Do not log repr, report, path, or exception text. + self.status = "source_unavailable" + finally: + if self._handle is not None and self.status != "validating": + self._handle.cancel() + if self._scope is not None: + self._scope.close() + self._scope = None + self.close() + return self.status + + async def _capture_report(self): + from livekit.agents.voice.recorder_io import RecorderIO + + session, job = self._session, self._job + reason = _native_denial(session, job, self._privacy_processors) + if reason: + self.status = reason + return + # The report's path is NOT authority. Only the exact primary session, + # its SDK recorder and fixed SDK-owned filename under the bound job's + # directory are eligible. No user-supplied root, LocalFile or URL. + if job._primary_agent_session is not session: + self.status = "source_unavailable" + return + recorder = session._recorder_io + if not isinstance(recorder, RecorderIO) or recorder.recording: + self.status = "source_unavailable" + return + expected = self._root / "audio.ogg" + if recorder.output_path != expected or job.session_directory != self._root: + self.status = "source_unavailable" + return + report = job.make_session_report(session) + if report.audio_recording_path != expected: + self.status = "source_unavailable" + return + # No root or file stat/open occurs on a policy-denied report. + reason = _admission(self._manager, self._capture, self._correlation) + if reason: + self.status = reason + return + scope = RecordingFileScope(str(self._root), enabled=True) + self._scope = scope + # Deny-only pre-check (hard links, foreign owner, non-regular files). + # The scope's no-follow open and identity comparison stay the authority. + info = os.lstat(self._root / "audio.ogg") + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_uid != os.getuid(): + self.status = "source_unavailable" + return + identity = scope.identify("audio.ogg") + if (identity.device, identity.inode, identity.size) != (info.st_dev, info.st_ino, info.st_size): + self.status = "source_unavailable" + return + source = scope.source("audio.ogg", identity) + # Encoded multi-channel report: tier ``single``, direction ``mixed`` and + # no track or role claim (design section 7), the same as TS #685. + handle = self._manager.attach_owned_recording( + self._correlation, source, mime_type="audio/ogg", direction="mixed", + purpose="conversation_recording", capture_policy=self._capture, + ) + self._handle = handle + self.status = handle.reason + if handle.tier != "denied" and handle.reason == "queued": + # Poll the content-free handle, not a blocking executor waiter. + # Cancelling finalization must not leave asyncio.run() waiting for + # a background Event.wait(timeout) during executor shutdown. + while (result := handle.result()).status == "unfinished": + await asyncio.sleep(0.01) + self.status = result.status + + def close(self): + self._closed = True + if self._handle is not None and self._handle.result().status == "unfinished": + self._handle.cancel() + # Do not overwrite a hook installed by somebody else after this binding. + if self._wrapper is not None and getattr(self._job, "_on_session_end", None) is self._wrapper: + self._job._on_session_end = self._original + if getattr(self._job, _BINDING, None) is self: + delattr(self._job, _BINDING) + self._session = self._job = self._root = None + self._original = self._wrapper = None + self._privacy_processors = () + + +def bind_session(session, job_context, *, manager, owned_recording_files=False, + finalization_timeout=30, capture_policy=None): + """Bind AFTER ``await session.start(...)``; never start a recorder. + + The actual session span is snapshotted now, before LiveKit ends/clears it. + ``job_context._on_session_end`` is chained, not discarded. The SDK awaits + this hook after session close and before its tempdir cleanup. User-provided + AgentServer ``on_session_end`` runs earlier and remains untouched; if it + deletes the source first, this adapter reports ``source_unavailable``. + + Configure the worker's session-end/shutdown grace to exceed this deadline + plus the application's own callbacks. Hard process termination is lossy. + """ + if type(owned_recording_files) is not bool: + raise ValueError("owned recording permission must be explicit") + if (type(finalization_timeout) not in (int, float) + or not math.isfinite(finalization_timeout) or not 0 < finalization_timeout <= 30): + raise ValueError("finalization deadline must be within 30 seconds") + capture = capture_policy if capture_policy is not None else CapturePolicy() + if not isinstance(capture, CapturePolicy) or capture.new_recording: + raise ValueError("session import does not create a recording") + previous = getattr(job_context, _BINDING, None) + if previous is not None: + if (previous._session is session and previous._manager is manager + and previous._enabled == owned_recording_files + and previous._timeout == finalization_timeout and previous._capture == capture): + return previous + raise ValueError("conflicting recording binding") + binding = SessionRecordingBinding(session, job_context, manager, enabled=owned_recording_files, + timeout=finalization_timeout, capture=capture) + if not owned_recording_files: + binding.status = "disabled" + binding.close() + return binding + try: + if version("livekit-agents") not in _SUPPORTED: + binding.status = "unsupported_version" + binding.close() + return binding + from livekit.agents import AgentSession, JobContext + if not isinstance(session, AgentSession) or not isinstance(job_context, JobContext): + binding.status = "unsupported_version" + binding.close() + return binding + # Never fall back to an ambient span from an unrelated late callback. + span = session._session_span + if span is None: + binding.status = "correlation_unavailable" + binding.close() + return binding + binding._correlation = CorrelationHandle.from_span(span) + from livekit.agents.telemetry.pii import _PIIFilteringSpanProcessor + processors = getattr(getattr(span, "_span_processor", None), "_span_processors", ()) + binding._privacy_processors = tuple(p for p in processors if isinstance(p, _PIIFilteringSpanProcessor)) + reason = _admission(manager, capture, binding._correlation, binding_time=True) + if reason: + binding.status = reason + binding.close() + return binding + if job_context._primary_agent_session is not session: + binding.status = "source_unavailable" + binding.close() + return binding + reason = _native_denial(session, job_context, binding._privacy_processors) + if reason: + binding.status = reason + binding.close() + return binding + binding._root = Path(job_context.session_directory) + original = job_context._on_session_end + if not inspect.iscoroutinefunction(original): + binding.status = "unsupported_version" + binding.close() + return binding + + async def chained(): + # Keep original cleanup/report delivery even if instrumentation is + # cancelled. Ordinary adapter failures are already content-free. + try: + await binding.finalize() + finally: + await original() + + binding._original, binding._wrapper = original, chained + job_context._on_session_end = chained + setattr(job_context, _BINDING, binding) + # Only a true parentless session span is marked as the Voice root. + binding.root_marked = mark_conversation_root(span) + except PackageNotFoundError: + binding.status = "unsupported_version" + except Exception: + binding.status = "source_unavailable" + if binding.status != "session_bound": + binding.close() + return binding diff --git a/python/frameworks/livekit/traceai_livekit/exporters/base_exporter.py b/python/frameworks/livekit/traceai_livekit/exporters/base_exporter.py index 5680b62fd..e004ac7f0 100644 --- a/python/frameworks/livekit/traceai_livekit/exporters/base_exporter.py +++ b/python/frameworks/livekit/traceai_livekit/exporters/base_exporter.py @@ -23,6 +23,7 @@ _withhold_recording_failures, ) from .._recording_boundary import _copy_span, _RecordingScope +from .._session_context import _SESSION, _CONVERSATION logger = logging.getLogger(__name__) @@ -289,9 +290,16 @@ def _map_attributes_to_fi_conventions( mapped[SpanAttributes.GEN_AI_TOOL_DEFINITIONS] = _ensure_json_string(attributes.get(LK_FUNCTION_TOOLS)) # 5. Session & User Info + # These sources survive only when LiveKit's privacy filter permits them. + # Map on our export copy, never on the shared span sent to LiveKit Cloud. + if not recording_snapshot: + for source, alias in ((_SESSION, SpanAttributes.SESSION_ID), + (_CONVERSATION, SpanAttributes.GEN_AI_CONVERSATION_ID)): + if source in attributes: + mapped.setdefault(alias, attributes[source]) found, room_name = _first(attributes, LK_ROOM_NAME) if found and not recording_snapshot: - mapped[SpanAttributes.GEN_AI_CONVERSATION_ID] = room_name + mapped.setdefault(SpanAttributes.GEN_AI_CONVERSATION_ID, room_name) # Only retained recording sources supply aliases; there is no room fallback, # including when a sampler or the SDK evicted the operation attribute. diff --git a/python/frameworks/livekit/traceai_livekit/integration.py b/python/frameworks/livekit/traceai_livekit/integration.py index a82a0bbae..02da91d9e 100644 --- a/python/frameworks/livekit/traceai_livekit/integration.py +++ b/python/frameworks/livekit/traceai_livekit/integration.py @@ -358,6 +358,9 @@ def enable_fi_attribute_mapping(transport: Transport = Transport.HTTP) -> bool: # Automatically set the tracer provider for LiveKit agents try: from livekit.agents.telemetry import set_tracer_provider + from ._session_context import _install_session_context + + _install_session_context(provider) set_tracer_provider(provider) logger.info("Automatically configured LiveKit agents tracer provider") except ImportError: diff --git a/python/frameworks/livekit/traceai_livekit/recording.py b/python/frameworks/livekit/traceai_livekit/recording.py index f95c26cdb..28d8d3d0a 100644 --- a/python/frameworks/livekit/traceai_livekit/recording.py +++ b/python/frameworks/livekit/traceai_livekit/recording.py @@ -6,6 +6,8 @@ import inspect import logging +import math +import re from typing import Any, Optional, Tuple from fi_instrumentation import FITracer, TraceConfig @@ -225,3 +227,142 @@ def wrap_egress_client( client, parent=parent, room_name=room_name, conversation_id=conversation_id, tracer=selected, trace_config=config, ) + + +_EGRESS_MIMES = {".ogg": "audio/ogg", ".oga": "audio/ogg", ".opus": "audio/ogg", ".m4a": "audio/mp4", + ".mp4": "audio/mp4", ".wav": "audio/wav", ".webm": "audio/webm"} +_UUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") +_TOKEN = re.compile(r"[a-zA-Z0-9_-]{1,128}") +_AGENTS_SUPPORTED = ("1.8.10", "1.8.11") + + +def _audio_only_request(info) -> bool: + """Audio-only is proven from the Egress REQUEST, never from a filename.""" + kind = info.WhichOneof("request") + if kind == "room_composite": + return bool(info.room_composite.audio_only) + if kind == "track_composite": + request = info.track_composite + return bool(request.audio_track_id) and not request.video_track_id + # Track, participant and web egress cannot prove an audio-only output. + return False + + +class LiveKitEgressResolver: + """Completed LiveKit Egress output -> opaque F01 ``ProviderFile`` facts. + + ``EgressInfo`` comes ONLY from the application: either an object it already + holds (``egress_info``) or ONE fixed-ID read through its existing client + (``egress_client.list_egress(ListEgressRequest(egress_id=...))``). No generic + listing, no polling, no Egress start/stop. ``registry`` maps the completed + output to the server-registered output id (backend B08; CQ-4). No location, + filename, signed URL or credential leaves this object. + """ + + def __init__(self, *, connection_id: str, registry, room_name: str, egress_info=None, + egress_client=None) -> None: + self._connection_id = connection_id + self._registry = registry + self._room_name = room_name + self._info = egress_info + self._client = egress_client + + def __repr__(self) -> str: + return "LiveKitEgressResolver()" + + async def _read(self, egress_id: str): + if self._info is not None: + return self._info + if self._client is None: + return None + from livekit import api + response = await self._client.list_egress(api.ListEgressRequest(egress_id=egress_id)) + items = list(response.items) + return items[0] if len(items) == 1 else None + + async def resolve(self, request, *, deadline: float): + from fi_instrumentation.attachments import ProviderFileSource, ResolvedCompletedRecording + from livekit import api + + try: + if request.provider != "livekit_egress" or not _UUID.fullmatch(self._connection_id or ""): + return "source_unavailable" + info = await self._read(request.provider_ref) + if not isinstance(info, api.EgressInfo): + return "source_unavailable" + if info.egress_id != request.provider_ref or info.room_name != self._room_name: + return "source_unavailable" + # Start/stop snapshots and failed/aborted jobs are never "ready". + if info.status != api.EGRESS_COMPLETE or info.ended_at <= 0: + return "source_unavailable" + if not _audio_only_request(info): + return "source_unavailable" + if (len(info.file_results) != 1 or len(info.stream_results) or len(info.segment_results) + or len(info.image_results)): + return "source_unavailable" + result = info.file_results[0] + name = result.filename.lower() + mime = next((m for ext, m in _EGRESS_MIMES.items() if name.endswith(ext)), None) + if mime is None or result.size <= 0 or result.duration <= 0: + return "source_unavailable" + registered = await self._registry.lookup(connection_id=self._connection_id, + egress_id=request.provider_ref, file_index=0) + if not isinstance(registered, str) or not _UUID.fullmatch(registered): + return "source_unavailable" + return ResolvedCompletedRecording( + source=ProviderFileSource(connection_id=self._connection_id, registered_output_id=registered), + declared_bytes=int(result.size), declared_duration_ms=math.ceil(int(result.duration) / 1e6), + mime_type=mime, direction="mixed", + ) + except Exception: + return "source_unavailable" + + +async def import_completed_egress(egress_info=None, *, session_context, egress_id, room_name, manager, + connection_id=None, registry=None, egress_client=None, + existing_source_authorized=False, capture_policy=None, timeout=30.0): + """Offer an explicitly authorized COMPLETED Egress to the private importer. + + The core's ``import_completed_recording`` preflights first (deny wins), then + this adapter's ``LiveKitEgressResolver`` turns the app-held ``EgressInfo`` + (or one fixed-ID read through ``egress_client``) into opaque ProviderFile + facts, and the core submits an F01 ``completed_recording`` import with the + original credentials. ``ImportHandle.reason == "queued"`` means the server + accepted the job (``importing``), never that media is playable. + + Without ``connection_id`` + ``registry`` (CQ-4) the result is + ``source_unavailable``. This never starts/stops Egress, polls, or sends a + location, filename, URL or credential. Metadata-only + ``record_egress()``/``wrap_egress_client()`` keep their behavior. + """ + # Lazy imports keep metadata-only installs compatible with released cores. + from fi_instrumentation.attachments import ( + CapturePolicy, CompletedRecordingRequest, CorrelationHandle, ImportHandle, UnavailableResolver, + ) + if existing_source_authorized is not True: + return ImportHandle("disabled") + try: + capture = capture_policy if capture_policy is not None else CapturePolicy() + if not isinstance(capture, CapturePolicy) or capture.new_recording: + return ImportHandle("policy_denied") + if session_context is None: + return ImportHandle("correlation_unavailable") + from importlib.metadata import version + if version("livekit-agents") not in _AGENTS_SUPPORTED: + return ImportHandle("unsupported_version") + if (not isinstance(egress_id, str) or not _TOKEN.fullmatch(egress_id) + or type(room_name) is not str or not 0 < len(room_name) <= 256): + return ImportHandle("source_unavailable") + target = session_context + if not isinstance(target, CorrelationHandle) and not hasattr(target, "is_recording"): + target = CorrelationHandle.from_span(target) + if connection_id is not None and registry is not None: + resolver = LiveKitEgressResolver(connection_id=connection_id, registry=registry, room_name=room_name, + egress_info=egress_info, egress_client=egress_client) + else: + resolver = UnavailableResolver() + return await manager.import_completed_recording( + target, CompletedRecordingRequest("livekit_egress", egress_id, "mixed"), + resolver=resolver, capture_policy=capture, timeout=timeout) + except Exception: + return ImportHandle("source_unavailable") diff --git a/python/frameworks/pipecat/README.md b/python/frameworks/pipecat/README.md index 607d10436..4d758e49a 100644 --- a/python/frameworks/pipecat/README.md +++ b/python/frameworks/pipecat/README.md @@ -183,3 +183,54 @@ Base class for mapped span exporters. 3. **Data not being sent to FutureAGI** - Ensure that your you have set the `FI_API_KEY` and `FI_SECRET_KEY` environment variables - Ensure that the `set_global_tracer_provider` in the `register` function is set to `True` + +## Call recordings as trace attachments (opt-in) + +Attachments are off by default. When they are enabled in `traceai.init(...)`, the adapter can upload the audio your own `AudioBufferProcessor` already produces. It does not start, stop or reconfigure the recorder. Requires `pipecat-ai==1.15.0` and the attachment-enabled `fi-instrumentation-otel`. + +```python +import traceai +from fi_instrumentation.attachments import AttachmentConfig +from pipecat.pipeline.worker import PipelineWorker +from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor +from traceai_pipecat.attachments import AudioSessionContext + +traceai.init(project_name="voice-agent", attachment_config=AttachmentConfig(enabled=True)) + +recorder = AudioBufferProcessor(num_channels=2, buffer_size=32 * 1024, auto_start_recording=True) +worker = PipelineWorker(pipeline, enable_tracing=True, enable_turn_tracking=True) +binding = traceai.bind_pipecat_audio_processor(recorder, AudioSessionContext.from_pipeline(worker)) +# ... run the worker ... +await binding.finish() # waits up to 30 s for the final segment and manifest +``` + +- The recording is attached to the `conversation` span that Pipecat's tracing starts. That span is marked as the conversation root (`fi.span.kind=CONVERSATION`) only when it has no parent. +- Short calls are kept. Pipecat delivers a call shorter than `buffer_size` in one chunk when recording stops, and it can run after the worker has ended the conversation span. The adapter remembers the conversation's identity while the span is live (at bind, when recording starts, and on every chunk), so that chunk still attaches to the same conversation. The root marker is set while the span is live. +- Layouts: mono gives one mixed track. Stereo gives user (left, input) and bot (right, output) tracks in one file. `layout="separate_tracks"` uploads each side separately, so hiding inputs drops only the user side. With mono or stereo, any hide flag blocks the whole recording. +- Audio is uploaded as WAV segments of up to 30 s, plus a manifest that records gaps. If uploads fall behind, the overflow becomes a `queue_full` gap rather than growing memory. +- `buffer_size` must be between 1 and 131072 bytes. `buffer_size=0` is refused (`unbounded_source`), and so is `enable_turn_audio=True` unless you opt in to per-turn audio (below). +- `binding.status` reports a short, content-free state such as `recording`, `published`, `partial`, `policy_denied` or `correlation_unavailable`. +- `traceai_pipecat.create_audio_capture(...)` builds a new, unstarted recorder only when both `AttachmentConfig(record_audio=True)` and the server's `record_audio` allow it. + +### Per-turn audio (separate opt-in) + +Per-turn uploads are off unless you pass explicit caps for the whole call. There are no defaults: + +```python +from traceai_pipecat import TurnAudioCaps + +recorder = AudioBufferProcessor(num_channels=1, buffer_size=32 * 1024, auto_start_recording=True, + enable_turn_audio=True) +binding = traceai.bind_pipecat_audio_processor( + recorder, AudioSessionContext.from_pipeline(worker), + turn_audio=TurnAudioCaps(max_turns=50, max_bytes=16 * 1024 * 1024, max_duration_ms=20 * 60 * 1000)) +``` + +- Each finished turn uploads once per side: `on_user_turn_audio` as `input` and `on_bot_turn_audio` as `output`, as a mono WAV with purpose `turn_recording`, `turn_id` = Pipecat's turn number and the call's `conversation_id`. +- Each upload is bound to that turn's own span, from Pipecat's `TurnTraceObserver.get_turn_context(turn_number)`. It never falls back to the conversation span or an ambient span. With `AudioSessionContext.from_span(...)` there are no turn spans, so turns end `correlation_unavailable`. +- A turn upload never writes `fi.attachment.refs` (or any `fi.attachment.*` attribute) on the turn span, even while that span is still open. Future AGI binds it through the upload's `association.parent`, which names the turn span. +- Caps apply to the whole call: `max_turns` uploads (at most 1000), `max_bytes` of WAV (at most 64 MiB) and `max_duration_ms` of audio (at most 2 h). A turn that would cross a cap is skipped whole, never truncated: `queue_full` for the count, `too_large` for bytes or duration. Caps are checked on the length before any copy. +- Privacy is checked per turn, per side. `FI_HIDE_INPUTS` drops the user's turns and keeps the bot's. Unsampled turn spans upload nothing. +- `enable_turn_audio=True` makes Pipecat keep each turn's audio in its own memory until the turn ends. The caps bound what the adapter copies and uploads, not Pipecat's own buffer. +- `binding.turn_status` is the last turn's admission (`queued` or a reason), `binding.turn_denials` counts skipped turns by reason, and `await binding.finish()` also waits for turn uploads within the same deadline. +- `create_audio_capture(..., turn_audio=TurnAudioCaps(...))` checks the per-turn policy before it builds the recorder. It builds it with `enable_turn_audio=True` only when at least one side may upload turns. Otherwise Pipecat keeps no turn audio, no turn handler is registered and `binding.turn_status` is the reason, such as `policy_denied`. After the recorder is built, each turn is still checked as a new recording, so turning the server's `record_audio` off denies later turns (`recording_disabled`). diff --git a/python/frameworks/pipecat/traceai_pipecat/__init__.py b/python/frameworks/pipecat/traceai_pipecat/__init__.py index ec1cc7526..d34875c38 100644 --- a/python/frameworks/pipecat/traceai_pipecat/__init__.py +++ b/python/frameworks/pipecat/traceai_pipecat/__init__.py @@ -18,9 +18,35 @@ enable_http_attribute_mapping, ) +def bind_audio_processor(processor, session_context, *, manager, **options): + """Observe an existing AudioBufferProcessor into a bounded core recording. + + See ``traceai_pipecat.attachments.bind_audio_processor``. Requires the + attachment-enabled core; never starts the recorder. + """ + from .attachments import bind_audio_processor as bind + return bind(processor, session_context, manager=manager, **options) + + +def create_audio_capture(session_context, *, manager, **options): + """Create an un-started recorder only with local and server record_audio.""" + from .attachments import create_audio_capture as create + return create(session_context, manager=manager, **options) + + +def __getattr__(name): + # Lazy, so importing the package never needs the attachment-enabled core. + if name in ("TurnAudioCaps", "AudioSessionContext"): + from . import attachments + return getattr(attachments, name) + raise AttributeError("module %r has no attribute %r" % (__name__, name)) + + __version__ = "0.1.2" # x-release-please-version __all__ = [ + "bind_audio_processor", + "create_audio_capture", "enable_fi_attribute_mapping", "enable_http_attribute_mapping", "enable_grpc_attribute_mapping", diff --git a/python/frameworks/pipecat/traceai_pipecat/attachments.py b/python/frameworks/pipecat/traceai_pipecat/attachments.py new file mode 100644 index 000000000..5fae837f9 --- /dev/null +++ b/python/frameworks/pipecat/traceai_pipecat/attachments.py @@ -0,0 +1,664 @@ +"""Bounded Pipecat AudioBufferProcessor capture bound to the real conversation. + +The adapter only OBSERVES audio the application's own ``AudioBufferProcessor`` +already emits (``on_audio_data`` or ``on_track_audio_data``). It never starts, +stops or reconfigures a recorder, never reads the processor's internal buffers +and never falls back to an ambient span. Every PCM chunk goes straight into a +core ``RecordingWriter`` (``start_recording``): segments, gaps and a recording +manifest on a sample-count clock, with the core's own memory bounds. + +Recording is OFF unless attachments are enabled. Creating a NEW recorder +(``create_audio_capture``) additionally needs the local and server +``record_audio`` permissions (``operation="create_recording"``); observing an +existing app recorder is ``operation="observe_consumed"``. + +Upstream bound: ``buffer_size`` must be within 1..128 KiB, so the processor +flushes bounded chunks on its own schedule. ``buffer_size=0`` keeps the whole +call in Pipecat memory until stop and is refused (``unbounded_source``). +``enable_turn_audio`` is refused the same way unless the caller opts in to +per-turn uploads with explicit ``TurnAudioCaps`` (count, bytes and duration per +call); each turn then uploads once, bound to Pipecat's own turn span. + +Short calls: the conversation identity is snapshotted while Pipecat's +conversation span is live (at bind, on ``on_recording_started`` and on every +chunk). Pipecat runs these handlers as tasks, so a call shorter than +``buffer_size`` delivers its only chunk on stop, possibly after the worker has +ended the conversation span; the recording then binds to the snapshot instead +of coming out ``unsampled`` or ``correlation_unavailable``. +""" + +import asyncio +import math +import re +import struct +import threading +import time +import weakref +from dataclasses import dataclass +from importlib.metadata import PackageNotFoundError, version +from typing import Optional + +from fi_instrumentation.attachments import ( + Bytes, CapturePolicy, CorrelationHandle, PcmFormat, RecordingOptions, TrackSpec, +) + +# Exact versions exercised by python/voice-tests on a real pipeline. 1.15.0 is +# the first qualified release with ``on_recording_stopped`` and the +# PipelineWorker ``turn_trace_observer`` surface used here. +_SUPPORTED = frozenset(("1.15.0",)) +_MAX_BUFFER = 128 * 1024 +_TOKEN = re.compile(r"[a-zA-Z0-9_-]{1,128}") +_SENTINEL = CorrelationHandle("0" * 31 + "1", "0" * 15 + "1", True) +_BINDINGS: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary() +_BINDINGS_LOCK = threading.Lock() + +CONVERSATION_KIND = "CONVERSATION" + +# Hard ceilings for TurnAudioCaps; a caller can only choose lower caps. +_TURN_CEILINGS = {"max_turns": 1000, "max_bytes": 64 * 1024 * 1024, "max_duration_ms": 2 * 60 * 60 * 1000} +_WAV_HEADER = 44 +_TURN_SIDES = {"on_user_turn_audio": ("input", "user"), "on_bot_turn_audio": ("output", "assistant")} + + +@dataclass(frozen=True) +class TurnAudioCaps: + """Explicit per-call caps for per-turn uploads. There are no defaults. + + ``max_turns`` caps how many turn uploads one call (one binding) admits, + ``max_bytes`` the summed WAV bytes and ``max_duration_ms`` the summed audio + duration. A turn that would cross a cap is skipped whole (never truncated): + ``queue_full`` for the count, ``too_large`` for bytes or duration. + """ + + max_turns: int + max_bytes: int + max_duration_ms: int + + def __post_init__(self): + for name, ceiling in _TURN_CEILINGS.items(): + value = getattr(self, name) + if type(value) is not int or not 0 < value <= ceiling: + raise ValueError("%s must be an int within 1..%d" % (name, ceiling)) + + +def _wav_header(data_bytes: int, sample_rate: int) -> bytes: + # Mono 16-bit PCM, the format Pipecat's TurnAudioData carries. + return b"".join((b"RIFF", struct.pack(" "AudioSessionContext": + correlation = CorrelationHandle.from_span(span) + live = span if hasattr(span, "is_recording") and hasattr(span, "set_attribute") else None + return cls(correlation=correlation, span=live, conversation_id=conversation_id) + + @classmethod + def from_pipeline(cls, worker) -> "AudioSessionContext": + if worker is None: + raise ValueError("pipeline worker required") + return cls(worker=worker) + + +def mark_conversation_root(span) -> bool: + """Mark ``span`` as the conversation root ONLY when it is a true root. + + A true root is a live SDK span with no parent at all (local or remote). + A ``SpanContext``/``NonRecordingSpan`` cannot prove root-ness: no marker. + """ + try: + from opentelemetry.sdk.trace import Span as SdkSpan + if not isinstance(span, SdkSpan) or span.parent is not None or not span.is_recording(): + return False + span.set_attribute("fi.span.kind", CONVERSATION_KIND) + span.set_attribute("gen_ai.span.kind", CONVERSATION_KIND) + return True + except Exception: + return False + + +def _span_id(span) -> Optional[str]: + try: + return format(span.get_span_context().span_id, "016x") + except Exception: + return None + + +def _token(value) -> Optional[str]: + # Opaque ids that are not F01 tokens are dropped, never rewritten. + return value if isinstance(value, str) and _TOKEN.fullmatch(value) else None + + +def _probe(manager, operation, directions, capture, purpose="conversation_recording") -> Optional[str]: + """Deny-wins policy check through the public preflight; retains nothing. + + The sentinel identity only lets preflight run past correlation; the ticket + is released at once. Returns None when at least one direction is admitted. + """ + reasons = [] + for direction in directions: + try: + ticket = manager.preflight(_SENTINEL, mime_type="audio/wav", direction=direction, max_bytes=1, + operation=operation, source_kind="Bytes", purpose=purpose, + capture_policy=capture) + except Exception: + return "source_unavailable" + reason = ticket.reason + ticket.release() + # Bootstrap may still be running at bind time; the core re-admits (and + # records ``bootstrap_pending`` gaps) once the session exists. + if reason in ("queued", "bootstrap_pending"): + return None + reasons.append(reason) + return reasons[0] if reasons else "policy_denied" + + +def _layout(processor, layout): + channels = processor.num_channels + if layout == "separate_tracks": + return "separate_tracks", (TrackSpec((0,), "input", "user"), TrackSpec((1,), "output", "assistant")) + if layout != "auto": + return None, None + if channels == 1: + return "mixed", (TrackSpec((0,), "mixed", "mixed"),) + if channels == 2: + # Pipecat's stereo output puts the user on the left, the bot on the right. + return "stereo", (TrackSpec((0,), "input", "user"), TrackSpec((1,), "output", "assistant")) + return None, None + + +class AudioCaptureBinding: + """Content-free status of one processor binding. + + ``status``: ``session_bound`` (waiting for audio), ``recording``, a denial + reason, or the last finished ``RecordingResult.state``. ``results`` holds + each finished recording cycle (one per start/stop of the app's recorder). + ``close()`` detaches; it never stops the application's recorder. + + Per-turn uploads (only with ``turn_audio=TurnAudioCaps(...)``) report + separately: ``turn_status`` is ``off``, ``session_bound``, the last turn's + admission (``queued`` or a denial reason); ``turn_handles`` holds + ``(turn_number, direction, AttachmentHandle)`` for each admitted turn (at + most ``max_turns``) and ``turn_denials`` counts skipped turns by reason. + """ + + def __init__(self, status: str, processor=None, *, manager=None, context=None, layout=None, tracks=None, + operation="observe_consumed", capture=None, timeout=30.0, turn_audio=None): + self.status = status + self.processor = processor + self.results = [] + self.root_marked = False + self.turn_status = "off" + self.turn_handles = [] + self.turn_denials = {} + self._manager = manager + self._context = context + self._layout = layout + self._tracks = tracks + self._operation = operation + self._capture = capture + self._timeout = timeout + self._turn_audio = turn_audio + self._turn_used = [0, 0, 0] # uploads, WAV bytes, duration ms admitted so far in this call + self._writer = None + self._format = None + self._cycle_denied = False + self._snapshot = None # (CorrelationHandle, span_id, conversation_id) of the live conversation + self._closed = status != "session_bound" + self._handlers = [] + self._pending = set() + + # ---------------------------------------------------------- handlers + def _on_audio_data(self, _processor, audio, sample_rate, num_channels): + try: + self._write(((audio, 0),), sample_rate, num_channels) + except Exception: + self._fail() + + def _on_track_audio_data(self, _processor, user_audio, bot_audio, sample_rate, _num_channels): + try: + # Each Pipecat track buffer is mono PCM at the processor rate. + self._write(((user_audio, 0), (bot_audio, 1)), sample_rate, 1) + except Exception: + self._fail() + + def _on_recording_started(self, _processor): + # The conversation span exists by now (the observer starts it on the + # StartFrame push); remember its identity while it is still live. + if not self._closed: + self._remember() + + def _on_recording_stopped(self, _processor): + try: + self._finish_cycle() + except Exception: + self._fail() + + def _on_user_turn_audio(self, _processor, data): + self._turn("on_user_turn_audio", data) + + def _on_bot_turn_audio(self, _processor, data): + self._turn("on_bot_turn_audio", data) + + # ---------------------------------------------------------- internals + def _fail(self) -> None: + # Never log the exception: it can carry caller objects. Drop retention. + writer, self._writer = self._writer, None + if writer is not None: + writer.abort() + self.status = "source_unavailable" + + def _observer(self): + return getattr(self._context.worker, "turn_trace_observer", None) + + def _remember(self) -> None: + """Snapshot the observer's conversation identity while its span is live.""" + try: + if self._context.correlation is not None: + return + observer = self._observer() + # Version-qualified seam (1.15.0): the observer's own conversation span. + span = getattr(observer, "_conversation_span", None) if observer is not None else None + if span is None or not span.is_recording(): + return + if self._snapshot is not None and self._snapshot[1] == _span_id(span): + return + correlation = CorrelationHandle.from_span(span) + self._snapshot = (correlation, correlation.span_id, _token(getattr(observer, "_conversation_id", None))) + # Admitted binding + live true root: mark it now, because a short + # call may only deliver audio after the span has ended. + if correlation.sampled and mark_conversation_root(span): + self.root_marked = True + except Exception: + return + + def _resolve(self): + context = self._context + if context.correlation is not None: + return None, context.correlation, context.span, _token(context.conversation_id) + observer = self._observer() + if observer is None: + return "correlation_unavailable", None, None, None + span = getattr(observer, "_conversation_span", None) + snapshot = self._snapshot + if span is not None and span.is_recording(): + self._remember() + snapshot = self._snapshot + if snapshot is not None: + return None, snapshot[0], span, snapshot[2] + if snapshot is not None and (span is None or _span_id(span) == snapshot[1]): + # The worker ended (and cleared) the conversation span before this + # chunk ran: bind to the identity captured while it was live. + return None, snapshot[0], None, snapshot[2] + if span is None: + return "correlation_unavailable", None, None, None + try: + correlation = CorrelationHandle.from_span(span) + except Exception: + return "correlation_unavailable", None, None, None + return None, correlation, span, _token(getattr(observer, "_conversation_id", None)) + + def _open(self, sample_rate, channels): + if self._cycle_denied: + return None + reason, correlation, span, conversation_id = self._resolve() + if reason is None and not correlation.sampled: + reason = "unsampled" + live = None + if reason is None: + options = RecordingOptions(format=PcmFormat(int(sample_rate)), layout=self._layout, tracks=self._tracks, + operation=self._operation, purpose="conversation_recording", + capture_policy=self._capture, conversation_id=conversation_id) + live = span if span is not None and span.is_recording() else None + writer = self._manager.start_recording(live if live is not None else correlation, options) + reason = writer.reason if writer.reason != "queued" else None + if reason is not None: + self._cycle_denied = True + self.status = reason + return None + if live is not None and mark_conversation_root(live): + self.root_marked = True + self._writer, self._format = writer, (int(sample_rate), channels) + self.status = "recording" + return writer + + def _turn(self, event, data) -> None: + """Upload one finished turn for one side, bound to Pipecat's turn span.""" + try: + if self._closed or self._turn_audio is None: + return + reason = self._turn_upload(event, data) + except Exception: + reason = "source_unavailable" + if reason != "queued": + self.turn_denials[reason] = self.turn_denials.get(reason, 0) + 1 + self.turn_status = reason + + def _turn_upload(self, event, data) -> str: + direction, role = _TURN_SIDES[event] + caps, used = self._turn_audio, self._turn_used + turn_number, rate = getattr(data, "turn_number", None), getattr(data, "sample_rate", None) + audio, channels = getattr(data, "audio", None), getattr(data, "num_channels", None) + if (type(turn_number) is not int or turn_number < 1 or type(rate) is not int or rate <= 0 + or channels != 1 or type(audio) is not bytes or not audio or len(audio) % 2): + return "source_unavailable" + # Caps first, on the length alone: an over-cap turn is never copied. + size = _WAV_HEADER + len(audio) + duration = math.ceil(len(audio) * 1000 / (2 * rate)) + if used[0] >= caps.max_turns: + return "queue_full" + if used[1] + size > caps.max_bytes or used[2] + duration > caps.max_duration_ms: + return "too_large" + # Pipecat's own turn span (public API); never the conversation or an ambient span. + observer = self._observer() + span_context = observer.get_turn_context(turn_number) if observer is not None else None + if span_context is None: + return "correlation_unavailable" + correlation = CorrelationHandle.from_span(span_context) + ticket = self._manager.preflight(correlation, mime_type="audio/wav", direction=direction, max_bytes=size, + operation=self._operation, source_kind="Bytes", purpose="turn_recording", + capture_policy=self._capture) + if not ticket.admitted: + ticket.release() + return ticket.reason + used[0], used[1], used[2] = used[0] + 1, used[1] + size, used[2] + duration + conversation_id = self._snapshot[2] if self._snapshot is not None else _token(self._context.conversation_id) + payload = _wav_header(len(audio), rate) + audio + handle = ticket.attach(Bytes(payload), role=role, conversation_id=conversation_id, turn_id=str(turn_number)) + self.turn_handles.append((turn_number, direction, handle)) + return handle.admission + + def _write(self, chunks, sample_rate, channels): + if self._closed: + return + expected = 2 if self._layout == "stereo" else 1 + writer = self._writer + if writer is None: + if type(sample_rate) is not int or channels != expected: + return + writer = self._open(sample_rate, channels) + if writer is None: + return + if (sample_rate, channels) != self._format: + # Format drift: keep the shared clock with an explicit gap. + size = max(len(chunk) for chunk, _ in chunks) + if type(sample_rate) is int and sample_rate > 0 and channels in (1, 2): + duration = math.ceil(size * 1000 / (2 * channels * sample_rate)) + if duration > 0: + writer.mark_gap(duration, reason="source_unavailable") + return + for chunk, track in chunks: + writer.try_write(chunk, track=track if self._layout == "separate_tracks" else 0) + + def _finish_cycle(self): + writer, self._writer = self._writer, None + self._format, self._cycle_denied = None, False + if writer is None: + return + try: + loop = asyncio.get_running_loop() + except RuntimeError: + self._record(writer.finish(self._timeout)) + return + task = loop.create_task(writer.finish_async(self._timeout)) + self._pending.add(task) + task.add_done_callback(self._finished) + + def _finished(self, task): + self._pending.discard(task) + if task.cancelled(): + self.status = "incomplete" + return + error = task.exception() + if error is not None: + self.status = "source_unavailable" + return + self._record(task.result()) + + def _record(self, result): + self.results.append(result) + self.status = result.state + + # ------------------------------------------------------------- public + async def finish(self, timeout: Optional[float] = None): + """Finish the open recording cycle and await uploads, within a deadline. + + Call before process exit (after the pipeline ends). Pipecat already + finishes a cycle on ``on_recording_stopped``; this only awaits it. + Returns the last ``RecordingResult`` or None. A hard kill is lossy. + Per-turn uploads are awaited within the same deadline (``turn_handles``). + """ + budget = self._timeout if timeout is None else min(max(float(timeout), 0.0), self._timeout) + deadline = time.monotonic() + budget + self._finish_cycle() + pending = list(self._pending) + if pending: + await asyncio.wait(pending, timeout=budget + 1) + # Turn handles complete on the core's workers; poll without a thread each. + while time.monotonic() < deadline and any( + handle.result(0).status == "unfinished" for _, _, handle in self.turn_handles): + await asyncio.sleep(0.02) + return self.results[-1] if self.results else None + + def close(self) -> None: + """Detach handlers and drop any unfinished cycle (no I/O).""" + self._closed = True + processor = self.processor + for event, handler in self._handlers: + try: + processor.remove_event_handler(event, handler) + except Exception: + pass + self._handlers = [] + writer, self._writer = self._writer, None + if writer is not None: + writer.abort() + self.status = "incomplete" + if processor is not None: + with _BINDINGS_LOCK: + if _BINDINGS.get(processor) is self: + del _BINDINGS[processor] + + def __repr__(self) -> str: + return "AudioCaptureBinding(status=%r)" % self.status + + +def _validate_options(layout, finalization_timeout, capture_policy): + if layout not in ("auto", "separate_tracks"): + raise ValueError("layout must be 'auto' or 'separate_tracks'") + if (type(finalization_timeout) not in (int, float) or not math.isfinite(finalization_timeout) + or not 0 < finalization_timeout <= 30): + raise ValueError("finalization deadline must be within 30 seconds") + capture = capture_policy if capture_policy is not None else CapturePolicy() + if not isinstance(capture, CapturePolicy): + raise ValueError("capture policy must be a CapturePolicy") + return capture + + +def _qualified_processor(processor, turn_audio): + if version("pipecat-ai") not in _SUPPORTED: + return "unsupported_version" + from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor + if not isinstance(processor, AudioBufferProcessor): + return "unsupported_version" + size = processor._buffer_size + if type(size) is not int or not 0 < size <= _MAX_BUFFER: + return "unbounded_source" + if bool(processor._enable_turn_audio) != (turn_audio is not None): + if turn_audio is None: + # Pipecat keeps each turn in memory for its turn events; without + # explicit caps that is an unbounded source we refuse to observe. + return "unbounded_source" + raise ValueError("turn_audio needs AudioBufferProcessor(enable_turn_audio=True)") + return None + + +def _validate_turn_audio(turn_audio): + if turn_audio is not None and not isinstance(turn_audio, TurnAudioCaps): + raise ValueError("turn_audio must be TurnAudioCaps(max_turns=, max_bytes=, max_duration_ms=)") + return turn_audio + + +def _bind(processor, session_context, manager, *, layout, timeout, capture, operation, turn_audio=None): + if not isinstance(session_context, AudioSessionContext): + return AudioCaptureBinding("correlation_unavailable") + try: + reason = _qualified_processor(processor, turn_audio) + if reason: + return AudioCaptureBinding(reason) + selected, tracks = _layout(processor, layout) + if selected is None: + return AudioCaptureBinding("unsupported_version") + directions = ("input", "output") if selected == "separate_tracks" else ("mixed",) + reason = _probe(manager, operation, directions, capture) + if reason: + return AudioCaptureBinding(reason) + # Per-turn uploads: admitted only if at least one side may upload a + # turn_recording; otherwise no turn handler is registered at all. + turn_reason = None + if turn_audio is not None: + turn_reason = _probe(manager, operation, ("input", "output"), capture, purpose="turn_recording") + with _BINDINGS_LOCK: + previous = _BINDINGS.get(processor) + if previous is not None and not previous._closed: + if (previous._manager is manager and previous._context == session_context + and previous._layout == selected and previous._operation == operation + and previous._capture == capture and previous._timeout == timeout + and previous._turn_audio == turn_audio): + return previous + raise ValueError("conflicting recording binding") + binding = AudioCaptureBinding("session_bound", processor, manager=manager, context=session_context, + layout=selected, tracks=tracks, operation=operation, capture=capture, + timeout=timeout, turn_audio=turn_audio) + _BINDINGS[processor] = binding + if selected == "separate_tracks": + handlers = (("on_track_audio_data", binding._on_track_audio_data),) + else: + handlers = (("on_audio_data", binding._on_audio_data),) + handlers += (("on_recording_started", binding._on_recording_started), + ("on_recording_stopped", binding._on_recording_stopped)) + if turn_audio is not None: + binding.turn_status = turn_reason or "session_bound" + if turn_reason is None: + handlers += (("on_user_turn_audio", binding._on_user_turn_audio), + ("on_bot_turn_audio", binding._on_bot_turn_audio)) + for event, handler in handlers: + processor.add_event_handler(event, handler) + binding._handlers.append((event, handler)) + binding._remember() + return binding + except ValueError: + raise + except PackageNotFoundError: + return AudioCaptureBinding("unsupported_version") + except Exception: + return AudioCaptureBinding("source_unavailable") + + +def bind_audio_processor(processor, session_context, *, manager, layout="auto", finalization_timeout=30, + capture_policy=None, turn_audio=None): + """Observe the app's EXISTING AudioBufferProcessor; never start or modify it. + + Bind before the pipeline runs. ``layout="auto"`` follows the processor: + mono -> ``mixed``; stereo -> ``stereo`` (user left/input, bot right/output). + ``layout="separate_tracks"`` observes ``on_track_audio_data`` and keeps one + track per side, so a hide flag on one side drops only that side. + + ``turn_audio=TurnAudioCaps(max_turns=, max_bytes=, max_duration_ms=)`` + additionally uploads each finished turn (``on_user_turn_audio`` as input, + ``on_bot_turn_audio`` as output, purpose ``turn_recording``) bound to that + turn's own Pipecat span via ``get_turn_context(turn_number)``. It is off by + default and needs ``AudioBufferProcessor(enable_turn_audio=True)`` plus a + worker with tracing and turn tracking; an explicit-span context has no turn + spans, so its turns end ``correlation_unavailable``. + + Policy is checked before the processor is inspected, and again by the core + on every chunk. Disabled attachments return ``disabled`` and register + nothing. Await ``binding.finish()`` after the pipeline ends. + """ + capture = _validate_options(layout, finalization_timeout, capture_policy) + turn_audio = _validate_turn_audio(turn_audio) + if capture.new_recording: + raise ValueError("observing an existing recorder does not create a recording") + try: + state = manager.status().state + except Exception: + return AudioCaptureBinding("source_unavailable") + if state == "disabled": + return AudioCaptureBinding("disabled") + reason = _probe(manager, "observe_consumed", ("input", "output", "mixed"), capture) + if reason: + return AudioCaptureBinding(reason) + return _bind(processor, session_context, manager, layout=layout, timeout=finalization_timeout, + capture=capture, operation="observe_consumed", turn_audio=turn_audio) + + +def create_audio_capture(session_context, *, manager, sample_rate=None, num_channels=1, buffer_size=32 * 1024, + layout="auto", finalization_timeout=30, capture_policy=None, turn_audio=None): + """Create a NEW, un-started AudioBufferProcessor only with both permissions. + + Needs local ``AttachmentConfig(record_audio=True)`` AND server + ``record_audio``. The returned ``binding.processor`` must be inserted into + the pipeline by the application, which also starts it (for example with + ``AudioBufferStartRecordingFrame``). Nothing is created when denied. + ``turn_audio`` (``TurnAudioCaps``) uploads turns as in + ``bind_audio_processor``. The ``turn_recording`` policy is checked BEFORE + the processor is built: only when at least one side is admitted is it + created with ``enable_turn_audio=True``. Otherwise Pipecat never buffers + turns, no turn handler is registered and ``turn_status`` is the reason. + """ + capture = _validate_options(layout, finalization_timeout, capture_policy) + turn_audio = _validate_turn_audio(turn_audio) + if (type(buffer_size) is not int or not 0 < buffer_size <= _MAX_BUFFER + or num_channels not in (1, 2)): + raise ValueError("buffer_size must be within 1..131072 and num_channels 1 or 2") + try: + state = manager.status().state + except Exception: + return AudioCaptureBinding("source_unavailable") + if state == "disabled": + return AudioCaptureBinding("disabled") + directions = ("input", "output") if layout == "separate_tracks" else ("mixed",) + reason = _probe(manager, "create_recording", directions, capture) + if reason: + return AudioCaptureBinding(reason) + if not isinstance(session_context, AudioSessionContext): + return AudioCaptureBinding("correlation_unavailable") + # Turn policy before construction: a recorder built with enable_turn_audio + # keeps every turn in Pipecat memory, so build it that way only when at + # least one side may upload a turn_recording. + turn_reason = None + if turn_audio is not None: + turn_reason = _probe(manager, "create_recording", ("input", "output"), capture, purpose="turn_recording") + turns = turn_audio if turn_reason is None else None + try: + if version("pipecat-ai") not in _SUPPORTED: + return AudioCaptureBinding("unsupported_version") + from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor + processor = AudioBufferProcessor(sample_rate=sample_rate, num_channels=num_channels, + buffer_size=buffer_size, enable_turn_audio=turns is not None) + except PackageNotFoundError: + return AudioCaptureBinding("unsupported_version") + except Exception: + return AudioCaptureBinding("source_unavailable") + binding = _bind(processor, session_context, manager, layout=layout, timeout=finalization_timeout, + capture=capture, operation="create_recording", turn_audio=turns) + if turn_reason is not None and binding.status == "session_bound": + binding.turn_status = turn_reason + return binding diff --git a/python/traceai/__init__.py b/python/traceai/__init__.py index 4888b539d..5408e9343 100644 --- a/python/traceai/__init__.py +++ b/python/traceai/__init__.py @@ -319,13 +319,26 @@ def _bind(library: str, entry_point: str, *args, **kwargs): return None # A binding alone is not proof of recorded media. The adapter owns reporting # when a real source becomes available; Q02 qualifies end-to-end playback. - _adapter_status[library] = "session_bound" if binding is not None else "source_unavailable" + status = getattr(binding, "status", None) + if binding is None: + _adapter_status[library] = "source_unavailable" + else: + # The adapter's own content-free status (for example a denial reason); + # a binding object without one is only "session_bound". + _adapter_status[library] = status if isinstance(status, str) and len(status) <= 64 else "session_bound" return binding -def bind_pipecat_audio_processor(processor, session_context): - """Observe an existing processor; never start a new recording implicitly.""" - return _bind("pipecat", "bind_audio_processor", processor, session_context) +def bind_pipecat_audio_processor(processor, session_context, **options): + """Observe an existing processor; never start a new recording implicitly. + + ``session_context`` is a ``traceai_pipecat.attachments.AudioSessionContext`` + (``from_pipeline(worker)`` or ``from_span(span)``). Options are passed to + ``traceai_pipecat.bind_audio_processor`` (``layout``, ``finalization_timeout``, + ``capture_policy``, and the opt-in per-turn caps + ``turn_audio=traceai_pipecat.TurnAudioCaps(max_turns=, max_bytes=, max_duration_ms=)``). + """ + return _bind("pipecat", "bind_audio_processor", processor, session_context, **options) def bind_livekit_session(session, job_context, *, owned_recording_files=False, @@ -336,12 +349,40 @@ def bind_livekit_session(session, job_context, *, owned_recording_files=False, finalization_timeout=finalization_timeout) +async def import_livekit_completed_egress(egress_info=None, *, session_context, egress_id, room_name, + connection_id=None, registry=None, egress_client=None, + existing_source_authorized=False, timeout=30.0): + """Offer an explicitly authorized completed LiveKit Egress to the importer. + + Uses the provider's shared attachment manager. Returns the core + ``ImportHandle`` (``queued`` = accepted for import, not playable), or None + when the runtime or adapter is unavailable. + """ + if not _ATTACHMENTS_SUPPORTED: + _adapter_status["livekit"] = "unsupported_version" + return None + manager = getattr(_tracer_provider, "attachment_manager", None) + if manager is None: + raise RuntimeError("initialize traceai before importing a recording") + try: + module = importlib.import_module(_INSTRUMENTOR_REGISTRY["livekit"]) + importer = getattr(module, "import_completed_egress") + except (ImportError, AttributeError): + _adapter_status["livekit"] = "unsupported_version" + return None + return await importer(egress_info, session_context=session_context, egress_id=egress_id, + room_name=room_name, manager=manager, connection_id=connection_id, + registry=registry, egress_client=egress_client, + existing_source_authorized=existing_source_authorized, timeout=timeout) + + __all__ = [ "init", "AttachmentConfig", "attachment_status", "bind_pipecat_audio_processor", "bind_livekit_session", + "import_livekit_completed_egress", "instrument", "get_tracer_provider", "SemanticConvention", diff --git a/python/traceai/tests/test_attachments.py b/python/traceai/tests/test_attachments.py index b4e18e306..f1ad6ded6 100644 --- a/python/traceai/tests/test_attachments.py +++ b/python/traceai/tests/test_attachments.py @@ -47,6 +47,41 @@ def test_explicit_session_binding_uses_shared_manager_without_recording_permissi assert "recording_source_ready" not in str(traceai.attachment_status()) +def test_pipecat_binding_passes_options_and_reports_adapter_status(monkeypatch): + manager = SimpleNamespace(status=lambda: SimpleNamespace(state="ready")) + monkeypatch.setattr(traceai, "_tracer_provider", SimpleNamespace(attachment_manager=manager)) + binder = Mock(return_value=SimpleNamespace(status="policy_denied")) + monkeypatch.setattr(traceai.importlib, "import_module", + Mock(return_value=SimpleNamespace(bind_audio_processor=binder))) + processor, context = object(), object() + result = traceai.bind_pipecat_audio_processor(processor, context, layout="separate_tracks") + assert result is binder.return_value + binder.assert_called_once_with(processor, context, manager=manager, layout="separate_tracks") + # A denied binding is never reported as bound. + assert traceai.attachment_status()["adapters"]["pipecat"] == "policy_denied" + + +def test_livekit_egress_import_uses_shared_manager(monkeypatch): + import asyncio + + manager = SimpleNamespace(status=lambda: SimpleNamespace(state="ready")) + monkeypatch.setattr(traceai, "_tracer_provider", SimpleNamespace(attachment_manager=manager)) + calls = [] + + async def importer(info, **kwargs): + calls.append((info, kwargs)) + return "handle" + + monkeypatch.setattr(traceai.importlib, "import_module", + Mock(return_value=SimpleNamespace(import_completed_egress=importer))) + info = object() + result = asyncio.run(traceai.import_livekit_completed_egress( + info, session_context="span", egress_id="EG_synthetic", room_name="room", existing_source_authorized=True)) + assert result == "handle" + assert calls[0][0] is info and calls[0][1]["manager"] is manager + assert calls[0][1]["existing_source_authorized"] is True and calls[0][1]["registry"] is None + + def test_old_optional_adapter_reports_unsupported_without_exception_body(monkeypatch, capsys): monkeypatch.setattr(traceai, "_tracer_provider", SimpleNamespace(attachment_manager=SimpleNamespace(status=lambda: SimpleNamespace(state="ready")))) monkeypatch.setattr(traceai.importlib, "import_module", Mock(return_value=SimpleNamespace())) diff --git a/python/voice-tests/conftest.py b/python/voice-tests/conftest.py new file mode 100644 index 000000000..ebf67d45b --- /dev/null +++ b/python/voice-tests/conftest.py @@ -0,0 +1,17 @@ +"""Voice fixtures reuse the core's synthetic transport and real TLS server.""" + +import importlib.util +import sys +from pathlib import Path + +_CORE_TESTS = Path(__file__).resolve().parents[1] / "tests" / "attachments" +sys.path.insert(0, str(_CORE_TESTS)) +_spec = importlib.util.spec_from_file_location("attachment_test_support", _CORE_TESTS / "conftest.py") +_support = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_support) +TransportDouble = _support.TransportDouble +clean_attachment_env = _support.clean_attachment_env +manager = _support.manager +transport = _support.transport +make_manager = _support.make_manager +from test_http_contract import server # noqa: E402,F401 diff --git a/python/voice-tests/process_exit_probe.py b/python/voice-tests/process_exit_probe.py new file mode 100644 index 000000000..694cbc2f9 --- /dev/null +++ b/python/voice-tests/process_exit_probe.py @@ -0,0 +1,44 @@ +"""Synthetic cancellation probe executed in a separate interpreter by pytest.""" + +import asyncio +import tempfile +from pathlib import Path + +from conftest import TransportDouble +from fi_instrumentation.attachments import AttachmentConfig, AttachmentManager +from test_livekit_attachments import owned_session +from traceai_livekit import bind_session + + +async def main(): + wire = TransportDouble() + wire.release.clear() + manager = AttachmentManager(config=AttachmentConfig(enabled=True), project_name="synthetic", transport=wire) + assert manager.ready(2) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + session, job, _, span, provider, calls = owned_session(root) + (root / "audio.ogg").write_bytes(b"synthetic") + binding = bind_session(session, job, manager=manager, owned_recording_files=True, finalization_timeout=30) + task = asyncio.create_task(job._on_session_end()) + for _ in range(200): + if wire.entered.is_set(): + break + await asyncio.sleep(.01) + assert wire.entered.is_set() + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + assert binding.status == "incomplete" and calls == ["original"] + assert binding._scope is None + wire.release.set() + manager.shutdown(.2) + span.end() + provider.shutdown() + print("bounded-cancel-exit") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/voice-tests/test_livekit_attachments.py b/python/voice-tests/test_livekit_attachments.py new file mode 100644 index 000000000..32d9d7faf --- /dev/null +++ b/python/voice-tests/test_livekit_attachments.py @@ -0,0 +1,477 @@ +"""Synthetic component tests, not a real SFU call or product-playback claim.""" + +import asyncio +import inspect +import io +import os +import subprocess +import time +from dataclasses import replace +from importlib.metadata import version +from types import SimpleNamespace + +import av +import pytest +from livekit import api +from livekit.agents import AgentSession, JobContext +from livekit.agents.voice.recorder_io import RecorderIO +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, attach, detach, set_value +from opentelemetry.sdk.trace import TracerProvider + +from fi_instrumentation.attachments import AttachmentConfig, AttachmentManager, CapturePolicy +from fi_instrumentation.instrumentation.config import TraceConfig +from traceai_livekit import bind_session, import_completed_egress + + +def owned_session(tmp_path, *, recording=True): + """Real AgentSession, RecorderIO and report method; no RTC/network startup.""" + provider = TracerProvider() + span = provider.get_tracer("synthetic").start_span("actual-conversation") + session = AgentSession() + session._session_span = span + session._opts.recording_options = {"audio": True, "redaction": False} + recorder = RecorderIO(agent_session=session) + recorder._output_path = tmp_path / "audio.ogg" + session._recorder_io = recorder if recording else None + # Build only the job shell that make_session_report needs. Constructors that + # connect to an SFU are not exercised by these deterministic components. + job = JobContext.__new__(JobContext) + job._session_directory = tmp_path + job._primary_agent_session = session + job._redaction_enabled = False + job._tagger = SimpleNamespace(tags=set()) + job._info = SimpleNamespace(job=api.Job(id="synthetic-job", room=api.Room(name="synthetic", sid="synthetic-room"))) + original_calls = [] + + async def original(): + original_calls.append("original") + + job._on_session_end = original + return session, job, recorder, span, provider, original_calls + + +def make_ogg(path): + """0.1 s stereo 440 Hz Opus in Ogg, encoded with PyAV's bundled codecs (no system ffmpeg).""" + import math + import struct + rate, count = 48000, 4800 + pcm = b"".join(struct.pack(" core RecordingWriter (segments + manifest). + +The pipeline tests run a real PipelineWorker with Pipecat's own tracing, so the +recording binds to the conversation span Pipecat itself started. Audio is a +synthetic tone; the transport is the core's synthetic F01 double. This is not a +real SFU call or a product-playback claim (Q02). +""" + +import asyncio +import io +import json +import math +import struct +import threading +import wave +from dataclasses import replace + +import pytest +from opentelemetry import trace +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, attach, detach, set_value +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from conftest import TransportDouble, make_manager +from fi_instrumentation.attachments import AttachmentConfig, AttachmentManager, CapturePolicy, CorrelationHandle +from fi_instrumentation.attachments._disabled import DisabledAttachmentManager +from pipecat.frames.frames import ( + BotStartedSpeakingFrame, BotStoppedSpeakingFrame, EndFrame, InputAudioRawFrame, OutputAudioRawFrame, + UserStartedSpeakingFrame, UserStoppedSpeakingFrame, +) +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.worker import PipelineParams, PipelineWorker +from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor, TurnAudioData +from pipecat.workers.runner import WorkerRunner +from traceai_pipecat import bind_audio_processor, create_audio_capture +from traceai_pipecat.attachments import AudioSessionContext, TurnAudioCaps, mark_conversation_root + +# Pipecat's TurnTraceObserver uses the GLOBAL tracer provider. +EXPORTER = InMemorySpanExporter() +PROVIDER = TracerProvider() +PROVIDER.add_span_processor(SimpleSpanProcessor(EXPORTER)) +trace.set_tracer_provider(PROVIDER) +TRACER = PROVIDER.get_tracer("synthetic") +RATE = 16000 + + +def tone(freq, ms, rate=RATE, amplitude=8000): + count = rate * ms // 1000 + return b"".join(struct.pack(" uploaded payload (requests and payloads are in lane order).""" + uploads = [r for r in transport.requests if "recording_id" in r] + assert len(uploads) == len(transport.payloads) + return {r["occurrence_id"]: (r, p) for r, p in zip(uploads, transport.payloads)} + + +def final_manifest(transport): + assert transport.manifests, "no manifest published" + return transport.manifests[-1][1] + + +async def run_pipeline(processor, frames, *, conversation_id="conv-synthetic", parent=None): + worker = PipelineWorker(Pipeline([processor]), + params=PipelineParams(audio_in_sample_rate=RATE, audio_out_sample_rate=RATE), + enable_tracing=True, enable_turn_tracking=True, conversation_id=conversation_id, + idle_timeout_secs=None, enable_rtvi=False) + return worker + + +async def drive(worker, frames, parent=None): + await worker.queue_frames(frames + [EndFrame()]) + runner = WorkerRunner(handle_sigint=False) + # run(worker) is the form that ends with the worker on 1.15.0 (it warns as + # deprecated); add_workers() + run() kept running after EndFrame here. + if parent is None: + await asyncio.wait_for(runner.run(worker), 60) + else: + with trace.use_span(parent, end_on_exit=False): + await asyncio.wait_for(runner.run(worker), 60) + + +def call_frames(turns=6, ms=500): + frames = [] + for i in range(turns): + for _ in range(ms // 20): + frames.append(InputAudioRawFrame(audio=tone(440 + i, 20), sample_rate=RATE, num_channels=1)) + for _ in range(ms // 20): + frames.append(OutputAudioRawFrame(audio=tone(880 + i, 20), sample_rate=RATE, num_channels=1)) + return frames + + +def conversation_span(): + spans = [s for s in EXPORTER.get_finished_spans() if s.name == "conversation"] + assert len(spans) == 1 + return spans[0] + + +@pytest.fixture(autouse=True) +def reset_spans(): + EXPORTER.clear() + yield + + +# ------------------------------------------------------------------ real pipeline +@pytest.mark.asyncio +@pytest.mark.parametrize("channels,layout", [(1, "mixed"), (2, "stereo")]) +async def test_real_pipeline_whole_call_recording(manager, transport, channels, layout): + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=channels, buffer_size=16000, + auto_start_recording=True) + emitted = [] + processor.add_event_handler("on_audio_data", lambda _p, audio, rate, nch: emitted.append((audio, rate, nch))) + worker = await run_pipeline(processor, None) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(worker), manager=manager) + assert binding.status == "session_bound" + await drive(worker, call_frames()) + result = await binding.finish(10) + assert result is not None and result.state == "published" and result.capture_complete, result + assert binding.status == "published" + assert {(r, n) for _, r, n in emitted} == {(RATE, channels)} + manifest = final_manifest(transport) + assert manifest["layout"] == layout and manifest["sample_rate_hz"] == RATE + assert manifest["channels"] == channels and not manifest["gaps"] and manifest["capture_complete"] + if layout == "stereo": + assert [(t["channels"], t["direction"], t["role"]) for t in manifest["tracks"]] == [ + ([0], "input", "user"), ([1], "output", "assistant")] + else: + assert [(t["direction"], t["role"]) for t in manifest["tracks"]] == [("mixed", "mixed")] + segments = segment_map(transport) + uploaded = b"" + for segment in sorted(manifest["segments"], key=lambda s: s["start_ms"]): + request, payload = segments[segment["occurrence_id"]] + assert request["direction"] == "mixed" and request["purpose"] == "conversation_recording" + nch, rate, frames = pcm(payload) + assert (nch, rate) == (channels, RATE) + uploaded += frames + # Byte identity: exactly what Pipecat emitted, in order, nothing more. + assert uploaded == b"".join(audio for audio, _, _ in emitted) + assert manifest["duration_ms"] == math.ceil(len(uploaded) * 1000 / (2 * channels * RATE)) + # Bound to the conversation span Pipecat started, with its conversation id. + root = conversation_span() + parent = transport.requests[0]["association"]["parent"] + assert parent["span_id"] == format(root.context.span_id, "016x") + assert all(r["conversation_id"] == "conv-synthetic" for r in transport.requests if "recording_id" in r) + assert root.parent is None and binding.root_marked + assert root.attributes["fi.span.kind"] == "CONVERSATION" + assert root.attributes["fi.attachment.schema_version"] == 1 + for span in EXPORTER.get_finished_spans(): + if span is not root: + assert span.attributes.get("fi.span.kind") != "CONVERSATION" + + +@pytest.mark.asyncio +async def test_real_pipeline_per_track_manifest(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=2, buffer_size=16000, + auto_start_recording=True) + emitted = [] + processor.add_event_handler("on_track_audio_data", + lambda _p, user, bot, rate, nch: emitted.append((user, bot))) + worker = await run_pipeline(processor, None) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(worker), manager=manager, + layout="separate_tracks") + await drive(worker, call_frames()) + result = await binding.finish(10) + assert result.state == "published" and result.capture_complete + manifest = final_manifest(transport) + assert manifest["layout"] == "separate_tracks" and manifest["channels"] == 2 + user_id, bot_id = (t["track_id"] for t in manifest["tracks"]) + segments = segment_map(transport) + per_track = {user_id: b"", bot_id: b""} + for segment in sorted(manifest["segments"], key=lambda s: (s["start_ms"], s["track_id"])): + request, payload = segments[segment["occurrence_id"]] + nch, rate, frames = pcm(payload) + assert (nch, rate) == (1, RATE) + assert request["direction"] == ("input" if segment["track_id"] == user_id else "output") + per_track[segment["track_id"]] += frames + assert per_track[user_id] == b"".join(u for u, _ in emitted) + assert per_track[bot_id] == b"".join(b for _, b in emitted) + assert conversation_span().attributes["fi.span.kind"] == "CONVERSATION" + + +@pytest.mark.asyncio +async def test_conversation_with_a_parent_is_never_marked_root(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=16000, + auto_start_recording=True) + worker = await run_pipeline(processor, None) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(worker), manager=manager) + app = TRACER.start_span("app-request") + try: + await drive(worker, call_frames(turns=2), parent=app) + finally: + app.end() + result = await binding.finish(10) + assert result.state == "published" + root = conversation_span() + assert root.parent is not None and root.parent.span_id == app.get_span_context().span_id + assert not binding.root_marked and "fi.span.kind" not in root.attributes + + +@pytest.mark.asyncio +async def test_pipeline_without_tracing_never_uses_ambient_span(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=16000, + auto_start_recording=True) + worker = PipelineWorker(Pipeline([processor]), params=PipelineParams(audio_in_sample_rate=RATE, + audio_out_sample_rate=RATE), enable_tracing=False, idle_timeout_secs=None, + enable_rtvi=False) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(worker), manager=manager) + ambient = TRACER.start_span("ambient") + with trace.use_span(ambient, end_on_exit=True): + await drive(worker, call_frames(turns=2)) + assert await binding.finish(5) is None + assert binding.status == "correlation_unavailable" + assert not transport.requests and not transport.payloads and manager.owned_bytes == 0 + + +# ------------------------------------------------------- direct event dispatch +async def dispatch(processor, event, *args): + """Pipecat's own dispatch: one task per handler, run to completion.""" + await processor._call_event_handler(event, *args) + for _ in range(3): + await asyncio.sleep(0) + + +def explicit_context(): + span = TRACER.start_span("explicit-conversation") + return span, AudioSessionContext.from_span(span, conversation_id="conv-explicit") + + +@pytest.mark.asyncio +async def test_stereo_one_side_hidden_denies_whole_recording(manager, transport, monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=2, buffer_size=16000) + before = handler_counts(processor) + span, context = explicit_context() + binding = bind_audio_processor(processor, context, manager=manager) + assert binding.status == "policy_denied" and handler_counts(processor) == before + span.end() + assert not transport.requests + + +@pytest.mark.asyncio +async def test_separate_tracks_hidden_side_becomes_gap_other_side_uploads(manager, transport, monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=16000) + span, context = explicit_context() + binding = bind_audio_processor(processor, context, manager=manager, layout="separate_tracks") + assert binding.status == "session_bound" + for i in range(4): + await dispatch(processor, "on_track_audio_data", tone(440, 500), tone(880, 500), RATE, 1) + await dispatch(processor, "on_recording_stopped") + result = await binding.finish(10) + span.end() + assert result.state == "partial" and not result.capture_complete + manifest = final_manifest(transport) + user_id, bot_id = (t["track_id"] for t in manifest["tracks"]) + assert {s["track_id"] for s in manifest["segments"]} == {bot_id} + assert {(g["track_id"], g["reason"]) for g in manifest["gaps"]} == {(user_id, "policy_denied")} + assert all(r["direction"] == "output" for r in transport.requests if "recording_id" in r) + assert b"".join(pcm(p)[2] for p in transport.payloads) == tone(880, 500) * 4 + + +@pytest.mark.parametrize("buffer_size,turns", [(0, False), (16000, True), (128 * 1024 + 1, False)]) +def test_unbounded_upstream_is_refused_and_untouched(manager, transport, buffer_size, turns): + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=buffer_size, enable_turn_audio=turns) + before = handler_counts(processor) + span, context = explicit_context() + assert bind_audio_processor(processor, context, manager=manager).status == "unbounded_source" + assert handler_counts(processor) == before and not processor._recording + span.end() + + +@pytest.mark.parametrize("flags", [("FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS")]) +def test_policy_denial_precedes_processor_access(manager, monkeypatch, flags): + class Poison: + def __getattribute__(self, name): + raise AssertionError("denied processor accessed") + for flag in flags: + monkeypatch.setenv(flag, "true") + span, context = explicit_context() + assert bind_audio_processor(Poison(), context, manager=manager).status == "policy_denied" + span.end() + + +def test_suppressed_instrumentation_binds_nothing(manager): + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + before = handler_counts(processor) + span, context = explicit_context() + token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) + try: + assert bind_audio_processor(processor, context, manager=manager).status == "policy_denied" + finally: + detach(token) + assert handler_counts(processor) == before + span.end() + + +def test_off_by_default_registers_nothing_and_writes_no_attributes(): + class Poison: + def __getattribute__(self, name): + raise AssertionError("disabled adapter touched its arguments") + for manager in (AttachmentManager(config=AttachmentConfig()), DisabledAttachmentManager()): + try: + assert bind_audio_processor(Poison(), Poison(), manager=manager).status == "disabled" + assert create_audio_capture(Poison(), manager=manager).status == "disabled" + assert manager.owned_bytes == 0 + finally: + manager.shutdown(0) + + +@pytest.mark.asyncio +async def test_unsampled_conversation_retains_nothing(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + context = AudioSessionContext(correlation=CorrelationHandle("1" * 32, "2" * 16, False)) + binding = bind_audio_processor(processor, context, manager=manager) + await dispatch(processor, "on_audio_data", tone(440, 500), RATE, 1) + assert binding.status == "unsampled" and manager.owned_bytes == 0 and not transport.requests + + +def test_create_capture_needs_local_and_server_recording_permission(transport): + span, context = explicit_context() + manager = make_manager(transport) + try: + assert create_audio_capture(context, manager=manager).status == "recording_disabled" + finally: + manager.shutdown(1) + transport.policy = replace(transport.policy, record_audio=False) + manager = make_manager(transport, record_audio=True) + try: + assert create_audio_capture(context, manager=manager).status == "recording_disabled" + finally: + manager.shutdown(1) + transport.policy = replace(transport.policy, record_audio=True) + manager = make_manager(transport, record_audio=True) + try: + binding = create_audio_capture(context, manager=manager, num_channels=2) + assert binding.status == "session_bound" + processor = binding.processor + assert isinstance(processor, AudioBufferProcessor) and processor.num_channels == 2 + assert not processor._recording and not processor._auto_start_recording + binding.close() + finally: + manager.shutdown(1) + span.end() + + +@pytest.mark.asyncio +async def test_blocked_uploader_turns_overflow_into_gaps_not_memory(transport): + manager = make_manager(transport) + transport.release.clear() + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=32000) + span, context = explicit_context() + binding = bind_audio_processor(processor, context, manager=manager) + peak = 0 + try: + for _ in range(150): # 150 s of mono audio while the uploader is stuck + await dispatch(processor, "on_audio_data", tone(440, 1000), RATE, 1) + peak = max(peak, manager.owned_bytes) + assert peak <= manager.config.max_owned_bytes + # Two 30 s segments may wait (window_segments=2); the rest is a gap. + assert peak <= 2 * (30 * RATE * 2) + 3 * RATE * 2 + transport.release.set() + await dispatch(processor, "on_recording_stopped") + result = await binding.finish(20) + finally: + transport.release.set() + manager.shutdown(2) + span.end() + assert result.state == "partial" and not result.capture_complete + manifest = final_manifest(transport) + assert any(g["reason"] == "queue_full" for g in manifest["gaps"]) + assert manifest["duration_ms"] == 150000 + + +@pytest.mark.asyncio +async def test_policy_tightening_mid_recording_stops_retention(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + span, context = explicit_context() + binding = bind_audio_processor(processor, context, manager=manager) + for _ in range(35): + await dispatch(processor, "on_audio_data", tone(440, 1000), RATE, 1) + manager.update_policy(replace(transport.policy, enabled=False)) + for _ in range(5): + await dispatch(processor, "on_audio_data", tone(440, 1000), RATE, 1) + await dispatch(processor, "on_recording_stopped") + result = await binding.finish(10) + span.end() + assert result.state in ("partial", "unpublished") and not result.capture_complete + assert sum(len(pcm(p)[2]) for p in transport.payloads) <= 35 * RATE * 2 + + +@pytest.mark.asyncio +async def test_format_drift_keeps_clock_with_explicit_gap(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + span, context = explicit_context() + binding = bind_audio_processor(processor, context, manager=manager) + await dispatch(processor, "on_audio_data", tone(440, 500), RATE, 1) + await dispatch(processor, "on_audio_data", tone(440, 500, rate=8000), 8000, 1) + await dispatch(processor, "on_audio_data", tone(440, 500), RATE, 1) + await dispatch(processor, "on_recording_stopped") + result = await binding.finish(10) + span.end() + manifest = final_manifest(transport) + assert result.state == "partial" + assert [(g["start_ms"], g["duration_ms"], g["reason"]) for g in manifest["gaps"]] == [ + (500, 500, "source_unavailable")] + assert manifest["duration_ms"] == 1500 + + +@pytest.mark.asyncio +async def test_each_start_stop_cycle_is_its_own_recording(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + span, context = explicit_context() + binding = bind_audio_processor(processor, context, manager=manager) + await dispatch(processor, "on_audio_data", tone(440, 500), RATE, 1) + await dispatch(processor, "on_recording_stopped") + # The stop closes the cycle without the app calling finish(); its upload + # runs in the background and the binding is ready for the next start. + for _ in range(200): + if binding.results: + break + await asyncio.sleep(0.01) + assert [r.state for r in binding.results] == ["published"] + await dispatch(processor, "on_audio_data", tone(660, 700), RATE, 1) + await dispatch(processor, "on_recording_stopped") + await binding.finish(10) + span.end() + assert [r.state for r in binding.results] == ["published", "published"] + first, second = binding.results + assert first.recording_id != second.recording_id + durations = {m["recording_id"]: m["duration_ms"] for _, m, _, _ in transport.manifests} + assert durations == {first.recording_id: 500, second.recording_id: 700} + + +@pytest.mark.asyncio +async def test_handler_failure_is_isolated_and_content_free(manager, transport, caplog): + class Exploding: + @property + def turn_trace_observer(self): + raise RuntimeError("CANARY_PRIVATE_DETAIL") + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(Exploding()), manager=manager) + await dispatch(processor, "on_audio_data", tone(440, 500), RATE, 1) + assert binding.status == "source_unavailable" + assert "CANARY" not in caplog.text and not transport.requests and manager.owned_bytes == 0 + + +def test_duplicate_conflicting_and_detach(manager): + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + before = handler_counts(processor) + span, context = explicit_context() + binding = bind_audio_processor(processor, context, manager=manager) + assert bind_audio_processor(processor, context, manager=manager) is binding + with pytest.raises(ValueError, match="conflicting"): + bind_audio_processor(processor, context, manager=manager, layout="separate_tracks") + assert handler_counts(processor) != before + binding.close() + assert handler_counts(processor) == before and not processor._recording + assert bind_audio_processor(processor, context, manager=manager) is not binding + span.end() + + +def test_unknown_upstream_fails_closed(manager, monkeypatch): + monkeypatch.setattr("traceai_pipecat.attachments.version", lambda _: "1.14.0") + span, context = explicit_context() + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + assert bind_audio_processor(processor, context, manager=manager).status == "unsupported_version" + span.end() + + +def test_root_marker_only_on_true_parentless_live_span(): + root = TRACER.start_span("root") + child = TRACER.start_span("child", context=trace.set_span_in_context(root)) + remote = TRACER.start_span("remote-child", context=trace.set_span_in_context(trace.NonRecordingSpan( + trace.SpanContext(int("1" * 32, 16), int("2" * 16, 16), True, trace.TraceFlags(1))))) + assert mark_conversation_root(root) and root.attributes["fi.span.kind"] == "CONVERSATION" + assert root.attributes["gen_ai.span.kind"] == "CONVERSATION" + assert not mark_conversation_root(child) and "fi.span.kind" not in child.attributes + assert not mark_conversation_root(remote) and "fi.span.kind" not in remote.attributes + assert not mark_conversation_root(trace.NonRecordingSpan(root.get_span_context())) + for span in (remote, child, root): + span.end() + ended = TRACER.start_span("ended") + ended.end() + assert not mark_conversation_root(ended) + + +def test_no_private_manager_internals_in_voice_adapters(): + import inspect + + import traceai_pipecat.attachments as pipecat + for module in (pipecat,): + source = inspect.getsource(module) + for private in ("manager._session", "manager.policy", "._fd", "manager._", "_manager._"): + assert private not in source, (module.__name__, private) + + +# ------------------------------------------------------------- short calls +@pytest.mark.asyncio +async def test_short_call_binds_to_snapshot_after_worker_ended_conversation(manager, transport): + """Pipecat runs on_audio_data as a task: a call shorter than buffer_size + delivers its only chunk on stop, which can run after the worker's cleanup + has ended and cleared the conversation span.""" + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=128 * 1024) + worker = await run_pipeline(processor, None) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(worker), manager=manager) + observer = worker.turn_trace_observer + observer.start_conversation_tracing("conv-short") + await dispatch(processor, "on_recording_started") + live_id = format(observer._conversation_span.get_span_context().span_id, "016x") + observer.end_conversation_tracing() + assert observer._conversation_span is None + await dispatch(processor, "on_audio_data", tone(440, 300), RATE, 1) + await dispatch(processor, "on_recording_stopped") + result = await binding.finish(10) + assert result is not None and result.state == "published", (binding.status, result) + uploads = [r for r in transport.requests if "recording_id" in r] + assert uploads and all(r["association"]["parent"]["span_id"] == live_id for r in uploads) + assert all(r["conversation_id"] == "conv-short" for r in uploads) + root = conversation_span() + assert format(root.context.span_id, "016x") == live_id + # Marked while it was still live, because the audio came after it ended. + assert binding.root_marked and root.attributes["fi.span.kind"] == "CONVERSATION" + assert pcm(transport.payloads[0])[2] == tone(440, 300) + + +@pytest.mark.asyncio +async def test_no_snapshot_and_no_span_stays_correlation_unavailable(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=16000) + worker = await run_pipeline(processor, None) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(worker), manager=manager) + ambient = TRACER.start_span("ambient") + with trace.use_span(ambient, end_on_exit=True): + await dispatch(processor, "on_audio_data", tone(440, 300), RATE, 1) + assert binding.status == "correlation_unavailable" + assert not transport.requests and manager.owned_bytes == 0 + + +@pytest.mark.asyncio +async def test_real_pipeline_short_call_under_buffer_size_is_recorded(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=128 * 1024, + auto_start_recording=True) + worker = await run_pipeline(processor, None) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(worker), manager=manager) + await drive(worker, call_frames(turns=1, ms=200)) + result = await binding.finish(10) + assert result is not None and result.state == "published", (binding.status, result) + root = conversation_span() + parent = transport.requests[0]["association"]["parent"] + assert parent["span_id"] == format(root.context.span_id, "016x") + assert binding.root_marked and root.attributes["fi.span.kind"] == "CONVERSATION" + + +# ------------------------------------------------------------ per-turn audio +CAPS = TurnAudioCaps(max_turns=100, max_bytes=8 * 1024 * 1024, max_duration_ms=600000) + + +def turn_frames(turns=3, ms=200): + frames = [] + for i in range(turns): + frames.append(UserStartedSpeakingFrame()) + frames += [InputAudioRawFrame(audio=tone(300 + i, 20), sample_rate=RATE, num_channels=1) + for _ in range(ms // 20)] + frames.append(UserStoppedSpeakingFrame()) + frames.append(BotStartedSpeakingFrame()) + frames += [OutputAudioRawFrame(audio=tone(900 + i, 20), sample_rate=RATE, num_channels=1) + for _ in range(ms // 20)] + frames.append(BotStoppedSpeakingFrame()) + return frames + + +def pair_uploads(transport): + """Record (UploadRequest, payload) per upload, paired on the worker thread that ran both. + + The core runs two upload workers, so the double's flat request and payload + lists can interleave; one job's initialize and content PUT share a thread. + """ + pairs, local = [], threading.local() + initialize = transport.initialize + + def paired_initialize(session, body, **kwargs): + upload = initialize(session, body, **kwargs) + local.request = json.loads(body) + return upload + + class Payloads(list): + def append(self, data): + super().append(data) + pairs.append((getattr(local, "request", None), data)) + + transport.initialize = paired_initialize + transport.payloads = Payloads(transport.payloads) + return pairs + + +def turn_pairs(pairs): + return [(r, p) for r, p in pairs if r is not None and r.get("purpose") == "turn_recording"] + + +@pytest.mark.asyncio +async def test_turn_audio_is_off_by_default_and_needs_explicit_caps(manager): + span, context = explicit_context() + turns = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000, enable_turn_audio=True) + before = handler_counts(turns) + assert bind_audio_processor(turns, context, manager=manager).status == "unbounded_source" + assert handler_counts(turns) == before + plain = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + with pytest.raises(ValueError, match="enable_turn_audio"): + bind_audio_processor(plain, context, manager=manager, turn_audio=CAPS) + with pytest.raises(ValueError): + bind_audio_processor(turns, context, manager=manager, turn_audio={"max_turns": 1}) + for bad in ((0, 1, 1), (1, 0, 1), (1, 1, 0), (1001, 1, 1), (1, 64 * 1024 * 1024 + 1, 1), + (1, 1, 2 * 60 * 60 * 1000 + 1), (1.0, 1, 1), (True, 1, 1)): + with pytest.raises(ValueError): + TurnAudioCaps(*bad) + binding = bind_audio_processor(plain, context, manager=manager) + assert binding.turn_status == "off" and "on_user_turn_audio" not in [e for e, _ in binding._handlers] + binding.close() + span.end() + + +@pytest.mark.asyncio +async def test_real_pipeline_uploads_each_turn_bound_to_its_turn_span(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=16000, + auto_start_recording=True, enable_turn_audio=True) + emitted = [] + processor.add_event_handler("on_user_turn_audio", lambda _p, d: emitted.append(("input", d.turn_number, d.audio))) + processor.add_event_handler("on_bot_turn_audio", lambda _p, d: emitted.append(("output", d.turn_number, d.audio))) + worker = await run_pipeline(processor, None) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(worker), manager=manager, + turn_audio=CAPS) + assert binding.turn_status == "session_bound" + pairs = pair_uploads(transport) + await drive(worker, turn_frames(turns=3)) + result = await binding.finish(10) + assert result is not None and result.state == "published" + assert emitted, "Pipecat emitted no turn audio" + turn_spans = {s.attributes["turn.number"]: s for s in EXPORTER.get_finished_spans() if s.name == "turn"} + uploads = turn_pairs(pairs) + assert len(uploads) == len(emitted) == len(binding.turn_handles) + got = {} + for request, payload in uploads: + number = int(request["turn_id"]) + assert request["association"]["parent"]["span_id"] == format(turn_spans[number].context.span_id, "016x") + assert request["conversation_id"] == "conv-synthetic" and request["mime_type"] == "audio/wav" + nch, rate, frames = pcm(payload) + assert (nch, rate) == (1, RATE) + got[(request["direction"], number)] = frames + assert got == {(direction, number): audio for direction, number, audio in emitted} + # Queued synthetic frames: Bot/User*SpeakingFrame are system frames and can + # overtake queued audio, so which turns Pipecat emits here depends on its + # scheduling. Every emitted turn must upload, byte-identical, on its own + # turn span; the bot side is covered by the dispatch tests below. + assert all(direction in ("input", "output") for direction, _ in got) + assert all(h.result(0).status != "unfinished" for _, _, h in binding.turn_handles) + # The whole-call recording is unchanged and still binds to the conversation. + root = conversation_span() + recording = [r for r in transport.requests if "recording_id" in r] + assert recording and recording[0]["association"]["parent"]["span_id"] == format(root.context.span_id, "016x") + assert manager.owned_bytes == 0 + + +async def turn_binding(manager, caps=CAPS, *, context=None, layout="auto", capture=None): + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=16000, enable_turn_audio=True) + worker = await run_pipeline(processor, None) + observer = worker.turn_trace_observer + observer.start_conversation_tracing("conv-turns") + binding = bind_audio_processor(processor, context or AudioSessionContext.from_pipeline(worker), + manager=manager, turn_audio=caps, layout=layout, capture_policy=capture) + return processor, observer, binding + + +async def run_turn(processor, observer, number, user_ms=500, bot_ms=500): + await observer._turn_tracker._call_event_handler("on_turn_started", number) + for _ in range(3): + await asyncio.sleep(0) + await observer._turn_tracker._call_event_handler("on_turn_ended", number, 1.0, False) + for _ in range(3): + await asyncio.sleep(0) + if user_ms: + await dispatch(processor, "on_user_turn_audio", TurnAudioData(number, tone(300, user_ms), RATE, 1)) + if bot_ms: + await dispatch(processor, "on_bot_turn_audio", TurnAudioData(number, tone(900, bot_ms), RATE, 1)) + + +@pytest.mark.asyncio +async def test_turn_count_cap_skips_whole_turns(manager, transport): + processor, observer, binding = await turn_binding( + manager, TurnAudioCaps(max_turns=2, max_bytes=8 * 1024 * 1024, max_duration_ms=600000)) + for number in (1, 2, 3): + await run_turn(processor, observer, number) + await binding.finish(10) + observer.end_conversation_tracing() + uploads = [r for r in transport.requests if r.get("purpose") == "turn_recording"] + assert sorted((r["turn_id"], r["direction"]) for r in uploads) == [("1", "input"), ("1", "output")] + assert binding.turn_denials == {"queue_full": 4} and binding.turn_status == "queue_full" + assert len(binding.turn_handles) == 2 + + +@pytest.mark.asyncio +async def test_turn_byte_and_duration_caps_never_truncate(manager, transport): + one_turn = 44 + len(tone(300, 500)) + pairs = pair_uploads(transport) + processor, observer, binding = await turn_binding( + manager, TurnAudioCaps(max_turns=100, max_bytes=one_turn + 10, max_duration_ms=600000)) + await run_turn(processor, observer, 1) + await binding.finish(10) + uploads = turn_pairs(pairs) + assert [(r["direction"], r["declared_bytes"]) for r, _ in uploads] == [("input", one_turn)] + assert pcm(uploads[0][1])[2] == tone(300, 500) + assert binding.turn_denials == {"too_large": 1} + binding.close() + observer.end_conversation_tracing() + # Duration cap: 500 ms fits under 700 ms, the bot's 500 ms more does not. + processor, observer, binding = await turn_binding( + manager, TurnAudioCaps(max_turns=100, max_bytes=8 * 1024 * 1024, max_duration_ms=700)) + await run_turn(processor, observer, 1) + await binding.finish(10) + uploads = turn_pairs(pairs)[1:] + assert [r["direction"] for r, _ in uploads] == ["input"] + assert binding.turn_denials == {"too_large": 1} + observer.end_conversation_tracing() + + +@pytest.mark.asyncio +async def test_hidden_side_turns_are_denied_other_side_uploads(manager, transport, monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + # A mono mixed recording is denied whole under a hide flag; per-side tracks are not. + processor, observer, binding = await turn_binding(manager, layout="separate_tracks") + assert binding.status == "session_bound" and binding.turn_status == "session_bound" + await run_turn(processor, observer, 1) + await binding.finish(10) + observer.end_conversation_tracing() + uploads = [r for r in transport.requests if r.get("purpose") == "turn_recording"] + assert [r["direction"] for r in uploads] == ["output"] + assert binding.turn_denials == {"policy_denied": 1} + + +@pytest.mark.asyncio +async def test_turn_never_falls_back_to_conversation_or_explicit_span(manager, transport): + span, context = explicit_context() + processor, observer, binding = await turn_binding(manager, context=context) + await run_turn(processor, observer, 1) + assert binding.turn_denials == {"correlation_unavailable": 2} + # Pipeline context, but a turn number Pipecat never traced. + binding.close() + observer.end_conversation_tracing() + processor, observer, binding = await turn_binding(manager) + await dispatch(processor, "on_user_turn_audio", TurnAudioData(7, tone(300, 200), RATE, 1)) + assert binding.turn_status == "correlation_unavailable" + observer.end_conversation_tracing() + span.end() + assert not [r for r in transport.requests if r.get("purpose") == "turn_recording"] + assert manager.owned_bytes == 0 + + +@pytest.mark.asyncio +async def test_unsampled_turn_retains_nothing(manager, transport): + processor, observer, binding = await turn_binding(manager) + unsampled = trace.SpanContext(int("3" * 32, 16), int("4" * 16, 16), False, trace.TraceFlags(0)) + observer._trace_context_map[1] = unsampled + await dispatch(processor, "on_user_turn_audio", TurnAudioData(1, tone(300, 200), RATE, 1)) + observer.end_conversation_tracing() + assert binding.turn_status == "unsampled" and manager.owned_bytes == 0 + assert not [r for r in transport.requests if r.get("purpose") == "turn_recording"] + + +def test_turns_denied_on_both_sides_register_no_turn_handler(transport): + # The server allows only mixed recordings: the call records, turns cannot. + transport.policy = replace(transport.policy, allowed_directions=frozenset(("mixed",))) + manager = make_manager(transport) + span, context = explicit_context() + try: + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000, enable_turn_audio=True) + binding = bind_audio_processor(processor, context, manager=manager, turn_audio=CAPS) + assert binding.status == "session_bound" and binding.turn_status == "policy_denied" + events = {e for e, _ in binding._handlers} + assert "on_audio_data" in events and not events & {"on_user_turn_audio", "on_bot_turn_audio"} + binding.close() + finally: + manager.shutdown(1) + span.end() + + +def test_turn_caps_are_part_of_binding_identity(manager): + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000, enable_turn_audio=True) + span, context = explicit_context() + binding = bind_audio_processor(processor, context, manager=manager, turn_audio=CAPS) + assert bind_audio_processor(processor, context, manager=manager, turn_audio=CAPS) is binding + with pytest.raises(ValueError, match="conflicting"): + bind_audio_processor(processor, context, manager=manager, + turn_audio=TurnAudioCaps(max_turns=1, max_bytes=1000, max_duration_ms=1000)) + binding.close() + span.end() + + +def test_disabled_turn_audio_touches_nothing(): + class Poison: + def __getattribute__(self, name): + raise AssertionError("disabled adapter touched its arguments") + manager = DisabledAttachmentManager() + assert bind_audio_processor(Poison(), Poison(), manager=manager, turn_audio=CAPS).status == "disabled" + assert create_audio_capture(Poison(), manager=manager, turn_audio=CAPS).status == "disabled" + + +def test_create_capture_with_turn_audio_enables_pipecat_turns(transport): + span, context = explicit_context() + transport.policy = replace(transport.policy, record_audio=True) + manager = make_manager(transport, record_audio=True) + try: + binding = create_audio_capture(context, manager=manager, turn_audio=CAPS) + assert binding.status == "session_bound" and binding.turn_status == "session_bound" + assert binding.processor._enable_turn_audio is True + assert {"on_user_turn_audio", "on_bot_turn_audio"} <= {e for e, _ in binding._handlers} + binding.close() + plain = create_audio_capture(context, manager=manager) + assert plain.processor._enable_turn_audio is False and plain.turn_status == "off" + plain.close() + finally: + manager.shutdown(1) + span.end() + + +# ------------------------------------------- per-turn audio: review follow-ups +@pytest.mark.asyncio +async def test_local_capture_policy_hiding_inputs_drops_only_user_turns(manager, transport): + # A LOCAL CapturePolicy (no env flag) reaches the per-turn preflight. + processor, observer, binding = await turn_binding(manager, layout="separate_tracks", + capture=CapturePolicy(record_inputs=False)) + assert binding.status == "session_bound" and binding.turn_status == "session_bound" + await run_turn(processor, observer, 1) + await binding.finish(10) + observer.end_conversation_tracing() + uploads = [r for r in transport.requests if r.get("purpose") == "turn_recording"] + assert [(r["direction"], r["turn_id"]) for r in uploads] == [("output", "1")] + assert binding.turn_denials == {"policy_denied": 1} + assert [direction for _, direction, _ in binding.turn_handles] == ["output"] + + +@pytest.mark.asyncio +async def test_created_recorder_turns_stop_when_server_record_audio_is_revoked(transport): + # create_audio_capture's turns preflight as operation="create_recording": + # once the server turns record_audio off, the next turn is denied. + transport.policy = replace(transport.policy, record_audio=True) + manager = make_manager(transport, record_audio=True) + try: + worker = await run_pipeline(AudioBufferProcessor(sample_rate=RATE, buffer_size=16000), None) + observer = worker.turn_trace_observer + observer.start_conversation_tracing("conv-created") + binding = create_audio_capture(AudioSessionContext.from_pipeline(worker), manager=manager, + sample_rate=RATE, turn_audio=CAPS) + assert binding.status == "session_bound" and binding.turn_status == "session_bound" + processor = binding.processor + await run_turn(processor, observer, 1) + assert binding.turn_denials == {} and len(binding.turn_handles) == 2 + # Let turn 1 finish first: a tightened policy also stops uploads that are still queued. + for _, _, handle in binding.turn_handles: + assert (await handle.wait(10)).status != "unfinished" + manager.update_policy(replace(transport.policy, record_audio=False)) + await run_turn(processor, observer, 2) + await binding.finish(10) + observer.end_conversation_tracing() + uploads = [r for r in transport.requests if r.get("purpose") == "turn_recording"] + assert sorted((r["turn_id"], r["direction"]) for r in uploads) == [("1", "input"), ("1", "output")] + assert binding.turn_denials == {"recording_disabled": 2} + assert binding.turn_status == "recording_disabled" and len(binding.turn_handles) == 2 + binding.close() + finally: + manager.shutdown(2) + + +def test_created_recorder_has_no_pipecat_turn_buffer_when_turns_are_denied(transport): + # The server records whole calls (mixed) but admits no per-side turn: the + # recorder is built WITHOUT enable_turn_audio and no turn handler exists. + transport.policy = replace(transport.policy, record_audio=True, allowed_directions=frozenset(("mixed",))) + manager = make_manager(transport, record_audio=True) + span, context = explicit_context() + try: + binding = create_audio_capture(context, manager=manager, turn_audio=CAPS) + assert binding.status == "session_bound" and binding.turn_status == "policy_denied" + assert binding.processor._enable_turn_audio is False + events = {e for e, _ in binding._handlers} + assert "on_audio_data" in events and not events & {"on_user_turn_audio", "on_bot_turn_audio"} + binding.close() + finally: + manager.shutdown(1) + span.end() + + +@pytest.mark.asyncio +async def test_turn_upload_binds_on_parent_and_never_projects_refs(manager, transport): + # Even when the turn span is still live at upload time, the turn span gets + # no fi.attachment.* attribute; the upload names it in association.parent. + pairs = pair_uploads(transport) + processor, observer, binding = await turn_binding(manager) + await observer._turn_tracker._call_event_handler("on_turn_started", 1) + for _ in range(3): + await asyncio.sleep(0) + await dispatch(processor, "on_user_turn_audio", TurnAudioData(1, tone(300, 200), RATE, 1)) + for _, _, handle in binding.turn_handles: + assert (await handle.wait(10)).status != "unfinished" + await observer._turn_tracker._call_event_handler("on_turn_ended", 1, 1.0, False) + for _ in range(3): + await asyncio.sleep(0) + await binding.finish(10) + observer.end_conversation_tracing() + turn = [s for s in EXPORTER.get_finished_spans() if s.name == "turn"] + assert len(turn) == 1 + assert not [k for k in turn[0].attributes if k.startswith("fi.attachment")] + uploads = turn_pairs(pairs) + assert len(uploads) == 1 + assert uploads[0][0]["association"]["parent"]["span_id"] == format(turn[0].context.span_id, "016x")