From a0cf3dc4363342e78820229235ab1d64186db6a4 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 18:57:32 -0800 Subject: [PATCH 1/9] feat(python): add opt-in voice attachment binding components Keep Pipecat streams and completed Egress imports fail-closed until the shared transport prerequisites are qualified. Co-Authored-By: Claude Code --- .../workflows/python-voice-attachments.yml | 53 +++ .../livekit/traceai_livekit/__init__.py | 13 +- .../livekit/traceai_livekit/attachments.py | 296 ++++++++++++++++ .../livekit/traceai_livekit/recording.py | 64 ++++ .../pipecat/traceai_pipecat/__init__.py | 14 + .../pipecat/traceai_pipecat/attachments.py | 95 +++++ python/voice-tests/conftest.py | 16 + python/voice-tests/process_exit_probe.py | 44 +++ .../voice-tests/test_livekit_attachments.py | 330 ++++++++++++++++++ .../voice-tests/test_pipecat_attachments.py | 63 ++++ 10 files changed, 987 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/python-voice-attachments.yml create mode 100644 python/frameworks/livekit/traceai_livekit/attachments.py create mode 100644 python/frameworks/pipecat/traceai_pipecat/attachments.py create mode 100644 python/voice-tests/conftest.py create mode 100644 python/voice-tests/process_exit_probe.py create mode 100644 python/voice-tests/test_livekit_attachments.py create mode 100644 python/voice-tests/test_pipecat_attachments.py diff --git a/.github/workflows/python-voice-attachments.yml b/.github/workflows/python-voice-attachments.yml new file mode 100644 index 000000000..3068391fa --- /dev/null +++ b/.github/workflows/python-voice-attachments.yml @@ -0,0 +1,53 @@ +name: Python Voice Attachments + +on: + pull_request: + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/livekit/**' + - 'python/frameworks/pipecat/**' + - 'python/voice-tests/**' + - 'python/tests/attachments/**' + - '.github/workflows/python-voice-attachments.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + components: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + python: ['3.10', '3.11', '3.12'] + adapter: + - {name: pipecat, upstream: pipecat-ai, version: '1.0.0'} + - {name: pipecat, upstream: pipecat-ai, version: '1.15.0'} + - {name: livekit, upstream: livekit-agents, version: '1.8.10'} + - {name: livekit, upstream: livekit-agents, version: '1.8.11'} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install system codec + run: sudo apt-get update -qq && sudo apt-get install -y ffmpeg libsndfile1 + - name: Install exact upstream with this checkout + run: | + python -m pip install -e python -e python/frameworks/${{ matrix.adapter.name }} \ + '${{ matrix.adapter.upstream }}==${{ matrix.adapter.version }}' \ + opentelemetry-exporter-otlp-proto-grpc pytest pytest-asyncio jsonschema + python -m pip check + python -m pip freeze > voice-versions.txt + - name: Run voice components and negative controls + run: python -m pytest python/voice-tests/test_${{ matrix.adapter.name }}_attachments.py -q --tb=short + - name: Run existing LiveKit regression tests + if: matrix.adapter.name == 'livekit' + run: python -m pytest python/frameworks/livekit/tests -q --tb=short + - uses: actions/upload-artifact@v4 + if: always() + with: + name: voice-versions-${{ matrix.python }}-${{ matrix.adapter.name }}-${{ matrix.adapter.version }} + path: voice-versions.txt diff --git a/python/frameworks/livekit/traceai_livekit/__init__.py b/python/frameworks/livekit/traceai_livekit/__init__.py index ce846a40d..e6c780f76 100644 --- a/python/frameworks/livekit/traceai_livekit/__init__.py +++ b/python/frameworks/livekit/traceai_livekit/__init__.py @@ -19,7 +19,16 @@ flush_on_job_shutdown, ) -from .recording import record_egress, wrap_egress_client +from .recording import import_completed_egress, record_egress, wrap_egress_client + + +def bind_session(session, job_context, *, manager, owned_recording_files=False, + finalization_timeout=30, capture_policy=None): + """Opt-in existing recording import; requires the attachment-enabled core.""" + from .attachments import bind_session as bind + return bind(session, job_context, manager=manager, owned_recording_files=owned_recording_files, + finalization_timeout=finalization_timeout, capture_policy=capture_policy) + __version__ = "0.2.1" # x-release-please-version @@ -30,6 +39,8 @@ "create_mapped_http_exporter", "create_mapped_grpc_exporter", "flush_on_job_shutdown", + "bind_session", + "import_completed_egress", "record_egress", "wrap_egress_client", # Exporters diff --git a/python/frameworks/livekit/traceai_livekit/attachments.py b/python/frameworks/livekit/traceai_livekit/attachments.py new file mode 100644 index 000000000..e4432732e --- /dev/null +++ b/python/frameworks/livekit/traceai_livekit/attachments.py @@ -0,0 +1,296 @@ +"""Opt-in import of a bound LiveKit session's existing local recording. + +This adapter never enables recording. It intentionally supports only the exact +Agents versions exercised by the component suite: 1.8.10 and 1.8.11. The private +SDK lifecycle seam is confined here and fails closed on unknown versions. +""" + +import asyncio +import inspect +import math +import os +import stat +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path + +from fi_instrumentation.attachments import ( + CapturePolicy, CorrelationHandle, RecordingFileScope, +) +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, get_value + +_SUPPORTED = frozenset(("1.8.10", "1.8.11")) +_BINDING = "_traceai_recording_binding" + + +def _admission(manager, capture, correlation): + """Use the core's policy, never a weaker adapter-local privacy lattice. + + P01 does not yet expose a public preflight. This versioned seam reads its + negotiated snapshot; attach() and the worker repeat admission before IO. + """ + try: + session = manager._session + reason = manager.policy.evaluate( + session.policy if session else None, capture, direction="mixed", + mime_type="audio/ogg", source_kind="OwnedRecordingFile", + ) + if reason: + return reason + if get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return "policy_denied" + if manager.status != "ready": + return manager.status + if correlation is None: + return "correlation_unavailable" + if not correlation.sampled: + return "unsampled" + except Exception: + return "source_unavailable" + return None + + +def _native_denial(session, job, processors): + # Native integration restrictions precede even recording-path inspection. + # Text redaction never sanitizes binary recordings. + from livekit.agents.telemetry.utils import allow_pii_from_env + if allow_pii_from_env() is False or any(p._allow_pii is not True for p in processors): + return "policy_denied" + options = session.options.recording_options + if (options.get("redaction", False) is not False + or getattr(job, "_redaction_enabled", True) is not False): + return "redaction_unavailable" + if options.get("audio") is not True: + return "policy_denied" + return None + + +class SessionRecordingBinding: + """A content-free status and awaitable, at-most-once finalizer. + + ``validating`` means the bounded file reached quarantine, NOT playable. + SessionReport files larger than the core's limit remain ``too_large``. + ``close()`` only detaches this adapter; it never stops the application's + session, shuts down the shared manager or enables/deletes a recording. + """ + + def __init__(self, session, job_context, manager, *, enabled, timeout, capture): + self._session = session + self._job = job_context + self._manager = manager + self._capture = capture + self._enabled = enabled + self._timeout = timeout + self._correlation = None + self._privacy_processors = () + self._root = None + self._original = None + self._wrapper = None + self._lock = asyncio.Lock() + self._finished = False + self._closed = False + self._scope = None + self._handle = None + self.status = "session_bound" + + @property + def handle(self): + return self._handle + + async def finalize(self): + """Await delivery before SDK tempdir cleanup, with an explicit deadline.""" + async with self._lock: + if self._finished or self._closed: + return self.status + self._finished = True + if not self._enabled: + self.status = "disabled" + self.close() + return self.status + reason = _admission(self._manager, self._capture, self._correlation) + if reason: + self.status = reason + self.close() + return self.status + try: + # All file work is in this finalization callback, never in an + # inference/audio callback. A bounded source is handed to the + # manager; no caller stream is read or advanced. + await asyncio.wait_for(self._capture_report(), timeout=self._timeout) + except asyncio.TimeoutError: + self.status = "incomplete" + except asyncio.CancelledError: + self.status = "incomplete" + raise + except Exception: + # Do not log repr, report, path, or exception text. + self.status = "source_unavailable" + finally: + if self._handle is not None and self.status != "validating": + self._manager.cancel(self._handle) + if self._scope is not None: + self._scope.close() + self._scope = None + self.close() + return self.status + + async def _capture_report(self): + from livekit.agents.voice.recorder_io import RecorderIO + + session, job = self._session, self._job + reason = _native_denial(session, job, self._privacy_processors) + if reason: + self.status = reason + return + # The report's path is NOT authority. Only the exact primary session, + # its SDK recorder and fixed SDK-owned filename under the bound job's + # directory are eligible. No user-supplied root, LocalFile or URL. + if job._primary_agent_session is not session: + self.status = "source_unavailable" + return + recorder = session._recorder_io + if not isinstance(recorder, RecorderIO) or recorder.recording: + self.status = "source_unavailable" + return + expected = self._root / "audio.ogg" + if recorder.output_path != expected or job.session_directory != self._root: + self.status = "source_unavailable" + return + report = job.make_session_report(session) + if report.audio_recording_path != expected: + self.status = "source_unavailable" + return + # No root or file stat/open occurs on a policy-denied report. + reason = _admission(self._manager, self._capture, self._correlation) + if reason: + self.status = reason + return + scope = RecordingFileScope(str(self._root), enabled=True) + self._scope = scope + # Scope's no-follow root FD is part of this P01-version-qualified seam. + # The worker reopens independently and compares identity and size. + info = os.stat("audio.ogg", dir_fd=scope._fd, follow_symlinks=False) + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_uid != os.getuid(): + self.status = "source_unavailable" + return + source = scope.source("audio.ogg", device=info.st_dev, inode=info.st_ino, size=info.st_size) + handle = self._manager.attach( + self._correlation, source, mime_type="audio/ogg", direction="mixed", + purpose="conversation_recording", capture_policy=self._capture, + ) + self._handle = handle + self.status = handle.admission + if handle.admission == "queued": + # Poll the content-free handle, not a blocking executor waiter. + # Cancelling finalization must not leave asyncio.run() waiting for + # a background Event.wait(timeout) during executor shutdown. + while (result := handle.result()).status == "unfinished": + await asyncio.sleep(0.01) + self.status = result.status + + def close(self): + self._closed = True + if self._handle is not None and self._handle.result().status == "unfinished": + self._manager.cancel(self._handle) + # Do not overwrite a hook installed by somebody else after this binding. + if self._wrapper is not None and getattr(self._job, "_on_session_end", None) is self._wrapper: + self._job._on_session_end = self._original + if getattr(self._job, _BINDING, None) is self: + delattr(self._job, _BINDING) + self._session = self._job = self._root = None + self._original = self._wrapper = None + self._privacy_processors = () + + +def bind_session(session, job_context, *, manager, owned_recording_files=False, + finalization_timeout=30, capture_policy=None): + """Bind AFTER ``await session.start(...)``; never start a recorder. + + The actual session span is snapshotted now, before LiveKit ends/clears it. + ``job_context._on_session_end`` is chained, not discarded. The SDK awaits + this hook after session close and before its tempdir cleanup. User-provided + AgentServer ``on_session_end`` runs earlier and remains untouched; if it + deletes the source first, this adapter reports ``source_unavailable``. + + Configure the worker's session-end/shutdown grace to exceed this deadline + plus the application's own callbacks. Hard process termination is lossy. + """ + if type(owned_recording_files) is not bool: + raise ValueError("owned recording permission must be explicit") + if (type(finalization_timeout) not in (int, float) + or not math.isfinite(finalization_timeout) or not 0 < finalization_timeout <= 30): + raise ValueError("finalization deadline must be within 30 seconds") + capture = capture_policy if capture_policy is not None else CapturePolicy() + if not isinstance(capture, CapturePolicy) or capture.new_recording: + raise ValueError("session import does not create a recording") + previous = getattr(job_context, _BINDING, None) + if previous is not None: + if (previous._session is session and previous._manager is manager + and previous._enabled == owned_recording_files + and previous._timeout == finalization_timeout and previous._capture == capture): + return previous + raise ValueError("conflicting recording binding") + binding = SessionRecordingBinding(session, job_context, manager, enabled=owned_recording_files, + timeout=finalization_timeout, capture=capture) + if not owned_recording_files: + binding.status = "disabled" + binding.close() + return binding + try: + if version("livekit-agents") not in _SUPPORTED: + binding.status = "unsupported_version" + binding.close() + return binding + from livekit.agents import AgentSession, JobContext + if not isinstance(session, AgentSession) or not isinstance(job_context, JobContext): + binding.status = "unsupported_version" + binding.close() + return binding + # Never fall back to an ambient span from an unrelated late callback. + span = session._session_span + if span is None: + binding.status = "correlation_unavailable" + binding.close() + return binding + binding._correlation = CorrelationHandle.from_span(span) + from livekit.agents.telemetry.pii import _PIIFilteringSpanProcessor + processors = getattr(getattr(span, "_span_processor", None), "_span_processors", ()) + binding._privacy_processors = tuple(p for p in processors if isinstance(p, _PIIFilteringSpanProcessor)) + reason = _admission(manager, capture, binding._correlation) + if reason: + binding.status = reason + binding.close() + return binding + if job_context._primary_agent_session is not session: + binding.status = "source_unavailable" + binding.close() + return binding + reason = _native_denial(session, job_context, binding._privacy_processors) + if reason: + binding.status = reason + binding.close() + return binding + binding._root = Path(job_context.session_directory) + original = job_context._on_session_end + if not inspect.iscoroutinefunction(original): + binding.status = "unsupported_version" + binding.close() + return binding + + async def chained(): + # Keep original cleanup/report delivery even if instrumentation is + # cancelled. Ordinary adapter failures are already content-free. + try: + await binding.finalize() + finally: + await original() + + binding._original, binding._wrapper = original, chained + job_context._on_session_end = chained + setattr(job_context, _BINDING, binding) + except PackageNotFoundError: + binding.status = "unsupported_version" + except Exception: + binding.status = "source_unavailable" + if binding.status != "session_bound": + binding.close() + return binding diff --git a/python/frameworks/livekit/traceai_livekit/recording.py b/python/frameworks/livekit/traceai_livekit/recording.py index f95c26cdb..b0e8728ac 100644 --- a/python/frameworks/livekit/traceai_livekit/recording.py +++ b/python/frameworks/livekit/traceai_livekit/recording.py @@ -225,3 +225,67 @@ def wrap_egress_client( client, parent=parent, room_name=room_name, conversation_id=conversation_id, tracer=selected, trace_config=config, ) + + +def import_completed_egress(egress_info, *, session_context, egress_id, room_name, + manager, existing_source_authorized=False, capture_policy=None): + """Offer an explicitly authorized completed job to the shared private importer. + + This does NOT start/stop Egress, poll, inspect output locations, or select a + server connection. Pass an actual EgressInfo and its independently bound + egress/room IDs, not IDs copied out of an untrusted callback. The SDK cannot + authenticate a webhook; that is the durable backend observer's job. + + The current core has no qualified LiveKit ProviderFile resolver, so completed + jobs return ``source_unavailable`` (or a stricter policy denial), never ready. + Start/stop snapshots alone do not establish completion. Metadata-only + record_egress()/wrap_egress_client() keep their existing behavior. + """ + # Lazy imports keep metadata-only installs compatible with released cores. + from fi_instrumentation.attachments import ( + AttachmentHandle, CapturePolicy, CorrelationHandle, ProviderFile, + ) + if existing_source_authorized is not True: + return AttachmentHandle("disabled") + try: + capture = capture_policy if capture_policy is not None else CapturePolicy() + if not isinstance(capture, CapturePolicy) or capture.new_recording: + return AttachmentHandle("policy_denied") + session = manager._session + reason = manager.policy.evaluate( + session.policy if session else None, capture, direction="mixed", + mime_type="audio/ogg", source_kind="ProviderFile", + ) + if reason: + return AttachmentHandle(reason) + if context.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return AttachmentHandle("policy_denied") + if manager.status != "ready": + return AttachmentHandle(manager.status) + if session_context is None: + return AttachmentHandle("correlation_unavailable") + parent = CorrelationHandle.from_span(session_context) + if not parent.sampled: + return AttachmentHandle("unsampled") + from importlib.metadata import version + if version("livekit-agents") not in ("1.8.10", "1.8.11"): + return AttachmentHandle("unsupported_version") + from livekit import api + if not isinstance(egress_info, api.EgressInfo): + return AttachmentHandle("unsupported_version") + if (type(egress_id) is not str or not 0 < len(egress_id) <= 256 + or type(room_name) is not str or not 0 < len(room_name) <= 256 + or egress_info.egress_id != egress_id or egress_info.room_name != room_name): + return AttachmentHandle("source_unavailable") + if egress_info.status in (api.EGRESS_STARTING, api.EGRESS_ACTIVE, api.EGRESS_ENDING): + return AttachmentHandle("waiting_for_completion") + if egress_info.status != api.EGRESS_COMPLETE or egress_info.ended_at <= 0: + return AttachmentHandle("source_unavailable") + # No URL, filename, credentials or output metadata crosses this seam. + # Fail closed until the core has a version-qualified fixed-ID resolver + # AND explicitly delegated private source authority from the broker. + return manager.attach(parent, ProviderFile(provider="livekit", file_id=egress_id), + mime_type="audio/ogg", direction="mixed", + purpose="conversation_recording", capture_policy=capture) + except Exception: + return AttachmentHandle("source_unavailable") diff --git a/python/frameworks/pipecat/traceai_pipecat/__init__.py b/python/frameworks/pipecat/traceai_pipecat/__init__.py index ec1cc7526..5efc76a62 100644 --- a/python/frameworks/pipecat/traceai_pipecat/__init__.py +++ b/python/frameworks/pipecat/traceai_pipecat/__init__.py @@ -18,9 +18,23 @@ enable_http_attribute_mapping, ) +def bind_audio_processor(processor, session_context, *, manager): + """Check an existing processor without starting recording or retaining PCM.""" + from .attachments import bind_audio_processor as bind + return bind(processor, session_context, manager=manager) + + +def create_audio_capture(session_context, *, manager): + """Check separate recording consent; no recorder until stream support exists.""" + from .attachments import create_audio_capture as create + return create(session_context, manager=manager) + + __version__ = "0.1.2" # x-release-please-version __all__ = [ + "bind_audio_processor", + "create_audio_capture", "enable_fi_attribute_mapping", "enable_http_attribute_mapping", "enable_grpc_attribute_mapping", diff --git a/python/frameworks/pipecat/traceai_pipecat/attachments.py b/python/frameworks/pipecat/traceai_pipecat/attachments.py new file mode 100644 index 000000000..b885d5122 --- /dev/null +++ b/python/frameworks/pipecat/traceai_pipecat/attachments.py @@ -0,0 +1,95 @@ +"""Fail-closed Pipecat recording binding until the shared stream API is ready. + +Nonzero AudioBufferProcessor.buffer_size is a flush threshold, not a hard memory +cap: oversized frames, wall-clock silence and turn buffers can exceed it. Do not +label a series of unmanifested clips as a complete conversation recording. +""" + +from dataclasses import dataclass +from importlib.metadata import PackageNotFoundError, version + +from fi_instrumentation.attachments import CapturePolicy, CorrelationHandle +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, get_value + +_SUPPORTED = frozenset(("1.0.0", "1.15.0")) + + +@dataclass(frozen=True) +class AudioSessionContext: + """The actual conversation identity, never a late ambient span lookup.""" + + correlation: CorrelationHandle + + @classmethod + def from_span(cls, span): + return cls(CorrelationHandle.from_span(span)) + + +@dataclass(frozen=True) +class AudioCaptureBinding: + """A status is not evidence of a recorder or a recording source.""" + + status: str + processor: object = None + + +def _admission(manager, session_context, *, new_recording): + try: + capture = CapturePolicy(new_recording=new_recording) + session = manager._session + reason = manager.policy.evaluate( + session.policy if session else None, capture, direction="mixed", + mime_type="audio/wav", source_kind="Bytes", + ) + if reason: + return reason + if get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return "policy_denied" + if manager.status != "ready": + return manager.status + if not isinstance(session_context, AudioSessionContext): + return "correlation_unavailable" + if not session_context.correlation.sampled: + return "unsampled" + except Exception: + return "source_unavailable" + return None + + +def bind_audio_processor(processor, session_context, *, manager): + """Inspect eligibility for an existing recorder; never start or mutate it. + + This component currently returns ``streaming_unavailable`` for otherwise + eligible processors. P02 still lacks stream/manifest transport. In + particular, it must not register a callback which retains PCM, uploads + unlinked chunks, or pretends the upstream flush threshold is a hard bound. + Both auto and explicit callers receive this same truthful state. + """ + reason = _admission(manager, session_context, new_recording=False) + if reason: + return AudioCaptureBinding(reason) + try: + if version("pipecat-ai") not in _SUPPORTED: + return AudioCaptureBinding("unsupported_version") + from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor + if not isinstance(processor, AudioBufferProcessor): + return AudioCaptureBinding("unsupported_version") + if (type(processor._buffer_size) is not int or not 0 < processor._buffer_size <= 128 * 1024 + or processor._enable_turn_audio): + return AudioCaptureBinding("unbounded_source") + except PackageNotFoundError: + return AudioCaptureBinding("unsupported_version") + except Exception: + return AudioCaptureBinding("source_unavailable") + return AudioCaptureBinding("streaming_unavailable") + + +def create_audio_capture(session_context, *, manager): + """Do not create/start a new recorder without both recording permissions. + + Even with permission, this release fails closed until bounded upstream + processing, stream segments, gaps and manifests can be transported. No + processor is returned for pipeline insertion; existing audio is untouched. + """ + reason = _admission(manager, session_context, new_recording=True) + return AudioCaptureBinding(reason or "streaming_unavailable") diff --git a/python/voice-tests/conftest.py b/python/voice-tests/conftest.py new file mode 100644 index 000000000..eb868b5ff --- /dev/null +++ b/python/voice-tests/conftest.py @@ -0,0 +1,16 @@ +"""Voice fixtures reuse the core's synthetic transport and real TLS server.""" + +import importlib.util +import sys +from pathlib import Path + +_CORE_TESTS = Path(__file__).resolve().parents[1] / "tests" / "attachments" +sys.path.insert(0, str(_CORE_TESTS)) +_spec = importlib.util.spec_from_file_location("attachment_test_support", _CORE_TESTS / "conftest.py") +_support = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_support) +TransportDouble = _support.TransportDouble +clean_attachment_env = _support.clean_attachment_env +manager = _support.manager +transport = _support.transport +from test_http_contract import server # noqa: E402,F401 diff --git a/python/voice-tests/process_exit_probe.py b/python/voice-tests/process_exit_probe.py new file mode 100644 index 000000000..694cbc2f9 --- /dev/null +++ b/python/voice-tests/process_exit_probe.py @@ -0,0 +1,44 @@ +"""Synthetic cancellation probe executed in a separate interpreter by pytest.""" + +import asyncio +import tempfile +from pathlib import Path + +from conftest import TransportDouble +from fi_instrumentation.attachments import AttachmentConfig, AttachmentManager +from test_livekit_attachments import owned_session +from traceai_livekit import bind_session + + +async def main(): + wire = TransportDouble() + wire.release.clear() + manager = AttachmentManager(config=AttachmentConfig(enabled=True), project_name="synthetic", transport=wire) + assert manager.ready(2) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + session, job, _, span, provider, calls = owned_session(root) + (root / "audio.ogg").write_bytes(b"synthetic") + binding = bind_session(session, job, manager=manager, owned_recording_files=True, finalization_timeout=30) + task = asyncio.create_task(job._on_session_end()) + for _ in range(200): + if wire.entered.is_set(): + break + await asyncio.sleep(.01) + assert wire.entered.is_set() + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + assert binding.status == "incomplete" and calls == ["original"] + assert binding._scope is None + wire.release.set() + manager.shutdown(.2) + span.end() + provider.shutdown() + print("bounded-cancel-exit") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/voice-tests/test_livekit_attachments.py b/python/voice-tests/test_livekit_attachments.py new file mode 100644 index 000000000..49076045c --- /dev/null +++ b/python/voice-tests/test_livekit_attachments.py @@ -0,0 +1,330 @@ +"""Synthetic component tests, not a real SFU call or product-playback claim.""" + +import asyncio +import inspect +import io +import os +import subprocess +import time +from dataclasses import replace +from importlib.metadata import version +from types import SimpleNamespace + +import av +import pytest +from livekit import api +from livekit.agents import AgentSession, JobContext +from livekit.agents.voice.recorder_io import RecorderIO +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, attach, detach, set_value +from opentelemetry.sdk.trace import TracerProvider + +from fi_instrumentation.attachments import AttachmentConfig, AttachmentManager, CapturePolicy +from fi_instrumentation.instrumentation.config import TraceConfig +from traceai_livekit import bind_session, import_completed_egress + + +def owned_session(tmp_path, *, recording=True): + """Real AgentSession, RecorderIO and report method; no RTC/network startup.""" + provider = TracerProvider() + span = provider.get_tracer("synthetic").start_span("actual-conversation") + session = AgentSession() + session._session_span = span + session._opts.recording_options = {"audio": True, "redaction": False} + recorder = RecorderIO(agent_session=session) + recorder._output_path = tmp_path / "audio.ogg" + session._recorder_io = recorder if recording else None + # Build only the job shell that make_session_report needs. Constructors that + # connect to an SFU are not exercised by these deterministic components. + job = JobContext.__new__(JobContext) + job._session_directory = tmp_path + job._primary_agent_session = session + job._redaction_enabled = False + job._tagger = SimpleNamespace(tags=set()) + job._info = SimpleNamespace(job=api.Job(id="synthetic-job", room=api.Room(name="synthetic", sid="synthetic-room"))) + original_calls = [] + + async def original(): + original_calls.append("original") + + job._on_session_end = original + return session, job, recorder, span, provider, original_calls + + +def make_ogg(path): + subprocess.run(["ffmpeg", "-nostdin", "-v", "error", "-f", "lavfi", "-i", + "sine=frequency=440:duration=0.1:sample_rate=48000", "-ac", "2", + "-c:a", "libopus", "-y", str(path)], check=True, capture_output=True) + return path.read_bytes() + + +@pytest.mark.asyncio +async def test_real_report_tls_upload_survives_span_end_before_cleanup(server, tmp_path): + wire, state = server + manager = AttachmentManager(config=AttachmentConfig(enabled=True), project_name="synthetic", transport=wire) + assert manager.ready(2) + session, job, recorder, span, provider, calls = owned_session(tmp_path) + raw = make_ogg(tmp_path / "audio.ogg") + # This is the real SDK report, not a report-path test double. + assert job.make_session_report(session).audio_recording_path == recorder.output_path + binding = bind_session(session, job, manager=manager, owned_recording_files=True) + assert binding.status == "session_bound" + expected_trace = format(span.get_span_context().trace_id, "032x") + span.end() + session._session_span = None + try: + with provider.get_tracer("unrelated").start_as_current_span("another-session"): + await job._on_session_end() + assert binding.status == "validating" + assert state["payloads"] == [raw] + request = next(body for path, body, _ in state["requests"] if path.endswith("/uploads")) + assert request["association"]["parent"]["trace_id"].replace("-", "") == expected_trace + assert request["direction"] == "mixed" + assert "audio.ogg" not in str(state["requests"]) + assert calls == ["original"] + # Independent codec decode verifies actual bytes and channel count. + with av.open(io.BytesIO(state["payloads"][0])) as media: + frames = list(media.decode(audio=0)) + assert sum(frame.samples for frame in frames) == 4800 + assert len(frames[0].layout.channels) == 2 + os.unlink(tmp_path / "audio.ogg") + assert await binding.finalize() == "validating" + assert binding._session is None and binding._job is None + finally: + manager.shutdown(1) + provider.shutdown() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("flag", ["FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", "FI_HIDE_INPUT_MESSAGES", + "FI_HIDE_OUTPUT_MESSAGES", "FI_HIDE_INPUT_TEXT", "FI_HIDE_OUTPUT_TEXT", + "FI_PII_REDACTION"]) +async def test_denial_precedes_sdk_report_or_filesystem(manager, tmp_path, monkeypatch, flag): + session, job, _, _, provider, _ = owned_session(tmp_path) + binding = bind_session(session, job, manager=manager, owned_recording_files=True) + monkeypatch.setenv(flag, "true") + def forbidden(*args, **kwargs): + pytest.fail("denied capture inspected a report or file") + monkeypatch.setattr(job, "make_session_report", forbidden) + monkeypatch.setattr("traceai_livekit.attachments.RecordingFileScope", forbidden) + assert await binding.finalize() in ("policy_denied", "redaction_unavailable") + assert binding.handle is None + provider.shutdown() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("case", ["absent", "missing", "symlink", "hardlink", "foreign_path", "active", "wrong_session", + "recording_off", "redaction", "server_redaction"]) +async def test_unavailable_or_denied_owned_recordings_never_transfer(manager, transport, tmp_path, case): + session, job, recorder, _, provider, _ = owned_session(tmp_path, recording=case != "absent") + path = tmp_path / "audio.ogg" + if case not in ("absent", "missing"): + path.write_bytes(b"synthetic") + if case == "symlink": + path.rename(tmp_path / "other") + path.symlink_to(tmp_path / "other") + elif case == "hardlink": + os.link(path, tmp_path / "other") + elif case == "foreign_path": + recorder._output_path = tmp_path / "other" + elif case == "active": + recorder._started = True + elif case == "recording_off": + session._opts.recording_options["audio"] = False + elif case == "redaction": + session._opts.recording_options["redaction"] = True + elif case == "server_redaction": + job._redaction_enabled = True + binding = bind_session(session, job, manager=manager, owned_recording_files=True) + if case == "wrong_session": + job._primary_agent_session = AgentSession() + assert await binding.finalize() in ("source_unavailable", "policy_denied", "redaction_unavailable") + assert not transport.payloads + provider.shutdown() + + +@pytest.mark.asyncio +async def test_delayed_transfer_deadline_and_original_callback(manager, transport, tmp_path): + session, job, _, _, provider, calls = owned_session(tmp_path) + (tmp_path / "audio.ogg").write_bytes(b"synthetic") + transport.release.clear() + binding = bind_session(session, job, manager=manager, owned_recording_files=True, finalization_timeout=.1) + start = time.monotonic() + await job._on_session_end() + assert time.monotonic() - start < 1 + assert binding.status == "incomplete" and calls == ["original"] + assert binding._scope is None + transport.release.set() + assert (await manager.flush_async(2)).unfinished == 0 + assert not transport.payloads + provider.shutdown() + + +@pytest.mark.asyncio +async def test_file_replacement_after_identity_snapshot_is_rejected(manager, transport, tmp_path, monkeypatch): + session, job, _, _, provider, _ = owned_session(tmp_path) + path = tmp_path / "audio.ogg" + path.write_bytes(b"owned") + original = manager.attach + def replace_before_attach(*args, **kwargs): + path.rename(tmp_path / "original") + path.write_bytes(b"rogue") + return original(*args, **kwargs) + monkeypatch.setattr(manager, "attach", replace_before_attach) + binding = bind_session(session, job, manager=manager, owned_recording_files=True) + assert await binding.finalize() == "source_unavailable" + assert not transport.payloads + provider.shutdown() + + +@pytest.mark.asyncio +async def test_oversize_file_preserves_caller_position(manager, transport, tmp_path): + session, job, _, _, provider, _ = owned_session(tmp_path) + path = tmp_path / "audio.ogg" + with path.open("wb") as file: + file.truncate(8 * 1024 * 1024 + 1) + with path.open("rb") as file: + file.seek(3) + binding = bind_session(session, job, manager=manager, owned_recording_files=True) + assert await binding.finalize() == "too_large" + assert file.tell() == 3 + assert not transport.payloads + provider.shutdown() + + +@pytest.mark.asyncio +async def test_disable_unstarted_and_duplicate_binding(manager, tmp_path): + session, job, _, span, provider, _ = owned_session(tmp_path) + original = job._on_session_end + assert bind_session(session, job, manager=manager).status == "disabled" + assert job._on_session_end == original + binding = bind_session(session, job, manager=manager, owned_recording_files=True) + assert bind_session(session, job, manager=manager, owned_recording_files=True) is binding + with pytest.raises(ValueError, match="conflicting"): + bind_session(session, job, manager=manager, owned_recording_files=True, finalization_timeout=1) + binding.close() + assert job._on_session_end == original + session._session_span = None + assert bind_session(session, job, manager=manager, owned_recording_files=True).status == "correlation_unavailable" + span.end() + provider.shutdown() + + +@pytest.mark.asyncio +async def test_cancellation_preserves_original_and_closes_scope(manager, transport, tmp_path): + session, job, _, _, provider, calls = owned_session(tmp_path) + (tmp_path / "audio.ogg").write_bytes(b"synthetic") + transport.release.clear() + binding = bind_session(session, job, manager=manager, owned_recording_files=True, finalization_timeout=.2) + task = asyncio.create_task(job._on_session_end()) + assert await asyncio.to_thread(transport.entered.wait, 2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert binding.status == "incomplete" and calls == ["original"] + assert binding._scope is None + transport.release.set() + await manager.flush_async(2) + provider.shutdown() + + +@pytest.mark.asyncio +async def test_policy_tightening_cancels_admitted_file(manager, transport, tmp_path): + session, job, _, _, provider, _ = owned_session(tmp_path) + (tmp_path / "audio.ogg").write_bytes(b"synthetic") + transport.release.clear() + binding = bind_session(session, job, manager=manager, owned_recording_files=True) + task = asyncio.create_task(binding.finalize()) + assert await asyncio.to_thread(transport.entered.wait, 2) + manager.update_policy(replace(transport.policy, enabled=False)) + transport.release.set() + assert await task == "policy_denied" + assert not transport.payloads + provider.shutdown() + + +def test_exact_worker_lifecycle_orders_close_hook_before_cleanup(): + # Version-locked source assertions complement (not replace) the synthetic + # hook exercise. A whole Worker/SFU shutdown is a separate Q02 gate. + from livekit.agents.ipc.job_proc_lazy_main import _JobProc + shutdown = inspect.getsource(_JobProc._shutdown_job) + runner = inspect.getsource(_JobProc._run_job_task) + assert shutdown.index("session.aclose()") < shutdown.index("await self._job_ctx._on_session_end()") + assert runner.index("await self._shutdown_job(") < runner.index("await self._job_ctx._on_cleanup()") + assert version("livekit-agents") in ("1.8.10", "1.8.11") + + +@pytest.mark.parametrize("state,expected", [(api.EGRESS_STARTING, "waiting_for_completion"), + (api.EGRESS_ACTIVE, "waiting_for_completion"), (api.EGRESS_ENDING, "waiting_for_completion"), + (api.EGRESS_COMPLETE, "source_unavailable"), (api.EGRESS_FAILED, "source_unavailable")]) +def test_egress_never_marks_start_stop_or_complete_as_playable(manager, transport, state, expected): + from fi_instrumentation.attachments import CorrelationHandle + parent = CorrelationHandle("1" * 32, "2" * 16, True) + manager.update_policy(replace(transport.policy, allowed_sources=frozenset(("ProviderFile",)))) + info = api.EgressInfo(egress_id="synthetic", room_name="room", status=state, ended_at=1) + info.file_results.add(location="https://private.invalid/CANARY_SIGNED_PATH?token=SECRET") + handle = import_completed_egress(info, session_context=parent, egress_id="synthetic", room_name="room", + manager=manager, existing_source_authorized=True) + assert handle.admission == expected + assert not transport.requests and not transport.payloads + assert "CANARY" not in repr(handle) + + +def test_egress_no_authority_does_not_inspect_response(manager): + class Poison: + def __getattribute__(self, name): + raise AssertionError("denied object inspected") + assert import_completed_egress(Poison(), session_context=None, egress_id="", room_name="", manager=manager).admission == "disabled" + + +@pytest.mark.asyncio +async def test_adapter_exception_never_logs_source_or_replaces_original(manager, tmp_path, monkeypatch, caplog): + session, job, _, _, provider, calls = owned_session(tmp_path) + binding = bind_session(session, job, manager=manager, owned_recording_files=True) + def fail(*args): + raise ValueError("CANARY_PRIVATE_RECORDING_PATH") + monkeypatch.setattr(job, "make_session_report", fail) + await job._on_session_end() + assert binding.status == "source_unavailable" and calls == ["original"] + assert "CANARY" not in caplog.text + provider.shutdown() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["environment", "provider"]) +async def test_native_livekit_pii_denial_wins(manager, transport, tmp_path, monkeypatch, mode): + from livekit.agents.telemetry.pii import _PIIFilteringSpanProcessor + session, job, _, span, provider, _ = owned_session(tmp_path) + processor = _PIIFilteringSpanProcessor(allow_pii=False) + if mode == "provider": + provider.add_span_processor(processor) + else: + monkeypatch.setenv("LIVEKIT_TELEMETRY_ALLOW_PII", "0") + job._session_directory = object() # Path conversion must never be reached. + binding = bind_session(session, job, manager=manager, owned_recording_files=True) + assert await binding.finalize() == "policy_denied" + assert not transport.payloads + span.end() + provider.shutdown() + + +@pytest.mark.asyncio +async def test_suppression_and_unsampled_context_never_open_files(manager, tmp_path): + from opentelemetry import trace + session, job, _, span, provider, _ = owned_session(tmp_path) + token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) + try: + assert bind_session(session, job, manager=manager, owned_recording_files=True).status == "policy_denied" + finally: + detach(token) + session._session_span = trace.NonRecordingSpan(trace.SpanContext(1, 2, False, trace.TraceFlags(0))) + assert bind_session(session, job, manager=manager, owned_recording_files=True).status == "unsampled" + span.end() + provider.shutdown() + + +def test_cancelled_finalization_does_not_hold_process_for_full_deadline(): + import sys + from pathlib import Path + result = subprocess.run([sys.executable, str(Path(__file__).with_name("process_exit_probe.py"))], + timeout=10, check=True, capture_output=True, text=True) + assert "bounded-cancel-exit" in result.stdout diff --git a/python/voice-tests/test_pipecat_attachments.py b/python/voice-tests/test_pipecat_attachments.py new file mode 100644 index 000000000..948bf1263 --- /dev/null +++ b/python/voice-tests/test_pipecat_attachments.py @@ -0,0 +1,63 @@ +"""Assert truthful fail-closed states against real AudioBufferProcessor types.""" + +from dataclasses import replace + +import pytest +from fi_instrumentation.attachments import AttachmentConfig, AttachmentManager, CorrelationHandle +from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor +from traceai_pipecat import bind_audio_processor, create_audio_capture +from traceai_pipecat.attachments import AudioSessionContext + +CONTEXT = AudioSessionContext(CorrelationHandle("1" * 32, "2" * 16, True)) + + +@pytest.mark.parametrize("buffer_size,turns,status", [(0, False, "unbounded_source"), + (1024, True, "unbounded_source"), (131073, False, "unbounded_source"), + (4096, False, "streaming_unavailable")]) +def test_existing_processor_never_started_or_modified(manager, transport, buffer_size, turns, status): + processor = AudioBufferProcessor(buffer_size=buffer_size, enable_turn_audio=turns) + before = dict(processor.__dict__) + binding = bind_audio_processor(processor, CONTEXT, manager=manager) + assert binding.status == status and binding.processor is None + assert processor.__dict__ == before + assert not processor._recording + assert not transport.payloads and not transport.requests + + +def test_creation_requires_separate_local_and_server_permission(manager, transport): + assert create_audio_capture(CONTEXT, manager=manager).status == "recording_disabled" + manager.config = replace(manager.config, record_audio=True) + manager.policy.config = manager.config + assert create_audio_capture(CONTEXT, manager=manager).status == "recording_disabled" + manager.update_policy(replace(transport.policy, record_audio=True)) + result = create_audio_capture(CONTEXT, manager=manager) + assert result.status == "streaming_unavailable" and result.processor is None + + +@pytest.mark.parametrize("flag", ["FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", "FI_HIDE_INPUT_TEXT", + "FI_HIDE_OUTPUT_TEXT", "FI_PII_REDACTION"]) +def test_policy_denial_precedes_processor_access(manager, monkeypatch, flag): + class Poison: + def __getattribute__(self, name): + raise AssertionError("denied processor accessed") + monkeypatch.setenv(flag, "true") + assert bind_audio_processor(Poison(), CONTEXT, manager=manager).status in ("policy_denied", "redaction_unavailable") + + +def test_off_by_default_does_not_import_or_touch_upstream(): + manager = AttachmentManager(config=AttachmentConfig()) + try: + assert bind_audio_processor(object(), object(), manager=manager).status == "disabled" + assert create_audio_capture(object(), manager=manager).status == "disabled" + assert manager.owned_bytes == 0 + finally: + manager.shutdown(0) + + +def test_unknown_upstream_fails_closed(manager, monkeypatch): + monkeypatch.setattr("traceai_pipecat.attachments.version", lambda _: "2.0.0") + assert bind_audio_processor(object(), CONTEXT, manager=manager).status == "unsupported_version" + + +def test_no_late_ambient_context_lookup(manager): + assert bind_audio_processor(object(), None, manager=manager).status == "correlation_unavailable" From 7f24ad2cb1c365f9d5dcf69a63bed6e678b12f46 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 19:15:21 -0800 Subject: [PATCH 2/9] ci(python): respect Pipecat published Python minimum Co-Authored-By: Claude Code --- .github/workflows/python-voice-attachments.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-voice-attachments.yml b/.github/workflows/python-voice-attachments.yml index 3068391fa..695258a8f 100644 --- a/.github/workflows/python-voice-attachments.yml +++ b/.github/workflows/python-voice-attachments.yml @@ -21,12 +21,18 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.10', '3.11', '3.12'] + # Both pinned Pipecat 1.x distributions require Python >=3.11. + python: ['3.11', '3.12'] adapter: - {name: pipecat, upstream: pipecat-ai, version: '1.0.0'} - {name: pipecat, upstream: pipecat-ai, version: '1.15.0'} - {name: livekit, upstream: livekit-agents, version: '1.8.10'} - {name: livekit, upstream: livekit-agents, version: '1.8.11'} + include: + - python: '3.10' + adapter: {name: livekit, upstream: livekit-agents, version: '1.8.10'} + - python: '3.10' + adapter: {name: livekit, upstream: livekit-agents, version: '1.8.11'} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 From 4d689f9803d30290dad6d1e0ae3c84ca968027db Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 06:47:50 -0800 Subject: [PATCH 3/9] feat(python): bounded Pipecat recorder, LiveKit owned-file and Egress import on the shared core V01: bind_audio_processor observes the app's AudioBufferProcessor (on_audio_data / on_track_audio_data) into a core RecordingWriter bound to the conversation span Pipecat's own TurnTraceObserver started; mixed, stereo and separate_tracks manifests; recording stays off unless enabled; create_audio_capture needs local and server record_audio. V02: LiveKit session binding uses public preflight, identify() and attach_owned_recording (tier single, direction mixed); no private manager or scope internals. V03: LiveKitEgressResolver + async import_completed_egress through the core import_completed_recording seam (completed audio-only Egress only). fi.span.kind=CONVERSATION is set only on a live, parentless SDK span. Co-Authored-By: Claude Opus 5.5 --- .../workflows/python-voice-attachments.yml | 4 +- .../livekit/traceai_livekit/__init__.py | 3 +- .../livekit/traceai_livekit/attachments.py | 80 +-- .../livekit/traceai_livekit/recording.py | 175 +++++-- .../pipecat/traceai_pipecat/__init__.py | 16 +- .../pipecat/traceai_pipecat/attachments.py | 474 ++++++++++++++--- python/traceai/__init__.py | 48 +- python/traceai/tests/test_attachments.py | 35 ++ python/voice-tests/conftest.py | 1 + .../voice-tests/test_livekit_attachments.py | 155 +++++- .../voice-tests/test_pipecat_attachments.py | 487 ++++++++++++++++-- 11 files changed, 1274 insertions(+), 204 deletions(-) diff --git a/.github/workflows/python-voice-attachments.yml b/.github/workflows/python-voice-attachments.yml index 695258a8f..317e8139e 100644 --- a/.github/workflows/python-voice-attachments.yml +++ b/.github/workflows/python-voice-attachments.yml @@ -7,6 +7,7 @@ on: - 'python/frameworks/livekit/**' - 'python/frameworks/pipecat/**' - 'python/voice-tests/**' + - 'python/traceai/**' - 'python/tests/attachments/**' - '.github/workflows/python-voice-attachments.yml' workflow_dispatch: @@ -21,10 +22,9 @@ jobs: strategy: fail-fast: false matrix: - # Both pinned Pipecat 1.x distributions require Python >=3.11. + # Pipecat 1.15.0 (the qualified recorder surface) requires Python >=3.11. python: ['3.11', '3.12'] adapter: - - {name: pipecat, upstream: pipecat-ai, version: '1.0.0'} - {name: pipecat, upstream: pipecat-ai, version: '1.15.0'} - {name: livekit, upstream: livekit-agents, version: '1.8.10'} - {name: livekit, upstream: livekit-agents, version: '1.8.11'} diff --git a/python/frameworks/livekit/traceai_livekit/__init__.py b/python/frameworks/livekit/traceai_livekit/__init__.py index e6c780f76..c39c0c497 100644 --- a/python/frameworks/livekit/traceai_livekit/__init__.py +++ b/python/frameworks/livekit/traceai_livekit/__init__.py @@ -19,7 +19,7 @@ flush_on_job_shutdown, ) -from .recording import import_completed_egress, record_egress, wrap_egress_client +from .recording import LiveKitEgressResolver, import_completed_egress, record_egress, wrap_egress_client def bind_session(session, job_context, *, manager, owned_recording_files=False, @@ -41,6 +41,7 @@ def bind_session(session, job_context, *, manager, owned_recording_files=False, "flush_on_job_shutdown", "bind_session", "import_completed_egress", + "LiveKitEgressResolver", "record_egress", "wrap_egress_client", # Exporters diff --git a/python/frameworks/livekit/traceai_livekit/attachments.py b/python/frameworks/livekit/traceai_livekit/attachments.py index e4432732e..e2856c098 100644 --- a/python/frameworks/livekit/traceai_livekit/attachments.py +++ b/python/frameworks/livekit/traceai_livekit/attachments.py @@ -16,37 +16,46 @@ from fi_instrumentation.attachments import ( CapturePolicy, CorrelationHandle, RecordingFileScope, ) -from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, get_value _SUPPORTED = frozenset(("1.8.10", "1.8.11")) _BINDING = "_traceai_recording_binding" -def _admission(manager, capture, correlation): - """Use the core's policy, never a weaker adapter-local privacy lattice. +def _admission(manager, capture, correlation, *, binding_time=False): + """The core's public preflight decides; this adapter adds no weaker lattice. - P01 does not yet expose a public preflight. This versioned seam reads its - negotiated snapshot; attach() and the worker repeat admission before IO. + Nothing is retained: the one-byte ticket is released at once. At bind time + a still-running bootstrap is not a denial (finalization re-checks). """ + if correlation is None: + return "correlation_unavailable" try: - session = manager._session - reason = manager.policy.evaluate( - session.policy if session else None, capture, direction="mixed", - mime_type="audio/ogg", source_kind="OwnedRecordingFile", - ) - if reason: - return reason - if get_value(_SUPPRESS_INSTRUMENTATION_KEY): - return "policy_denied" - if manager.status != "ready": - return manager.status - if correlation is None: - return "correlation_unavailable" - if not correlation.sampled: - return "unsampled" + ticket = manager.preflight(correlation, mime_type="audio/ogg", direction="mixed", max_bytes=1, + operation="import_existing", source_kind="OwnedRecordingFile", + purpose="conversation_recording", capture_policy=capture) + reason = ticket.reason + ticket.release() except Exception: return "source_unavailable" - return None + if reason == "queued" or (binding_time and reason == "bootstrap_pending"): + return None + return reason + + +def mark_conversation_root(span) -> bool: + """Mark ``span`` as the conversation root ONLY when it is a true root. + + A true root is a live SDK span with no parent at all (local or remote). + """ + try: + from opentelemetry.sdk.trace import Span as SdkSpan + if not isinstance(span, SdkSpan) or span.parent is not None or not span.is_recording(): + return False + span.set_attribute("fi.span.kind", "CONVERSATION") + span.set_attribute("gen_ai.span.kind", "CONVERSATION") + return True + except Exception: + return False def _native_denial(session, job, processors): @@ -90,6 +99,7 @@ def __init__(self, session, job_context, manager, *, enabled, timeout, capture): self._closed = False self._scope = None self._handle = None + self.root_marked = False self.status = "session_bound" @property @@ -126,7 +136,7 @@ async def finalize(self): self.status = "source_unavailable" finally: if self._handle is not None and self.status != "validating": - self._manager.cancel(self._handle) + self._handle.cancel() if self._scope is not None: self._scope.close() self._scope = None @@ -166,20 +176,26 @@ async def _capture_report(self): return scope = RecordingFileScope(str(self._root), enabled=True) self._scope = scope - # Scope's no-follow root FD is part of this P01-version-qualified seam. - # The worker reopens independently and compares identity and size. - info = os.stat("audio.ogg", dir_fd=scope._fd, follow_symlinks=False) + # Deny-only pre-check (hard links, foreign owner, non-regular files). + # The scope's no-follow open and identity comparison stay the authority. + info = os.lstat(self._root / "audio.ogg") if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_uid != os.getuid(): self.status = "source_unavailable" return - source = scope.source("audio.ogg", device=info.st_dev, inode=info.st_ino, size=info.st_size) - handle = self._manager.attach( + identity = scope.identify("audio.ogg") + if (identity.device, identity.inode, identity.size) != (info.st_dev, info.st_ino, info.st_size): + self.status = "source_unavailable" + return + source = scope.source("audio.ogg", identity) + # Encoded multi-channel report: tier ``single``, direction ``mixed`` and + # no track or role claim (design section 7), the same as TS #685. + handle = self._manager.attach_owned_recording( self._correlation, source, mime_type="audio/ogg", direction="mixed", purpose="conversation_recording", capture_policy=self._capture, ) self._handle = handle - self.status = handle.admission - if handle.admission == "queued": + self.status = handle.reason + if handle.tier != "denied" and handle.reason == "queued": # Poll the content-free handle, not a blocking executor waiter. # Cancelling finalization must not leave asyncio.run() waiting for # a background Event.wait(timeout) during executor shutdown. @@ -190,7 +206,7 @@ async def _capture_report(self): def close(self): self._closed = True if self._handle is not None and self._handle.result().status == "unfinished": - self._manager.cancel(self._handle) + self._handle.cancel() # Do not overwrite a hook installed by somebody else after this binding. if self._wrapper is not None and getattr(self._job, "_on_session_end", None) is self._wrapper: self._job._on_session_end = self._original @@ -255,7 +271,7 @@ def bind_session(session, job_context, *, manager, owned_recording_files=False, from livekit.agents.telemetry.pii import _PIIFilteringSpanProcessor processors = getattr(getattr(span, "_span_processor", None), "_span_processors", ()) binding._privacy_processors = tuple(p for p in processors if isinstance(p, _PIIFilteringSpanProcessor)) - reason = _admission(manager, capture, binding._correlation) + reason = _admission(manager, capture, binding._correlation, binding_time=True) if reason: binding.status = reason binding.close() @@ -287,6 +303,8 @@ async def chained(): binding._original, binding._wrapper = original, chained job_context._on_session_end = chained setattr(job_context, _BINDING, binding) + # Only a true parentless session span is marked as the Voice root. + binding.root_marked = mark_conversation_root(span) except PackageNotFoundError: binding.status = "unsupported_version" except Exception: diff --git a/python/frameworks/livekit/traceai_livekit/recording.py b/python/frameworks/livekit/traceai_livekit/recording.py index b0e8728ac..28d8d3d0a 100644 --- a/python/frameworks/livekit/traceai_livekit/recording.py +++ b/python/frameworks/livekit/traceai_livekit/recording.py @@ -6,6 +6,8 @@ import inspect import logging +import math +import re from typing import Any, Optional, Tuple from fi_instrumentation import FITracer, TraceConfig @@ -227,65 +229,140 @@ def wrap_egress_client( ) -def import_completed_egress(egress_info, *, session_context, egress_id, room_name, - manager, existing_source_authorized=False, capture_policy=None): - """Offer an explicitly authorized completed job to the shared private importer. +_EGRESS_MIMES = {".ogg": "audio/ogg", ".oga": "audio/ogg", ".opus": "audio/ogg", ".m4a": "audio/mp4", + ".mp4": "audio/mp4", ".wav": "audio/wav", ".webm": "audio/webm"} +_UUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") +_TOKEN = re.compile(r"[a-zA-Z0-9_-]{1,128}") +_AGENTS_SUPPORTED = ("1.8.10", "1.8.11") - This does NOT start/stop Egress, poll, inspect output locations, or select a - server connection. Pass an actual EgressInfo and its independently bound - egress/room IDs, not IDs copied out of an untrusted callback. The SDK cannot - authenticate a webhook; that is the durable backend observer's job. - The current core has no qualified LiveKit ProviderFile resolver, so completed - jobs return ``source_unavailable`` (or a stricter policy denial), never ready. - Start/stop snapshots alone do not establish completion. Metadata-only - record_egress()/wrap_egress_client() keep their existing behavior. +def _audio_only_request(info) -> bool: + """Audio-only is proven from the Egress REQUEST, never from a filename.""" + kind = info.WhichOneof("request") + if kind == "room_composite": + return bool(info.room_composite.audio_only) + if kind == "track_composite": + request = info.track_composite + return bool(request.audio_track_id) and not request.video_track_id + # Track, participant and web egress cannot prove an audio-only output. + return False + + +class LiveKitEgressResolver: + """Completed LiveKit Egress output -> opaque F01 ``ProviderFile`` facts. + + ``EgressInfo`` comes ONLY from the application: either an object it already + holds (``egress_info``) or ONE fixed-ID read through its existing client + (``egress_client.list_egress(ListEgressRequest(egress_id=...))``). No generic + listing, no polling, no Egress start/stop. ``registry`` maps the completed + output to the server-registered output id (backend B08; CQ-4). No location, + filename, signed URL or credential leaves this object. + """ + + def __init__(self, *, connection_id: str, registry, room_name: str, egress_info=None, + egress_client=None) -> None: + self._connection_id = connection_id + self._registry = registry + self._room_name = room_name + self._info = egress_info + self._client = egress_client + + def __repr__(self) -> str: + return "LiveKitEgressResolver()" + + async def _read(self, egress_id: str): + if self._info is not None: + return self._info + if self._client is None: + return None + from livekit import api + response = await self._client.list_egress(api.ListEgressRequest(egress_id=egress_id)) + items = list(response.items) + return items[0] if len(items) == 1 else None + + async def resolve(self, request, *, deadline: float): + from fi_instrumentation.attachments import ProviderFileSource, ResolvedCompletedRecording + from livekit import api + + try: + if request.provider != "livekit_egress" or not _UUID.fullmatch(self._connection_id or ""): + return "source_unavailable" + info = await self._read(request.provider_ref) + if not isinstance(info, api.EgressInfo): + return "source_unavailable" + if info.egress_id != request.provider_ref or info.room_name != self._room_name: + return "source_unavailable" + # Start/stop snapshots and failed/aborted jobs are never "ready". + if info.status != api.EGRESS_COMPLETE or info.ended_at <= 0: + return "source_unavailable" + if not _audio_only_request(info): + return "source_unavailable" + if (len(info.file_results) != 1 or len(info.stream_results) or len(info.segment_results) + or len(info.image_results)): + return "source_unavailable" + result = info.file_results[0] + name = result.filename.lower() + mime = next((m for ext, m in _EGRESS_MIMES.items() if name.endswith(ext)), None) + if mime is None or result.size <= 0 or result.duration <= 0: + return "source_unavailable" + registered = await self._registry.lookup(connection_id=self._connection_id, + egress_id=request.provider_ref, file_index=0) + if not isinstance(registered, str) or not _UUID.fullmatch(registered): + return "source_unavailable" + return ResolvedCompletedRecording( + source=ProviderFileSource(connection_id=self._connection_id, registered_output_id=registered), + declared_bytes=int(result.size), declared_duration_ms=math.ceil(int(result.duration) / 1e6), + mime_type=mime, direction="mixed", + ) + except Exception: + return "source_unavailable" + + +async def import_completed_egress(egress_info=None, *, session_context, egress_id, room_name, manager, + connection_id=None, registry=None, egress_client=None, + existing_source_authorized=False, capture_policy=None, timeout=30.0): + """Offer an explicitly authorized COMPLETED Egress to the private importer. + + The core's ``import_completed_recording`` preflights first (deny wins), then + this adapter's ``LiveKitEgressResolver`` turns the app-held ``EgressInfo`` + (or one fixed-ID read through ``egress_client``) into opaque ProviderFile + facts, and the core submits an F01 ``completed_recording`` import with the + original credentials. ``ImportHandle.reason == "queued"`` means the server + accepted the job (``importing``), never that media is playable. + + Without ``connection_id`` + ``registry`` (CQ-4) the result is + ``source_unavailable``. This never starts/stops Egress, polls, or sends a + location, filename, URL or credential. Metadata-only + ``record_egress()``/``wrap_egress_client()`` keep their behavior. """ # Lazy imports keep metadata-only installs compatible with released cores. from fi_instrumentation.attachments import ( - AttachmentHandle, CapturePolicy, CorrelationHandle, ProviderFile, + CapturePolicy, CompletedRecordingRequest, CorrelationHandle, ImportHandle, UnavailableResolver, ) if existing_source_authorized is not True: - return AttachmentHandle("disabled") + return ImportHandle("disabled") try: capture = capture_policy if capture_policy is not None else CapturePolicy() if not isinstance(capture, CapturePolicy) or capture.new_recording: - return AttachmentHandle("policy_denied") - session = manager._session - reason = manager.policy.evaluate( - session.policy if session else None, capture, direction="mixed", - mime_type="audio/ogg", source_kind="ProviderFile", - ) - if reason: - return AttachmentHandle(reason) - if context.get_value(_SUPPRESS_INSTRUMENTATION_KEY): - return AttachmentHandle("policy_denied") - if manager.status != "ready": - return AttachmentHandle(manager.status) + return ImportHandle("policy_denied") if session_context is None: - return AttachmentHandle("correlation_unavailable") - parent = CorrelationHandle.from_span(session_context) - if not parent.sampled: - return AttachmentHandle("unsampled") + return ImportHandle("correlation_unavailable") from importlib.metadata import version - if version("livekit-agents") not in ("1.8.10", "1.8.11"): - return AttachmentHandle("unsupported_version") - from livekit import api - if not isinstance(egress_info, api.EgressInfo): - return AttachmentHandle("unsupported_version") - if (type(egress_id) is not str or not 0 < len(egress_id) <= 256 - or type(room_name) is not str or not 0 < len(room_name) <= 256 - or egress_info.egress_id != egress_id or egress_info.room_name != room_name): - return AttachmentHandle("source_unavailable") - if egress_info.status in (api.EGRESS_STARTING, api.EGRESS_ACTIVE, api.EGRESS_ENDING): - return AttachmentHandle("waiting_for_completion") - if egress_info.status != api.EGRESS_COMPLETE or egress_info.ended_at <= 0: - return AttachmentHandle("source_unavailable") - # No URL, filename, credentials or output metadata crosses this seam. - # Fail closed until the core has a version-qualified fixed-ID resolver - # AND explicitly delegated private source authority from the broker. - return manager.attach(parent, ProviderFile(provider="livekit", file_id=egress_id), - mime_type="audio/ogg", direction="mixed", - purpose="conversation_recording", capture_policy=capture) + if version("livekit-agents") not in _AGENTS_SUPPORTED: + return ImportHandle("unsupported_version") + if (not isinstance(egress_id, str) or not _TOKEN.fullmatch(egress_id) + or type(room_name) is not str or not 0 < len(room_name) <= 256): + return ImportHandle("source_unavailable") + target = session_context + if not isinstance(target, CorrelationHandle) and not hasattr(target, "is_recording"): + target = CorrelationHandle.from_span(target) + if connection_id is not None and registry is not None: + resolver = LiveKitEgressResolver(connection_id=connection_id, registry=registry, room_name=room_name, + egress_info=egress_info, egress_client=egress_client) + else: + resolver = UnavailableResolver() + return await manager.import_completed_recording( + target, CompletedRecordingRequest("livekit_egress", egress_id, "mixed"), + resolver=resolver, capture_policy=capture, timeout=timeout) except Exception: - return AttachmentHandle("source_unavailable") + return ImportHandle("source_unavailable") diff --git a/python/frameworks/pipecat/traceai_pipecat/__init__.py b/python/frameworks/pipecat/traceai_pipecat/__init__.py index 5efc76a62..66e71552d 100644 --- a/python/frameworks/pipecat/traceai_pipecat/__init__.py +++ b/python/frameworks/pipecat/traceai_pipecat/__init__.py @@ -18,16 +18,20 @@ enable_http_attribute_mapping, ) -def bind_audio_processor(processor, session_context, *, manager): - """Check an existing processor without starting recording or retaining PCM.""" +def bind_audio_processor(processor, session_context, *, manager, **options): + """Observe an existing AudioBufferProcessor into a bounded core recording. + + See ``traceai_pipecat.attachments.bind_audio_processor``. Requires the + attachment-enabled core; never starts the recorder. + """ from .attachments import bind_audio_processor as bind - return bind(processor, session_context, manager=manager) + return bind(processor, session_context, manager=manager, **options) -def create_audio_capture(session_context, *, manager): - """Check separate recording consent; no recorder until stream support exists.""" +def create_audio_capture(session_context, *, manager, **options): + """Create an un-started recorder only with local and server record_audio.""" from .attachments import create_audio_capture as create - return create(session_context, manager=manager) + return create(session_context, manager=manager, **options) __version__ = "0.1.2" # x-release-please-version diff --git a/python/frameworks/pipecat/traceai_pipecat/attachments.py b/python/frameworks/pipecat/traceai_pipecat/attachments.py index b885d5122..14714a5f7 100644 --- a/python/frameworks/pipecat/traceai_pipecat/attachments.py +++ b/python/frameworks/pipecat/traceai_pipecat/attachments.py @@ -1,95 +1,453 @@ -"""Fail-closed Pipecat recording binding until the shared stream API is ready. +"""Bounded Pipecat AudioBufferProcessor capture bound to the real conversation. -Nonzero AudioBufferProcessor.buffer_size is a flush threshold, not a hard memory -cap: oversized frames, wall-clock silence and turn buffers can exceed it. Do not -label a series of unmanifested clips as a complete conversation recording. +The adapter only OBSERVES audio the application's own ``AudioBufferProcessor`` +already emits (``on_audio_data`` or ``on_track_audio_data``). It never starts, +stops or reconfigures a recorder, never reads the processor's internal buffers +and never falls back to an ambient span. Every PCM chunk goes straight into a +core ``RecordingWriter`` (``start_recording``): segments, gaps and a recording +manifest on a sample-count clock, with the core's own memory bounds. + +Recording is OFF unless attachments are enabled. Creating a NEW recorder +(``create_audio_capture``) additionally needs the local and server +``record_audio`` permissions (``operation="create_recording"``); observing an +existing app recorder is ``operation="observe_consumed"``. + +Upstream bound: ``buffer_size`` must be within 1..128 KiB, so the processor +flushes bounded chunks on its own schedule. ``buffer_size=0`` keeps the whole +call in Pipecat memory until stop and is refused (``unbounded_source``), as is +``enable_turn_audio`` (per-turn buffers are not observed by this adapter). """ +import asyncio +import math +import re +import threading +import weakref from dataclasses import dataclass from importlib.metadata import PackageNotFoundError, version +from typing import Optional + +from fi_instrumentation.attachments import ( + CapturePolicy, CorrelationHandle, PcmFormat, RecordingOptions, TrackSpec, +) -from fi_instrumentation.attachments import CapturePolicy, CorrelationHandle -from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, get_value +# Exact versions exercised by python/voice-tests on a real pipeline. 1.15.0 is +# the first qualified release with ``on_recording_stopped`` and the +# PipelineWorker ``turn_trace_observer`` surface used here. +_SUPPORTED = frozenset(("1.15.0",)) +_MAX_BUFFER = 128 * 1024 +_TOKEN = re.compile(r"[a-zA-Z0-9_-]{1,128}") +_SENTINEL = CorrelationHandle("0" * 31 + "1", "0" * 15 + "1", True) +_BINDINGS: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary() +_BINDINGS_LOCK = threading.Lock() -_SUPPORTED = frozenset(("1.0.0", "1.15.0")) +CONVERSATION_KIND = "CONVERSATION" @dataclass(frozen=True) class AudioSessionContext: - """The actual conversation identity, never a late ambient span lookup.""" + """The conversation a recording belongs to; never a late ambient lookup. + + ``from_pipeline(worker)`` resolves the conversation span that Pipecat's own + ``TurnTraceObserver`` started (``PipelineWorker(enable_tracing=True, + enable_turn_tracking=True)``) when the first audio chunk arrives. + ``from_span(span)`` binds an explicit span or ``CorrelationHandle``. + """ - correlation: CorrelationHandle + correlation: Optional[CorrelationHandle] = None + span: object = None + worker: object = None + conversation_id: Optional[str] = None @classmethod - def from_span(cls, span): - return cls(CorrelationHandle.from_span(span)) + def from_span(cls, span, *, conversation_id: Optional[str] = None) -> "AudioSessionContext": + correlation = CorrelationHandle.from_span(span) + live = span if hasattr(span, "is_recording") and hasattr(span, "set_attribute") else None + return cls(correlation=correlation, span=live, conversation_id=conversation_id) + + @classmethod + def from_pipeline(cls, worker) -> "AudioSessionContext": + if worker is None: + raise ValueError("pipeline worker required") + return cls(worker=worker) + + +def mark_conversation_root(span) -> bool: + """Mark ``span`` as the conversation root ONLY when it is a true root. + + A true root is a live SDK span with no parent at all (local or remote). + A ``SpanContext``/``NonRecordingSpan`` cannot prove root-ness: no marker. + """ + try: + from opentelemetry.sdk.trace import Span as SdkSpan + if not isinstance(span, SdkSpan) or span.parent is not None or not span.is_recording(): + return False + span.set_attribute("fi.span.kind", CONVERSATION_KIND) + span.set_attribute("gen_ai.span.kind", CONVERSATION_KIND) + return True + except Exception: + return False + + +def _token(value) -> Optional[str]: + # Opaque ids that are not F01 tokens are dropped, never rewritten. + return value if isinstance(value, str) and _TOKEN.fullmatch(value) else None + + +def _probe(manager, operation, directions, capture) -> Optional[str]: + """Deny-wins policy check through the public preflight; retains nothing. + + The sentinel identity only lets preflight run past correlation; the ticket + is released at once. Returns None when at least one direction is admitted. + """ + reasons = [] + for direction in directions: + try: + ticket = manager.preflight(_SENTINEL, mime_type="audio/wav", direction=direction, max_bytes=1, + operation=operation, source_kind="Bytes", purpose="conversation_recording", + capture_policy=capture) + except Exception: + return "source_unavailable" + reason = ticket.reason + ticket.release() + # Bootstrap may still be running at bind time; the core re-admits (and + # records ``bootstrap_pending`` gaps) once the session exists. + if reason in ("queued", "bootstrap_pending"): + return None + reasons.append(reason) + return reasons[0] if reasons else "policy_denied" + + +def _layout(processor, layout): + channels = processor.num_channels + if layout == "separate_tracks": + return "separate_tracks", (TrackSpec((0,), "input", "user"), TrackSpec((1,), "output", "assistant")) + if layout != "auto": + return None, None + if channels == 1: + return "mixed", (TrackSpec((0,), "mixed", "mixed"),) + if channels == 2: + # Pipecat's stereo output puts the user on the left, the bot on the right. + return "stereo", (TrackSpec((0,), "input", "user"), TrackSpec((1,), "output", "assistant")) + return None, None -@dataclass(frozen=True) class AudioCaptureBinding: - """A status is not evidence of a recorder or a recording source.""" + """Content-free status of one processor binding. + + ``status``: ``session_bound`` (waiting for audio), ``recording``, a denial + reason, or the last finished ``RecordingResult.state``. ``results`` holds + each finished recording cycle (one per start/stop of the app's recorder). + ``close()`` detaches; it never stops the application's recorder. + """ + + def __init__(self, status: str, processor=None, *, manager=None, context=None, layout=None, tracks=None, + operation="observe_consumed", capture=None, timeout=30.0): + self.status = status + self.processor = processor + self.results = [] + self.root_marked = False + self._manager = manager + self._context = context + self._layout = layout + self._tracks = tracks + self._operation = operation + self._capture = capture + self._timeout = timeout + self._writer = None + self._format = None + self._cycle_denied = False + self._closed = status != "session_bound" + self._handlers = [] + self._pending = set() + + # ---------------------------------------------------------- handlers + def _on_audio_data(self, _processor, audio, sample_rate, num_channels): + try: + self._write(((audio, 0),), sample_rate, num_channels) + except Exception: + self._fail() + + def _on_track_audio_data(self, _processor, user_audio, bot_audio, sample_rate, _num_channels): + try: + # Each Pipecat track buffer is mono PCM at the processor rate. + self._write(((user_audio, 0), (bot_audio, 1)), sample_rate, 1) + except Exception: + self._fail() + + def _on_recording_stopped(self, _processor): + try: + self._finish_cycle() + except Exception: + self._fail() + + # ---------------------------------------------------------- internals + def _fail(self) -> None: + # Never log the exception: it can carry caller objects. Drop retention. + writer, self._writer = self._writer, None + if writer is not None: + writer.abort() + self.status = "source_unavailable" + + def _resolve(self): + context = self._context + if context.correlation is not None: + return None, context.correlation, context.span, _token(context.conversation_id) + observer = getattr(context.worker, "turn_trace_observer", None) + if observer is None: + return "correlation_unavailable", None, None, None + # Version-qualified seam (1.15.0): the observer's own conversation span. + span = getattr(observer, "_conversation_span", None) + if span is None: + return "correlation_unavailable", None, None, None + try: + correlation = CorrelationHandle.from_span(span) + except Exception: + return "correlation_unavailable", None, None, None + return None, correlation, span, _token(getattr(observer, "_conversation_id", None)) + + def _open(self, sample_rate, channels): + if self._cycle_denied: + return None + reason, correlation, span, conversation_id = self._resolve() + if reason is None and not correlation.sampled: + reason = "unsampled" + if reason is None: + options = RecordingOptions(format=PcmFormat(int(sample_rate)), layout=self._layout, tracks=self._tracks, + operation=self._operation, purpose="conversation_recording", + capture_policy=self._capture, conversation_id=conversation_id) + live = span if span is not None and span.is_recording() else None + writer = self._manager.start_recording(live if live is not None else correlation, options) + reason = writer.reason if writer.reason != "queued" else None + if reason is not None: + self._cycle_denied = True + self.status = reason + return None + if live is not None and mark_conversation_root(live): + self.root_marked = True + self._writer, self._format = writer, (int(sample_rate), channels) + self.status = "recording" + return writer + + def _write(self, chunks, sample_rate, channels): + if self._closed: + return + expected = 2 if self._layout == "stereo" else 1 + writer = self._writer + if writer is None: + if type(sample_rate) is not int or channels != expected: + return + writer = self._open(sample_rate, channels) + if writer is None: + return + if (sample_rate, channels) != self._format: + # Format drift: keep the shared clock with an explicit gap. + size = max(len(chunk) for chunk, _ in chunks) + if type(sample_rate) is int and sample_rate > 0 and channels in (1, 2): + duration = math.ceil(size * 1000 / (2 * channels * sample_rate)) + if duration > 0: + writer.mark_gap(duration, reason="source_unavailable") + return + for chunk, track in chunks: + writer.try_write(chunk, track=track if self._layout == "separate_tracks" else 0) + + def _finish_cycle(self): + writer, self._writer = self._writer, None + self._format, self._cycle_denied = None, False + if writer is None: + return + try: + loop = asyncio.get_running_loop() + except RuntimeError: + self._record(writer.finish(self._timeout)) + return + task = loop.create_task(writer.finish_async(self._timeout)) + self._pending.add(task) + task.add_done_callback(self._finished) + + def _finished(self, task): + self._pending.discard(task) + if task.cancelled(): + self.status = "incomplete" + return + error = task.exception() + if error is not None: + self.status = "source_unavailable" + return + self._record(task.result()) - status: str - processor: object = None + def _record(self, result): + self.results.append(result) + self.status = result.state + # ------------------------------------------------------------- public + async def finish(self, timeout: Optional[float] = None): + """Finish the open recording cycle and await uploads, within a deadline. -def _admission(manager, session_context, *, new_recording): + Call before process exit (after the pipeline ends). Pipecat already + finishes a cycle on ``on_recording_stopped``; this only awaits it. + Returns the last ``RecordingResult`` or None. A hard kill is lossy. + """ + budget = self._timeout if timeout is None else min(max(float(timeout), 0.0), self._timeout) + self._finish_cycle() + pending = list(self._pending) + if pending: + await asyncio.wait(pending, timeout=budget + 1) + return self.results[-1] if self.results else None + + def close(self) -> None: + """Detach handlers and drop any unfinished cycle (no I/O).""" + self._closed = True + processor = self.processor + for event, handler in self._handlers: + try: + processor.remove_event_handler(event, handler) + except Exception: + pass + self._handlers = [] + writer, self._writer = self._writer, None + if writer is not None: + writer.abort() + self.status = "incomplete" + if processor is not None: + with _BINDINGS_LOCK: + if _BINDINGS.get(processor) is self: + del _BINDINGS[processor] + + def __repr__(self) -> str: + return "AudioCaptureBinding(status=%r)" % self.status + + +def _validate_options(layout, finalization_timeout, capture_policy): + if layout not in ("auto", "separate_tracks"): + raise ValueError("layout must be 'auto' or 'separate_tracks'") + if (type(finalization_timeout) not in (int, float) or not math.isfinite(finalization_timeout) + or not 0 < finalization_timeout <= 30): + raise ValueError("finalization deadline must be within 30 seconds") + capture = capture_policy if capture_policy is not None else CapturePolicy() + if not isinstance(capture, CapturePolicy): + raise ValueError("capture policy must be a CapturePolicy") + return capture + + +def _qualified_processor(processor): + if version("pipecat-ai") not in _SUPPORTED: + return "unsupported_version" + from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor + if not isinstance(processor, AudioBufferProcessor): + return "unsupported_version" + size = processor._buffer_size + if type(size) is not int or not 0 < size <= _MAX_BUFFER or processor._enable_turn_audio: + return "unbounded_source" + return None + + +def _bind(processor, session_context, manager, *, layout, timeout, capture, operation): + if not isinstance(session_context, AudioSessionContext): + return AudioCaptureBinding("correlation_unavailable") try: - capture = CapturePolicy(new_recording=new_recording) - session = manager._session - reason = manager.policy.evaluate( - session.policy if session else None, capture, direction="mixed", - mime_type="audio/wav", source_kind="Bytes", - ) + reason = _qualified_processor(processor) if reason: - return reason - if get_value(_SUPPRESS_INSTRUMENTATION_KEY): - return "policy_denied" - if manager.status != "ready": - return manager.status - if not isinstance(session_context, AudioSessionContext): - return "correlation_unavailable" - if not session_context.correlation.sampled: - return "unsampled" + return AudioCaptureBinding(reason) + selected, tracks = _layout(processor, layout) + if selected is None: + return AudioCaptureBinding("unsupported_version") + directions = ("input", "output") if selected == "separate_tracks" else ("mixed",) + reason = _probe(manager, operation, directions, capture) + if reason: + return AudioCaptureBinding(reason) + with _BINDINGS_LOCK: + previous = _BINDINGS.get(processor) + if previous is not None and not previous._closed: + if (previous._manager is manager and previous._context == session_context + and previous._layout == selected and previous._operation == operation + and previous._capture == capture and previous._timeout == timeout): + return previous + raise ValueError("conflicting recording binding") + binding = AudioCaptureBinding("session_bound", processor, manager=manager, context=session_context, + layout=selected, tracks=tracks, operation=operation, capture=capture, + timeout=timeout) + _BINDINGS[processor] = binding + if selected == "separate_tracks": + handlers = (("on_track_audio_data", binding._on_track_audio_data),) + else: + handlers = (("on_audio_data", binding._on_audio_data),) + handlers += (("on_recording_stopped", binding._on_recording_stopped),) + for event, handler in handlers: + processor.add_event_handler(event, handler) + binding._handlers.append((event, handler)) + return binding + except ValueError: + raise + except PackageNotFoundError: + return AudioCaptureBinding("unsupported_version") except Exception: - return "source_unavailable" - return None + return AudioCaptureBinding("source_unavailable") -def bind_audio_processor(processor, session_context, *, manager): - """Inspect eligibility for an existing recorder; never start or mutate it. +def bind_audio_processor(processor, session_context, *, manager, layout="auto", finalization_timeout=30, + capture_policy=None): + """Observe the app's EXISTING AudioBufferProcessor; never start or modify it. - This component currently returns ``streaming_unavailable`` for otherwise - eligible processors. P02 still lacks stream/manifest transport. In - particular, it must not register a callback which retains PCM, uploads - unlinked chunks, or pretends the upstream flush threshold is a hard bound. - Both auto and explicit callers receive this same truthful state. + Bind before the pipeline runs. ``layout="auto"`` follows the processor: + mono -> ``mixed``; stereo -> ``stereo`` (user left/input, bot right/output). + ``layout="separate_tracks"`` observes ``on_track_audio_data`` and keeps one + track per side, so a hide flag on one side drops only that side. + + Policy is checked before the processor is inspected, and again by the core + on every chunk. Disabled attachments return ``disabled`` and register + nothing. Await ``binding.finish()`` after the pipeline ends. """ - reason = _admission(manager, session_context, new_recording=False) + capture = _validate_options(layout, finalization_timeout, capture_policy) + if capture.new_recording: + raise ValueError("observing an existing recorder does not create a recording") + try: + state = manager.status().state + except Exception: + return AudioCaptureBinding("source_unavailable") + if state == "disabled": + return AudioCaptureBinding("disabled") + reason = _probe(manager, "observe_consumed", ("input", "output", "mixed"), capture) + if reason: + return AudioCaptureBinding(reason) + return _bind(processor, session_context, manager, layout=layout, timeout=finalization_timeout, + capture=capture, operation="observe_consumed") + + +def create_audio_capture(session_context, *, manager, sample_rate=None, num_channels=1, buffer_size=32 * 1024, + layout="auto", finalization_timeout=30, capture_policy=None): + """Create a NEW, un-started AudioBufferProcessor only with both permissions. + + Needs local ``AttachmentConfig(record_audio=True)`` AND server + ``record_audio``. The returned ``binding.processor`` must be inserted into + the pipeline by the application, which also starts it (for example with + ``AudioBufferStartRecordingFrame``). Nothing is created when denied. + """ + capture = _validate_options(layout, finalization_timeout, capture_policy) + if (type(buffer_size) is not int or not 0 < buffer_size <= _MAX_BUFFER + or num_channels not in (1, 2)): + raise ValueError("buffer_size must be within 1..131072 and num_channels 1 or 2") + try: + state = manager.status().state + except Exception: + return AudioCaptureBinding("source_unavailable") + if state == "disabled": + return AudioCaptureBinding("disabled") + directions = ("input", "output") if layout == "separate_tracks" else ("mixed",) + reason = _probe(manager, "create_recording", directions, capture) if reason: return AudioCaptureBinding(reason) + if not isinstance(session_context, AudioSessionContext): + return AudioCaptureBinding("correlation_unavailable") try: if version("pipecat-ai") not in _SUPPORTED: return AudioCaptureBinding("unsupported_version") from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor - if not isinstance(processor, AudioBufferProcessor): - return AudioCaptureBinding("unsupported_version") - if (type(processor._buffer_size) is not int or not 0 < processor._buffer_size <= 128 * 1024 - or processor._enable_turn_audio): - return AudioCaptureBinding("unbounded_source") + processor = AudioBufferProcessor(sample_rate=sample_rate, num_channels=num_channels, + buffer_size=buffer_size, enable_turn_audio=False) except PackageNotFoundError: return AudioCaptureBinding("unsupported_version") except Exception: return AudioCaptureBinding("source_unavailable") - return AudioCaptureBinding("streaming_unavailable") - - -def create_audio_capture(session_context, *, manager): - """Do not create/start a new recorder without both recording permissions. - - Even with permission, this release fails closed until bounded upstream - processing, stream segments, gaps and manifests can be transported. No - processor is returned for pipeline insertion; existing audio is untouched. - """ - reason = _admission(manager, session_context, new_recording=True) - return AudioCaptureBinding(reason or "streaming_unavailable") + return _bind(processor, session_context, manager, layout=layout, timeout=finalization_timeout, + capture=capture, operation="create_recording") diff --git a/python/traceai/__init__.py b/python/traceai/__init__.py index 4888b539d..c04374de0 100644 --- a/python/traceai/__init__.py +++ b/python/traceai/__init__.py @@ -319,13 +319,25 @@ def _bind(library: str, entry_point: str, *args, **kwargs): return None # A binding alone is not proof of recorded media. The adapter owns reporting # when a real source becomes available; Q02 qualifies end-to-end playback. - _adapter_status[library] = "session_bound" if binding is not None else "source_unavailable" + status = getattr(binding, "status", None) + if binding is None: + _adapter_status[library] = "source_unavailable" + else: + # The adapter's own content-free status (for example a denial reason); + # a binding object without one is only "session_bound". + _adapter_status[library] = status if isinstance(status, str) and len(status) <= 64 else "session_bound" return binding -def bind_pipecat_audio_processor(processor, session_context): - """Observe an existing processor; never start a new recording implicitly.""" - return _bind("pipecat", "bind_audio_processor", processor, session_context) +def bind_pipecat_audio_processor(processor, session_context, **options): + """Observe an existing processor; never start a new recording implicitly. + + ``session_context`` is a ``traceai_pipecat.attachments.AudioSessionContext`` + (``from_pipeline(worker)`` or ``from_span(span)``). Options are passed to + ``traceai_pipecat.bind_audio_processor`` (``layout``, ``finalization_timeout``, + ``capture_policy``). + """ + return _bind("pipecat", "bind_audio_processor", processor, session_context, **options) def bind_livekit_session(session, job_context, *, owned_recording_files=False, @@ -336,12 +348,40 @@ def bind_livekit_session(session, job_context, *, owned_recording_files=False, finalization_timeout=finalization_timeout) +async def import_livekit_completed_egress(egress_info=None, *, session_context, egress_id, room_name, + connection_id=None, registry=None, egress_client=None, + existing_source_authorized=False, timeout=30.0): + """Offer an explicitly authorized completed LiveKit Egress to the importer. + + Uses the provider's shared attachment manager. Returns the core + ``ImportHandle`` (``queued`` = accepted for import, not playable), or None + when the runtime or adapter is unavailable. + """ + if not _ATTACHMENTS_SUPPORTED: + _adapter_status["livekit"] = "unsupported_version" + return None + manager = getattr(_tracer_provider, "attachment_manager", None) + if manager is None: + raise RuntimeError("initialize traceai before importing a recording") + try: + module = importlib.import_module(_INSTRUMENTOR_REGISTRY["livekit"]) + importer = getattr(module, "import_completed_egress") + except (ImportError, AttributeError): + _adapter_status["livekit"] = "unsupported_version" + return None + return await importer(egress_info, session_context=session_context, egress_id=egress_id, + room_name=room_name, manager=manager, connection_id=connection_id, + registry=registry, egress_client=egress_client, + existing_source_authorized=existing_source_authorized, timeout=timeout) + + __all__ = [ "init", "AttachmentConfig", "attachment_status", "bind_pipecat_audio_processor", "bind_livekit_session", + "import_livekit_completed_egress", "instrument", "get_tracer_provider", "SemanticConvention", diff --git a/python/traceai/tests/test_attachments.py b/python/traceai/tests/test_attachments.py index b4e18e306..f1ad6ded6 100644 --- a/python/traceai/tests/test_attachments.py +++ b/python/traceai/tests/test_attachments.py @@ -47,6 +47,41 @@ def test_explicit_session_binding_uses_shared_manager_without_recording_permissi assert "recording_source_ready" not in str(traceai.attachment_status()) +def test_pipecat_binding_passes_options_and_reports_adapter_status(monkeypatch): + manager = SimpleNamespace(status=lambda: SimpleNamespace(state="ready")) + monkeypatch.setattr(traceai, "_tracer_provider", SimpleNamespace(attachment_manager=manager)) + binder = Mock(return_value=SimpleNamespace(status="policy_denied")) + monkeypatch.setattr(traceai.importlib, "import_module", + Mock(return_value=SimpleNamespace(bind_audio_processor=binder))) + processor, context = object(), object() + result = traceai.bind_pipecat_audio_processor(processor, context, layout="separate_tracks") + assert result is binder.return_value + binder.assert_called_once_with(processor, context, manager=manager, layout="separate_tracks") + # A denied binding is never reported as bound. + assert traceai.attachment_status()["adapters"]["pipecat"] == "policy_denied" + + +def test_livekit_egress_import_uses_shared_manager(monkeypatch): + import asyncio + + manager = SimpleNamespace(status=lambda: SimpleNamespace(state="ready")) + monkeypatch.setattr(traceai, "_tracer_provider", SimpleNamespace(attachment_manager=manager)) + calls = [] + + async def importer(info, **kwargs): + calls.append((info, kwargs)) + return "handle" + + monkeypatch.setattr(traceai.importlib, "import_module", + Mock(return_value=SimpleNamespace(import_completed_egress=importer))) + info = object() + result = asyncio.run(traceai.import_livekit_completed_egress( + info, session_context="span", egress_id="EG_synthetic", room_name="room", existing_source_authorized=True)) + assert result == "handle" + assert calls[0][0] is info and calls[0][1]["manager"] is manager + assert calls[0][1]["existing_source_authorized"] is True and calls[0][1]["registry"] is None + + def test_old_optional_adapter_reports_unsupported_without_exception_body(monkeypatch, capsys): monkeypatch.setattr(traceai, "_tracer_provider", SimpleNamespace(attachment_manager=SimpleNamespace(status=lambda: SimpleNamespace(state="ready")))) monkeypatch.setattr(traceai.importlib, "import_module", Mock(return_value=SimpleNamespace())) diff --git a/python/voice-tests/conftest.py b/python/voice-tests/conftest.py index eb868b5ff..ebf67d45b 100644 --- a/python/voice-tests/conftest.py +++ b/python/voice-tests/conftest.py @@ -13,4 +13,5 @@ clean_attachment_env = _support.clean_attachment_env manager = _support.manager transport = _support.transport +make_manager = _support.make_manager from test_http_contract import server # noqa: E402,F401 diff --git a/python/voice-tests/test_livekit_attachments.py b/python/voice-tests/test_livekit_attachments.py index 49076045c..cbe75982a 100644 --- a/python/voice-tests/test_livekit_attachments.py +++ b/python/voice-tests/test_livekit_attachments.py @@ -68,6 +68,8 @@ async def test_real_report_tls_upload_survives_span_end_before_cleanup(server, t assert job.make_session_report(session).audio_recording_path == recorder.output_path binding = bind_session(session, job, manager=manager, owned_recording_files=True) assert binding.status == "session_bound" + # The synthetic session span has no parent: it is the true conversation root. + assert binding.root_marked and span.attributes["fi.span.kind"] == "CONVERSATION" expected_trace = format(span.get_span_context().trace_id, "032x") span.end() session._session_span = None @@ -164,12 +166,12 @@ async def test_file_replacement_after_identity_snapshot_is_rejected(manager, tra session, job, _, _, provider, _ = owned_session(tmp_path) path = tmp_path / "audio.ogg" path.write_bytes(b"owned") - original = manager.attach + original = manager.attach_owned_recording def replace_before_attach(*args, **kwargs): path.rename(tmp_path / "original") path.write_bytes(b"rogue") return original(*args, **kwargs) - monkeypatch.setattr(manager, "attach", replace_before_attach) + monkeypatch.setattr(manager, "attach_owned_recording", replace_before_attach) binding = bind_session(session, job, manager=manager, owned_recording_files=True) assert await binding.finalize() == "source_unavailable" assert not transport.payloads @@ -253,27 +255,120 @@ def test_exact_worker_lifecycle_orders_close_hook_before_cleanup(): assert version("livekit-agents") in ("1.8.10", "1.8.11") -@pytest.mark.parametrize("state,expected", [(api.EGRESS_STARTING, "waiting_for_completion"), - (api.EGRESS_ACTIVE, "waiting_for_completion"), (api.EGRESS_ENDING, "waiting_for_completion"), - (api.EGRESS_COMPLETE, "source_unavailable"), (api.EGRESS_FAILED, "source_unavailable")]) -def test_egress_never_marks_start_stop_or_complete_as_playable(manager, transport, state, expected): +CONNECTION = "00000000-0000-4000-8000-00000000c0de" +REGISTERED = "00000000-0000-4000-8000-0000000f11e5" +CANARY = "https://private.invalid/CANARY_SIGNED_PATH?token=SECRET" + + +class Registry: + def __init__(self, value=REGISTERED): + self.value, self.calls = value, [] + + async def lookup(self, *, connection_id, egress_id, file_index): + self.calls.append((connection_id, egress_id, file_index)) + return self.value + + +def egress(status=api.EGRESS_COMPLETE, *, audio_only=True, files=1, name="call.ogg", size=48213, + duration=12_345_678_901, egress_id="EG_synthetic", room="room"): + info = api.EgressInfo(egress_id=egress_id, room_name=room, status=status, ended_at=2 if status else 0) + info.room_composite.CopyFrom(api.RoomCompositeEgressRequest(room_name=room, audio_only=audio_only)) + for _ in range(files): + info.file_results.add(filename="/private/CANARY_DIR/" + name, location=CANARY, size=size, + duration=duration) + return info + + +def import_args(manager, info, **overrides): from fi_instrumentation.attachments import CorrelationHandle - parent = CorrelationHandle("1" * 32, "2" * 16, True) - manager.update_policy(replace(transport.policy, allowed_sources=frozenset(("ProviderFile",)))) - info = api.EgressInfo(egress_id="synthetic", room_name="room", status=state, ended_at=1) - info.file_results.add(location="https://private.invalid/CANARY_SIGNED_PATH?token=SECRET") - handle = import_completed_egress(info, session_context=parent, egress_id="synthetic", room_name="room", - manager=manager, existing_source_authorized=True) - assert handle.admission == expected - assert not transport.requests and not transport.payloads + args = dict(session_context=CorrelationHandle("1" * 32, "2" * 16, True), egress_id="EG_synthetic", + room_name="room", manager=manager, connection_id=CONNECTION, registry=Registry(), + existing_source_authorized=True) + args.update(overrides) + return info, args + + +@pytest.mark.asyncio +async def test_completed_audio_egress_imports_opaque_provider_file(manager, transport): + info, args = import_args(manager, egress()) + handle = await import_completed_egress(info, **args) + assert handle.reason == "queued" + assert (await handle.wait(5)).status == "importing" and handle.job_id + [(body, key)] = transport.imports + assert key.startswith("im-") + assert body["mode"] == "completed_recording" + assert body["source"] == {"kind": "ProviderFile", "connection_id": CONNECTION, + "registered_output_id": REGISTERED} + recording = body["recording"] + assert recording["mime_type"] == "audio/ogg" and recording["direction"] == "mixed" + assert recording["declared_bytes"] == 48213 and recording["declared_duration_ms"] == 12346 + assert recording["association"]["parent"]["span_id"] == "2" * 16 + assert args["registry"].calls == [(CONNECTION, "EG_synthetic", 0)] + assert "CANARY" not in repr(transport.imports) and "CANARY" not in repr(handle) + assert await handle.status() == ("importing", "binding_pending") + assert len(transport.statuses) == 1 # one GET, no polling + + +@pytest.mark.asyncio +@pytest.mark.parametrize("case", ["starting", "active", "ending", "failed", "video", "two_files", "unknown_ext", + "no_size", "no_duration", "wrong_id", "wrong_room", "unregistered", "no_registry"]) +async def test_incomplete_or_unprovable_egress_never_imports(manager, transport, case): + status = {"starting": api.EGRESS_STARTING, "active": api.EGRESS_ACTIVE, "ending": api.EGRESS_ENDING, + "failed": api.EGRESS_FAILED}.get(case, api.EGRESS_COMPLETE) + info = egress(status, audio_only=case != "video", files=2 if case == "two_files" else 1, + name="call.flv" if case == "unknown_ext" else "call.ogg", size=0 if case == "no_size" else 9, + duration=0 if case == "no_duration" else 9, egress_id="EG_other" if case == "wrong_id" else + "EG_synthetic", room="other" if case == "wrong_room" else "room") + overrides = {} + if case == "unregistered": + overrides["registry"] = Registry(None) + if case == "no_registry": + overrides["registry"] = None + info, args = import_args(manager, info, **overrides) + handle = await import_completed_egress(info, **args) + assert handle.reason == "source_unavailable" + assert not transport.imports and not transport.payloads assert "CANARY" not in repr(handle) -def test_egress_no_authority_does_not_inspect_response(manager): +@pytest.mark.asyncio +async def test_fixed_id_read_through_application_client(manager, transport): + class Client: + def __init__(self): + self.requests = [] + + async def list_egress(self, request): + self.requests.append(request) + return api.ListEgressResponse(items=[egress()]) + client = Client() + _, args = import_args(manager, None, egress_client=client) + handle = await import_completed_egress(None, **args) + assert handle.reason == "queued" and (await handle.wait(5)).status == "importing" + assert len(transport.imports) == 1 + assert [r.egress_id for r in client.requests] == ["EG_synthetic"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("flag", ["FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", "FI_PII_REDACTION"]) +async def test_privacy_denial_precedes_resolver(manager, transport, monkeypatch, flag): + class Poison: + async def lookup(self, **kwargs): + raise AssertionError("resolver consulted after denial") + monkeypatch.setenv(flag, "true") + info, args = import_args(manager, egress(), registry=Poison()) + handle = await import_completed_egress(info, **args) + assert handle.reason in ("policy_denied", "redaction_unavailable") + assert not transport.imports + + +@pytest.mark.asyncio +async def test_egress_no_authority_does_not_inspect_response(manager): class Poison: def __getattribute__(self, name): raise AssertionError("denied object inspected") - assert import_completed_egress(Poison(), session_context=None, egress_id="", room_name="", manager=manager).admission == "disabled" + handle = await import_completed_egress(Poison(), session_context=None, egress_id="", room_name="", + manager=manager) + assert handle.reason == "disabled" @pytest.mark.asyncio @@ -328,3 +423,31 @@ def test_cancelled_finalization_does_not_hold_process_for_full_deadline(): result = subprocess.run([sys.executable, str(Path(__file__).with_name("process_exit_probe.py"))], timeout=10, check=True, capture_output=True, text=True) assert "bounded-cancel-exit" in result.stdout + + +@pytest.mark.asyncio +async def test_child_session_span_is_never_marked_root(manager, tmp_path): + from opentelemetry import trace + session, job, _, span, provider, _ = owned_session(tmp_path) + span.end() + parent = provider.get_tracer("synthetic").start_span("job_entrypoint") + child = provider.get_tracer("synthetic").start_span("agent_session", context=trace.set_span_in_context(parent)) + session._session_span = child + binding = bind_session(session, job, manager=manager, owned_recording_files=True) + assert binding.status == "session_bound" and not binding.root_marked + assert "fi.span.kind" not in child.attributes + binding.close() + child.end() + parent.end() + provider.shutdown() + + +def test_no_private_manager_internals_in_livekit_adapter(): + import inspect + + import traceai_livekit.attachments as attachments + import traceai_livekit.recording as recording + for module in (attachments, recording): + source = inspect.getsource(module) + for private in ("manager._session", "manager.policy", "._fd", "manager._", "_manager._", "manager.status"): + assert private not in source, (module.__name__, private) diff --git a/python/voice-tests/test_pipecat_attachments.py b/python/voice-tests/test_pipecat_attachments.py index 948bf1263..045832164 100644 --- a/python/voice-tests/test_pipecat_attachments.py +++ b/python/voice-tests/test_pipecat_attachments.py @@ -1,63 +1,476 @@ -"""Assert truthful fail-closed states against real AudioBufferProcessor types.""" +"""V01: real Pipecat AudioBufferProcessor -> core RecordingWriter (segments + manifest). +The pipeline tests run a real PipelineWorker with Pipecat's own tracing, so the +recording binds to the conversation span Pipecat itself started. Audio is a +synthetic tone; the transport is the core's synthetic F01 double. This is not a +real SFU call or a product-playback claim (Q02). +""" + +import asyncio +import io +import math +import struct +import wave from dataclasses import replace import pytest +from opentelemetry import trace +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, attach, detach, set_value +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from conftest import TransportDouble, make_manager from fi_instrumentation.attachments import AttachmentConfig, AttachmentManager, CorrelationHandle +from fi_instrumentation.attachments._disabled import DisabledAttachmentManager +from pipecat.frames.frames import EndFrame, InputAudioRawFrame, OutputAudioRawFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.worker import PipelineParams, PipelineWorker from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor +from pipecat.workers.runner import WorkerRunner from traceai_pipecat import bind_audio_processor, create_audio_capture -from traceai_pipecat.attachments import AudioSessionContext +from traceai_pipecat.attachments import AudioSessionContext, mark_conversation_root + +# Pipecat's TurnTraceObserver uses the GLOBAL tracer provider. +EXPORTER = InMemorySpanExporter() +PROVIDER = TracerProvider() +PROVIDER.add_span_processor(SimpleSpanProcessor(EXPORTER)) +trace.set_tracer_provider(PROVIDER) +TRACER = PROVIDER.get_tracer("synthetic") +RATE = 16000 + + +def tone(freq, ms, rate=RATE, amplitude=8000): + count = rate * ms // 1000 + return b"".join(struct.pack(" uploaded payload (requests and payloads are in lane order).""" + uploads = [r for r in transport.requests if "recording_id" in r] + assert len(uploads) == len(transport.payloads) + return {r["occurrence_id"]: (r, p) for r, p in zip(uploads, transport.payloads)} + + +def final_manifest(transport): + assert transport.manifests, "no manifest published" + return transport.manifests[-1][1] + + +async def run_pipeline(processor, frames, *, conversation_id="conv-synthetic", parent=None): + worker = PipelineWorker(Pipeline([processor]), + params=PipelineParams(audio_in_sample_rate=RATE, audio_out_sample_rate=RATE), + enable_tracing=True, enable_turn_tracking=True, conversation_id=conversation_id, + idle_timeout_secs=None, enable_rtvi=False) + return worker + + +async def drive(worker, frames, parent=None): + await worker.queue_frames(frames + [EndFrame()]) + runner = WorkerRunner(handle_sigint=False) + # run(worker) is the form that ends with the worker on 1.15.0 (it warns as + # deprecated); add_workers() + run() kept running after EndFrame here. + if parent is None: + await asyncio.wait_for(runner.run(worker), 60) + else: + with trace.use_span(parent, end_on_exit=False): + await asyncio.wait_for(runner.run(worker), 60) + + +def call_frames(turns=6, ms=500): + frames = [] + for i in range(turns): + for _ in range(ms // 20): + frames.append(InputAudioRawFrame(audio=tone(440 + i, 20), sample_rate=RATE, num_channels=1)) + for _ in range(ms // 20): + frames.append(OutputAudioRawFrame(audio=tone(880 + i, 20), sample_rate=RATE, num_channels=1)) + return frames + + +def conversation_span(): + spans = [s for s in EXPORTER.get_finished_spans() if s.name == "conversation"] + assert len(spans) == 1 + return spans[0] + + +@pytest.fixture(autouse=True) +def reset_spans(): + EXPORTER.clear() + yield + + +# ------------------------------------------------------------------ real pipeline +@pytest.mark.asyncio +@pytest.mark.parametrize("channels,layout", [(1, "mixed"), (2, "stereo")]) +async def test_real_pipeline_whole_call_recording(manager, transport, channels, layout): + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=channels, buffer_size=16000, + auto_start_recording=True) + emitted = [] + processor.add_event_handler("on_audio_data", lambda _p, audio, rate, nch: emitted.append((audio, rate, nch))) + worker = await run_pipeline(processor, None) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(worker), manager=manager) + assert binding.status == "session_bound" + await drive(worker, call_frames()) + result = await binding.finish(10) + assert result is not None and result.state == "published" and result.capture_complete, result + assert binding.status == "published" + assert {(r, n) for _, r, n in emitted} == {(RATE, channels)} + manifest = final_manifest(transport) + assert manifest["layout"] == layout and manifest["sample_rate_hz"] == RATE + assert manifest["channels"] == channels and not manifest["gaps"] and manifest["capture_complete"] + if layout == "stereo": + assert [(t["channels"], t["direction"], t["role"]) for t in manifest["tracks"]] == [ + ([0], "input", "user"), ([1], "output", "assistant")] + else: + assert [(t["direction"], t["role"]) for t in manifest["tracks"]] == [("mixed", "mixed")] + segments = segment_map(transport) + uploaded = b"" + for segment in sorted(manifest["segments"], key=lambda s: s["start_ms"]): + request, payload = segments[segment["occurrence_id"]] + assert request["direction"] == "mixed" and request["purpose"] == "conversation_recording" + nch, rate, frames = pcm(payload) + assert (nch, rate) == (channels, RATE) + uploaded += frames + # Byte identity: exactly what Pipecat emitted, in order, nothing more. + assert uploaded == b"".join(audio for audio, _, _ in emitted) + assert manifest["duration_ms"] == math.ceil(len(uploaded) * 1000 / (2 * channels * RATE)) + # Bound to the conversation span Pipecat started, with its conversation id. + root = conversation_span() + parent = transport.requests[0]["association"]["parent"] + assert parent["span_id"] == format(root.context.span_id, "016x") + assert all(r["conversation_id"] == "conv-synthetic" for r in transport.requests if "recording_id" in r) + assert root.parent is None and binding.root_marked + assert root.attributes["fi.span.kind"] == "CONVERSATION" + assert root.attributes["fi.attachment.schema_version"] == 1 + for span in EXPORTER.get_finished_spans(): + if span is not root: + assert span.attributes.get("fi.span.kind") != "CONVERSATION" + + +@pytest.mark.asyncio +async def test_real_pipeline_per_track_manifest(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=2, buffer_size=16000, + auto_start_recording=True) + emitted = [] + processor.add_event_handler("on_track_audio_data", + lambda _p, user, bot, rate, nch: emitted.append((user, bot))) + worker = await run_pipeline(processor, None) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(worker), manager=manager, + layout="separate_tracks") + await drive(worker, call_frames()) + result = await binding.finish(10) + assert result.state == "published" and result.capture_complete + manifest = final_manifest(transport) + assert manifest["layout"] == "separate_tracks" and manifest["channels"] == 2 + user_id, bot_id = (t["track_id"] for t in manifest["tracks"]) + segments = segment_map(transport) + per_track = {user_id: b"", bot_id: b""} + for segment in sorted(manifest["segments"], key=lambda s: (s["start_ms"], s["track_id"])): + request, payload = segments[segment["occurrence_id"]] + nch, rate, frames = pcm(payload) + assert (nch, rate) == (1, RATE) + assert request["direction"] == ("input" if segment["track_id"] == user_id else "output") + per_track[segment["track_id"]] += frames + assert per_track[user_id] == b"".join(u for u, _ in emitted) + assert per_track[bot_id] == b"".join(b for _, b in emitted) + assert conversation_span().attributes["fi.span.kind"] == "CONVERSATION" + + +@pytest.mark.asyncio +async def test_conversation_with_a_parent_is_never_marked_root(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=16000, + auto_start_recording=True) + worker = await run_pipeline(processor, None) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(worker), manager=manager) + app = TRACER.start_span("app-request") + try: + await drive(worker, call_frames(turns=2), parent=app) + finally: + app.end() + result = await binding.finish(10) + assert result.state == "published" + root = conversation_span() + assert root.parent is not None and root.parent.span_id == app.get_span_context().span_id + assert not binding.root_marked and "fi.span.kind" not in root.attributes + + +@pytest.mark.asyncio +async def test_pipeline_without_tracing_never_uses_ambient_span(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=16000, + auto_start_recording=True) + worker = PipelineWorker(Pipeline([processor]), params=PipelineParams(audio_in_sample_rate=RATE, + audio_out_sample_rate=RATE), enable_tracing=False, idle_timeout_secs=None, + enable_rtvi=False) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(worker), manager=manager) + ambient = TRACER.start_span("ambient") + with trace.use_span(ambient, end_on_exit=True): + await drive(worker, call_frames(turns=2)) + assert await binding.finish(5) is None + assert binding.status == "correlation_unavailable" + assert not transport.requests and not transport.payloads and manager.owned_bytes == 0 + -CONTEXT = AudioSessionContext(CorrelationHandle("1" * 32, "2" * 16, True)) +# ------------------------------------------------------- direct event dispatch +async def dispatch(processor, event, *args): + """Pipecat's own dispatch: one task per handler, run to completion.""" + await processor._call_event_handler(event, *args) + for _ in range(3): + await asyncio.sleep(0) -@pytest.mark.parametrize("buffer_size,turns,status", [(0, False, "unbounded_source"), - (1024, True, "unbounded_source"), (131073, False, "unbounded_source"), - (4096, False, "streaming_unavailable")]) -def test_existing_processor_never_started_or_modified(manager, transport, buffer_size, turns, status): - processor = AudioBufferProcessor(buffer_size=buffer_size, enable_turn_audio=turns) - before = dict(processor.__dict__) - binding = bind_audio_processor(processor, CONTEXT, manager=manager) - assert binding.status == status and binding.processor is None - assert processor.__dict__ == before - assert not processor._recording - assert not transport.payloads and not transport.requests +def explicit_context(): + span = TRACER.start_span("explicit-conversation") + return span, AudioSessionContext.from_span(span, conversation_id="conv-explicit") -def test_creation_requires_separate_local_and_server_permission(manager, transport): - assert create_audio_capture(CONTEXT, manager=manager).status == "recording_disabled" - manager.config = replace(manager.config, record_audio=True) - manager.policy.config = manager.config - assert create_audio_capture(CONTEXT, manager=manager).status == "recording_disabled" - manager.update_policy(replace(transport.policy, record_audio=True)) - result = create_audio_capture(CONTEXT, manager=manager) - assert result.status == "streaming_unavailable" and result.processor is None +@pytest.mark.asyncio +async def test_stereo_one_side_hidden_denies_whole_recording(manager, transport, monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=2, buffer_size=16000) + before = handler_counts(processor) + span, context = explicit_context() + binding = bind_audio_processor(processor, context, manager=manager) + assert binding.status == "policy_denied" and handler_counts(processor) == before + span.end() + assert not transport.requests -@pytest.mark.parametrize("flag", ["FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", "FI_HIDE_INPUT_TEXT", - "FI_HIDE_OUTPUT_TEXT", "FI_PII_REDACTION"]) -def test_policy_denial_precedes_processor_access(manager, monkeypatch, flag): +@pytest.mark.asyncio +async def test_separate_tracks_hidden_side_becomes_gap_other_side_uploads(manager, transport, monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=16000) + span, context = explicit_context() + binding = bind_audio_processor(processor, context, manager=manager, layout="separate_tracks") + assert binding.status == "session_bound" + for i in range(4): + await dispatch(processor, "on_track_audio_data", tone(440, 500), tone(880, 500), RATE, 1) + await dispatch(processor, "on_recording_stopped") + result = await binding.finish(10) + span.end() + assert result.state == "partial" and not result.capture_complete + manifest = final_manifest(transport) + user_id, bot_id = (t["track_id"] for t in manifest["tracks"]) + assert {s["track_id"] for s in manifest["segments"]} == {bot_id} + assert {(g["track_id"], g["reason"]) for g in manifest["gaps"]} == {(user_id, "policy_denied")} + assert all(r["direction"] == "output" for r in transport.requests if "recording_id" in r) + assert b"".join(pcm(p)[2] for p in transport.payloads) == tone(880, 500) * 4 + + +@pytest.mark.parametrize("buffer_size,turns", [(0, False), (16000, True), (128 * 1024 + 1, False)]) +def test_unbounded_upstream_is_refused_and_untouched(manager, transport, buffer_size, turns): + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=buffer_size, enable_turn_audio=turns) + before = handler_counts(processor) + span, context = explicit_context() + assert bind_audio_processor(processor, context, manager=manager).status == "unbounded_source" + assert handler_counts(processor) == before and not processor._recording + span.end() + + +@pytest.mark.parametrize("flags", [("FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS")]) +def test_policy_denial_precedes_processor_access(manager, monkeypatch, flags): class Poison: def __getattribute__(self, name): raise AssertionError("denied processor accessed") - monkeypatch.setenv(flag, "true") - assert bind_audio_processor(Poison(), CONTEXT, manager=manager).status in ("policy_denied", "redaction_unavailable") + for flag in flags: + monkeypatch.setenv(flag, "true") + span, context = explicit_context() + assert bind_audio_processor(Poison(), context, manager=manager).status == "policy_denied" + span.end() -def test_off_by_default_does_not_import_or_touch_upstream(): - manager = AttachmentManager(config=AttachmentConfig()) +def test_suppressed_instrumentation_binds_nothing(manager): + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + before = handler_counts(processor) + span, context = explicit_context() + token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) try: - assert bind_audio_processor(object(), object(), manager=manager).status == "disabled" - assert create_audio_capture(object(), manager=manager).status == "disabled" - assert manager.owned_bytes == 0 + assert bind_audio_processor(processor, context, manager=manager).status == "policy_denied" finally: - manager.shutdown(0) + detach(token) + assert handler_counts(processor) == before + span.end() + + +def test_off_by_default_registers_nothing_and_writes_no_attributes(): + class Poison: + def __getattribute__(self, name): + raise AssertionError("disabled adapter touched its arguments") + for manager in (AttachmentManager(config=AttachmentConfig()), DisabledAttachmentManager()): + try: + assert bind_audio_processor(Poison(), Poison(), manager=manager).status == "disabled" + assert create_audio_capture(Poison(), manager=manager).status == "disabled" + assert manager.owned_bytes == 0 + finally: + manager.shutdown(0) + + +@pytest.mark.asyncio +async def test_unsampled_conversation_retains_nothing(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + context = AudioSessionContext(correlation=CorrelationHandle("1" * 32, "2" * 16, False)) + binding = bind_audio_processor(processor, context, manager=manager) + await dispatch(processor, "on_audio_data", tone(440, 500), RATE, 1) + assert binding.status == "unsampled" and manager.owned_bytes == 0 and not transport.requests + + +def test_create_capture_needs_local_and_server_recording_permission(transport): + span, context = explicit_context() + manager = make_manager(transport) + try: + assert create_audio_capture(context, manager=manager).status == "recording_disabled" + finally: + manager.shutdown(1) + transport.policy = replace(transport.policy, record_audio=False) + manager = make_manager(transport, record_audio=True) + try: + assert create_audio_capture(context, manager=manager).status == "recording_disabled" + finally: + manager.shutdown(1) + transport.policy = replace(transport.policy, record_audio=True) + manager = make_manager(transport, record_audio=True) + try: + binding = create_audio_capture(context, manager=manager, num_channels=2) + assert binding.status == "session_bound" + processor = binding.processor + assert isinstance(processor, AudioBufferProcessor) and processor.num_channels == 2 + assert not processor._recording and not processor._auto_start_recording + binding.close() + finally: + manager.shutdown(1) + span.end() + + +@pytest.mark.asyncio +async def test_blocked_uploader_turns_overflow_into_gaps_not_memory(transport): + manager = make_manager(transport) + transport.release.clear() + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=32000) + span, context = explicit_context() + binding = bind_audio_processor(processor, context, manager=manager) + peak = 0 + try: + for _ in range(150): # 150 s of mono audio while the uploader is stuck + await dispatch(processor, "on_audio_data", tone(440, 1000), RATE, 1) + peak = max(peak, manager.owned_bytes) + assert peak <= manager.config.max_owned_bytes + # Two 30 s segments may wait (window_segments=2); the rest is a gap. + assert peak <= 2 * (30 * RATE * 2) + 3 * RATE * 2 + transport.release.set() + await dispatch(processor, "on_recording_stopped") + result = await binding.finish(20) + finally: + transport.release.set() + manager.shutdown(2) + span.end() + assert result.state == "partial" and not result.capture_complete + manifest = final_manifest(transport) + assert any(g["reason"] == "queue_full" for g in manifest["gaps"]) + assert manifest["duration_ms"] == 150000 + + +@pytest.mark.asyncio +async def test_policy_tightening_mid_recording_stops_retention(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + span, context = explicit_context() + binding = bind_audio_processor(processor, context, manager=manager) + for _ in range(35): + await dispatch(processor, "on_audio_data", tone(440, 1000), RATE, 1) + manager.update_policy(replace(transport.policy, enabled=False)) + for _ in range(5): + await dispatch(processor, "on_audio_data", tone(440, 1000), RATE, 1) + await dispatch(processor, "on_recording_stopped") + result = await binding.finish(10) + span.end() + assert result.state in ("partial", "unpublished") and not result.capture_complete + assert sum(len(pcm(p)[2]) for p in transport.payloads) <= 35 * RATE * 2 + + +@pytest.mark.asyncio +async def test_format_drift_keeps_clock_with_explicit_gap(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + span, context = explicit_context() + binding = bind_audio_processor(processor, context, manager=manager) + await dispatch(processor, "on_audio_data", tone(440, 500), RATE, 1) + await dispatch(processor, "on_audio_data", tone(440, 500, rate=8000), 8000, 1) + await dispatch(processor, "on_audio_data", tone(440, 500), RATE, 1) + await dispatch(processor, "on_recording_stopped") + result = await binding.finish(10) + span.end() + manifest = final_manifest(transport) + assert result.state == "partial" + assert [(g["start_ms"], g["duration_ms"], g["reason"]) for g in manifest["gaps"]] == [ + (500, 500, "source_unavailable")] + assert manifest["duration_ms"] == 1500 + + +@pytest.mark.asyncio +async def test_handler_failure_is_isolated_and_content_free(manager, transport, caplog): + class Exploding: + @property + def turn_trace_observer(self): + raise RuntimeError("CANARY_PRIVATE_DETAIL") + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(Exploding()), manager=manager) + await dispatch(processor, "on_audio_data", tone(440, 500), RATE, 1) + assert binding.status == "source_unavailable" + assert "CANARY" not in caplog.text and not transport.requests and manager.owned_bytes == 0 + + +def test_duplicate_conflicting_and_detach(manager): + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + before = handler_counts(processor) + span, context = explicit_context() + binding = bind_audio_processor(processor, context, manager=manager) + assert bind_audio_processor(processor, context, manager=manager) is binding + with pytest.raises(ValueError, match="conflicting"): + bind_audio_processor(processor, context, manager=manager, layout="separate_tracks") + assert handler_counts(processor) != before + binding.close() + assert handler_counts(processor) == before and not processor._recording + assert bind_audio_processor(processor, context, manager=manager) is not binding + span.end() def test_unknown_upstream_fails_closed(manager, monkeypatch): - monkeypatch.setattr("traceai_pipecat.attachments.version", lambda _: "2.0.0") - assert bind_audio_processor(object(), CONTEXT, manager=manager).status == "unsupported_version" + monkeypatch.setattr("traceai_pipecat.attachments.version", lambda _: "1.14.0") + span, context = explicit_context() + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + assert bind_audio_processor(processor, context, manager=manager).status == "unsupported_version" + span.end() + + +def test_root_marker_only_on_true_parentless_live_span(): + root = TRACER.start_span("root") + child = TRACER.start_span("child", context=trace.set_span_in_context(root)) + remote = TRACER.start_span("remote-child", context=trace.set_span_in_context(trace.NonRecordingSpan( + trace.SpanContext(int("1" * 32, 16), int("2" * 16, 16), True, trace.TraceFlags(1))))) + assert mark_conversation_root(root) and root.attributes["fi.span.kind"] == "CONVERSATION" + assert root.attributes["gen_ai.span.kind"] == "CONVERSATION" + assert not mark_conversation_root(child) and "fi.span.kind" not in child.attributes + assert not mark_conversation_root(remote) and "fi.span.kind" not in remote.attributes + assert not mark_conversation_root(trace.NonRecordingSpan(root.get_span_context())) + for span in (remote, child, root): + span.end() + ended = TRACER.start_span("ended") + ended.end() + assert not mark_conversation_root(ended) + +def test_no_private_manager_internals_in_voice_adapters(): + import inspect -def test_no_late_ambient_context_lookup(manager): - assert bind_audio_processor(object(), None, manager=manager).status == "correlation_unavailable" + import traceai_pipecat.attachments as pipecat + for module in (pipecat,): + source = inspect.getsource(module) + for private in ("manager._session", "manager.policy", "._fd", "manager._", "_manager._"): + assert private not in source, (module.__name__, private) From 878db07f65dbe4eac6718527d4681d90632ba529 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 06:53:50 -0800 Subject: [PATCH 4/9] test(python): one recording per Pipecat start/stop cycle; document opt-in voice recordings Co-Authored-By: Claude Opus 5.5 --- python/frameworks/livekit/README.md | 26 ++++++++++++++++++ python/frameworks/pipecat/README.md | 27 +++++++++++++++++++ .../voice-tests/test_pipecat_attachments.py | 25 +++++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/python/frameworks/livekit/README.md b/python/frameworks/livekit/README.md index ea3feb2b5..53d0ac0fb 100644 --- a/python/frameworks/livekit/README.md +++ b/python/frameworks/livekit/README.md @@ -188,3 +188,29 @@ The pinned development lock contains LiveKit Agents 1.8.6, API 1.2.1, protocol 1 Check the `examples/` directory for agent examples (validate recording SDK compatibility before deployment): * `simple_agent.py`: A basic voice assistant agent. * `egress_agent.py`: An agent with opt-in Egress metadata capture, an explicit session envelope and bounded stop/close cleanup. The modern `LiveKitAPI().egress` request shape and native-session linkage require the deployment's SDK/mode validation. + +## Call recordings as trace attachments (opt-in) + +Attachments are off by default. Both paths below need the attachment-enabled `fi-instrumentation-otel` and `livekit-agents` 1.8.10 or 1.8.11. Neither starts, stops or enables recording. + +**Local session recording.** If your session already records audio (`record=True`), bind it after `await session.start(...)`: + +```python +binding = traceai.bind_livekit_session(session, ctx, owned_recording_files=True) +``` + +When the session ends, the SDK's `audio.ogg` is uploaded as one `mixed` file. This happens before LiveKit deletes its temporary directory, with a deadline of up to 30 s. Hard links, symlinks, foreign owners and replaced files are refused. LiveKit redaction, `LIVEKIT_TELEMETRY_ALLOW_PII` and FI hide flags all deny. The session span is marked `fi.span.kind=CONVERSATION` only if it has no parent. In a LiveKit worker, `agent_session` is a child of `job_entrypoint`, so it is not marked. + +**Completed Egress.** Pass a completed audio-only Egress that your app is authorized to use: + +```python +handle = await traceai.import_livekit_completed_egress( + egress_info, session_context=session_span, egress_id=egress_info.egress_id, + room_name=room, connection_id=CONNECTION_ID, registry=registry, existing_source_authorized=True) +``` + +- `egress_info` is an object you already have. Alternatively, pass `egress_client=lkapi.egress`, which makes one read by that ID. +- `connection_id` and `registry` identify the output that Future AGI has registered for your LiveKit connection. +- Only `EGRESS_COMPLETE` jobs from audio-only room or track composites with exactly one file are accepted. Anything else returns `source_unavailable`. +- No location, filename, URL or credential is ever sent. +- `handle.reason == "queued"` means the server accepted the import. It does not mean the recording is playable yet. diff --git a/python/frameworks/pipecat/README.md b/python/frameworks/pipecat/README.md index 607d10436..3f383a6b2 100644 --- a/python/frameworks/pipecat/README.md +++ b/python/frameworks/pipecat/README.md @@ -183,3 +183,30 @@ Base class for mapped span exporters. 3. **Data not being sent to FutureAGI** - Ensure that your you have set the `FI_API_KEY` and `FI_SECRET_KEY` environment variables - Ensure that the `set_global_tracer_provider` in the `register` function is set to `True` + +## Call recordings as trace attachments (opt-in) + +Attachments are off by default. When they are enabled in `traceai.init(...)`, the adapter can upload the audio your own `AudioBufferProcessor` already produces. It does not start, stop or reconfigure the recorder. Requires `pipecat-ai==1.15.0` and the attachment-enabled `fi-instrumentation-otel`. + +```python +import traceai +from fi_instrumentation.attachments import AttachmentConfig +from pipecat.pipeline.worker import PipelineWorker +from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor +from traceai_pipecat.attachments import AudioSessionContext + +traceai.init(project_name="voice-agent", attachment_config=AttachmentConfig(enabled=True)) + +recorder = AudioBufferProcessor(num_channels=2, buffer_size=32 * 1024, auto_start_recording=True) +worker = PipelineWorker(pipeline, enable_tracing=True, enable_turn_tracking=True) +binding = traceai.bind_pipecat_audio_processor(recorder, AudioSessionContext.from_pipeline(worker)) +# ... run the worker ... +await binding.finish() # waits up to 30 s for the final segment and manifest +``` + +- The recording is attached to the `conversation` span that Pipecat's tracing starts. That span is marked as the conversation root (`fi.span.kind=CONVERSATION`) only when it has no parent. +- Layouts: mono gives one mixed track. Stereo gives user (left, input) and bot (right, output) tracks in one file. `layout="separate_tracks"` uploads each side separately, so hiding inputs drops only the user side. With mono or stereo, any hide flag blocks the whole recording. +- Audio is uploaded as WAV segments of up to 30 s, plus a manifest that records gaps. If uploads fall behind, the overflow becomes a `queue_full` gap rather than growing memory. +- `buffer_size` must be between 1 and 131072 bytes. `buffer_size=0` and `enable_turn_audio=True` are refused (`unbounded_source`). +- `binding.status` reports a short, content-free state such as `recording`, `published`, `partial`, `policy_denied` or `correlation_unavailable`. +- `traceai_pipecat.create_audio_capture(...)` builds a new, unstarted recorder only when both `AttachmentConfig(record_audio=True)` and the server's `record_audio` allow it. diff --git a/python/voice-tests/test_pipecat_attachments.py b/python/voice-tests/test_pipecat_attachments.py index 045832164..093ad8700 100644 --- a/python/voice-tests/test_pipecat_attachments.py +++ b/python/voice-tests/test_pipecat_attachments.py @@ -413,6 +413,31 @@ async def test_format_drift_keeps_clock_with_explicit_gap(manager, transport): assert manifest["duration_ms"] == 1500 +@pytest.mark.asyncio +async def test_each_start_stop_cycle_is_its_own_recording(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + span, context = explicit_context() + binding = bind_audio_processor(processor, context, manager=manager) + await dispatch(processor, "on_audio_data", tone(440, 500), RATE, 1) + await dispatch(processor, "on_recording_stopped") + # The stop closes the cycle without the app calling finish(); its upload + # runs in the background and the binding is ready for the next start. + for _ in range(200): + if binding.results: + break + await asyncio.sleep(0.01) + assert [r.state for r in binding.results] == ["published"] + await dispatch(processor, "on_audio_data", tone(660, 700), RATE, 1) + await dispatch(processor, "on_recording_stopped") + await binding.finish(10) + span.end() + assert [r.state for r in binding.results] == ["published", "published"] + first, second = binding.results + assert first.recording_id != second.recording_id + durations = {m["recording_id"]: m["duration_ms"] for _, m, _, _ in transport.manifests} + assert durations == {first.recording_id: 500, second.recording_id: 700} + + @pytest.mark.asyncio async def test_handler_failure_is_isolated_and_content_free(manager, transport, caplog): class Exploding: From 74eb15b15b338a3dff42ebfb6fe5b62dc7c9f7df Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 07:45:21 -0800 Subject: [PATCH 5/9] ci(python): voice components need no system ffmpeg The runner apt mirror failed DNS on both attempts. The synthetic Ogg/Opus fixture is now encoded with PyAV's bundled codecs (already a LiveKit Agents dependency), so the apt step is removed. Co-Authored-By: Claude Opus 5.5 --- .../workflows/python-voice-attachments.yml | 2 -- .../voice-tests/test_livekit_attachments.py | 31 ++++++++++++++----- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/.github/workflows/python-voice-attachments.yml b/.github/workflows/python-voice-attachments.yml index 317e8139e..967921ed3 100644 --- a/.github/workflows/python-voice-attachments.yml +++ b/.github/workflows/python-voice-attachments.yml @@ -38,8 +38,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - - name: Install system codec - run: sudo apt-get update -qq && sudo apt-get install -y ffmpeg libsndfile1 - name: Install exact upstream with this checkout run: | python -m pip install -e python -e python/frameworks/${{ matrix.adapter.name }} \ diff --git a/python/voice-tests/test_livekit_attachments.py b/python/voice-tests/test_livekit_attachments.py index cbe75982a..a41495333 100644 --- a/python/voice-tests/test_livekit_attachments.py +++ b/python/voice-tests/test_livekit_attachments.py @@ -51,12 +51,31 @@ async def original(): def make_ogg(path): - subprocess.run(["ffmpeg", "-nostdin", "-v", "error", "-f", "lavfi", "-i", - "sine=frequency=440:duration=0.1:sample_rate=48000", "-ac", "2", - "-c:a", "libopus", "-y", str(path)], check=True, capture_output=True) + """0.1 s stereo 440 Hz Opus in Ogg, encoded with PyAV's bundled codecs (no system ffmpeg).""" + import math + import struct + rate, count = 48000, 4800 + pcm = b"".join(struct.pack(" Date: Wed, 11 Nov 2026 18:58:32 -0800 Subject: [PATCH 6/9] feat(python): Pipecat per-turn audio, short-call binding, LiveKit Voice root docs - Pipecat per-turn uploads (opt-in): `turn_audio=TurnAudioCaps(max_turns=, max_bytes=, max_duration_ms=)` on bind_audio_processor/create_audio_capture. on_user_turn_audio -> input, on_bot_turn_audio -> output, purpose turn_recording, bound to Pipecat's own turn span via TurnTraceObserver.get_turn_context(turn_number). Caps are per call, checked on length before any copy; an over-cap turn is skipped whole (queue_full for the count, too_large for bytes/duration). enable_turn_audio without caps is still refused as unbounded_source. No fallback to the conversation or an ambient span. Turn handlers are not registered when both sides are denied. - Short calls: the conversation identity is snapshotted while Pipecat's conversation span is live (bind, on_recording_started, every chunk), so a call shorter than buffer_size whose only chunk runs after the worker ended the span still binds to that conversation instead of coming out unsampled/correlation_unavailable. The root marker is set while live. - LiveKit: document how Voice finds a call without an SDK CONVERSATION root (ingest maps the untyped job_entrypoint root carrying lk.job_id); pin that the adapter marks neither job_entrypoint nor agent_session. Co-Authored-By: Claude Opus 5.5 --- python/frameworks/livekit/README.md | 6 + python/frameworks/pipecat/README.md | 25 +- .../pipecat/traceai_pipecat/__init__.py | 8 + .../pipecat/traceai_pipecat/attachments.py | 238 +++++++++++-- python/traceai/__init__.py | 3 +- .../voice-tests/test_livekit_attachments.py | 9 +- .../voice-tests/test_pipecat_attachments.py | 337 +++++++++++++++++- 7 files changed, 599 insertions(+), 27 deletions(-) diff --git a/python/frameworks/livekit/README.md b/python/frameworks/livekit/README.md index 53d0ac0fb..6ec070dba 100644 --- a/python/frameworks/livekit/README.md +++ b/python/frameworks/livekit/README.md @@ -201,6 +201,12 @@ binding = traceai.bind_livekit_session(session, ctx, owned_recording_files=True) When the session ends, the SDK's `audio.ogg` is uploaded as one `mixed` file. This happens before LiveKit deletes its temporary directory, with a deadline of up to 30 s. Hard links, symlinks, foreign owners and replaced files are refused. LiveKit redaction, `LIVEKIT_TELEMETRY_ALLOW_PII` and FI hide flags all deny. The session span is marked `fi.span.kind=CONVERSATION` only if it has no parent. In a LiveKit worker, `agent_session` is a child of `job_entrypoint`, so it is not marked. +**How Future AGI Voice finds a LiveKit call.** The SDK never invents a conversation root. In a real worker the trace's true root is LiveKit's own `job_entrypoint` span, which carries `lk.job_id`, and the recording is attached to `agent_session`, its child. When attachments are enabled on the Future AGI server, ingest treats an untyped, parentless `job_entrypoint` span with a non-empty `lk.job_id` as the call's conversation root, so the call is listed in Voice and its recording is found on the child span. + +- This needs tracing set up before the job starts (for example at worker start-up, see "Flushing at the end of a job"). With set-up inside the entrypoint, LiveKit starts `job_entrypoint` before tracing is on, the trace is rooted at `job_shutdown` instead, and the call is not listed in Voice. +- Traces ingested before attachments were enabled on the server are not re-classified. +- Do not set `fi.span.kind` on `job_entrypoint` or `agent_session` yourself to force a root; that changes how the whole trace is classified. + **Completed Egress.** Pass a completed audio-only Egress that your app is authorized to use: ```python diff --git a/python/frameworks/pipecat/README.md b/python/frameworks/pipecat/README.md index 3f383a6b2..905d12067 100644 --- a/python/frameworks/pipecat/README.md +++ b/python/frameworks/pipecat/README.md @@ -205,8 +205,31 @@ await binding.finish() # waits up to 30 s for the final segment and manifest ``` - The recording is attached to the `conversation` span that Pipecat's tracing starts. That span is marked as the conversation root (`fi.span.kind=CONVERSATION`) only when it has no parent. +- Short calls are kept. Pipecat delivers a call shorter than `buffer_size` in one chunk when recording stops, and it can run after the worker has ended the conversation span. The adapter remembers the conversation's identity while the span is live (at bind, when recording starts, and on every chunk), so that chunk still attaches to the same conversation. The root marker is set while the span is live. - Layouts: mono gives one mixed track. Stereo gives user (left, input) and bot (right, output) tracks in one file. `layout="separate_tracks"` uploads each side separately, so hiding inputs drops only the user side. With mono or stereo, any hide flag blocks the whole recording. - Audio is uploaded as WAV segments of up to 30 s, plus a manifest that records gaps. If uploads fall behind, the overflow becomes a `queue_full` gap rather than growing memory. -- `buffer_size` must be between 1 and 131072 bytes. `buffer_size=0` and `enable_turn_audio=True` are refused (`unbounded_source`). +- `buffer_size` must be between 1 and 131072 bytes. `buffer_size=0` is refused (`unbounded_source`), and so is `enable_turn_audio=True` unless you opt in to per-turn audio (below). - `binding.status` reports a short, content-free state such as `recording`, `published`, `partial`, `policy_denied` or `correlation_unavailable`. - `traceai_pipecat.create_audio_capture(...)` builds a new, unstarted recorder only when both `AttachmentConfig(record_audio=True)` and the server's `record_audio` allow it. + +### Per-turn audio (separate opt-in) + +Per-turn uploads are off unless you pass explicit caps for the whole call. There are no defaults: + +```python +from traceai_pipecat import TurnAudioCaps + +recorder = AudioBufferProcessor(num_channels=1, buffer_size=32 * 1024, auto_start_recording=True, + enable_turn_audio=True) +binding = traceai.bind_pipecat_audio_processor( + recorder, AudioSessionContext.from_pipeline(worker), + turn_audio=TurnAudioCaps(max_turns=50, max_bytes=16 * 1024 * 1024, max_duration_ms=20 * 60 * 1000)) +``` + +- Each finished turn uploads once per side: `on_user_turn_audio` as `input` and `on_bot_turn_audio` as `output`, as a mono WAV with purpose `turn_recording`, `turn_id` = Pipecat's turn number and the call's `conversation_id`. +- Each upload is bound to that turn's own span, from Pipecat's `TurnTraceObserver.get_turn_context(turn_number)`. It never falls back to the conversation span or an ambient span. With `AudioSessionContext.from_span(...)` there are no turn spans, so turns end `correlation_unavailable`. +- Caps apply to the whole call: `max_turns` uploads (at most 1000), `max_bytes` of WAV (at most 64 MiB) and `max_duration_ms` of audio (at most 2 h). A turn that would cross a cap is skipped whole, never truncated: `queue_full` for the count, `too_large` for bytes or duration. Caps are checked on the length before any copy. +- Privacy is checked per turn, per side. `FI_HIDE_INPUTS` drops the user's turns and keeps the bot's. Unsampled turn spans upload nothing. +- `enable_turn_audio=True` makes Pipecat keep each turn's audio in its own memory until the turn ends. The caps bound what the adapter copies and uploads, not Pipecat's own buffer. +- `binding.turn_status` is the last turn's admission (`queued` or a reason), `binding.turn_denials` counts skipped turns by reason, and `await binding.finish()` also waits for turn uploads within the same deadline. +- `create_audio_capture(..., turn_audio=TurnAudioCaps(...))` builds the recorder with `enable_turn_audio=True`. diff --git a/python/frameworks/pipecat/traceai_pipecat/__init__.py b/python/frameworks/pipecat/traceai_pipecat/__init__.py index 66e71552d..d34875c38 100644 --- a/python/frameworks/pipecat/traceai_pipecat/__init__.py +++ b/python/frameworks/pipecat/traceai_pipecat/__init__.py @@ -34,6 +34,14 @@ def create_audio_capture(session_context, *, manager, **options): return create(session_context, manager=manager, **options) +def __getattr__(name): + # Lazy, so importing the package never needs the attachment-enabled core. + if name in ("TurnAudioCaps", "AudioSessionContext"): + from . import attachments + return getattr(attachments, name) + raise AttributeError("module %r has no attribute %r" % (__name__, name)) + + __version__ = "0.1.2" # x-release-please-version __all__ = [ diff --git a/python/frameworks/pipecat/traceai_pipecat/attachments.py b/python/frameworks/pipecat/traceai_pipecat/attachments.py index 14714a5f7..817f22c1c 100644 --- a/python/frameworks/pipecat/traceai_pipecat/attachments.py +++ b/python/frameworks/pipecat/traceai_pipecat/attachments.py @@ -14,21 +14,32 @@ Upstream bound: ``buffer_size`` must be within 1..128 KiB, so the processor flushes bounded chunks on its own schedule. ``buffer_size=0`` keeps the whole -call in Pipecat memory until stop and is refused (``unbounded_source``), as is -``enable_turn_audio`` (per-turn buffers are not observed by this adapter). +call in Pipecat memory until stop and is refused (``unbounded_source``). +``enable_turn_audio`` is refused the same way unless the caller opts in to +per-turn uploads with explicit ``TurnAudioCaps`` (count, bytes and duration per +call); each turn then uploads once, bound to Pipecat's own turn span. + +Short calls: the conversation identity is snapshotted while Pipecat's +conversation span is live (at bind, on ``on_recording_started`` and on every +chunk). Pipecat runs these handlers as tasks, so a call shorter than +``buffer_size`` delivers its only chunk on stop, possibly after the worker has +ended the conversation span; the recording then binds to the snapshot instead +of coming out ``unsampled`` or ``correlation_unavailable``. """ import asyncio import math import re +import struct import threading +import time import weakref from dataclasses import dataclass from importlib.metadata import PackageNotFoundError, version from typing import Optional from fi_instrumentation.attachments import ( - CapturePolicy, CorrelationHandle, PcmFormat, RecordingOptions, TrackSpec, + Bytes, CapturePolicy, CorrelationHandle, PcmFormat, RecordingOptions, TrackSpec, ) # Exact versions exercised by python/voice-tests on a real pipeline. 1.15.0 is @@ -43,6 +54,39 @@ CONVERSATION_KIND = "CONVERSATION" +# Hard ceilings for TurnAudioCaps; a caller can only choose lower caps. +_TURN_CEILINGS = {"max_turns": 1000, "max_bytes": 64 * 1024 * 1024, "max_duration_ms": 2 * 60 * 60 * 1000} +_WAV_HEADER = 44 +_TURN_SIDES = {"on_user_turn_audio": ("input", "user"), "on_bot_turn_audio": ("output", "assistant")} + + +@dataclass(frozen=True) +class TurnAudioCaps: + """Explicit per-call caps for per-turn uploads. There are no defaults. + + ``max_turns`` caps how many turn uploads one call (one binding) admits, + ``max_bytes`` the summed WAV bytes and ``max_duration_ms`` the summed audio + duration. A turn that would cross a cap is skipped whole (never truncated): + ``queue_full`` for the count, ``too_large`` for bytes or duration. + """ + + max_turns: int + max_bytes: int + max_duration_ms: int + + def __post_init__(self): + for name, ceiling in _TURN_CEILINGS.items(): + value = getattr(self, name) + if type(value) is not int or not 0 < value <= ceiling: + raise ValueError("%s must be an int within 1..%d" % (name, ceiling)) + + +def _wav_header(data_bytes: int, sample_rate: int) -> bytes: + # Mono 16-bit PCM, the format Pipecat's TurnAudioData carries. + return b"".join((b"RIFF", struct.pack(" bool: return False +def _span_id(span) -> Optional[str]: + try: + return format(span.get_span_context().span_id, "016x") + except Exception: + return None + + def _token(value) -> Optional[str]: # Opaque ids that are not F01 tokens are dropped, never rewritten. return value if isinstance(value, str) and _TOKEN.fullmatch(value) else None -def _probe(manager, operation, directions, capture) -> Optional[str]: +def _probe(manager, operation, directions, capture, purpose="conversation_recording") -> Optional[str]: """Deny-wins policy check through the public preflight; retains nothing. The sentinel identity only lets preflight run past correlation; the ticket @@ -104,7 +155,7 @@ def _probe(manager, operation, directions, capture) -> Optional[str]: for direction in directions: try: ticket = manager.preflight(_SENTINEL, mime_type="audio/wav", direction=direction, max_bytes=1, - operation=operation, source_kind="Bytes", purpose="conversation_recording", + operation=operation, source_kind="Bytes", purpose=purpose, capture_policy=capture) except Exception: return "source_unavailable" @@ -139,14 +190,23 @@ class AudioCaptureBinding: reason, or the last finished ``RecordingResult.state``. ``results`` holds each finished recording cycle (one per start/stop of the app's recorder). ``close()`` detaches; it never stops the application's recorder. + + Per-turn uploads (only with ``turn_audio=TurnAudioCaps(...)``) report + separately: ``turn_status`` is ``off``, ``session_bound``, the last turn's + admission (``queued`` or a denial reason); ``turn_handles`` holds + ``(turn_number, direction, AttachmentHandle)`` for each admitted turn (at + most ``max_turns``) and ``turn_denials`` counts skipped turns by reason. """ def __init__(self, status: str, processor=None, *, manager=None, context=None, layout=None, tracks=None, - operation="observe_consumed", capture=None, timeout=30.0): + operation="observe_consumed", capture=None, timeout=30.0, turn_audio=None): self.status = status self.processor = processor self.results = [] self.root_marked = False + self.turn_status = "off" + self.turn_handles = [] + self.turn_denials = {} self._manager = manager self._context = context self._layout = layout @@ -154,9 +214,12 @@ def __init__(self, status: str, processor=None, *, manager=None, context=None, l self._operation = operation self._capture = capture self._timeout = timeout + self._turn_audio = turn_audio + self._turn_used = [0, 0, 0] # uploads, WAV bytes, duration ms admitted so far in this call self._writer = None self._format = None self._cycle_denied = False + self._snapshot = None # (CorrelationHandle, span_id, conversation_id) of the live conversation self._closed = status != "session_bound" self._handlers = [] self._pending = set() @@ -175,12 +238,24 @@ def _on_track_audio_data(self, _processor, user_audio, bot_audio, sample_rate, _ except Exception: self._fail() + def _on_recording_started(self, _processor): + # The conversation span exists by now (the observer starts it on the + # StartFrame push); remember its identity while it is still live. + if not self._closed: + self._remember() + def _on_recording_stopped(self, _processor): try: self._finish_cycle() except Exception: self._fail() + def _on_user_turn_audio(self, _processor, data): + self._turn("on_user_turn_audio", data) + + def _on_bot_turn_audio(self, _processor, data): + self._turn("on_bot_turn_audio", data) + # ---------------------------------------------------------- internals def _fail(self) -> None: # Never log the exception: it can carry caller objects. Drop retention. @@ -189,15 +264,48 @@ def _fail(self) -> None: writer.abort() self.status = "source_unavailable" + def _observer(self): + return getattr(self._context.worker, "turn_trace_observer", None) + + def _remember(self) -> None: + """Snapshot the observer's conversation identity while its span is live.""" + try: + if self._context.correlation is not None: + return + observer = self._observer() + # Version-qualified seam (1.15.0): the observer's own conversation span. + span = getattr(observer, "_conversation_span", None) if observer is not None else None + if span is None or not span.is_recording(): + return + if self._snapshot is not None and self._snapshot[1] == _span_id(span): + return + correlation = CorrelationHandle.from_span(span) + self._snapshot = (correlation, correlation.span_id, _token(getattr(observer, "_conversation_id", None))) + # Admitted binding + live true root: mark it now, because a short + # call may only deliver audio after the span has ended. + if correlation.sampled and mark_conversation_root(span): + self.root_marked = True + except Exception: + return + def _resolve(self): context = self._context if context.correlation is not None: return None, context.correlation, context.span, _token(context.conversation_id) - observer = getattr(context.worker, "turn_trace_observer", None) + observer = self._observer() if observer is None: return "correlation_unavailable", None, None, None - # Version-qualified seam (1.15.0): the observer's own conversation span. span = getattr(observer, "_conversation_span", None) + snapshot = self._snapshot + if span is not None and span.is_recording(): + self._remember() + snapshot = self._snapshot + if snapshot is not None: + return None, snapshot[0], span, snapshot[2] + if snapshot is not None and (span is None or _span_id(span) == snapshot[1]): + # The worker ended (and cleared) the conversation span before this + # chunk ran: bind to the identity captured while it was live. + return None, snapshot[0], None, snapshot[2] if span is None: return "correlation_unavailable", None, None, None try: @@ -212,6 +320,7 @@ def _open(self, sample_rate, channels): reason, correlation, span, conversation_id = self._resolve() if reason is None and not correlation.sampled: reason = "unsampled" + live = None if reason is None: options = RecordingOptions(format=PcmFormat(int(sample_rate)), layout=self._layout, tracks=self._tracks, operation=self._operation, purpose="conversation_recording", @@ -229,6 +338,52 @@ def _open(self, sample_rate, channels): self.status = "recording" return writer + def _turn(self, event, data) -> None: + """Upload one finished turn for one side, bound to Pipecat's turn span.""" + try: + if self._closed or self._turn_audio is None: + return + reason = self._turn_upload(event, data) + except Exception: + reason = "source_unavailable" + if reason != "queued": + self.turn_denials[reason] = self.turn_denials.get(reason, 0) + 1 + self.turn_status = reason + + def _turn_upload(self, event, data) -> str: + direction, role = _TURN_SIDES[event] + caps, used = self._turn_audio, self._turn_used + turn_number, rate = getattr(data, "turn_number", None), getattr(data, "sample_rate", None) + audio, channels = getattr(data, "audio", None), getattr(data, "num_channels", None) + if (type(turn_number) is not int or turn_number < 1 or type(rate) is not int or rate <= 0 + or channels != 1 or type(audio) is not bytes or not audio or len(audio) % 2): + return "source_unavailable" + # Caps first, on the length alone: an over-cap turn is never copied. + size = _WAV_HEADER + len(audio) + duration = math.ceil(len(audio) * 1000 / (2 * rate)) + if used[0] >= caps.max_turns: + return "queue_full" + if used[1] + size > caps.max_bytes or used[2] + duration > caps.max_duration_ms: + return "too_large" + # Pipecat's own turn span (public API); never the conversation or an ambient span. + observer = self._observer() + span_context = observer.get_turn_context(turn_number) if observer is not None else None + if span_context is None: + return "correlation_unavailable" + correlation = CorrelationHandle.from_span(span_context) + ticket = self._manager.preflight(correlation, mime_type="audio/wav", direction=direction, max_bytes=size, + operation=self._operation, source_kind="Bytes", purpose="turn_recording", + capture_policy=self._capture) + if not ticket.admitted: + ticket.release() + return ticket.reason + used[0], used[1], used[2] = used[0] + 1, used[1] + size, used[2] + duration + conversation_id = self._snapshot[2] if self._snapshot is not None else _token(self._context.conversation_id) + payload = _wav_header(len(audio), rate) + audio + handle = ticket.attach(Bytes(payload), role=role, conversation_id=conversation_id, turn_id=str(turn_number)) + self.turn_handles.append((turn_number, direction, handle)) + return handle.admission + def _write(self, chunks, sample_rate, channels): if self._closed: return @@ -287,12 +442,18 @@ async def finish(self, timeout: Optional[float] = None): Call before process exit (after the pipeline ends). Pipecat already finishes a cycle on ``on_recording_stopped``; this only awaits it. Returns the last ``RecordingResult`` or None. A hard kill is lossy. + Per-turn uploads are awaited within the same deadline (``turn_handles``). """ budget = self._timeout if timeout is None else min(max(float(timeout), 0.0), self._timeout) + deadline = time.monotonic() + budget self._finish_cycle() pending = list(self._pending) if pending: await asyncio.wait(pending, timeout=budget + 1) + # Turn handles complete on the core's workers; poll without a thread each. + while time.monotonic() < deadline and any( + handle.result(0).status == "unfinished" for _, _, handle in self.turn_handles): + await asyncio.sleep(0.02) return self.results[-1] if self.results else None def close(self) -> None: @@ -330,23 +491,35 @@ def _validate_options(layout, finalization_timeout, capture_policy): return capture -def _qualified_processor(processor): +def _qualified_processor(processor, turn_audio): if version("pipecat-ai") not in _SUPPORTED: return "unsupported_version" from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor if not isinstance(processor, AudioBufferProcessor): return "unsupported_version" size = processor._buffer_size - if type(size) is not int or not 0 < size <= _MAX_BUFFER or processor._enable_turn_audio: + if type(size) is not int or not 0 < size <= _MAX_BUFFER: return "unbounded_source" + if bool(processor._enable_turn_audio) != (turn_audio is not None): + if turn_audio is None: + # Pipecat keeps each turn in memory for its turn events; without + # explicit caps that is an unbounded source we refuse to observe. + return "unbounded_source" + raise ValueError("turn_audio needs AudioBufferProcessor(enable_turn_audio=True)") return None -def _bind(processor, session_context, manager, *, layout, timeout, capture, operation): +def _validate_turn_audio(turn_audio): + if turn_audio is not None and not isinstance(turn_audio, TurnAudioCaps): + raise ValueError("turn_audio must be TurnAudioCaps(max_turns=, max_bytes=, max_duration_ms=)") + return turn_audio + + +def _bind(processor, session_context, manager, *, layout, timeout, capture, operation, turn_audio=None): if not isinstance(session_context, AudioSessionContext): return AudioCaptureBinding("correlation_unavailable") try: - reason = _qualified_processor(processor) + reason = _qualified_processor(processor, turn_audio) if reason: return AudioCaptureBinding(reason) selected, tracks = _layout(processor, layout) @@ -356,26 +529,39 @@ def _bind(processor, session_context, manager, *, layout, timeout, capture, oper reason = _probe(manager, operation, directions, capture) if reason: return AudioCaptureBinding(reason) + # Per-turn uploads: admitted only if at least one side may upload a + # turn_recording; otherwise no turn handler is registered at all. + turn_reason = None + if turn_audio is not None: + turn_reason = _probe(manager, operation, ("input", "output"), capture, purpose="turn_recording") with _BINDINGS_LOCK: previous = _BINDINGS.get(processor) if previous is not None and not previous._closed: if (previous._manager is manager and previous._context == session_context and previous._layout == selected and previous._operation == operation - and previous._capture == capture and previous._timeout == timeout): + and previous._capture == capture and previous._timeout == timeout + and previous._turn_audio == turn_audio): return previous raise ValueError("conflicting recording binding") binding = AudioCaptureBinding("session_bound", processor, manager=manager, context=session_context, layout=selected, tracks=tracks, operation=operation, capture=capture, - timeout=timeout) + timeout=timeout, turn_audio=turn_audio) _BINDINGS[processor] = binding if selected == "separate_tracks": handlers = (("on_track_audio_data", binding._on_track_audio_data),) else: handlers = (("on_audio_data", binding._on_audio_data),) - handlers += (("on_recording_stopped", binding._on_recording_stopped),) + handlers += (("on_recording_started", binding._on_recording_started), + ("on_recording_stopped", binding._on_recording_stopped)) + if turn_audio is not None: + binding.turn_status = turn_reason or "session_bound" + if turn_reason is None: + handlers += (("on_user_turn_audio", binding._on_user_turn_audio), + ("on_bot_turn_audio", binding._on_bot_turn_audio)) for event, handler in handlers: processor.add_event_handler(event, handler) binding._handlers.append((event, handler)) + binding._remember() return binding except ValueError: raise @@ -386,7 +572,7 @@ def _bind(processor, session_context, manager, *, layout, timeout, capture, oper def bind_audio_processor(processor, session_context, *, manager, layout="auto", finalization_timeout=30, - capture_policy=None): + capture_policy=None, turn_audio=None): """Observe the app's EXISTING AudioBufferProcessor; never start or modify it. Bind before the pipeline runs. ``layout="auto"`` follows the processor: @@ -394,11 +580,20 @@ def bind_audio_processor(processor, session_context, *, manager, layout="auto", ``layout="separate_tracks"`` observes ``on_track_audio_data`` and keeps one track per side, so a hide flag on one side drops only that side. + ``turn_audio=TurnAudioCaps(max_turns=, max_bytes=, max_duration_ms=)`` + additionally uploads each finished turn (``on_user_turn_audio`` as input, + ``on_bot_turn_audio`` as output, purpose ``turn_recording``) bound to that + turn's own Pipecat span via ``get_turn_context(turn_number)``. It is off by + default and needs ``AudioBufferProcessor(enable_turn_audio=True)`` plus a + worker with tracing and turn tracking; an explicit-span context has no turn + spans, so its turns end ``correlation_unavailable``. + Policy is checked before the processor is inspected, and again by the core on every chunk. Disabled attachments return ``disabled`` and register nothing. Await ``binding.finish()`` after the pipeline ends. """ capture = _validate_options(layout, finalization_timeout, capture_policy) + turn_audio = _validate_turn_audio(turn_audio) if capture.new_recording: raise ValueError("observing an existing recorder does not create a recording") try: @@ -411,19 +606,22 @@ def bind_audio_processor(processor, session_context, *, manager, layout="auto", if reason: return AudioCaptureBinding(reason) return _bind(processor, session_context, manager, layout=layout, timeout=finalization_timeout, - capture=capture, operation="observe_consumed") + capture=capture, operation="observe_consumed", turn_audio=turn_audio) def create_audio_capture(session_context, *, manager, sample_rate=None, num_channels=1, buffer_size=32 * 1024, - layout="auto", finalization_timeout=30, capture_policy=None): + layout="auto", finalization_timeout=30, capture_policy=None, turn_audio=None): """Create a NEW, un-started AudioBufferProcessor only with both permissions. Needs local ``AttachmentConfig(record_audio=True)`` AND server ``record_audio``. The returned ``binding.processor`` must be inserted into the pipeline by the application, which also starts it (for example with ``AudioBufferStartRecordingFrame``). Nothing is created when denied. + ``turn_audio`` (``TurnAudioCaps``) creates the processor with + ``enable_turn_audio=True`` and uploads turns as in ``bind_audio_processor``. """ capture = _validate_options(layout, finalization_timeout, capture_policy) + turn_audio = _validate_turn_audio(turn_audio) if (type(buffer_size) is not int or not 0 < buffer_size <= _MAX_BUFFER or num_channels not in (1, 2)): raise ValueError("buffer_size must be within 1..131072 and num_channels 1 or 2") @@ -444,10 +642,10 @@ def create_audio_capture(session_context, *, manager, sample_rate=None, num_chan return AudioCaptureBinding("unsupported_version") from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor processor = AudioBufferProcessor(sample_rate=sample_rate, num_channels=num_channels, - buffer_size=buffer_size, enable_turn_audio=False) + buffer_size=buffer_size, enable_turn_audio=turn_audio is not None) except PackageNotFoundError: return AudioCaptureBinding("unsupported_version") except Exception: return AudioCaptureBinding("source_unavailable") return _bind(processor, session_context, manager, layout=layout, timeout=finalization_timeout, - capture=capture, operation="create_recording") + capture=capture, operation="create_recording", turn_audio=turn_audio) diff --git a/python/traceai/__init__.py b/python/traceai/__init__.py index c04374de0..5408e9343 100644 --- a/python/traceai/__init__.py +++ b/python/traceai/__init__.py @@ -335,7 +335,8 @@ def bind_pipecat_audio_processor(processor, session_context, **options): ``session_context`` is a ``traceai_pipecat.attachments.AudioSessionContext`` (``from_pipeline(worker)`` or ``from_span(span)``). Options are passed to ``traceai_pipecat.bind_audio_processor`` (``layout``, ``finalization_timeout``, - ``capture_policy``). + ``capture_policy``, and the opt-in per-turn caps + ``turn_audio=traceai_pipecat.TurnAudioCaps(max_turns=, max_bytes=, max_duration_ms=)``). """ return _bind("pipecat", "bind_audio_processor", processor, session_context, **options) diff --git a/python/voice-tests/test_livekit_attachments.py b/python/voice-tests/test_livekit_attachments.py index a41495333..aabbeba21 100644 --- a/python/voice-tests/test_livekit_attachments.py +++ b/python/voice-tests/test_livekit_attachments.py @@ -447,12 +447,17 @@ async def test_child_session_span_is_never_marked_root(manager, tmp_path): from opentelemetry import trace session, job, _, span, provider, _ = owned_session(tmp_path) span.end() - parent = provider.get_tracer("synthetic").start_span("job_entrypoint") + # The real worker shape: LiveKit's own root carries lk.job_id; the session is its child. + parent = provider.get_tracer("synthetic").start_span("job_entrypoint", attributes={"lk.job_id": "synthetic-job"}) child = provider.get_tracer("synthetic").start_span("agent_session", context=trace.set_span_in_context(parent)) session._session_span = child binding = bind_session(session, job, manager=manager, owned_recording_files=True) assert binding.status == "session_bound" and not binding.root_marked - assert "fi.span.kind" not in child.attributes + # No fake root: the SDK marks neither span. Ingest (I04) lists the call + # from the untyped job_entrypoint root that carries lk.job_id. + for span in (child, parent): + assert "fi.span.kind" not in span.attributes and "gen_ai.span.kind" not in span.attributes + assert dict(parent.attributes) == {"lk.job_id": "synthetic-job"} binding.close() child.end() parent.end() diff --git a/python/voice-tests/test_pipecat_attachments.py b/python/voice-tests/test_pipecat_attachments.py index 093ad8700..6721da5a7 100644 --- a/python/voice-tests/test_pipecat_attachments.py +++ b/python/voice-tests/test_pipecat_attachments.py @@ -8,8 +8,10 @@ import asyncio import io +import json import math import struct +import threading import wave from dataclasses import replace @@ -23,13 +25,16 @@ from conftest import TransportDouble, make_manager from fi_instrumentation.attachments import AttachmentConfig, AttachmentManager, CorrelationHandle from fi_instrumentation.attachments._disabled import DisabledAttachmentManager -from pipecat.frames.frames import EndFrame, InputAudioRawFrame, OutputAudioRawFrame +from pipecat.frames.frames import ( + BotStartedSpeakingFrame, BotStoppedSpeakingFrame, EndFrame, InputAudioRawFrame, OutputAudioRawFrame, + UserStartedSpeakingFrame, UserStoppedSpeakingFrame, +) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.worker import PipelineParams, PipelineWorker -from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor +from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor, TurnAudioData from pipecat.workers.runner import WorkerRunner from traceai_pipecat import bind_audio_processor, create_audio_capture -from traceai_pipecat.attachments import AudioSessionContext, mark_conversation_root +from traceai_pipecat.attachments import AudioSessionContext, TurnAudioCaps, mark_conversation_root # Pipecat's TurnTraceObserver uses the GLOBAL tracer provider. EXPORTER = InMemorySpanExporter() @@ -499,3 +504,329 @@ def test_no_private_manager_internals_in_voice_adapters(): source = inspect.getsource(module) for private in ("manager._session", "manager.policy", "._fd", "manager._", "_manager._"): assert private not in source, (module.__name__, private) + + +# ------------------------------------------------------------- short calls +@pytest.mark.asyncio +async def test_short_call_binds_to_snapshot_after_worker_ended_conversation(manager, transport): + """Pipecat runs on_audio_data as a task: a call shorter than buffer_size + delivers its only chunk on stop, which can run after the worker's cleanup + has ended and cleared the conversation span.""" + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=128 * 1024) + worker = await run_pipeline(processor, None) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(worker), manager=manager) + observer = worker.turn_trace_observer + observer.start_conversation_tracing("conv-short") + await dispatch(processor, "on_recording_started") + live_id = format(observer._conversation_span.get_span_context().span_id, "016x") + observer.end_conversation_tracing() + assert observer._conversation_span is None + await dispatch(processor, "on_audio_data", tone(440, 300), RATE, 1) + await dispatch(processor, "on_recording_stopped") + result = await binding.finish(10) + assert result is not None and result.state == "published", (binding.status, result) + uploads = [r for r in transport.requests if "recording_id" in r] + assert uploads and all(r["association"]["parent"]["span_id"] == live_id for r in uploads) + assert all(r["conversation_id"] == "conv-short" for r in uploads) + root = conversation_span() + assert format(root.context.span_id, "016x") == live_id + # Marked while it was still live, because the audio came after it ended. + assert binding.root_marked and root.attributes["fi.span.kind"] == "CONVERSATION" + assert pcm(transport.payloads[0])[2] == tone(440, 300) + + +@pytest.mark.asyncio +async def test_no_snapshot_and_no_span_stays_correlation_unavailable(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=16000) + worker = await run_pipeline(processor, None) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(worker), manager=manager) + ambient = TRACER.start_span("ambient") + with trace.use_span(ambient, end_on_exit=True): + await dispatch(processor, "on_audio_data", tone(440, 300), RATE, 1) + assert binding.status == "correlation_unavailable" + assert not transport.requests and manager.owned_bytes == 0 + + +@pytest.mark.asyncio +async def test_real_pipeline_short_call_under_buffer_size_is_recorded(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=128 * 1024, + auto_start_recording=True) + worker = await run_pipeline(processor, None) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(worker), manager=manager) + await drive(worker, call_frames(turns=1, ms=200)) + result = await binding.finish(10) + assert result is not None and result.state == "published", (binding.status, result) + root = conversation_span() + parent = transport.requests[0]["association"]["parent"] + assert parent["span_id"] == format(root.context.span_id, "016x") + assert binding.root_marked and root.attributes["fi.span.kind"] == "CONVERSATION" + + +# ------------------------------------------------------------ per-turn audio +CAPS = TurnAudioCaps(max_turns=100, max_bytes=8 * 1024 * 1024, max_duration_ms=600000) + + +def turn_frames(turns=3, ms=200): + frames = [] + for i in range(turns): + frames.append(UserStartedSpeakingFrame()) + frames += [InputAudioRawFrame(audio=tone(300 + i, 20), sample_rate=RATE, num_channels=1) + for _ in range(ms // 20)] + frames.append(UserStoppedSpeakingFrame()) + frames.append(BotStartedSpeakingFrame()) + frames += [OutputAudioRawFrame(audio=tone(900 + i, 20), sample_rate=RATE, num_channels=1) + for _ in range(ms // 20)] + frames.append(BotStoppedSpeakingFrame()) + return frames + + +def pair_uploads(transport): + """Record (UploadRequest, payload) per upload, paired on the worker thread that ran both. + + The core runs two upload workers, so the double's flat request and payload + lists can interleave; one job's initialize and content PUT share a thread. + """ + pairs, local = [], threading.local() + initialize = transport.initialize + + def paired_initialize(session, body, **kwargs): + upload = initialize(session, body, **kwargs) + local.request = json.loads(body) + return upload + + class Payloads(list): + def append(self, data): + super().append(data) + pairs.append((getattr(local, "request", None), data)) + + transport.initialize = paired_initialize + transport.payloads = Payloads(transport.payloads) + return pairs + + +def turn_pairs(pairs): + return [(r, p) for r, p in pairs if r is not None and r.get("purpose") == "turn_recording"] + + +@pytest.mark.asyncio +async def test_turn_audio_is_off_by_default_and_needs_explicit_caps(manager): + span, context = explicit_context() + turns = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000, enable_turn_audio=True) + before = handler_counts(turns) + assert bind_audio_processor(turns, context, manager=manager).status == "unbounded_source" + assert handler_counts(turns) == before + plain = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000) + with pytest.raises(ValueError, match="enable_turn_audio"): + bind_audio_processor(plain, context, manager=manager, turn_audio=CAPS) + with pytest.raises(ValueError): + bind_audio_processor(turns, context, manager=manager, turn_audio={"max_turns": 1}) + for bad in ((0, 1, 1), (1, 0, 1), (1, 1, 0), (1001, 1, 1), (1, 64 * 1024 * 1024 + 1, 1), + (1, 1, 2 * 60 * 60 * 1000 + 1), (1.0, 1, 1), (True, 1, 1)): + with pytest.raises(ValueError): + TurnAudioCaps(*bad) + binding = bind_audio_processor(plain, context, manager=manager) + assert binding.turn_status == "off" and "on_user_turn_audio" not in [e for e, _ in binding._handlers] + binding.close() + span.end() + + +@pytest.mark.asyncio +async def test_real_pipeline_uploads_each_turn_bound_to_its_turn_span(manager, transport): + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=16000, + auto_start_recording=True, enable_turn_audio=True) + emitted = [] + processor.add_event_handler("on_user_turn_audio", lambda _p, d: emitted.append(("input", d.turn_number, d.audio))) + processor.add_event_handler("on_bot_turn_audio", lambda _p, d: emitted.append(("output", d.turn_number, d.audio))) + worker = await run_pipeline(processor, None) + binding = bind_audio_processor(processor, AudioSessionContext.from_pipeline(worker), manager=manager, + turn_audio=CAPS) + assert binding.turn_status == "session_bound" + pairs = pair_uploads(transport) + await drive(worker, turn_frames(turns=3)) + result = await binding.finish(10) + assert result is not None and result.state == "published" + assert emitted, "Pipecat emitted no turn audio" + turn_spans = {s.attributes["turn.number"]: s for s in EXPORTER.get_finished_spans() if s.name == "turn"} + uploads = turn_pairs(pairs) + assert len(uploads) == len(emitted) == len(binding.turn_handles) + got = {} + for request, payload in uploads: + number = int(request["turn_id"]) + assert request["association"]["parent"]["span_id"] == format(turn_spans[number].context.span_id, "016x") + assert request["conversation_id"] == "conv-synthetic" and request["mime_type"] == "audio/wav" + nch, rate, frames = pcm(payload) + assert (nch, rate) == (1, RATE) + got[(request["direction"], number)] = frames + assert got == {(direction, number): audio for direction, number, audio in emitted} + # Queued synthetic frames: Bot/User*SpeakingFrame are system frames and can + # overtake queued audio, so which turns Pipecat emits here depends on its + # scheduling. Every emitted turn must upload, byte-identical, on its own + # turn span; the bot side is covered by the dispatch tests below. + assert all(direction in ("input", "output") for direction, _ in got) + assert all(h.result(0).status != "unfinished" for _, _, h in binding.turn_handles) + # The whole-call recording is unchanged and still binds to the conversation. + root = conversation_span() + recording = [r for r in transport.requests if "recording_id" in r] + assert recording and recording[0]["association"]["parent"]["span_id"] == format(root.context.span_id, "016x") + assert manager.owned_bytes == 0 + + +async def turn_binding(manager, caps=CAPS, *, context=None, layout="auto"): + processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=16000, enable_turn_audio=True) + worker = await run_pipeline(processor, None) + observer = worker.turn_trace_observer + observer.start_conversation_tracing("conv-turns") + binding = bind_audio_processor(processor, context or AudioSessionContext.from_pipeline(worker), + manager=manager, turn_audio=caps, layout=layout) + return processor, observer, binding + + +async def run_turn(processor, observer, number, user_ms=500, bot_ms=500): + await observer._turn_tracker._call_event_handler("on_turn_started", number) + for _ in range(3): + await asyncio.sleep(0) + await observer._turn_tracker._call_event_handler("on_turn_ended", number, 1.0, False) + for _ in range(3): + await asyncio.sleep(0) + if user_ms: + await dispatch(processor, "on_user_turn_audio", TurnAudioData(number, tone(300, user_ms), RATE, 1)) + if bot_ms: + await dispatch(processor, "on_bot_turn_audio", TurnAudioData(number, tone(900, bot_ms), RATE, 1)) + + +@pytest.mark.asyncio +async def test_turn_count_cap_skips_whole_turns(manager, transport): + processor, observer, binding = await turn_binding( + manager, TurnAudioCaps(max_turns=2, max_bytes=8 * 1024 * 1024, max_duration_ms=600000)) + for number in (1, 2, 3): + await run_turn(processor, observer, number) + await binding.finish(10) + observer.end_conversation_tracing() + uploads = [r for r in transport.requests if r.get("purpose") == "turn_recording"] + assert sorted((r["turn_id"], r["direction"]) for r in uploads) == [("1", "input"), ("1", "output")] + assert binding.turn_denials == {"queue_full": 4} and binding.turn_status == "queue_full" + assert len(binding.turn_handles) == 2 + + +@pytest.mark.asyncio +async def test_turn_byte_and_duration_caps_never_truncate(manager, transport): + one_turn = 44 + len(tone(300, 500)) + pairs = pair_uploads(transport) + processor, observer, binding = await turn_binding( + manager, TurnAudioCaps(max_turns=100, max_bytes=one_turn + 10, max_duration_ms=600000)) + await run_turn(processor, observer, 1) + await binding.finish(10) + uploads = turn_pairs(pairs) + assert [(r["direction"], r["declared_bytes"]) for r, _ in uploads] == [("input", one_turn)] + assert pcm(uploads[0][1])[2] == tone(300, 500) + assert binding.turn_denials == {"too_large": 1} + binding.close() + observer.end_conversation_tracing() + # Duration cap: 500 ms fits under 700 ms, the bot's 500 ms more does not. + processor, observer, binding = await turn_binding( + manager, TurnAudioCaps(max_turns=100, max_bytes=8 * 1024 * 1024, max_duration_ms=700)) + await run_turn(processor, observer, 1) + await binding.finish(10) + uploads = turn_pairs(pairs)[1:] + assert [r["direction"] for r, _ in uploads] == ["input"] + assert binding.turn_denials == {"too_large": 1} + observer.end_conversation_tracing() + + +@pytest.mark.asyncio +async def test_hidden_side_turns_are_denied_other_side_uploads(manager, transport, monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + # A mono mixed recording is denied whole under a hide flag; per-side tracks are not. + processor, observer, binding = await turn_binding(manager, layout="separate_tracks") + assert binding.status == "session_bound" and binding.turn_status == "session_bound" + await run_turn(processor, observer, 1) + await binding.finish(10) + observer.end_conversation_tracing() + uploads = [r for r in transport.requests if r.get("purpose") == "turn_recording"] + assert [r["direction"] for r in uploads] == ["output"] + assert binding.turn_denials == {"policy_denied": 1} + + +@pytest.mark.asyncio +async def test_turn_never_falls_back_to_conversation_or_explicit_span(manager, transport): + span, context = explicit_context() + processor, observer, binding = await turn_binding(manager, context=context) + await run_turn(processor, observer, 1) + assert binding.turn_denials == {"correlation_unavailable": 2} + # Pipeline context, but a turn number Pipecat never traced. + binding.close() + observer.end_conversation_tracing() + processor, observer, binding = await turn_binding(manager) + await dispatch(processor, "on_user_turn_audio", TurnAudioData(7, tone(300, 200), RATE, 1)) + assert binding.turn_status == "correlation_unavailable" + observer.end_conversation_tracing() + span.end() + assert not [r for r in transport.requests if r.get("purpose") == "turn_recording"] + assert manager.owned_bytes == 0 + + +@pytest.mark.asyncio +async def test_unsampled_turn_retains_nothing(manager, transport): + processor, observer, binding = await turn_binding(manager) + unsampled = trace.SpanContext(int("3" * 32, 16), int("4" * 16, 16), False, trace.TraceFlags(0)) + observer._trace_context_map[1] = unsampled + await dispatch(processor, "on_user_turn_audio", TurnAudioData(1, tone(300, 200), RATE, 1)) + observer.end_conversation_tracing() + assert binding.turn_status == "unsampled" and manager.owned_bytes == 0 + assert not [r for r in transport.requests if r.get("purpose") == "turn_recording"] + + +def test_turns_denied_on_both_sides_register_no_turn_handler(transport): + # The server allows only mixed recordings: the call records, turns cannot. + transport.policy = replace(transport.policy, allowed_directions=frozenset(("mixed",))) + manager = make_manager(transport) + span, context = explicit_context() + try: + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000, enable_turn_audio=True) + binding = bind_audio_processor(processor, context, manager=manager, turn_audio=CAPS) + assert binding.status == "session_bound" and binding.turn_status == "policy_denied" + events = {e for e, _ in binding._handlers} + assert "on_audio_data" in events and not events & {"on_user_turn_audio", "on_bot_turn_audio"} + binding.close() + finally: + manager.shutdown(1) + span.end() + + +def test_turn_caps_are_part_of_binding_identity(manager): + processor = AudioBufferProcessor(sample_rate=RATE, buffer_size=16000, enable_turn_audio=True) + span, context = explicit_context() + binding = bind_audio_processor(processor, context, manager=manager, turn_audio=CAPS) + assert bind_audio_processor(processor, context, manager=manager, turn_audio=CAPS) is binding + with pytest.raises(ValueError, match="conflicting"): + bind_audio_processor(processor, context, manager=manager, + turn_audio=TurnAudioCaps(max_turns=1, max_bytes=1000, max_duration_ms=1000)) + binding.close() + span.end() + + +def test_disabled_turn_audio_touches_nothing(): + class Poison: + def __getattribute__(self, name): + raise AssertionError("disabled adapter touched its arguments") + manager = DisabledAttachmentManager() + assert bind_audio_processor(Poison(), Poison(), manager=manager, turn_audio=CAPS).status == "disabled" + assert create_audio_capture(Poison(), manager=manager, turn_audio=CAPS).status == "disabled" + + +def test_create_capture_with_turn_audio_enables_pipecat_turns(transport): + span, context = explicit_context() + transport.policy = replace(transport.policy, record_audio=True) + manager = make_manager(transport, record_audio=True) + try: + binding = create_audio_capture(context, manager=manager, turn_audio=CAPS) + assert binding.status == "session_bound" and binding.turn_status == "session_bound" + assert binding.processor._enable_turn_audio is True + assert {"on_user_turn_audio", "on_bot_turn_audio"} <= {e for e, _ in binding._handlers} + binding.close() + plain = create_audio_capture(context, manager=manager) + assert plain.processor._enable_turn_audio is False and plain.turn_status == "off" + plain.close() + finally: + manager.shutdown(1) + span.end() From 1704cf4edd5516d69644e14de9ef0580d9f83513 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 02:32:04 -0800 Subject: [PATCH 7/9] fix(python): Pipecat turn policy before recorder construction; pin local policy and create_recording on turns - create_audio_capture checks the turn_recording policy before it builds the AudioBufferProcessor and sets enable_turn_audio only when at least one side is admitted; otherwise Pipecat keeps no turn audio, no turn handler is registered and turn_status carries the reason. - Tests: a local CapturePolicy(record_inputs=False) reaches the per-turn preflight (output turns only, policy_denied for input); a created recorder's turns preflight as create_recording, so revoking server record_audio after bind denies later turns (recording_disabled); turn uploads never project fi.attachment.refs and bind on association.parent. - README: per-turn refs and the create-time probe. Co-Authored-By: Claude Opus 5.5 --- python/frameworks/pipecat/README.md | 3 +- .../pipecat/traceai_pipecat/attachments.py | 23 ++++- .../voice-tests/test_pipecat_attachments.py | 97 ++++++++++++++++++- 3 files changed, 114 insertions(+), 9 deletions(-) diff --git a/python/frameworks/pipecat/README.md b/python/frameworks/pipecat/README.md index 905d12067..4d758e49a 100644 --- a/python/frameworks/pipecat/README.md +++ b/python/frameworks/pipecat/README.md @@ -228,8 +228,9 @@ binding = traceai.bind_pipecat_audio_processor( - Each finished turn uploads once per side: `on_user_turn_audio` as `input` and `on_bot_turn_audio` as `output`, as a mono WAV with purpose `turn_recording`, `turn_id` = Pipecat's turn number and the call's `conversation_id`. - Each upload is bound to that turn's own span, from Pipecat's `TurnTraceObserver.get_turn_context(turn_number)`. It never falls back to the conversation span or an ambient span. With `AudioSessionContext.from_span(...)` there are no turn spans, so turns end `correlation_unavailable`. +- A turn upload never writes `fi.attachment.refs` (or any `fi.attachment.*` attribute) on the turn span, even while that span is still open. Future AGI binds it through the upload's `association.parent`, which names the turn span. - Caps apply to the whole call: `max_turns` uploads (at most 1000), `max_bytes` of WAV (at most 64 MiB) and `max_duration_ms` of audio (at most 2 h). A turn that would cross a cap is skipped whole, never truncated: `queue_full` for the count, `too_large` for bytes or duration. Caps are checked on the length before any copy. - Privacy is checked per turn, per side. `FI_HIDE_INPUTS` drops the user's turns and keeps the bot's. Unsampled turn spans upload nothing. - `enable_turn_audio=True` makes Pipecat keep each turn's audio in its own memory until the turn ends. The caps bound what the adapter copies and uploads, not Pipecat's own buffer. - `binding.turn_status` is the last turn's admission (`queued` or a reason), `binding.turn_denials` counts skipped turns by reason, and `await binding.finish()` also waits for turn uploads within the same deadline. -- `create_audio_capture(..., turn_audio=TurnAudioCaps(...))` builds the recorder with `enable_turn_audio=True`. +- `create_audio_capture(..., turn_audio=TurnAudioCaps(...))` checks the per-turn policy before it builds the recorder. It builds it with `enable_turn_audio=True` only when at least one side may upload turns. Otherwise Pipecat keeps no turn audio, no turn handler is registered and `binding.turn_status` is the reason, such as `policy_denied`. After the recorder is built, each turn is still checked as a new recording, so turning the server's `record_audio` off denies later turns (`recording_disabled`). diff --git a/python/frameworks/pipecat/traceai_pipecat/attachments.py b/python/frameworks/pipecat/traceai_pipecat/attachments.py index 817f22c1c..5fae837f9 100644 --- a/python/frameworks/pipecat/traceai_pipecat/attachments.py +++ b/python/frameworks/pipecat/traceai_pipecat/attachments.py @@ -617,8 +617,11 @@ def create_audio_capture(session_context, *, manager, sample_rate=None, num_chan ``record_audio``. The returned ``binding.processor`` must be inserted into the pipeline by the application, which also starts it (for example with ``AudioBufferStartRecordingFrame``). Nothing is created when denied. - ``turn_audio`` (``TurnAudioCaps``) creates the processor with - ``enable_turn_audio=True`` and uploads turns as in ``bind_audio_processor``. + ``turn_audio`` (``TurnAudioCaps``) uploads turns as in + ``bind_audio_processor``. The ``turn_recording`` policy is checked BEFORE + the processor is built: only when at least one side is admitted is it + created with ``enable_turn_audio=True``. Otherwise Pipecat never buffers + turns, no turn handler is registered and ``turn_status`` is the reason. """ capture = _validate_options(layout, finalization_timeout, capture_policy) turn_audio = _validate_turn_audio(turn_audio) @@ -637,15 +640,25 @@ def create_audio_capture(session_context, *, manager, sample_rate=None, num_chan return AudioCaptureBinding(reason) if not isinstance(session_context, AudioSessionContext): return AudioCaptureBinding("correlation_unavailable") + # Turn policy before construction: a recorder built with enable_turn_audio + # keeps every turn in Pipecat memory, so build it that way only when at + # least one side may upload a turn_recording. + turn_reason = None + if turn_audio is not None: + turn_reason = _probe(manager, "create_recording", ("input", "output"), capture, purpose="turn_recording") + turns = turn_audio if turn_reason is None else None try: if version("pipecat-ai") not in _SUPPORTED: return AudioCaptureBinding("unsupported_version") from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor processor = AudioBufferProcessor(sample_rate=sample_rate, num_channels=num_channels, - buffer_size=buffer_size, enable_turn_audio=turn_audio is not None) + buffer_size=buffer_size, enable_turn_audio=turns is not None) except PackageNotFoundError: return AudioCaptureBinding("unsupported_version") except Exception: return AudioCaptureBinding("source_unavailable") - return _bind(processor, session_context, manager, layout=layout, timeout=finalization_timeout, - capture=capture, operation="create_recording", turn_audio=turn_audio) + binding = _bind(processor, session_context, manager, layout=layout, timeout=finalization_timeout, + capture=capture, operation="create_recording", turn_audio=turns) + if turn_reason is not None and binding.status == "session_bound": + binding.turn_status = turn_reason + return binding diff --git a/python/voice-tests/test_pipecat_attachments.py b/python/voice-tests/test_pipecat_attachments.py index 6721da5a7..1640b21b8 100644 --- a/python/voice-tests/test_pipecat_attachments.py +++ b/python/voice-tests/test_pipecat_attachments.py @@ -23,7 +23,7 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from conftest import TransportDouble, make_manager -from fi_instrumentation.attachments import AttachmentConfig, AttachmentManager, CorrelationHandle +from fi_instrumentation.attachments import AttachmentConfig, AttachmentManager, CapturePolicy, CorrelationHandle from fi_instrumentation.attachments._disabled import DisabledAttachmentManager from pipecat.frames.frames import ( BotStartedSpeakingFrame, BotStoppedSpeakingFrame, EndFrame, InputAudioRawFrame, OutputAudioRawFrame, @@ -671,13 +671,13 @@ async def test_real_pipeline_uploads_each_turn_bound_to_its_turn_span(manager, t assert manager.owned_bytes == 0 -async def turn_binding(manager, caps=CAPS, *, context=None, layout="auto"): +async def turn_binding(manager, caps=CAPS, *, context=None, layout="auto", capture=None): processor = AudioBufferProcessor(sample_rate=RATE, num_channels=1, buffer_size=16000, enable_turn_audio=True) worker = await run_pipeline(processor, None) observer = worker.turn_trace_observer observer.start_conversation_tracing("conv-turns") binding = bind_audio_processor(processor, context or AudioSessionContext.from_pipeline(worker), - manager=manager, turn_audio=caps, layout=layout) + manager=manager, turn_audio=caps, layout=layout, capture_policy=capture) return processor, observer, binding @@ -830,3 +830,94 @@ def test_create_capture_with_turn_audio_enables_pipecat_turns(transport): finally: manager.shutdown(1) span.end() + + +# ------------------------------------------- per-turn audio: review follow-ups +@pytest.mark.asyncio +async def test_local_capture_policy_hiding_inputs_drops_only_user_turns(manager, transport): + # A LOCAL CapturePolicy (no env flag) reaches the per-turn preflight. + processor, observer, binding = await turn_binding(manager, layout="separate_tracks", + capture=CapturePolicy(record_inputs=False)) + assert binding.status == "session_bound" and binding.turn_status == "session_bound" + await run_turn(processor, observer, 1) + await binding.finish(10) + observer.end_conversation_tracing() + uploads = [r for r in transport.requests if r.get("purpose") == "turn_recording"] + assert [(r["direction"], r["turn_id"]) for r in uploads] == [("output", "1")] + assert binding.turn_denials == {"policy_denied": 1} + assert [direction for _, direction, _ in binding.turn_handles] == ["output"] + + +@pytest.mark.asyncio +async def test_created_recorder_turns_stop_when_server_record_audio_is_revoked(transport): + # create_audio_capture's turns preflight as operation="create_recording": + # once the server turns record_audio off, the next turn is denied. + transport.policy = replace(transport.policy, record_audio=True) + manager = make_manager(transport, record_audio=True) + try: + worker = await run_pipeline(AudioBufferProcessor(sample_rate=RATE, buffer_size=16000), None) + observer = worker.turn_trace_observer + observer.start_conversation_tracing("conv-created") + binding = create_audio_capture(AudioSessionContext.from_pipeline(worker), manager=manager, + sample_rate=RATE, turn_audio=CAPS) + assert binding.status == "session_bound" and binding.turn_status == "session_bound" + processor = binding.processor + await run_turn(processor, observer, 1) + assert binding.turn_denials == {} and len(binding.turn_handles) == 2 + # Let turn 1 finish first: a tightened policy also stops uploads that are still queued. + for _, _, handle in binding.turn_handles: + assert (await handle.wait(10)).status != "unfinished" + manager.update_policy(replace(transport.policy, record_audio=False)) + await run_turn(processor, observer, 2) + await binding.finish(10) + observer.end_conversation_tracing() + uploads = [r for r in transport.requests if r.get("purpose") == "turn_recording"] + assert sorted((r["turn_id"], r["direction"]) for r in uploads) == [("1", "input"), ("1", "output")] + assert binding.turn_denials == {"recording_disabled": 2} + assert binding.turn_status == "recording_disabled" and len(binding.turn_handles) == 2 + binding.close() + finally: + manager.shutdown(2) + + +def test_created_recorder_has_no_pipecat_turn_buffer_when_turns_are_denied(transport): + # The server records whole calls (mixed) but admits no per-side turn: the + # recorder is built WITHOUT enable_turn_audio and no turn handler exists. + transport.policy = replace(transport.policy, record_audio=True, allowed_directions=frozenset(("mixed",))) + manager = make_manager(transport, record_audio=True) + span, context = explicit_context() + try: + binding = create_audio_capture(context, manager=manager, turn_audio=CAPS) + assert binding.status == "session_bound" and binding.turn_status == "policy_denied" + assert binding.processor._enable_turn_audio is False + events = {e for e, _ in binding._handlers} + assert "on_audio_data" in events and not events & {"on_user_turn_audio", "on_bot_turn_audio"} + binding.close() + finally: + manager.shutdown(1) + span.end() + + +@pytest.mark.asyncio +async def test_turn_upload_binds_on_parent_and_never_projects_refs(manager, transport): + # Even when the turn span is still live at upload time, the turn span gets + # no fi.attachment.* attribute; the upload names it in association.parent. + pairs = pair_uploads(transport) + processor, observer, binding = await turn_binding(manager) + await observer._turn_tracker._call_event_handler("on_turn_started", 1) + for _ in range(3): + await asyncio.sleep(0) + await dispatch(processor, "on_user_turn_audio", TurnAudioData(1, tone(300, 200), RATE, 1)) + for _, _, handle in binding.turn_handles: + assert (await handle.wait(10)).status != "unfinished" + await observer._turn_tracker._call_event_handler("on_turn_ended", 1, 1.0, False) + for _ in range(3): + await asyncio.sleep(0) + await binding.finish(10) + observer.end_conversation_tracing() + turn = [s for s in EXPORTER.get_finished_spans() if s.name == "turn"] + assert len(turn) == 1 + assert not [k for k in turn[0].attributes if k.startswith("fi.attachment")] + uploads = turn_pairs(pairs) + assert len(uploads) == 1 + assert uploads[0][0]["association"]["parent"]["span_id"] == format(turn[0].context.span_id, "016x") From 1d02a4617d4d79c2bf72c2e353e0f4455e3c2bb6 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 02:47:01 -0800 Subject: [PATCH 8/9] docs(python): LiveKit root depends on where tracing is set up (real 1.8.10/1.8.11 worker) A real livekit-agents worker (loopback livekit-server, explicit dispatch, the recording binding admitted) shows: - tracing set up inside the entrypoint (the Quick Start): job_entrypoint is not recorded, agent_session is the parentless root of the call's trace and the binding marks it CONVERSATION; job_shutdown and room_connect are separate root traces. - tracing set up before the job (setup_fnc): job_entrypoint is the root with lk.job_id; agent_session and job_shutdown are its children; not marked. The README said the inside-entrypoint trace was rooted at job_shutdown and the call was never listed in Voice; both statements are corrected. Co-Authored-By: Claude Opus 5.5 --- python/frameworks/livekit/README.md | 12 +++++++----- python/voice-tests/test_livekit_attachments.py | 4 +++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/python/frameworks/livekit/README.md b/python/frameworks/livekit/README.md index 6ec070dba..ac388f7fb 100644 --- a/python/frameworks/livekit/README.md +++ b/python/frameworks/livekit/README.md @@ -57,8 +57,9 @@ them twice over: trace, `agent_session` included, as soon as the call ends. - **When the process exits**, just before it stops: this sends what LiveKit ends after the shutdown callbacks, its own `job_shutdown` span, and the job's root `job_entrypoint` span when tracing is set up before the job starts. With - set-up inside the entrypoint, LiveKit starts `job_entrypoint` before tracing is on, so it is never recorded and the - job trace arrives rooted at `job_shutdown`. + set-up inside the entrypoint, LiveKit starts `job_entrypoint` before tracing is on, so it is never recorded. The call + then arrives as separate traces: the session trace rooted at `agent_session`, and `job_shutdown` as a root of its own + (see "How Future AGI Voice finds a LiveKit call"). Each waits at most 3 seconds for the collector, well inside the 10 seconds LiveKit gives a job process to exit, so its own clean-up still runs when the collector is slow. @@ -199,11 +200,12 @@ Attachments are off by default. Both paths below need the attachment-enabled `fi binding = traceai.bind_livekit_session(session, ctx, owned_recording_files=True) ``` -When the session ends, the SDK's `audio.ogg` is uploaded as one `mixed` file. This happens before LiveKit deletes its temporary directory, with a deadline of up to 30 s. Hard links, symlinks, foreign owners and replaced files are refused. LiveKit redaction, `LIVEKIT_TELEMETRY_ALLOW_PII` and FI hide flags all deny. The session span is marked `fi.span.kind=CONVERSATION` only if it has no parent. In a LiveKit worker, `agent_session` is a child of `job_entrypoint`, so it is not marked. +When the session ends, the SDK's `audio.ogg` is uploaded as one `mixed` file. This happens before LiveKit deletes its temporary directory, with a deadline of up to 30 s. Hard links, symlinks, foreign owners and replaced files are refused. LiveKit redaction, `LIVEKIT_TELEMETRY_ALLOW_PII` and FI hide flags all deny. The session span is marked `fi.span.kind=CONVERSATION` only if it has no parent, which depends on where tracing is set up (below). -**How Future AGI Voice finds a LiveKit call.** The SDK never invents a conversation root. In a real worker the trace's true root is LiveKit's own `job_entrypoint` span, which carries `lk.job_id`, and the recording is attached to `agent_session`, its child. When attachments are enabled on the Future AGI server, ingest treats an untyped, parentless `job_entrypoint` span with a non-empty `lk.job_id` as the call's conversation root, so the call is listed in Voice and its recording is found on the child span. +**How Future AGI Voice finds a LiveKit call.** The SDK never invents a conversation root: it marks the session span only when that span is already the root of its trace. Which span is the root depends on where tracing is set up (checked with a real livekit-agents 1.8.10 and 1.8.11 worker): -- This needs tracing set up before the job starts (for example at worker start-up, see "Flushing at the end of a job"). With set-up inside the entrypoint, LiveKit starts `job_entrypoint` before tracing is on, the trace is rooted at `job_shutdown` instead, and the call is not listed in Voice. +- **Tracing set up inside the entrypoint** (the Quick Start above). LiveKit starts `job_entrypoint` before tracing is on, so it is not recorded and `agent_session` is the root of the call's trace (it carries `lk.job_id`). An admitted `bind_livekit_session(..., owned_recording_files=True)` marks `agent_session` as `CONVERSATION`, the recording is attached to it, and the call is listed in Voice. Without an admitted binding (attachments off, a denial, or `owned_recording_files=False`) nothing is marked, and the call is not listed. `job_shutdown` (with the session-end upload span) and `room_connect` arrive as separate root traces. +- **Tracing set up before the job starts** (for example in `setup_fnc`, or at worker start-up). LiveKit's `job_entrypoint` span is the trace root and carries `lk.job_id`; `agent_session` and `job_shutdown` are its children, so the binding does not mark `agent_session`. When attachments are enabled on the Future AGI server, ingest treats an untyped, parentless `job_entrypoint` span with a non-empty `lk.job_id` as the call's conversation root, so the call is listed in Voice and its recording is found on the child `agent_session` span. - Traces ingested before attachments were enabled on the server are not re-classified. - Do not set `fi.span.kind` on `job_entrypoint` or `agent_session` yourself to force a root; that changes how the whole trace is classified. diff --git a/python/voice-tests/test_livekit_attachments.py b/python/voice-tests/test_livekit_attachments.py index aabbeba21..32d9d7faf 100644 --- a/python/voice-tests/test_livekit_attachments.py +++ b/python/voice-tests/test_livekit_attachments.py @@ -447,7 +447,9 @@ async def test_child_session_span_is_never_marked_root(manager, tmp_path): from opentelemetry import trace session, job, _, span, provider, _ = owned_session(tmp_path) span.end() - # The real worker shape: LiveKit's own root carries lk.job_id; the session is its child. + # The real worker shape when tracing is set up before the job: LiveKit's own root carries + # lk.job_id and the session is its child. (Set up inside the entrypoint, agent_session is + # itself the root and is marked; see the parentless-span test above.) parent = provider.get_tracer("synthetic").start_span("job_entrypoint", attributes={"lk.job_id": "synthetic-job"}) child = provider.get_tracer("synthetic").start_span("agent_session", context=trace.set_span_in_context(parent)) session._session_span = child From fd8ea4508b7f1994ed647598c2386ab1ee3550ac Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 14 Nov 2026 20:48:11 -0800 Subject: [PATCH 9/9] fix(livekit): propagate call-local session context into native spans Capture context at native span creation, retain privacy-qualified sources until export, and preserve explicit identifiers. Keep attachment capture independent and avoid process-wide session state. Cover overlapping calls, late children, no-context behavior, both transports and privacy denial. Part of TH-2379 (Linear). Co-Authored-By: Claude Code --- python/frameworks/livekit/README.md | 33 +++ .../livekit/tests/test_integration.py | 58 ++++- .../livekit/tests/test_session_context.py | 235 ++++++++++++++++++ .../traceai_livekit/_session_context.py | 54 ++++ .../exporters/base_exporter.py | 10 +- .../livekit/traceai_livekit/integration.py | 3 + 6 files changed, 391 insertions(+), 2 deletions(-) create mode 100644 python/frameworks/livekit/tests/test_session_context.py create mode 100644 python/frameworks/livekit/traceai_livekit/_session_context.py diff --git a/python/frameworks/livekit/README.md b/python/frameworks/livekit/README.md index ac388f7fb..5c7b173bc 100644 --- a/python/frameworks/livekit/README.md +++ b/python/frameworks/livekit/README.md @@ -47,6 +47,39 @@ drops the Future AGI exporter. LiveKit installs its privacy filter once per prov ignored. To keep conversation content out of your traces, set `LIVEKIT_TELEMETRY_ALLOW_PII=false` before calling `enable_*` (livekit-agents 1.8 and later). +### Grouping a call into a session + +After enabling attribute mapping, use `using_session` inside each job entrypoint. Wrap the call's +`AgentSession` startup and work, not just tracing setup at worker import time: + +```python +from fi_instrumentation import using_session +from livekit.agents import Agent, AgentSession + +# Inside entrypoint(ctx), after register() and enable_http_attribute_mapping(): +with using_session(session_id=ctx.room.name): + await ctx.connect() + session = AgentSession() + await session.start(agent=Agent(instructions="Answer briefly."), room=ctx.room) + # Run your conversation here. +``` + +The native spans capture `session.id` when they start, so concurrent calls can use different IDs +on the same provider. An explicit `session.id` takes precedence; an existing `gen_ai.conversation.id` +(including LiveKit's own conversation ID) is preserved. Grouping does not enable attachments or +recording. With no session context, no `session.id` is added. + +On livekit-agents 1.8, context-derived IDs follow `LIVEKIT_TELEMETRY_ALLOW_PII` and project +redaction. Set your privacy choice before enabling mapping; changing it afterward does not +replace LiveKit's installed filter. The IDs are mapped only on the Future AGI export copy, so +they cannot bypass LiveKit Cloud redaction. + +A child can inherit from its in-process parent span even after leaving the context block. A +bare `SpanContext` or `NonRecordingSpan` carries only tracing IDs, not session metadata: watchdog +diagnostics such as `event_loop_blocked` may lack a session when emitted with only those IDs. Likewise, +a span started before entering the block is not retroactively grouped. Pass the context through +your own asynchronous work instead of assigning one session globally to the worker. + ### Flushing at the end of a job On Linux a LiveKit job process ends without running `atexit`, and LiveKit flushes only its own processors, so the spans diff --git a/python/frameworks/livekit/tests/test_integration.py b/python/frameworks/livekit/tests/test_integration.py index 74e469924..59349226d 100644 --- a/python/frameworks/livekit/tests/test_integration.py +++ b/python/frameworks/livekit/tests/test_integration.py @@ -86,7 +86,10 @@ def test_enabling_http_mapping_keeps_the_exporter_alongside_livekits_privacy_fil assert enable_http_attribute_mapping() is True processors = _processors(provider) - assert len(processors) == 2 + from traceai_livekit._session_context import _SessionContextProcessor + + assert len(processors) == 3 + assert isinstance(processors[2], _SessionContextProcessor) assert isinstance(processors[0], pii._PIIFilteringSpanProcessor) assert isinstance(processors[1], BatchSpanProcessor) assert isinstance(processors[1].span_exporter, MappedHTTPSpanExporter) @@ -406,6 +409,59 @@ def test_the_allow_pii_opt_out_keeps_content_out_of_the_export( assert [k for k in attributes if k.startswith("lk.pii.")] == [] +@pytest.mark.parametrize("allow_pii", [True, False]) +def test_overlapping_native_sessions_reach_collector_with_isolated_grouping( + monkeypatch: pytest.MonkeyPatch, collector: _Collector, make_provider: Any, allow_pii: bool, +) -> None: + from fi_instrumentation import using_session + from livekit.agents.telemetry import tracer + + monkeypatch.setenv("LIVEKIT_TELEMETRY_ALLOW_PII", str(allow_pii).lower()) + provider = make_provider() + enable_http_attribute_mapping() + roots = {} + + async def run(): + entered = [asyncio.Event(), asyncio.Event()] + + async def call(index): + with using_session("native-call-" + str(index)): + async with AgentSession(llm=_FakeLLM()) as session: + await session.start(Agent(instructions="Synthetic session isolation test.")) + roots[session._session_span.get_span_context().trace_id] = "native-call-" + str(index) + entered[index].set() + await entered[1 - index].wait() + await session.run(user_input=USER_TEXT) + + await asyncio.gather(call(0), call(1)) + tracer.start_span("outside-session").end() + + asyncio.run(run()) + provider.force_flush() + assert len(roots) == 2 + names = {trace_id: set() for trace_id in roots} + outside = [] + for body in collector.bodies: + request = ExportTraceServiceRequest.FromString(body) + for resource in request.resource_spans: + for scope in resource.scope_spans: + for span in scope.spans: + attrs = {a.key: _any_value(a.value) for a in span.attributes} + trace_id = int.from_bytes(span.trace_id, "big") + if trace_id in roots: + names[trace_id].add(span.name) + if allow_pii: + assert attrs[SpanAttributes.SESSION_ID] == roots[trace_id] + else: + assert SpanAttributes.SESSION_ID not in attrs + elif span.name == "outside-session": + outside.append(attrs) + for seen in names.values(): + assert {"agent_session", "agent_turn", "llm_node", "session_close"} <= seen + assert len(outside) == 1 + assert SpanAttributes.SESSION_ID not in outside[0] + + # Recording rows use real LiveKit telemetry, but never start an RPC/collector. @pytest.mark.parametrize('allow_pii', [True, False]) def test_recording_metadata_respects_livekit_pii_opt_out_and_cloud_redaction( diff --git a/python/frameworks/livekit/tests/test_session_context.py b/python/frameworks/livekit/tests/test_session_context.py new file mode 100644 index 000000000..0196f8fdb --- /dev/null +++ b/python/frameworks/livekit/tests/test_session_context.py @@ -0,0 +1,235 @@ +"""Call-local native LiveKit grouping, independent of attachment capture.""" + +import asyncio + +import pytest +from fi_instrumentation import using_session +from fi_instrumentation.fi_types import SpanAttributes +from fi_instrumentation.otel import GRPCSpanExporter, HTTPSpanExporter, Transport +from opentelemetry import context, trace +from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_livekit import enable_grpc_attribute_mapping, enable_http_attribute_mapping +from traceai_livekit._session_context import _CONVERSATION, _SESSION, _SessionContextProcessor + +SID = SpanAttributes.SESSION_ID +CID = SpanAttributes.GEN_AI_CONVERSATION_ID + + +@pytest.fixture +def mapped_spans(monkeypatch): + spans = [] + + def export(self, batch): + spans.extend(batch) + return SpanExportResult.SUCCESS + + # Exercise mapping, privacy and batch execution; replace only the HTTP send. + monkeypatch.setattr(HTTPSpanExporter, "export", export) + monkeypatch.setattr(GRPCSpanExporter, "export", export) + return spans + + +@pytest.mark.parametrize("transport,enable", [ + (Transport.HTTP, enable_http_attribute_mapping), + (Transport.GRPC, enable_grpc_attribute_mapping), +]) +def test_capture_at_start_not_setup_or_batch_export(make_provider, mapped_spans, transport, enable): + with using_session("setup-only"): + provider = make_provider(transport) + assert enable() + # Context grouping does not depend on (or turn on) attachment capture. + manager = getattr(provider, "attachment_manager", None) + assert manager is None or manager.status().state == "disabled" + tracer = provider.get_tracer("livekit-agents") + with using_session("call"): + span = tracer.start_span("agent_session") + span.end() + tracer.start_span("outside").end() + with using_session("export-only"): + provider.force_flush() + by_name = {s.name: s.attributes for s in mapped_spans} + assert by_name["agent_session"][SID] == "call" + assert by_name["agent_session"][CID] == "call" + assert SID not in by_name["outside"] + assert CID not in by_name["outside"] + + +def test_installation_is_idempotent_and_native_scope_only(make_provider, mapped_spans): + provider = make_provider() + enable_http_attribute_mapping() + enable_http_attribute_mapping() + assert sum(isinstance(p, _SessionContextProcessor) for p in + provider._active_span_processor._span_processors) == 1 + with using_session("native-only"): + provider.get_tracer("application").start_span("unrelated").end() + provider.force_flush() + assert SID not in mapped_spans[0].attributes + assert _SESSION not in mapped_spans[0].attributes + + +@pytest.mark.parametrize("separate_providers", [False, True]) +def test_overlapping_calls_and_no_context_are_isolated( + make_provider, mapped_spans, separate_providers, +): + first = make_provider() + enable_http_attribute_mapping() + second = make_provider() if separate_providers else first + if separate_providers: + enable_http_attribute_mapping() + tracers = [p.get_tracer("livekit-agents") for p in (first, second)] + + async def run(): + entered = [asyncio.Event(), asyncio.Event()] + + async def call(index): + with using_session("call-" + str(index)): + with tracers[index].start_as_current_span("root-" + str(index)): + entered[index].set() + await entered[1 - index].wait() + await asyncio.sleep(0) + tracers[index].start_span("child-" + str(index)).end() + + async def outside(): + await asyncio.gather(*(event.wait() for event in entered)) + tracers[0].start_span("outside").end() + + await asyncio.gather(call(0), call(1), outside()) + + asyncio.run(run()) + first.force_flush() + second.force_flush() + by_name = {s.name: s for s in mapped_spans} + for index in (0, 1): + root, child = (by_name[prefix + str(index)] for prefix in ("root-", "child-")) + assert root.attributes[SID] == child.attributes[SID] == "call-" + str(index) + assert root.attributes[CID] == child.attributes[CID] == "call-" + str(index) + assert child.parent.span_id == root.context.span_id + assert SID not in by_name["outside"].attributes + assert by_name["root-0"].context.trace_id != by_name["root-1"].context.trace_id + + +def test_explicit_parent_context_and_late_children(make_provider, mapped_spans): + provider = make_provider() + enable_http_attribute_mapping() + tracer = provider.get_tracer("livekit-agents") + with using_session("call"): + root = tracer.start_span("agent_session") + saved = context.get_current() + # LiveKit starts shutdown work outside the entrypoint's Python context. + with trace.use_span(root, end_on_exit=False): + tracer.start_span("session_close").end() + with using_session("unrelated"): + tracer.start_span("explicit-context", context=saved).end() + tracer.start_span("isolated-root", context=context.Context()).end() + # A bare span id carries no session metadata; do not look it up globally. + with trace.use_span(trace.NonRecordingSpan(root.context), end_on_exit=False): + tracer.start_span("id-only-parent").end() + root.end() + provider.force_flush() + by_name = {s.name: s.attributes for s in mapped_spans} + for name in ("agent_session", "session_close", "explicit-context"): + assert by_name[name][SID] == "call" + for name in ("isolated-root", "id-only-parent"): + assert SID not in by_name[name] + + +@pytest.mark.parametrize("attributes,expected_conversation", [ + ({SID: "explicit"}, "explicit"), + ({SID: "explicit", CID: "explicit-conversation"}, "explicit-conversation"), +]) +def test_explicit_ids_win_over_context_and_room( + make_provider, mapped_spans, attributes, expected_conversation, +): + provider = make_provider() + enable_http_attribute_mapping() + original = dict(attributes, **{"lk.pii.room_name": "room"}) + with using_session("ambient"): + provider.get_tracer("livekit-agents").start_span("agent_session", attributes=original).end() + provider.force_flush() + assert mapped_spans[0].attributes[SID] == "explicit" + assert mapped_spans[0].attributes[CID] == expected_conversation + assert original == dict(attributes, **{"lk.pii.room_name": "room"}) + + +@pytest.mark.parametrize("allow_pii", [True, False]) +def test_privacy_filter_and_cloud_gate_do_not_receive_unqualified_copies( + monkeypatch, make_provider, mapped_spans, allow_pii, +): + from livekit.agents.telemetry.traces import _GatedSpanExporter + + monkeypatch.setenv("LIVEKIT_TELEMETRY_ALLOW_PII", str(allow_pii).lower()) + provider = make_provider(batch=False) + enable_http_attribute_mapping() + raw = InMemorySpanExporter() + cloud = InMemorySpanExporter() + gate = _GatedSpanExporter(cloud, export_jobs={}) + gate.open_job("job") + provider.add_span_processor(SimpleSpanProcessor(raw)) + provider.add_span_processor(SimpleSpanProcessor(gate)) + with using_session("private-call"): + provider.get_tracer("livekit-agents").start_span("agent_session", attributes={"job_id": "job"}).end() + gate.job_registered("job", traces_enabled=True, redacted=True) + provider.force_flush() + for span in (*raw.get_finished_spans(), *cloud.get_finished_spans()): + assert SID not in span.attributes + assert CID not in span.attributes + cloud_attributes = cloud.get_finished_spans()[0].attributes + assert _SESSION not in cloud_attributes + assert _CONVERSATION not in cloud_attributes + if allow_pii: + assert mapped_spans[0].attributes[SID] == "private-call" + assert raw.get_finished_spans()[0].attributes[_SESSION] == "private-call" + else: + assert SID not in mapped_spans[0].attributes + assert CID not in mapped_spans[0].attributes + assert _SESSION not in raw.get_finished_spans()[0].attributes + + +def test_project_redaction_wins_over_provider_permission(monkeypatch, make_provider, mapped_spans): + from livekit.agents.types import ATTRIBUTE_REDACTION_ENABLED + + monkeypatch.setenv("LIVEKIT_TELEMETRY_ALLOW_PII", "true") + provider = make_provider() + enable_http_attribute_mapping() + with using_session("project-withheld"): + span = provider.get_tracer("livekit-agents").start_span( + "agent_session", attributes={ATTRIBUTE_REDACTION_ENABLED: True}, + ) + # End off the call context; the span's own project stamp still wins. + span.end() + provider.force_flush() + assert SID not in mapped_spans[0].attributes + assert CID not in mapped_spans[0].attributes + assert not any("pii" in key.split(".") for key in mapped_spans[0].attributes) + + +def test_nested_context_restores_outer_session(make_provider, mapped_spans): + provider = make_provider() + enable_http_attribute_mapping() + tracer = provider.get_tracer("livekit-agents") + with using_session("outer"): + with tracer.start_as_current_span("outer-root"): + with using_session("inner"): + tracer.start_span("inner-child").end() + tracer.start_span("outer-child").end() + tracer.start_span("outside").end() + provider.force_flush() + by_name = {s.name: s.attributes for s in mapped_spans} + assert by_name["outer-root"][SID] == by_name["outer-child"][SID] == "outer" + assert by_name["inner-child"][SID] == "inner" + assert SID not in by_name["outside"] + + +def test_first_privacy_denial_wins_over_later_enable(monkeypatch, make_provider, mapped_spans): + monkeypatch.setenv("LIVEKIT_TELEMETRY_ALLOW_PII", "false") + provider = make_provider() + enable_http_attribute_mapping() + monkeypatch.setenv("LIVEKIT_TELEMETRY_ALLOW_PII", "true") + enable_http_attribute_mapping() + with using_session("withheld"): + provider.get_tracer("livekit-agents").start_span("agent_session").end() + provider.force_flush() + assert SID not in mapped_spans[0].attributes + assert CID not in mapped_spans[0].attributes diff --git a/python/frameworks/livekit/traceai_livekit/_session_context.py b/python/frameworks/livekit/traceai_livekit/_session_context.py new file mode 100644 index 000000000..ea79b0405 --- /dev/null +++ b/python/frameworks/livekit/traceai_livekit/_session_context.py @@ -0,0 +1,54 @@ +"""Bridge call-local FI session context into LiveKit's native SDK spans. + +Keep identifiers under LiveKit's PII namespace until our exporter maps a copy. +There is no provider/process-wide session value or trace-id registry: a late child +can inherit only from its actual in-process parent, not from another call. +""" + +from typing import Optional + +from opentelemetry import context, trace +from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor, TracerProvider + +from fi_instrumentation.fi_types import SpanAttributes + +_SESSION = "lk.pii.traceai.session_id" +_CONVERSATION = "lk.pii.traceai.conversation_id" + + +class _SessionContextProcessor(SpanProcessor): + def on_start(self, span: Span, parent_context: Optional[context.Context] = None) -> None: + scope = span.instrumentation_scope + if scope is None or scope.name != "livekit-agents": + return + + attributes = span.attributes or {} + parent = trace.get_current_span(parent_context) + inherited = getattr(parent, "attributes", None) or {} + # An explicit OTel context is authoritative, including an empty context. + # Read on creation, never on export (which may run in a batch thread). + session = attributes.get(SpanAttributes.SESSION_ID) + if session is None: + session = context.get_value(SpanAttributes.SESSION_ID, parent_context) + if session is None: + session = inherited.get(_SESSION, inherited.get(SpanAttributes.SESSION_ID)) + if session is None: + return + + # Do not copy an explicit unqualified attribute: the application owns it. + # Its presence also prevents an ambient session from supplying an alias. + if SpanAttributes.SESSION_ID not in attributes and _SESSION not in attributes: + span.set_attribute(_SESSION, session) + if SpanAttributes.GEN_AI_CONVERSATION_ID not in attributes and _CONVERSATION not in attributes: + span.set_attribute(_CONVERSATION, session) + + def on_end(self, span: ReadableSpan) -> None: + pass + + +def _install_session_context(provider: TracerProvider) -> None: + """Install once per provider without retaining any call's context.""" + active = getattr(provider, "_active_span_processor", None) + processors = getattr(active, "_span_processors", (active,)) + if not any(isinstance(p, _SessionContextProcessor) for p in processors): + provider.add_span_processor(_SessionContextProcessor()) diff --git a/python/frameworks/livekit/traceai_livekit/exporters/base_exporter.py b/python/frameworks/livekit/traceai_livekit/exporters/base_exporter.py index 5680b62fd..e004ac7f0 100644 --- a/python/frameworks/livekit/traceai_livekit/exporters/base_exporter.py +++ b/python/frameworks/livekit/traceai_livekit/exporters/base_exporter.py @@ -23,6 +23,7 @@ _withhold_recording_failures, ) from .._recording_boundary import _copy_span, _RecordingScope +from .._session_context import _SESSION, _CONVERSATION logger = logging.getLogger(__name__) @@ -289,9 +290,16 @@ def _map_attributes_to_fi_conventions( mapped[SpanAttributes.GEN_AI_TOOL_DEFINITIONS] = _ensure_json_string(attributes.get(LK_FUNCTION_TOOLS)) # 5. Session & User Info + # These sources survive only when LiveKit's privacy filter permits them. + # Map on our export copy, never on the shared span sent to LiveKit Cloud. + if not recording_snapshot: + for source, alias in ((_SESSION, SpanAttributes.SESSION_ID), + (_CONVERSATION, SpanAttributes.GEN_AI_CONVERSATION_ID)): + if source in attributes: + mapped.setdefault(alias, attributes[source]) found, room_name = _first(attributes, LK_ROOM_NAME) if found and not recording_snapshot: - mapped[SpanAttributes.GEN_AI_CONVERSATION_ID] = room_name + mapped.setdefault(SpanAttributes.GEN_AI_CONVERSATION_ID, room_name) # Only retained recording sources supply aliases; there is no room fallback, # including when a sampler or the SDK evicted the operation attribute. diff --git a/python/frameworks/livekit/traceai_livekit/integration.py b/python/frameworks/livekit/traceai_livekit/integration.py index a82a0bbae..02da91d9e 100644 --- a/python/frameworks/livekit/traceai_livekit/integration.py +++ b/python/frameworks/livekit/traceai_livekit/integration.py @@ -358,6 +358,9 @@ def enable_fi_attribute_mapping(transport: Transport = Transport.HTTP) -> bool: # Automatically set the tracer provider for LiveKit agents try: from livekit.agents.telemetry import set_tracer_provider + from ._session_context import _install_session_context + + _install_session_context(provider) set_tracer_provider(provider) logger.info("Automatically configured LiveKit agents tracer provider") except ImportError: