diff --git a/.github/workflows/livekit-typescript-tests.yml b/.github/workflows/livekit-typescript-tests.yml index a546d66b9..a7b111fcc 100644 --- a/.github/workflows/livekit-typescript-tests.yml +++ b/.github/workflows/livekit-typescript-tests.yml @@ -9,6 +9,7 @@ on: - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/pnpm-lock.yaml' - '.github/workflows/livekit-typescript-tests.yml' + workflow_dispatch: permissions: contents: read @@ -85,3 +86,40 @@ jobs: env "LIVEKIT_COMPAT_DIR=$RUNNER_TEMP/livekit-sdk" \ "LIVEKIT_INSTRUMENTATION_DIR=$RUNNER_TEMP/livekit-sdk/node_modules/@traceai/fi-instrumentation-livekit" \ "$(command -v node)" --test packages/traceai_livekit/test/readme-shutdown.cjs + + agents-recording: + name: LiveKit Agents recording binding (${{ matrix.agents }}) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # Floor and latest of the exact versions the binding supports. + agents: ['1.8.0', '1.9.5'] + defaults: + run: + shell: bash + working-directory: typescript + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22.23.4' + - name: Install locked dependencies and the real Agents SDK + run: | + corepack enable + pnpm --filter @traceai/fi-instrumentation-livekit... install --frozen-lockfile --ignore-scripts + npm install --prefix "$RUNNER_TEMP/livekit-agents" --ignore-scripts --no-audit --no-fund \ + "@livekit/agents@${{ matrix.agents }}" "zod@^3.25.76" + - name: Build + run: | + pnpm run build:deps + pnpm --filter @traceai/fi-instrumentation-livekit build + - name: Real Agents session recording with loopback-only networking + run: | + sudo unshare --net -- sh -c 'ip link set lo up; exec "$@"' sh \ + env "LIVEKIT_AGENTS_DIR=$RUNNER_TEMP/livekit-agents" "$(command -v node)" \ + --test packages/traceai_livekit/test/agents-recording.cjs + git diff --exit-code -- pnpm-lock.yaml diff --git a/typescript/packages/traceai_elevenlabs/README.md b/typescript/packages/traceai_elevenlabs/README.md index 56772dc9a..20c920098 100644 --- a/typescript/packages/traceai_elevenlabs/README.md +++ b/typescript/packages/traceai_elevenlabs/README.md @@ -107,9 +107,20 @@ The optional `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` and `ELEVENLABS_BASE_URL` over | `textToSpeech.streamWithTimestamps` | You finish or stop iterating timestamp chunks | | `speechToText.convert` | Synchronous transcription completes, including multi-channel output | -Use these methods on the wrapped client as usual. `withRawResponse()`, `then`, `catch`, `finally`, byte-stream readers (including BYOB when the upstream stream supports it), iteration and piping remain available. Timestamp responses retain audio and alignment data in your application, but that data is not exported. Requests that set `webhook: true`, realtime/WebSocket APIs and other SDK namespaces pass through without integration spans. +Use these methods on the wrapped client as usual. `withRawResponse()`, `then`, `catch`, `finally`, byte-stream readers (including BYOB when the upstream stream supports it), iteration and piping remain available. Timestamp responses retain audio and alignment data in your application. By default, neither is exported. Requests that set `webhook: true`, realtime/WebSocket APIs and other SDK namespaces pass through without integration spans. -Hosted ElevenLabs Agents conversations are a separate integration. This package does not export conversation webhooks, audio attachments, usage tokens or estimated costs. +Hosted ElevenLabs Agents conversations are a separate integration. This package does not export conversation webhooks, usage tokens or estimated costs. + +## Optional binary attachments + +Binary capture is off by default. If your destination supports the attachment service, explicitly enable your provider's attachment manager and pass that **same manager** as `attachments: provider.attachments` to `ElevenLabsInstrumentation`. Await `provider.attachments.ready()` at startup to avoid missing early calls; continue to use the provider's `shutdown()` for delivery. The adapter never creates another uploader or enables recording. + +- `textToSpeech.convertWithTimestamps` can attach a complete MP3 response up to 256 KiB. The output format must be the SDK default or a supported MP3 format. Generated audio is not proof of playback. +- `speechToText.convert` can attach an existing `Blob`/`File` when you also set `attachmentInputMimeType` to its exact MIME type: `audio/wav`, `audio/mpeg`, `audio/ogg`, `audio/mp4`, `audio/webm` or `audio/flac`. Filenames are not exported; mismatched or missing types are not sniffed. +- `textToSpeech.convert`, `textToSpeech.stream` and `textToSpeech.streamWithTimestamps` can attach the MP3 your application **reads to the end**, up to 4 MiB per call (about 4 minutes at the default 128 kbit/s), or up to the server's single-upload limit when that is smaller. Tracing copies only the chunks you already read; it never reads ahead, tees or drains the stream. If you cancel, stop iterating, hit an error or exceed that cap, nothing is uploaded: a partial file is never stored as if complete. While a stream is being read, the manager reserves the whole cap for it (4 MiB, or the smaller server limit) out of its shared in-memory budget of about 31 MiB. With 4 MiB reservations about 7 streams can be captured at once on an otherwise idle manager; further concurrent streams report `queue_full` and pass through uncaptured. +- PCM/ulaw output, local paths and remote URLs are not captured. Alignment data stays in your SDK response, not in attachment metadata. + +Attachments obey the manager's default-off, server, sampling and PII policies in addition to the instrumentation's directional hide settings. Without a qualified speech sanitizer, `piiRedaction` denies capture. Existing raw audio may contain embedded metadata; a policy requiring metadata removal also denies it. The span's `elevenlabs.attachment.input` / `elevenlabs.attachment.output` value is the admission result; `queued` means admitted for transfer, **not** validated or playable. Content is out of band, never base64 in OTLP. A call made through this bound instrumentation carries the shared media-owner context marker (`traceai.auto.attachment_owner.v1`), so wire-level capture that honours the marker does not upload the same audio again. ## Privacy @@ -123,7 +134,7 @@ export FI_HIDE_ERROR_DETAILS=true `FI_HIDE_INPUT_MESSAGES` / `FI_HIDE_INPUT_TEXT` and their output equivalents also hide the corresponding speech text. You can pass core `traceConfig` options to the instrumentation instead. These settings affect traces, not SDK request or response values. -The package never exports raw audio, base64, alignment text, upload paths, source URLs, request headers or provider error bodies. Errors record an error status, a fixed error type, and an HTTP status when available, not exception messages or stack traces—even if `FI_HIDE_ERROR_DETAILS` is false. Your application is responsible for content it adds to parent spans and context metadata. +Trace attributes never contain raw audio, base64, alignment text, upload paths, source URLs, request headers or provider error bodies. Binary audio leaves the process only through the optional attachment manager described above. Errors record an error status, a fixed error type, and an HTTP status when available, not exception messages or stack traces—even if `FI_HIDE_ERROR_DETAILS` is false. Your application is responsible for content it adds to parent spans and context metadata. ## Stream lifecycle @@ -139,5 +150,5 @@ For module-wide registration, call `instrumentation.manuallyInstrument(sdk)` wit - **No span after receiving audio:** consume or cancel the returned stream. Receiving headers is not successful completion. Use `shutdown()` before exiting a short script. - **Spans appear separately from your application trace:** initialize an async context manager once and call the wrapped client within the active application span. -- **Text is missing:** check the `FI_HIDE_*` variables and the instrumentation's `traceConfig`. Audio and provider error text are intentionally never captured. +- **Text is missing:** check the `FI_HIDE_*` variables and the instrumentation's `traceConfig`. Binary audio needs the separate attachment opt-in; provider error text is never captured. - **Transcription spans are missing:** `webhook: true` submits an asynchronous job; this package does not trace its later completion. diff --git a/typescript/packages/traceai_elevenlabs/src/attachments.ts b/typescript/packages/traceai_elevenlabs/src/attachments.ts new file mode 100644 index 000000000..5e6b3aeaf --- /dev/null +++ b/typescript/packages/traceai_elevenlabs/src/attachments.ts @@ -0,0 +1,190 @@ +import type { Span } from "@opentelemetry/api"; +import type { TraceConfig } from "@traceai/fi-core"; +import type { AttachmentManager, AttachmentStream } from "@traceai/fi-core/attachments"; +import { record, safely } from "./operation.js"; + +export type ElevenLabsInputAudioMimeType = + | "audio/wav" + | "audio/mpeg" + | "audio/ogg" + | "audio/mp4" + | "audio/webm" + | "audio/flac"; + +// A bounded complete-response path for convertWithTimestamps (one JSON value). +const MAX_DECODED_BYTES = 256 * 1024; +/** + * Requested consumed-stream cap: about 4.4 minutes of the SDK's default 128 kbit/s MP3. The + * manager reserves min(this, the server's single-upload limit, its owned budget) when a stream is + * admitted, so a server whose `max_upload_bytes` is below 4 MiB still admits the stream with a + * smaller cap. The reservation bounds per-call memory. A response above the effective cap uploads + * nothing (`too_large`); a truncated object is never uploaded as complete. + */ +export const ELEVENLABS_STREAM_MAX_BYTES = 4 * 1024 * 1024; +const INPUT_TYPES: readonly string[] = [ + "audio/wav", "audio/mpeg", "audio/ogg", "audio/mp4", "audio/webm", "audio/flac", +]; +const MP3_FORMAT = /^mp3_(22050_32|24000_48|44100_(32|64|96|128|192))$/; +const BASE64 = /^[A-Za-z0-9+/]*={0,2}$/; + +/** Canonical, bounded base64 only. Buffer.from tolerates malformed input, so check first. */ +function decode(encoded: unknown, maxBytes: number): Uint8Array | undefined { + if (typeof encoded !== "string" || encoded.length === 0 || + encoded.length > Math.ceil(maxBytes / 3) * 4) return undefined; + if (encoded.length % 4 !== 0 || !BASE64.test(encoded)) return undefined; + const decoded = Buffer.from(encoded, "base64"); + if (decoded.length === 0 || decoded.length > maxBytes) return undefined; + if (decoded.toString("base64") !== encoded) return undefined; + return decoded; +} + +/** + * Adapters never own an uploader or change the provider manager's lifecycle. Every capture + * goes through the shared manager's admission before any read, decode or copy. + */ +export class SpeechAttachments { + private stream?: AttachmentStream; + private streamState: "idle" | "open" | "closed" = "idle"; + + constructor( + private readonly manager: AttachmentManager, + private readonly span: Span, + private readonly policy: TraceConfig, + private readonly outputFormat: string | undefined, + ) {} + + private allowed(direction: "input" | "output"): boolean { + // Adapter-local restrictions can only tighten the shared manager's policy. + // No qualified speech sanitizer is available here. + return !this.policy.piiRedaction && (direction === "input" + ? !(this.policy.hideInputs || this.policy.hideInputMessages || this.policy.hideInputText) + : !(this.policy.hideOutputs || this.policy.hideOutputMessages || this.policy.hideOutputText)); + } + + /** Raw PCM/ulaw lack sample-format metadata in v1; never label those bytes MP3. */ + private mp3(): boolean { + return this.outputFormat === undefined || MP3_FORMAT.test(this.outputFormat); + } + + input(request: unknown, mimeType: ElevenLabsInputAudioMimeType): void { + safely(() => { + if (!INPUT_TYPES.includes(mimeType)) return; + const result = this.manager.attach(this.span, { + kind: "Blob", + // Core checks default-off, server policy, PII, scope and sampling BEFORE + // this getter. No file/stream property is inspected on a denied capture. + get data(): Blob { + const file = record(request).file; + if (!(file instanceof Blob) || file.type !== mimeType) { + throw new Error("unsupported_audio_source"); + } + return file; + }, + }, { + mimeType, + direction: "input", + purpose: "model_input", + capture: { captureContent: this.allowed("input") }, + }); + this.span.setAttribute("elevenlabs.attachment.input", result.admission); + }); + } + + timestamped(value: unknown): void { + safely(() => { + if (!this.mp3()) { + this.span.setAttribute("elevenlabs.attachment.output", "source_unavailable"); + return; + } + const result = this.manager.attach(this.span, { + kind: "Bytes", + get data(): Uint8Array { + const decoded = decode(record(value).audioBase64, MAX_DECODED_BYTES); + if (!decoded) throw new Error("unsupported_audio_source"); + return decoded; + }, + }, { + mimeType: "audio/mpeg", + direction: "output", + purpose: "model_output", + capture: { captureContent: this.allowed("output") }, + }); + this.span.setAttribute("elevenlabs.attachment.output", result.admission); + }); + } + + /** + * Admit the consumed stream once per call, before any byte is copied or decoded. A denied + * admission retains nothing and is never retried for this call. + */ + private open(): AttachmentStream | undefined { + if (this.streamState === "idle") { + this.streamState = "closed"; + if (!this.mp3()) { + this.span.setAttribute("elevenlabs.attachment.output", "source_unavailable"); + return undefined; + } + const stream = this.manager.startAttachmentStream(this.span, { + mimeType: "audio/mpeg", + direction: "output", + purpose: "model_output", + maxBytes: ELEVENLABS_STREAM_MAX_BYTES, + capture: { captureContent: this.allowed("output") }, + }); + if (stream.reason !== "queued") { + this.span.setAttribute("elevenlabs.attachment.output", stream.reason); + return undefined; + } + this.stream = stream; + this.streamState = "open"; + } + return this.streamState === "open" ? this.stream : undefined; + } + + /** + * Observe one chunk the APPLICATION already pulled through the instrumentation-owned bridge + * (design 6.4). tryWrite copies synchronously, so a BYOB view may be handed back right after. + */ + chunk(bytes: Uint8Array): void { + safely(() => { + if (this.streamState === "closed" || !(bytes instanceof Uint8Array) || bytes.byteLength === 0) return; + const stream = this.open(); + if (stream && !stream.tryWrite(bytes)) this.close("incomplete"); + }); + } + + /** One base64 chunk of a consumed timestamp stream; decoded only after admission. */ + timestampChunk(value: unknown): void { + safely(() => { + if (this.streamState === "closed") return; + const encoded = record(value).audioBase64; + if (typeof encoded !== "string" || encoded.length === 0) return; + const stream = this.open(); + if (!stream) return; + const decoded = decode(encoded, ELEVENLABS_STREAM_MAX_BYTES); + // A malformed or oversized chunk means the object would be incomplete: upload nothing. + if (!decoded || !stream.tryWrite(decoded)) this.close("incomplete"); + }); + } + + /** + * Called by the operation BEFORE its span ends, so a finished stream's reference is + * projected on a recording span. Anything but a complete read aborts: nothing uploads. + */ + close(completion: "complete" | "incomplete"): void { + safely(() => { + const stream = this.stream; + const wasOpen = this.streamState === "open"; + this.stream = undefined; + this.streamState = "closed"; + if (!stream || !wasOpen) return; + if (completion === "complete") { + const handle = stream.finish(); + this.span.setAttribute("elevenlabs.attachment.output", handle.admission); + } else { + stream.abort("incomplete"); + this.span.setAttribute("elevenlabs.attachment.output", stream.reason); + } + }); + } +} diff --git a/typescript/packages/traceai_elevenlabs/src/index.ts b/typescript/packages/traceai_elevenlabs/src/index.ts index 6c95040eb..3c116683c 100644 --- a/typescript/packages/traceai_elevenlabs/src/index.ts +++ b/typescript/packages/traceai_elevenlabs/src/index.ts @@ -1,3 +1,4 @@ export { ElevenLabsInstrumentation } from "./instrumentation.js"; export type { ElevenLabsInstrumentationConfig } from "./instrumentation.js"; export { VERSION } from "./version.js"; +export type { ElevenLabsInputAudioMimeType } from "./attachments.js"; diff --git a/typescript/packages/traceai_elevenlabs/src/instrumentation.ts b/typescript/packages/traceai_elevenlabs/src/instrumentation.ts index 116dbd1ba..c662cf840 100644 --- a/typescript/packages/traceai_elevenlabs/src/instrumentation.ts +++ b/typescript/packages/traceai_elevenlabs/src/instrumentation.ts @@ -1,6 +1,7 @@ import { Attributes, context, + createContextKey, SpanKind, trace, TracerProvider, @@ -28,6 +29,11 @@ import { safely, TEXT_LIMIT, } from "./operation.js"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import { SpeechAttachments, ElevenLabsInputAudioMimeType } from "./attachments.js"; + +/** Context marker read by `@traceai/auto` wire capture; `createContextKey` is `Symbol.for`. */ +const ATTACHMENT_OWNER = createContextKey("traceai.auto.attachment_owner.v1"); import { observeResponse } from "./responses.js"; import { VERSION } from "./version.js"; @@ -68,6 +74,10 @@ export interface ElevenLabsInstrumentationConfig { instrumentationConfig?: InstrumentationConfig; tracerProvider?: TracerProvider; traceConfig?: TraceConfigOptions; + /** The configured provider's shared manager. Omitted means no binary capture. */ + attachments?: AttachmentManager; + /** Explicit MIME for STT Blob/File inputs; no sniffing or stream reads. */ + attachmentInputMimeType?: ElevenLabsInputAudioMimeType; /** Absolute span lifetime, including request and consumption. Default 300000 ms. */ maxSpanDurationMs?: number; /** Bound active telemetry, not SDK calls. Excess calls pass through. Default 1024. */ @@ -272,9 +282,8 @@ export class ElevenLabsInstrumentation extends InstrumentationBase { if (voice) attributes["elevenlabs.voice.id"] = voice; // Unlike JSON body overrides, the SDK skips nullish query values. const query = stt ? {} : record(options.queryParams); - const format = identifier( - bodyField(query, "output_format", undefined) ?? request.outputFormat, - ); + const formatValue = bodyField(query, "output_format", undefined) ?? request.outputFormat; + const format = identifier(formatValue); if (format) attributes["elevenlabs.output_format"] = format; if ( !stt && @@ -295,15 +304,26 @@ export class ElevenLabsInstrumentation extends InstrumentationBase { { kind: SpanKind.CLIENT, attributes }, callContext, ); + const attachments = this.options.attachments + ? new SpeechAttachments(this.options.attachments, span, policy, + formatValue === undefined ? undefined : format ?? "unsupported") + : undefined; operation = new Operation( span, policy, bounded(this.options.maxSpanDurationMs, 300_000, 86_400_000), (ended) => this.active.delete(ended), + attachments, ); + if (stt && this.options.attachmentInputMimeType && + !(options.abortSignal instanceof AbortSignal && options.abortSignal.aborted)) + attachments?.input(request, this.options.attachmentInputMimeType); this.active.add(operation); operation.watchAbort(record(args[stt ? 1 : 2]).abortSignal); callContext = trace.setSpan(callContext, span); + // This adapter owns the call's media, so wire-level auto capture must not upload the + // same bytes again (shared key; no dependency on the auto package). + if (attachments) callContext = callContext.setValue(ATTACHMENT_OWNER, true); } } catch { diagnose(); diff --git a/typescript/packages/traceai_elevenlabs/src/operation.ts b/typescript/packages/traceai_elevenlabs/src/operation.ts index 38d29ded2..2ec233916 100644 --- a/typescript/packages/traceai_elevenlabs/src/operation.ts +++ b/typescript/packages/traceai_elevenlabs/src/operation.ts @@ -1,6 +1,7 @@ import { diag, Span, SpanStatusCode } from "@opentelemetry/api"; import { SemanticConventions as S } from "@traceai/fi-semantic-conventions"; import { TraceConfig } from "@traceai/fi-core"; +import type { SpeechAttachments } from "./attachments.js"; let diagnosed = false; @@ -58,6 +59,7 @@ export class Operation { private readonly policy: TraceConfig, timeoutMs: number, private readonly remove: (operation: Operation) => void, + private attachments?: SpeechAttachments, ) { this.span = span; this.timer = setTimeout(() => this.end("incomplete"), timeoutMs); @@ -74,6 +76,20 @@ export class Operation { } } + timestampedAttachment(value: unknown): void { + if (this.span) this.attachments?.timestamped(value); + } + + /** A chunk the application consumed from an instrumentation-owned stream. */ + audioChunk(value: Uint8Array): void { + if (this.span) this.attachments?.chunk(value); + } + + /** A timestamp chunk the application consumed; audio is decoded only after admission. */ + timestampChunk(value: unknown): void { + if (this.span) this.attachments?.timestampChunk(value); + } + audio(value: unknown, timestamps = false): void { safely(() => { if (!this.span) return; @@ -151,6 +167,11 @@ export class Operation { end(completion: Completion): void { const span = this.span; if (!span) return; + // Finish or abort a consumed stream while the span still records, so a complete + // object projects its reference and anything else uploads nothing. + const attachments = this.attachments; + this.attachments = undefined; + safely(() => attachments?.close(completion === "complete" ? "complete" : "incomplete")); this.span = undefined; if (this.timer) clearTimeout(this.timer); this.timer = undefined; diff --git a/typescript/packages/traceai_elevenlabs/src/responses.ts b/typescript/packages/traceai_elevenlabs/src/responses.ts index 4af9cf43d..80e7eb369 100644 --- a/typescript/packages/traceai_elevenlabs/src/responses.ts +++ b/typescript/packages/traceai_elevenlabs/src/responses.ts @@ -55,6 +55,8 @@ function audioStream( reader.value.releaseLock(); } else { operation.audio(next.value); + // Copy before the view goes back to the consumer (BYOB) or downstream. + operation.audioChunk(next.value); if (request) request.respondWithNewView(next.value); else controller.enqueue(next.value); } @@ -94,11 +96,11 @@ function observeIterator( try { const next = await iterator.next(...args); if (next.done) operation.end("complete"); - else - operation.audio( - metadata ? record(next.value).data : next.value, - true, - ); + else { + const value = metadata ? record(next.value).data : next.value; + operation.audio(value, true); + operation.timestampChunk(value); + } return next; } catch (error) { operation.error(error); @@ -191,7 +193,10 @@ export function observeResponse( data = timestampStream(data as AsyncIterable, operation); } else { if (method === "speechToText.convert") operation.output(data); - else operation.audio(data, true); + else { + operation.timestampedAttachment(data); + operation.audio(data, true); + } operation.end("complete"); } }); diff --git a/typescript/packages/traceai_elevenlabs/test/attachments-fixture.cjs b/typescript/packages/traceai_elevenlabs/test/attachments-fixture.cjs new file mode 100644 index 000000000..23a7f78e7 --- /dev/null +++ b/typescript/packages/traceai_elevenlabs/test/attachments-fixture.cjs @@ -0,0 +1,72 @@ +const http = require('node:http'); +const { once } = require('node:events'); +const { randomUUID } = require('node:crypto'); +const { AttachmentManager, createAttachmentHttpTransport } = require('@traceai/fi-core/attachments'); +const ids = [1, 2, 3].map(n => `10000000-0000-4000-8000-00000000000${n}`); +// Sessions of at least 1 h, so the core's near-expiry renewal never adds a bootstrap here. +const expiry = () => new Date(Date.now() + 3600000).toISOString(); + +async function attachmentFixture(config = {}, serverPolicy = {}, serverLimits = {}) { + const calls = [], uploaded = [], requests = [], uploads = new Map(); + const policy = { version: 'v1', enabled: true, allowed_sources: ['Bytes', 'Blob'], allowed_modalities: ['audio'], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, + record_audio: false, record_video: false, ...serverPolicy }; + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const chunk of req) chunks.push(chunk); + const bytes = Buffer.concat(chunks); + calls.push({ method: req.method, path: req.url }); + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('/attachment-sessions')) { + const body = JSON.parse(bytes); + res.end(JSON.stringify({ schema_version: 1, scope: { organization_id: ids[0], workspace_id: ids[1], project_id: ids[2] }, + capture_session_id: body.capture_session_id, policy, limits: { max_upload_bytes: 8388608, segment_bytes: 8388608, + segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, + active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, + initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, + playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5, ...serverLimits }, + expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); + } else if (req.url.endsWith('/uploads')) { + const body = JSON.parse(bytes); requests.push(body); + const upload = { upload_id: randomUUID(), attachment_id: body.attachment_id, occurrence_id: body.occurrence_id, + state: 'pending_upload', reserved_bytes: body.declared_bytes, expires_at: expiry(), + transfer_mode: 'authenticated_stream', write_capability: 'synthetic-write-capability' }; + uploads.set(upload.upload_id, upload); res.statusCode = 201; res.end(JSON.stringify(upload)); + } else { + const id = req.url.split('/').at(-2); + if (!uploads.has(id) || req.headers.authorization !== 'Bearer synthetic-write-capability') { res.statusCode = 403; res.end('{}'); return; } + if (req.url.endsWith('/content')) { + uploaded.push(bytes); res.end(JSON.stringify({ upload_id: id, state: 'pending_upload', reason: 'queued' })); + } else { + res.statusCode = 202; res.end(JSON.stringify({ upload_id: id, state: 'validating', reason: 'binding_pending' })); + } + } + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const client = async req => { + const body = req.body instanceof Blob ? Buffer.from(await req.body.arrayBuffer()) : req.body; + return new Promise((resolve, reject) => { + const r = http.request(`http://127.0.0.1:${server.address().port}${new URL(req.url).pathname}`, + { method: req.method, headers: req.headers, signal: req.signal }, response => { + const chunks = []; response.on('data', c => chunks.push(c)); + response.on('end', () => resolve({ status: response.statusCode, body: Buffer.concat(chunks) })); + }); + r.on('error', reject); r.end(body); + }); + }; + const manager = new AttachmentManager({ enabled: true, projectName: 'synthetic-voice', projectType: 'observe', + transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic' }, client }), ...config }); + await manager.ready(); + return { manager, calls, uploaded, requests, + async close() { await manager.shutdown(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } }; +} + +function wav() { + const samples = 2400, out = Buffer.alloc(44 + samples * 2); + out.write('RIFF'); out.writeUInt32LE(out.length - 8, 4); out.write('WAVEfmt ', 8); + out.writeUInt32LE(16, 16); out.writeUInt16LE(1, 20); out.writeUInt16LE(1, 22); + out.writeUInt32LE(24000, 24); out.writeUInt32LE(48000, 28); out.writeUInt16LE(2, 32); out.writeUInt16LE(16, 34); + out.write('data', 36); out.writeUInt32LE(samples * 2, 40); + for (let i = 0; i < samples; i++) out.writeInt16LE(Math.round(Math.sin(i * 2 * Math.PI * 440 / 24000) * 2000), 44 + i * 2); + return out; +} +module.exports = { attachmentFixture, wav }; diff --git a/typescript/packages/traceai_elevenlabs/test/attachments.test.cjs b/typescript/packages/traceai_elevenlabs/test/attachments.test.cjs new file mode 100644 index 000000000..91f0b3f11 --- /dev/null +++ b/typescript/packages/traceai_elevenlabs/test/attachments.test.cjs @@ -0,0 +1,269 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const { fixture, bytes } = require('./fixture.cjs'); +const { attachmentFixture, wav } = require('./attachments-fixture.cjs'); +const { SpeechAttachments } = require('../dist/src/attachments.js'); +const { generateTraceConfig } = require('@traceai/fi-core'); +const tone = readFileSync(require('node:path').join(__dirname, 'fixtures/tone.mp3')); +const MiB = 1024 * 1024; + +for (const sdkName of ['@elevenlabs/sdk-floor', '@elevenlabs/elevenlabs-js']) { + const SDK = require(sdkName); + test(`${sdkName}: timestamp response uploads once; cached awaits retain data and parent`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager }, { base64: tone.toString('base64') }); t.after(() => f.close()); + const response = f.client.textToSpeech.convertWithTimestamps('synthetic-voice', { text: 'synthetic', outputFormat: 'mp3_44100_128' }); + const first = await response, second = await response.withRawResponse(); + assert.equal(first, second.data); + assert.deepEqual(Buffer.from(first.audioBase64, 'base64'), tone); + assert.equal(first.alignment.characterStartTimesSeconds[0], 0); + await a.manager.flush(); + assert.equal(a.uploaded.length, 1); assert.deepEqual(a.uploaded[0], tone); + const span = f.spans()[0]; + const refs = JSON.parse(span.attributes['fi.attachment.refs']); + assert.equal(refs.length, 1); assert.equal(refs[0].occurrence_id, a.requests[0].occurrence_id); + assert.equal(a.requests[0].association.parent.span_id, span.spanContext().spanId); + assert.equal(a.requests[0].association.parent.trace_id.replaceAll('-', ''), span.spanContext().traceId); + assert.equal(a.requests[0].mime_type, 'audio/mpeg'); + const wire = JSON.stringify(await f.flush()); + assert.ok(!wire.includes(tone.toString('base64'))); + assert.ok(!wire.includes('synthetic-write-capability')); + }); + + test(`${sdkName}: immutable typed STT Blob uploads independently without filename`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager, attachmentInputMimeType: 'audio/wav' }); t.after(() => f.close()); + const audio = wav(); const blob = new File([audio], 'synthetic-private-name.wav', { type: 'audio/wav' }); + const result = await f.client.speechToText.convert({ file: blob, modelId: 'scribe_v1' }); + assert.equal(result.text, 'synthetic-transcript-elevenlabs'); + await a.manager.flush(); assert.equal(a.uploaded.length, 1); assert.deepEqual(a.uploaded[0], audio); + assert.deepEqual(Buffer.from(await blob.arrayBuffer()), audio); + assert.equal(a.requests[0].direction, 'input'); assert.equal(a.requests[0].mime_type, 'audio/wav'); + assert.ok(!JSON.stringify(a.requests).includes('synthetic-private-name')); + assert.ok(!JSON.stringify(await f.flush()).includes('synthetic-private-name')); + }); + + for (const [name, managerConfig, localConfig, policy] of [ + ['disabled', { enabled: false }, {}], + ['core hidden input/output', { traceConfig: { hideInputs: true, hideOutputs: true } }, {}], + ['local hidden messages', {}, { hideInputMessages: true, hideOutputMessages: true }], + ['local hidden text', {}, { hideInputText: true, hideOutputText: true }], + ['local PII', {}, { piiRedaction: true }], + ['core PII', { traceConfig: { piiRedaction: true } }, {}], + ['server redaction', {}, {}, { require_media_redaction: true }], + ['environment denial', { environment: () => ({ FI_ATTACHMENTS_ENABLED: 'false' }) }, {}], + ]) test(`${sdkName}: ${name} prevents all attachment upload traffic`, async t => { + const a = await attachmentFixture(managerConfig, policy); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager, attachmentInputMimeType: 'audio/wav', traceConfig: localConfig }, + { base64: tone.toString('base64') }); t.after(() => f.close()); + await f.client.textToSpeech.convertWithTimestamps('voice', { text: 'synthetic' }); + await f.client.speechToText.convert({ file: new Blob([wav()], { type: 'audio/wav' }), modelId: 'scribe_v1' }); + await bytes(await f.client.textToSpeech.convert('voice', { text: 'synthetic' })); + await bytes(await f.client.textToSpeech.stream('voice', { text: 'synthetic' })); + for await (const chunk of await f.client.textToSpeech.streamWithTimestamps('voice', { text: 'synthetic' })) assert.ok(chunk); + await a.manager.flush(); + assert.equal(a.uploaded.length, 0); assert.equal(a.requests.length, 0); + assert.ok(a.calls.every(c => c.path.endsWith('/attachment-sessions'))); + assert.ok(f.spans().every(s => s.attributes['fi.attachment.refs'] === undefined)); + }); + + for (const method of ['convert', 'stream']) { + test(`${sdkName}: fully consumed ${method} uploads the exact MP3 once, before the span ends`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager }, { audio: tone }); t.after(() => f.close()); + const received = await bytes(await f.client.textToSpeech[method]('voice', { text: 'synthetic' })); + assert.deepEqual(Buffer.from(received), tone); + await a.manager.flush(); + assert.equal(a.uploaded.length, 1); assert.deepEqual(a.uploaded[0], tone); + const span = f.spans()[0]; + assert.equal(span.attributes['elevenlabs.completion'], 'complete'); + assert.equal(span.attributes['elevenlabs.attachment.output'], 'queued'); + const refs = JSON.parse(span.attributes['fi.attachment.refs']); + assert.equal(refs.length, 1); assert.equal(refs[0].occurrence_id, a.requests[0].occurrence_id); + assert.equal(a.requests[0].mime_type, 'audio/mpeg'); assert.equal(a.requests[0].direction, 'output'); + assert.equal(a.requests[0].declared_bytes, tone.length); + assert.equal(a.requests[0].association.parent.span_id, span.spanContext().spanId); + }); + } + + for (const method of ['convert', 'stream']) { + test(`${sdkName}: a server single-upload limit below 4 MiB still admits and uploads a consumed ${method}`, async t => { + // The stream reserves min(4 MiB, max_upload_bytes); before the core clamp this was denied too_large. + const a = await attachmentFixture({}, {}, { max_upload_bytes: MiB }); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager }, { audio: tone }); t.after(() => f.close()); + const received = await bytes(await f.client.textToSpeech[method]('voice', { text: 'synthetic' })); + assert.deepEqual(Buffer.from(received), tone); + await a.manager.flush(); + assert.equal(f.spans()[0].attributes['elevenlabs.attachment.output'], 'queued'); + assert.equal(a.uploaded.length, 1); assert.deepEqual(a.uploaded[0], tone); + assert.equal(a.requests[0].declared_bytes, tone.length); + assert.equal(a.manager.status().ownedBytes, 0); + }); + } + + test(`${sdkName}: BYOB reads keep consumer buffers and upload identical bytes`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager }, { audio: tone }); t.after(() => f.close()); + const stream = await f.client.textToSpeech.stream('voice', { text: 'synthetic' }); + let reader; + try { reader = stream.getReader({ mode: 'byob' }); } catch { t.skip('upstream stream is not a byte stream'); return; } + const out = []; let buffer = new ArrayBuffer(1024); + for (;;) { + const { done, value } = await reader.read(new Uint8Array(buffer, 0, 1024)); + if (done) break; + out.push(Buffer.from(value)); buffer = value.buffer; + } + assert.deepEqual(Buffer.concat(out), tone); + await a.manager.flush(); assert.equal(a.uploaded.length, 1); assert.deepEqual(a.uploaded[0], tone); + }); + + test(`${sdkName}: cancelled or broken streams upload nothing and keep cancellation`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager }, { audio: tone }); t.after(() => f.close()); + const stream = await f.client.textToSpeech.stream('voice', { text: 'synthetic' }); + const reader = stream.getReader(); + assert.equal((await reader.read()).done, false); + await reader.cancel('synthetic-cancel'); + const broken = await f.client.textToSpeech.convert('voice', { text: 'synthetic' }, { headers: { 'x-case': 'broken' } }); + await assert.rejects(bytes(broken)); + for await (const chunk of await f.client.textToSpeech.streamWithTimestamps('voice', { text: 'synthetic' })) { + assert.equal(typeof chunk.audioBase64, 'string'); break; + } + await a.manager.flush(); assert.equal(a.uploaded.length, 0); assert.equal(a.requests.length, 0); + const spans = f.spans(); + assert.ok(spans.some(s => s.attributes['elevenlabs.completion'] === 'cancelled')); + assert.ok(spans.every(s => s.attributes['fi.attachment.refs'] === undefined)); + assert.equal(a.manager.status().openStreams, 0); + }); + + test(`${sdkName}: fully iterated timestamp stream uploads the decoded MP3 chunks as one object`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager }, { base64: tone.toString('base64') }); t.after(() => f.close()); + let chunks = 0; + for await (const chunk of await f.client.textToSpeech.streamWithTimestamps('voice', { text: 'synthetic' })) { + assert.equal(chunk.audioBase64, tone.toString('base64')); chunks++; + } + assert.equal(chunks, 2); + await a.manager.flush(); + assert.equal(a.uploaded.length, 1); assert.deepEqual(a.uploaded[0], Buffer.concat([tone, tone])); + const span = f.spans()[0]; + assert.equal(JSON.parse(span.attributes['fi.attachment.refs']).length, 1); + assert.ok(!JSON.stringify(await f.flush()).includes(tone.toString('base64'))); + }); + + test(`${sdkName}: raw PCM streams are never labelled MP3`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager }, { audio: tone }); t.after(() => f.close()); + await bytes(await f.client.textToSpeech.stream('voice', { text: 'synthetic', outputFormat: 'pcm_24000' })); + await a.manager.flush(); assert.equal(a.uploaded.length, 0); + assert.equal(f.spans()[0].attributes['elevenlabs.attachment.output'], 'source_unavailable'); + }); + + test(`${sdkName}: the bound adapter marks its call context as the media owner`, async t => { + const { AsyncLocalStorageContextManager } = require('@opentelemetry/context-async-hooks'); + const { context } = require('@opentelemetry/api'); + const manager = new AsyncLocalStorageContextManager().enable(); + context.setGlobalContextManager(manager); + t.after(() => { context.disable(); }); + const key = Symbol.for('traceai.auto.attachment_owner.v1'); + const seen = []; + const original = globalThis.fetch; + globalThis.fetch = (...args) => { seen.push(context.active().getValue(key)); return original(...args); }; + t.after(() => { globalThis.fetch = original; }); + const a = await attachmentFixture(); t.after(() => a.close()); + const bound = await fixture(SDK, { attachments: a.manager }, { audio: tone }); t.after(() => bound.close()); + const plain = await fixture(SDK, {}, { audio: tone }); t.after(() => plain.close()); + await bytes(await bound.client.textToSpeech.convert('voice', { text: 'synthetic' })); + await bytes(await plain.client.textToSpeech.convert('voice', { text: 'synthetic' })); + assert.deepEqual(seen, [true, undefined]); + }); + + for (const [name, base64, format] of [ + ['oversized complete response', Buffer.alloc(256 * 1024 + 1).toString('base64'), 'mp3_44100_128'], + ['malformed base64', 'a!==', 'mp3_44100_128'], + ['raw PCM is not MP3', tone.toString('base64'), 'pcm_24000'], + ['unrecognized format cannot become the default', tone.toString('base64'), 'bad/format'], + ]) test(`${sdkName}: ${name} fails closed without changing SDK response`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager }, { base64 }); t.after(() => f.close()); + const value = await f.client.textToSpeech.convertWithTimestamps('voice', { text: 'synthetic', outputFormat: 'mp3_44100_128' }, + { queryParams: { output_format: format } }); + assert.equal(value.audioBase64, base64); await a.manager.flush(); assert.equal(a.uploaded.length, 0); + }); +} + +for (const config of [{ enabled: false }, { traceConfig: { piiRedaction: true } }, { traceConfig: { hideInputs: true, hideOutputs: true } }]) { + test(`denied sources are never inspected: ${JSON.stringify(config)}`, async t => { + const a = await attachmentFixture(config); t.after(() => a.close()); + let reads = 0; + const span = { spanContext: () => ({ traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }), + isRecording: () => true, setAttribute() {} }; + const capture = new SpeechAttachments(a.manager, span, generateTraceConfig(), undefined); + capture.timestamped({ get audioBase64() { reads++; throw Error('synthetic-private'); } }); + capture.input({ get file() { reads++; throw Error('synthetic-private'); } }, 'audio/wav'); + await a.manager.flush(); assert.equal(reads, 0); assert.equal(a.uploaded.length, 0); + }); +} + +test('a stream above the cap uploads nothing (too_large) and returns its reservation', async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const attrs = {}; + const span = { spanContext: () => ({ traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }), + isRecording: () => true, setAttribute(k, v) { attrs[k] = v; } }; + const capture = new SpeechAttachments(a.manager, span, generateTraceConfig(), undefined); + const { ELEVENLABS_STREAM_MAX_BYTES } = require('../dist/src/attachments.js'); + capture.chunk(new Uint8Array(ELEVENLABS_STREAM_MAX_BYTES)); + capture.chunk(new Uint8Array(1)); + capture.close('complete'); + await a.manager.flush(); + assert.equal(a.uploaded.length, 0); assert.equal(attrs['elevenlabs.attachment.output'], 'too_large'); + assert.equal(a.manager.status().ownedBytes, 0); +}); + +let spans = 0; +function syntheticSpan(attrs = {}) { + const spanId = (++spans).toString(16).padStart(16, '0'); + return { spanContext: () => ({ traceId: '0123456789abcdef0123456789abcdef', spanId, traceFlags: 1 }), + isRecording: () => true, setAttribute(k, v) { attrs[k] = v; } }; +} + +test('a server limit below 4 MiB caps the stream there: exactly the limit uploads, one byte more is too_large', async t => { + const a = await attachmentFixture({}, {}, { max_upload_bytes: MiB }); t.after(() => a.close()); + const fits = {}, over = {}; + const ok = new SpeechAttachments(a.manager, syntheticSpan(fits), generateTraceConfig(), undefined); + ok.chunk(new Uint8Array(MiB - 1)); ok.chunk(new Uint8Array(1)); ok.close('complete'); + const big = new SpeechAttachments(a.manager, syntheticSpan(over), generateTraceConfig(), undefined); + big.chunk(new Uint8Array(MiB)); big.chunk(new Uint8Array(1)); big.close('complete'); + await a.manager.flush(); + assert.equal(fits['elevenlabs.attachment.output'], 'queued'); + assert.equal(over['elevenlabs.attachment.output'], 'too_large'); + assert.equal(a.uploaded.length, 1); assert.equal(a.uploaded[0].length, MiB); + assert.equal(a.requests.length, 1); + assert.equal(a.manager.status().ownedBytes, 0); +}); + +test('concurrency: about 7 open 4 MiB streams fit the owned budget; the 8th is queue_full; a 1 MiB server limit fits more', async t => { + const { ELEVENLABS_STREAM_MAX_BYTES } = require('../dist/src/attachments.js'); + const open = async (serverLimits, n) => { + const a = await attachmentFixture({}, {}, serverLimits); t.after(() => a.close()); + const out = []; + for (let i = 0; i < n; i++) { + const attrs = {}; + const capture = new SpeechAttachments(a.manager, syntheticSpan(attrs), generateTraceConfig(), undefined); + capture.chunk(new Uint8Array(1)); // admission happens at the first observed chunk + out.push({ capture, attrs }); + } + return { a, out }; + }; + const wide = await open({}, 8); + assert.equal(ELEVENLABS_STREAM_MAX_BYTES, 4 * MiB); + assert.deepEqual(wide.out.map(s => s.attrs['elevenlabs.attachment.output'] ?? 'open'), [...Array(7).fill('open'), 'queue_full']); + for (const s of wide.out) s.capture.close('incomplete'); + assert.equal(wide.a.manager.status().ownedBytes, 0); + const narrow = await open({ max_upload_bytes: MiB }, 16); + assert.ok(narrow.out.every(s => s.attrs['elevenlabs.attachment.output'] === undefined), 'every 1 MiB reservation admitted'); + for (const s of narrow.out) s.capture.close('incomplete'); + await narrow.a.manager.flush(); + assert.equal(narrow.a.uploaded.length, 0); assert.equal(narrow.a.manager.status().ownedBytes, 0); +}); diff --git a/typescript/packages/traceai_elevenlabs/test/fixture.cjs b/typescript/packages/traceai_elevenlabs/test/fixture.cjs index 11b0e4bd6..3230b2eb8 100644 --- a/typescript/packages/traceai_elevenlabs/test/fixture.cjs +++ b/typescript/packages/traceai_elevenlabs/test/fixture.cjs @@ -44,7 +44,8 @@ const attributes = (span) => ]), ); -async function fixture(SDK, options = {}) { +async function fixture(SDK, options = {}, media = {}) { + const speech = { ...timestamp, audio_base64: media.base64 ?? BASE64 }; const requests = [], wire = []; const attempts = new Map(); @@ -122,25 +123,27 @@ async function fixture(SDK, options = {}) { if (req.url.includes("with-timestamps")) { res.setHeader("content-type", "application/json"); if (req.url.includes("/stream/")) { - res.write(JSON.stringify(timestamp) + "\n"); + res.write(JSON.stringify(speech) + "\n"); await delay(15); res.end( mode === "bad-stream" ? "{bad-json}\n" - : JSON.stringify(timestamp) + "\n", + : JSON.stringify(speech) + "\n", ); - } else res.end(JSON.stringify(timestamp)); + } else res.end(JSON.stringify(speech)); return; } res.setHeader("content-type", "audio/mpeg"); res.flushHeaders(); await delay(mode === "slow" ? 120 : 10); if (res.destroyed) return; - res.write(AUDIO.subarray(0, 4)); + const audio = media.audio ?? AUDIO; + const split = Math.min(4, audio.length); + res.write(audio.subarray(0, split)); await delay(mode === "slow" ? 120 : 10); if (res.destroyed) return; if (mode === "broken") res.destroy(); - else res.end(AUDIO.subarray(4)); + else res.end(audio.subarray(split)); } catch { res.destroy(); } diff --git a/typescript/packages/traceai_elevenlabs/test/fixtures/tone.mp3 b/typescript/packages/traceai_elevenlabs/test/fixtures/tone.mp3 new file mode 100644 index 000000000..0e538fc9a Binary files /dev/null and b/typescript/packages/traceai_elevenlabs/test/fixtures/tone.mp3 differ diff --git a/typescript/packages/traceai_livekit/README.md b/typescript/packages/traceai_livekit/README.md index 673f3f35c..94c6b82d2 100644 --- a/typescript/packages/traceai_livekit/README.md +++ b/typescript/packages/traceai_livekit/README.md @@ -77,6 +77,35 @@ const room = new client.Room(); `livekit-client` implements room methods on each instance. If you already have a room, use `instrumentation.instrumentRoom(room)` instead. Repeated registration does not add duplicate spans. Existing instrumented rooms resume tracing when you call `enable()` after `disable()`. Manually registered SDK modules should also be registered again to restore their participant and media hooks. +## Upload an existing LiveKit Agents recording (optional) + +LiveKit Agents for Node can record a session to `audio.ogg` in the job's session directory when you start the session with recording on (`session.start({ ..., record: true })`, or the project's LiveKit Cloud setting). This package can upload **that existing file** to Future AGI at session end. It never turns recording on, never starts or stops the recorder, and never deletes the file. Attachments stay off unless you enable them on your provider. + +Supported `@livekit/agents` versions: 1.8.0, 1.8.1 and 1.9.0 to 1.9.5. Any other version reports `unsupported_version` and uploads nothing. Qualified on Linux. On macOS the default session directory is under `/var`, which is a symlink, so the binding reports `source_unavailable` there. Windows is not supported. + +```javascript +const agents = require("@livekit/agents"); +const { bindLiveKitSessionRecording } = require("@traceai/fi-instrumentation-livekit/agents"); + +// Inside your agent entry, after `await session.start({ agent, room: ctx.room, record: true })`: +const recording = bindLiveKitSessionRecording(session, ctx, { + manager: provider.attachments, // the same manager your Future AGI provider created + agents, // the @livekit/agents module your application uses + ownedRecordingFiles: true, // explicit opt-in; anything else reports "disabled" + finalizationTimeoutMs: 30000, // upload deadline at session end, at most 30 s +}); +// recording.status is "session_bound" when it will run at session end. +``` + +How it works: + +- Bind right after `session.start(...)` returns. The session's `agent_session` span is captured then, and the upload is linked to it. Point LiveKit's telemetry at your Future AGI provider (`agents.telemetry.setTracerProvider(provider)`) so that span is exported; if the span is not recording, the binding reports `correlation_unavailable`. +- At session end, LiveKit closes the recorder, runs your `onSessionEnd`, then calls its own session-end step. The binding runs at the start of that step, uploads `audio.ogg` (mixed user and agent audio, up to the attachment service's single-upload limit), and then lets LiveKit continue. If your `onSessionEnd` moves or deletes the file, the binding reports `source_unavailable`. +- Only the job's primary session, its own recorder and the fixed `audio.ogg` in the job's session directory qualify. The file must be owned by the worker's own user. A symlink, a hard link, a file owned by another user, or a file swapped during upload fails closed. +- It uploads nothing when attachments are off, when your trace config hides inputs or outputs or redacts PII, when the server policy denies it, when LiveKit project or session redaction is on, when `LIVEKIT_TELEMETRY_ALLOW_PII` is false, or when session audio recording is off. These checks run again at session end, before the file is opened or even checked, so a policy tightened during the session still wins. +- Give your worker a shutdown grace longer than `finalizationTimeoutMs` plus your own session-end callbacks. A process killed before the upload finishes loses it. +- `recording.status` is content-free. `validating` means the file reached Future AGI and is being checked; it is not yet playable. `recording.close()` detaches the binding without touching the session. + ## Control sensitive data Tokens, URL credentials, URL paths, query strings, and raw audio/video are not captured as span attributes. Room names, participant identifiers, track names, publish options, and SDK error messages can still contain sensitive information. Use your collector's attribute filtering for identifying metadata, and configure content/error redaction when constructing the instrumentor: diff --git a/typescript/packages/traceai_livekit/package.json b/typescript/packages/traceai_livekit/package.json index feb18db86..a84d5e3e2 100644 --- a/typescript/packages/traceai_livekit/package.json +++ b/typescript/packages/traceai_livekit/package.json @@ -9,6 +9,18 @@ "types": "./dist/src/index.d.ts", "import": "./dist/src/index.js", "require": "./dist/src/index.js" + }, + "./agents": { + "types": "./dist/src/agents.d.ts", + "import": "./dist/src/agents.js", + "require": "./dist/src/agents.js" + } + }, + "typesVersions": { + "*": { + "agents": [ + "dist/src/agents.d.ts" + ] } }, "files": [ diff --git a/typescript/packages/traceai_livekit/src/agents.ts b/typescript/packages/traceai_livekit/src/agents.ts new file mode 100644 index 000000000..453e7a6d2 --- /dev/null +++ b/typescript/packages/traceai_livekit/src/agents.ts @@ -0,0 +1,267 @@ +/** + * Opt-in upload of a bound LiveKit Agents (Node) session's EXISTING local recording. + * + * Node only; import it from `@traceai/fi-instrumentation-livekit/agents`, never from the root entry. + * This binding never enables, starts or stops a recording, and never deletes the file. It supports + * only the exact `@livekit/agents` versions the component suite qualifies; the private lifecycle + * seams it uses are confined to this file and fail closed on any other version. + */ +import { lstat } from "node:fs/promises"; +import * as path from "node:path"; +import { trace, type Context } from "@opentelemetry/api"; +import type { + AttachmentAdmission, + AttachmentCapturePolicy, + AttachmentCompletion, + AttachmentCorrelation, + AttachmentManager, + AttachmentOwnedRecordingHandle, + AttachmentRecordingFileScope, +} from "@traceai/fi-core/attachments"; +import { createRecordingFileScope } from "@traceai/fi-core/attachments/node"; + +/** `@livekit/agents` versions whose session-end ordering and recorder seams were qualified. */ +export const LIVEKIT_AGENTS_SUPPORTED_VERSIONS: readonly string[] = Object.freeze([ + "1.8.0", "1.8.1", "1.9.0", "1.9.1", "1.9.2", "1.9.3", "1.9.4", "1.9.5", +]); +const RECORDING_FILE = "audio.ogg"; +const MAX_TIMEOUT_MS = 30_000; +const PII_FALSY = new Set(["0", "false", "no", "off"]); + +/** Content-free outcome. `validating` means the file reached quarantine, NOT that it is playable. */ +export type LiveKitSessionRecordingStatus = + | "session_bound" + | "disabled" + | "unsupported_version" + | "conflict" + | AttachmentAdmission + | AttachmentCompletion["status"]; + +export interface LiveKitSessionRecordingOptions { + /** The provider's shared manager (`provider.attachments`). This binding never creates another. */ + manager: AttachmentManager; + /** The `@livekit/agents` module your application uses (`import * as agents from "@livekit/agents"`). */ + agents: { version?: unknown; JobContext?: unknown; voice?: { AgentSession?: unknown } }; + /** Must be exactly `true` to upload an existing recording. Anything else reports `disabled`. */ + ownedRecordingFiles: boolean; + /** Upload deadline at session end, at most 30 000 ms (default 30 000). */ + finalizationTimeoutMs?: number; + /** Additional restrictions only; a binding never relaxes the manager's policy. */ + capture?: AttachmentCapturePolicy; +} + +interface RecorderLike { readonly recording?: unknown; readonly outputPath?: unknown } +interface SessionLike { + rootSpanContext?: Context; + _recorderIO?: RecorderLike; + _redactionEnabled?: unknown; + sessionOptions?: { recordingOptions?: { audio?: unknown; redaction?: unknown } }; +} +interface JobLike { + _primaryAgentSession?: unknown; + readonly sessionDirectory?: unknown; + _redactionEnabled?: unknown; + readonly job?: { enableRedaction?: unknown }; + makeSessionReport?: (session?: unknown) => { audioRecordingPath?: unknown }; + _onSessionEnd?: () => Promise; +} + +const BINDING = Symbol.for("traceai.livekit.agents.recording_binding.v1"); +const isFn = (value: unknown): value is (...args: unknown[]) => unknown => typeof value === "function"; + +function nativeDenial(session: SessionLike, job: JobLike, env: NodeJS.ProcessEnv): LiveKitSessionRecordingStatus | undefined { + // Native integration restrictions come first, before any recording-path inspection. + const pii = env.LIVEKIT_TELEMETRY_ALLOW_PII; + if (pii !== undefined && PII_FALSY.has(pii.trim().toLowerCase())) return "policy_denied"; + const options = session.sessionOptions?.recordingOptions; + // Text redaction never sanitizes a binary recording; any redaction requirement denies it. + if (options?.redaction || session._redactionEnabled !== false || job._redactionEnabled !== false || job.job?.enableRedaction) + return "redaction_unavailable"; + if (options?.audio !== true) return "policy_denied"; + return undefined; +} + +/** + * The binding's state is content-free. `close()` only detaches this adapter: it never stops the + * application's session, shuts down the shared manager, or enables or deletes a recording. + */ +export class LiveKitSessionRecording { + status: LiveKitSessionRecordingStatus = "session_bound"; + private session?: SessionLike; + private job?: JobLike; + private correlation?: AttachmentCorrelation; + private root?: string; + private wrapper?: () => Promise; + private hadOwnHook = false; + private ownHook?: () => Promise; + private handle?: AttachmentOwnedRecordingHandle; + private finalizing?: Promise; + private closed = false; + /** Set when the deadline wins; a late step must not open, upload or keep anything. */ + private stopped = false; + + /** @internal */ + constructor( + private readonly manager: AttachmentManager, + private readonly timeoutMs: number, + private readonly capture: AttachmentCapturePolicy | undefined, + private readonly env: NodeJS.ProcessEnv, + ) {} + + /** @internal */ + attach(session: SessionLike, job: JobLike, correlation: AttachmentCorrelation, root: string): void { + this.session = session; this.job = job; this.correlation = correlation; this.root = root; + const original = job._onSessionEnd!; + this.hadOwnHook = Object.prototype.hasOwnProperty.call(job, "_onSessionEnd"); + this.ownHook = this.hadOwnHook ? original : undefined; + const wrapper = async () => { + // The SDK awaits this after AgentSession.close() (recorder closed) and the application's + // onSessionEnd, and before its own report upload. Its own work always still runs. + try { await this.finalize(); } finally { await original.call(job); } + }; + this.wrapper = wrapper; + job._onSessionEnd = wrapper; + (job as Record)[BINDING] = this; + } + + /** @internal Same arguments as a previous bind. */ + same(manager: AttachmentManager, session: unknown, timeoutMs: number, capture: AttachmentCapturePolicy | undefined): boolean { + return manager === this.manager && session === this.session && timeoutMs === this.timeoutMs && + JSON.stringify(capture ?? {}) === JSON.stringify(this.capture ?? {}); + } + + /** The bounded transfer handle, once a file was handed to the manager. */ + get recording(): AttachmentOwnedRecordingHandle | undefined { return this.handle; } + + /** At most once; bounded by the finalization deadline. Never rejects. */ + finalize(): Promise { + this.finalizing ??= this.run(); + return this.finalizing; + } + + private async run(): Promise { + if (this.closed || this.status !== "session_bound") return this.status; + let scope: AttachmentRecordingFileScope | undefined; + let timer: ReturnType | undefined; + const deadline = Date.now() + this.timeoutMs; + try { + const outcome = await Promise.race([ + this.capture_(s => { scope = s; }, deadline), + new Promise(resolve => { timer = setTimeout(() => resolve("incomplete"), this.timeoutMs); }), + ]); + this.status = outcome; + } catch { + // Never log the report, path or error text. + this.status = "source_unavailable"; + } finally { + this.stopped = true; + if (timer) clearTimeout(timer); + if (this.handle && this.status !== "validating") this.handle.cancel(); + if (scope) await scope.close().catch(() => undefined); + this.close(); + } + return this.status; + } + + private async capture_(keep: (scope: AttachmentRecordingFileScope) => void, deadline: number): Promise { + const session = this.session!, job = this.job!, root = this.root!; + const denial = nativeDenial(session, job, this.env); + if (denial) return denial; + // The manager policy is checked again FIRST at session end, before the SDK report is built and + // before any stat or open of the session directory: a policy tightened mid-session wins. + const ticket = this.manager.preflight(this.correlation, { mimeType: "audio/ogg", direction: "mixed", maxBytes: 1, + operation: "import_existing", sourceKind: "OwnedRecordingFile", purpose: "conversation_recording", capture: this.capture }); + ticket.release(); + if (!ticket.admitted) return ticket.reason; + // The report path is NOT authority. Only the job's primary session, its SDK recorder (closed) + // and the SDK's fixed filename under the bound job's session directory are eligible. + if (job._primaryAgentSession !== session || job.sessionDirectory !== root) return "source_unavailable"; + const recorder = session._recorderIO; + const expected = path.join(root, RECORDING_FILE); + if (!recorder || recorder.recording !== false || recorder.outputPath !== expected) return "source_unavailable"; + const report = job.makeSessionReport!(session); + if (report?.audioRecordingPath !== expected) return "source_unavailable"; + if (this.stopped) return "incomplete"; + const info = await lstat(expected); + if (this.stopped) return "incomplete"; + if (!info.isFile() || info.nlink !== 1 || (typeof process.getuid === "function" && info.uid !== process.getuid())) return "source_unavailable"; + const scope = await createRecordingFileScope(root, { enabled: true }); + if (this.stopped) { await scope.close(); return "incomplete"; } + keep(scope); + const identity = await scope.identify(RECORDING_FILE); + if (this.stopped) return "incomplete"; + const handle = this.manager.attachOwnedRecording(this.correlation, scope.source(RECORDING_FILE, identity), { + mimeType: "audio/ogg", direction: "mixed", purpose: "conversation_recording", capture: this.capture, + }); + this.handle = handle; + if (handle.reason !== "queued") return handle.reason; + const result = await handle.wait(Math.max(1, deadline - Date.now())); + return result.status; + } + + /** Detach. Restores the SDK hook only if nobody replaced this binding's wrapper since. */ + close(): void { + if (this.closed) return; + this.closed = true; + const job = this.job; + if (job && this.wrapper && job._onSessionEnd === this.wrapper) { + if (this.hadOwnHook) job._onSessionEnd = this.ownHook; + else delete job._onSessionEnd; + } + if (job && (job as Record)[BINDING] === this) delete (job as Record)[BINDING]; + this.session = undefined; this.job = undefined; this.root = undefined; this.wrapper = undefined; this.ownHook = undefined; + } +} + +/** + * Bind AFTER `await session.start(...)`. The session span is snapshotted now, before the SDK ends it. + * At session end the SDK's own `_onSessionEnd` is chained, not replaced: it runs after + * `AgentSession.close()` and your `onSessionEnd`, and before LiveKit's report upload. If your + * `onSessionEnd` deletes or replaces `audio.ogg`, this binding reports `source_unavailable`. + * + * Give the worker a shutdown grace longer than this deadline plus your own callbacks. Hard process + * termination loses the upload. + */ +export function bindLiveKitSessionRecording(session: unknown, jobContext: unknown, options: LiveKitSessionRecordingOptions): LiveKitSessionRecording { + if (!options || typeof options.ownedRecordingFiles !== "boolean") throw new TypeError("ownedRecordingFiles must be an explicit boolean"); + const timeoutMs = options.finalizationTimeoutMs ?? MAX_TIMEOUT_MS; + if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > MAX_TIMEOUT_MS) + throw new RangeError("finalizationTimeoutMs must be within (0, 30000]"); + if (options.capture?.newRecording) throw new TypeError("a session upload never creates a recording"); + const job = jobContext as JobLike | undefined; + const previous = job && typeof job === "object" ? (job as Record)[BINDING] : undefined; + const binding = new LiveKitSessionRecording(options.manager, timeoutMs, options.capture, process.env); + if (previous instanceof LiveKitSessionRecording) { + if (previous.same(options.manager, session, timeoutMs, options.capture) && options.ownedRecordingFiles) return previous; + binding.status = "conflict"; + return binding; + } + const fail = (status: LiveKitSessionRecordingStatus) => { binding.status = status; binding.close(); return binding; }; + if (options.ownedRecordingFiles !== true) return fail("disabled"); + try { + const agents = options.agents; + if (!agents || typeof agents.version !== "string" || !LIVEKIT_AGENTS_SUPPORTED_VERSIONS.includes(agents.version)) return fail("unsupported_version"); + const JobContext = agents.JobContext, AgentSession = agents.voice?.AgentSession; + if (!isFn(JobContext) || !isFn(AgentSession) || !(jobContext instanceof JobContext) || !(session instanceof AgentSession)) return fail("unsupported_version"); + const s = session as SessionLike; + if (!isFn(job!._onSessionEnd) || !isFn(job!.makeSessionReport)) return fail("unsupported_version"); + // Never fall back to an ambient span from an unrelated late callback. + const span = s.rootSpanContext ? trace.getSpan(s.rootSpanContext) : undefined; + if (!span || !span.isRecording()) return fail("correlation_unavailable"); + const correlation = options.manager.correlation(span); + // Public admission only; the file is neither stat'ed nor opened while bound. + const ticket = options.manager.preflight(correlation, { mimeType: "audio/ogg", direction: "mixed", maxBytes: 1, + operation: "import_existing", sourceKind: "OwnedRecordingFile", purpose: "conversation_recording", capture: options.capture }); + ticket.release(); + if (!ticket.admitted) return fail(ticket.reason); + if (job!._primaryAgentSession !== session) return fail("source_unavailable"); + const denial = nativeDenial(s, job!, process.env); + if (denial) return fail(denial); + const root = job!.sessionDirectory; + if (typeof root !== "string" || !path.isAbsolute(root)) return fail("source_unavailable"); + binding.attach(s, job!, correlation, root); + } catch { + return fail("source_unavailable"); + } + return binding; +} diff --git a/typescript/packages/traceai_livekit/test/agents-fixture.cjs b/typescript/packages/traceai_livekit/test/agents-fixture.cjs new file mode 100644 index 000000000..f39116f90 --- /dev/null +++ b/typescript/packages/traceai_livekit/test/agents-fixture.cjs @@ -0,0 +1,144 @@ +/* Loopback attachment control plane plus real LiveKit Agents (Node) objects. Synthetic data only. */ +const http = require('node:http'); +const fs = require('node:fs'); +const path = require('node:path'); +const { once } = require('node:events'); +const { randomUUID } = require('node:crypto'); +const { AttachmentManager, createAttachmentHttpTransport } = require('@traceai/fi-core/attachments'); + +const ids = [1, 2, 3].map(n => `10000000-0000-4000-8000-00000000000${n}`); +// Sessions of at least 1 h, so the core's near-expiry renewal never adds a bootstrap here. +const expiry = () => new Date(Date.now() + 3600000).toISOString(); +const SLICE = 256 * 1024; + +async function readBody(body) { + if (body === undefined || typeof body === 'string' || body instanceof Uint8Array) return body; + if (typeof Blob !== 'undefined' && body instanceof Blob) return Buffer.from(await body.arrayBuffer()); + // Ranged owned-file source: check before every slice and after the last, like the Node client. + const out = Buffer.alloc(body.size); + for (let offset = 0; offset < body.size; offset += SLICE) { + await body.check(); + const chunk = await body.read(offset, Math.min(SLICE, body.size - offset)); + Buffer.from(chunk).copy(out, offset); + } + await body.check(); + return out; +} + +async function attachmentFixture(config = {}, serverPolicy = {}) { + const calls = [], uploaded = [], requests = [], uploads = new Map(); + let hold; const held = []; + const policy = { version: 'v1', enabled: true, allowed_sources: ['Bytes', 'Blob', 'OwnedRecordingFile'], allowed_modalities: ['audio'], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, + record_audio: false, record_video: false, ...serverPolicy }; + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const chunk of req) chunks.push(chunk); + const bytes = Buffer.concat(chunks); + calls.push({ method: req.method, path: req.url }); + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('/attachment-sessions')) { + const body = JSON.parse(bytes); + res.end(JSON.stringify({ schema_version: 1, scope: { organization_id: ids[0], workspace_id: ids[1], project_id: ids[2] }, + capture_session_id: body.capture_session_id, policy, limits: { max_upload_bytes: 8388608, segment_bytes: 8388608, + segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, + active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, + initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, + playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }, + expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); + } else if (req.url.endsWith('/uploads')) { + const body = JSON.parse(bytes); requests.push(body); + const upload = { upload_id: randomUUID(), attachment_id: body.attachment_id, occurrence_id: body.occurrence_id, + state: 'pending_upload', reserved_bytes: body.declared_bytes, expires_at: expiry(), + transfer_mode: 'authenticated_stream', write_capability: 'synthetic-write-capability' }; + uploads.set(upload.upload_id, upload); res.statusCode = 201; res.end(JSON.stringify(upload)); + } else { + const id = req.url.split('/').at(-2); + if (!uploads.has(id) || req.headers.authorization !== 'Bearer synthetic-write-capability') { res.statusCode = 403; res.end('{}'); return; } + if (req.url.endsWith('/content')) { + if (hold) { const gate = hold; hold = undefined; held.push(id); await gate; } + uploaded.push(bytes); res.end(JSON.stringify({ upload_id: id, state: 'pending_upload', reason: 'queued' })); + } else { + res.statusCode = 202; res.end(JSON.stringify({ upload_id: id, state: 'validating', reason: 'binding_pending' })); + } + } + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const client = async req => { + const body = await readBody(req.body); + return new Promise((resolve, reject) => { + const r = http.request(`http://127.0.0.1:${server.address().port}${new URL(req.url).pathname}`, + { method: req.method, headers: req.headers, signal: req.signal }, response => { + const chunks = []; response.on('data', c => chunks.push(c)); + response.on('end', () => resolve({ status: response.statusCode, body: Buffer.concat(chunks) })); + }); + r.on('error', reject); r.end(body); + }); + }; + const manager = new AttachmentManager({ enabled: true, projectName: 'synthetic-voice', projectType: 'observe', + transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic' }, client }), ...config }); + await manager.ready(); + return { manager, calls, uploaded, requests, held, + /** Hold the next content PUT until the returned release() is called. */ + holdNext() { let release; hold = new Promise(r => { release = r; }); return () => release(); }, + async close() { await manager.shutdown(1000); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } }; +} + +/** Real `@livekit/agents` objects from an installed copy (LIVEKIT_AGENTS_DIR). */ +function loadAgents(dir) { + const main = require.resolve('@livekit/agents', { paths: [dir] }); + const agents = require(main); + try { agents.initializeLogger({ pretty: false, level: 'error' }); } catch { /* already initialized */ } + // RecorderIO is internal to the SDK; load the real class from the installed copy's dist. + const dist = main.slice(0, main.lastIndexOf(`${path.sep}dist${path.sep}`) + 6); + const { RecorderIO } = require(path.join(dist, 'voice', 'recorder_io', 'index.cjs')); + const rtc = require(require.resolve('@livekit/rtc-node', { paths: [path.dirname(main)] })); + return { agents, RecorderIO, AudioFrame: rtc.AudioFrame, rtc }; +} + +let jobs = 0; +/** + * A real JobContext and AgentSession in the state `AgentSession.start({ record: true })` leaves them: + * primary session, audio recording option on, session span context, and the SDK's own RecorderIO + * writing `${ctx.sessionDirectory}/audio.ogg`. + */ +async function startedSession(sdk, tracer, { redaction = false, audio = true, record = true } = {}) { + const { agents, RecorderIO } = sdk; + const { EventEmitter } = require('node:events'); + const room = new EventEmitter(); room.isConnected = false; + const jobId = `AJ_synthetic_${process.pid}_${++jobs}`; + const info = { job: { id: jobId, room: { name: 'synthetic-room', sid: 'RM_synthetic' }, enableRecording: false, enableRedaction: redaction, agentName: '' }, + url: 'ws://127.0.0.1:9', acceptArguments: {}, token: 'synthetic', workerId: 'synthetic' }; + const ctx = new agents.JobContext({ userData: {} }, info, room, () => {}, () => {}, {}); + const session = new agents.voice.AgentSession({}); + session.sessionOptions.recordingOptions = { ...session.sessionOptions.recordingOptions, audio, redaction: false }; + ctx._primaryAgentSession = session; + const span = tracer.startSpan('agent_session'); + const { trace, ROOT_CONTEXT } = require('@opentelemetry/api'); + session.rootSpanContext = trace.setSpan(ROOT_CONTEXT, span); + let recorder; + if (record) { + class In extends agents.voice.AudioInput {} + class Out extends agents.voice.AudioOutput { + constructor() { super(48000); } + flush() { super.flush(); setTimeout(() => this.onPlaybackFinished({ playbackPosition: 0.5, interrupted: false }), 20); } + clearBuffer() {} + } + recorder = new RecorderIO({ agentSession: session }); + recorder.recordInput(new In()); const out = recorder.recordOutput(new Out()); + await recorder.start(path.join(ctx.sessionDirectory, 'audio.ogg')); + session._recorderIO = recorder; + const n = 24000, data = new Int16Array(n); + for (let i = 0; i < n; i++) data[i] = Math.round(8000 * Math.sin(2 * Math.PI * 1000 * i / 48000)); + await out.captureFrame(new sdk.AudioFrame(data, 48000, 1, n)); out.flush(); + await new Promise(resolve => setTimeout(resolve, 700)); + } + return { ctx, session, span, recorder, file: path.join(ctx.sessionDirectory, 'audio.ogg'), + // Runs even when an assertion failed, so a still-running recorder cannot keep the process alive. + async cleanup() { + try { if (recorder?.recording) await recorder.close(); } catch { /* best effort */ } + try { span.end(); } catch { /* ended */ } + fs.rmSync(ctx.sessionDirectory, { recursive: true, force: true }); + } }; +} + +module.exports = { attachmentFixture, loadAgents, startedSession }; diff --git a/typescript/packages/traceai_livekit/test/agents-recording.cjs b/typescript/packages/traceai_livekit/test/agents-recording.cjs new file mode 100644 index 000000000..5954887ac --- /dev/null +++ b/typescript/packages/traceai_livekit/test/agents-recording.cjs @@ -0,0 +1,303 @@ +/* + * V04: the session-recording binding against a REAL installed `@livekit/agents` (LIVEKIT_AGENTS_DIR): + * real JobContext._onSessionEnd and makeSessionReport, real AgentSession, and the SDK's own RecorderIO + * writing a real Ogg/Opus file. The control plane is a loopback HTTP double. Synthetic data only. + */ +const { test, after } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); +const { BasicTracerProvider } = require('@opentelemetry/sdk-trace-base'); +const { attachmentFixture, loadAgents, startedSession } = require('./agents-fixture.cjs'); + +const dir = process.env.LIVEKIT_AGENTS_DIR; +if (!dir) throw new Error('LIVEKIT_AGENTS_DIR must point at an install of @livekit/agents'); +const pkg = process.env.LIVEKIT_INSTRUMENTATION_DIR || path.join(__dirname, '..'); +const { bindLiveKitSessionRecording, LIVEKIT_AGENTS_SUPPORTED_VERSIONS, LiveKitSessionRecording } = + require(path.join(pkg, 'dist', 'src', 'agents.js')); +const sdk = loadAgents(dir); +const tracer = new BasicTracerProvider().getTracer('synthetic'); +// The RTC runtime the SDK loads keeps native threads alive until disposed. +after(async () => { await sdk.rtc.dispose?.(); }); +const ffprobe = (() => { try { execFileSync('ffprobe', ['-version'], { stdio: 'ignore' }); return true; } catch { return false; } })(); + +/** What the SDK's finalizeSession does: AgentSession.close() closes the recorder, then ctx._onSessionEnd(). */ +async function endSession(s, { closeRecorder = true } = {}) { + if (closeRecorder && s.recorder) await s.recorder.close(); + s.span.end(); + s.session._recordedEvents = ['synthetic-event']; + await s.ctx._onSessionEnd(); +} + +test(`real @livekit/agents ${sdk.agents.version} is a qualified version`, () => { + assert.ok(LIVEKIT_AGENTS_SUPPORTED_VERSIONS.includes(sdk.agents.version)); + assert.equal(typeof sdk.agents.JobContext.prototype._onSessionEnd, 'function'); + assert.equal(typeof sdk.agents.JobContext.prototype.makeSessionReport, 'function'); +}); + +test('uploads the SDK-written audio.ogg once, byte-identical, after the recorder closes; SDK hook still runs', async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + assert.equal(binding.status, 'session_bound'); + assert.equal(a.requests.length, 0, 'binding opens nothing'); + await endSession(s); + assert.equal(binding.status, 'validating'); + const file = fs.readFileSync(s.file); + assert.equal(a.uploaded.length, 1); assert.ok(a.uploaded[0].equals(file)); + const request = a.requests[0]; + assert.equal(request.mime_type, 'audio/ogg'); assert.equal(request.direction, 'mixed'); + assert.equal(request.source_kind, 'OwnedRecordingFile'); assert.equal(request.purpose, 'conversation_recording'); + assert.equal(request.declared_bytes, file.length); + assert.equal(request.association.parent.span_id, s.span.spanContext().spanId); + assert.equal(request.association.parent.trace_id.replaceAll('-', ''), s.span.spanContext().traceId); + assert.equal('channel' in request || 'role' in request || 'recording_id' in request, false); + assert.deepEqual(s.session._recordedEvents, [], 'the SDK _onSessionEnd ran after the binding'); + assert.equal(Object.prototype.hasOwnProperty.call(s.ctx, '_onSessionEnd'), false, 'hook restored'); + assert.ok(fs.existsSync(s.file), 'the binding never deletes the recording'); + if (ffprobe) { + const out = path.join(s.ctx.sessionDirectory, 'uploaded-copy.ogg'); fs.writeFileSync(out, a.uploaded[0]); + const probe = execFileSync('ffprobe', ['-v', 'error', '-show_entries', 'stream=codec_name,channels', '-of', 'csv=p=0', out]).toString().trim(); + assert.match(probe, /^opus,[12]$/); + } +}); + +test('a recorder still recording at finalization is source_unavailable; the SDK then closes it as before', async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + await endSession(s, { closeRecorder: false }); + assert.equal(binding.status, 'source_unavailable'); assert.equal(a.requests.length, 0); + assert.equal(s.recorder.recording, false, 'SDK _onSessionEnd closed the recorder'); +}); + +for (const [name, options, expected] of [ + ['ownedRecordingFiles false', { ownedRecordingFiles: false }, 'disabled'], + ['unsupported agents version', { agents: { ...sdk.agents, version: '1.7.0' } }, 'unsupported_version'], + ['foreign session object', { foreignSession: true }, 'unsupported_version'], +]) test(`${name} binds nothing (${expected})`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer, { record: false }); t.after(() => s.cleanup()); + const session = options.foreignSession ? {} : s.session; + const binding = bindLiveKitSessionRecording(session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true, ...options }); + assert.equal(binding.status, expected); + assert.equal(Object.prototype.hasOwnProperty.call(s.ctx, '_onSessionEnd'), false); +}); + +for (const [name, setup, expected, config, policy] of [ + ['attachments disabled', {}, 'disabled', { enabled: false }], + ['environment kill switch', {}, 'disabled', { environment: () => ({ FI_ATTACHMENTS_ENABLED: 'false' }) }], + ['core PII redaction', {}, 'redaction_unavailable', { traceConfig: { piiRedaction: true } }], + ['core hidden input', {}, 'policy_denied', { traceConfig: { hideInputs: true } }], + ['core hidden output', {}, 'policy_denied', { traceConfig: { hideOutputs: true } }], + ['server requires media redaction', {}, 'redaction_unavailable', {}, { require_media_redaction: true }], + ['server disallows owned files', {}, 'policy_denied', {}, { allowed_sources: ['Bytes'] }], + ['LiveKit project redaction', { redaction: true }, 'redaction_unavailable'], + ['session audio recording off', { audio: false }, 'policy_denied'], +]) test(`${name}: denied at bind (${expected}), no stat, no hook, no traffic`, async t => { + const a = await attachmentFixture(config, policy); t.after(() => a.close()); + const s = await startedSession(sdk, tracer, setup); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + assert.equal(binding.status, expected); + assert.equal(Object.prototype.hasOwnProperty.call(s.ctx, '_onSessionEnd'), false); + await endSession(s); + await a.manager.flush(); + assert.equal(a.requests.length, 0); assert.equal(a.uploaded.length, 0); +}); + +test('LIVEKIT_TELEMETRY_ALLOW_PII=0 denies the recording (deny wins)', async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const saved = process.env.LIVEKIT_TELEMETRY_ALLOW_PII; process.env.LIVEKIT_TELEMETRY_ALLOW_PII = '0'; + t.after(() => { if (saved === undefined) delete process.env.LIVEKIT_TELEMETRY_ALLOW_PII; else process.env.LIVEKIT_TELEMETRY_ALLOW_PII = saved; }); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + assert.equal(binding.status, 'policy_denied'); + await endSession(s); assert.equal(a.requests.length, 0); +}); + +test('policy tightened after bind: finalization re-checks before any stat or open', async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + s.ctx._redactionEnabled = true; // e.g. the project mandated redaction mid-session + await endSession(s); + assert.equal(binding.status, 'redaction_unavailable'); assert.equal(a.requests.length, 0); +}); + +/** + * Records every fs call on the session directory made while the binding is still finalizing + * (status `session_bound`). The binding assigns its outcome before the SDK's own `_onSessionEnd` + * runs, so the SDK's later work is excluded. Covers the binding's lstat and the core scope's + * realpath/open (both go through the shared `node:fs` promises object). + */ +function watchSessionFs(t, s, binding) { + const fsp = require('node:fs/promises'); + const seen = []; + const inDir = p => { try { return String(p).startsWith(s.ctx.sessionDirectory); } catch { return false; } }; + const wrap = (target, names) => { + for (const name of names) { + const original = target[name]; + if (typeof original !== 'function') continue; + target[name] = function (p, ...rest) { + if (binding.status === 'session_bound' && inDir(p)) seen.push(`${name}:${path.relative(s.ctx.sessionDirectory, String(p)) || '.'}`); + return original.call(this, p, ...rest); + }; + t.after(() => { target[name] = original; }); + } + }; + wrap(fsp, ['lstat', 'stat', 'open', 'realpath', 'readFile', 'readdir', 'opendir', 'access']); + wrap(fs, ['lstatSync', 'statSync', 'openSync', 'realpathSync', 'readFileSync', 'readdirSync', 'accessSync', 'createReadStream']); + return seen; +} + +test('control: an admitted session end does stat and open audio.ogg (the fs watch sees the binding)', async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + const seen = watchSessionFs(t, s, binding); + await endSession(s); + assert.equal(binding.status, 'validating'); + assert.ok(seen.includes('lstat:audio.ogg'), `seen: ${seen.join(',')}`); + assert.ok(seen.includes('open:audio.ogg'), `seen: ${seen.join(',')}`); + assert.equal(a.uploaded.length, 1); +}); + +for (const [name, tighten, expected] of [ + ['manager PII redaction enabled', a => a.manager.updatePolicy({ traceConfig: { piiRedaction: true } }), 'redaction_unavailable'], + ['manager output hidden', a => a.manager.updatePolicy({ traceConfig: { hideOutputs: true } }), 'policy_denied'], + ['manager disabled', a => a.manager.updatePolicy({ enabled: false }), 'disabled'], + ['environment kill switch flipped', (a, env) => { env.FI_ATTACHMENTS_ENABLED = 'false'; }, 'disabled'], + ['manager invalidated (revoked)', a => a.manager.invalidate('policy_denied'), 'policy_denied'], +]) { + for (const removeFile of [false, true]) { + test(`session-end deny before any stat or open: ${name}${removeFile ? ', recording removed' : ''} -> ${expected}`, async t => { + const env = {}; + const a = await attachmentFixture({ environment: () => env }); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + assert.equal(binding.status, 'session_bound', 'admitted at bind'); + await s.recorder.close(); + tighten(a, env); + // A removed file would make a stat-first order report source_unavailable instead of the policy reason. + if (removeFile) fs.rmSync(s.file); + // Watch only the session end; the test's own recorder close and rmSync (which lstats) stay outside. + const seen = watchSessionFs(t, s, binding); + await endSession(s, { closeRecorder: false }); + assert.equal(binding.status, expected); + assert.deepEqual(seen, [], 'no stat, open or realpath of the session directory before the deny'); + await a.manager.flush().catch(() => undefined); + assert.equal(a.requests.length, 0); assert.equal(a.uploaded.length, 0); + assert.deepEqual(s.session._recordedEvents, [], 'the SDK _onSessionEnd still ran'); + }); + } +} + +test('a recording file owned by another user is rejected (source_unavailable), nothing uploaded', async t => { + // Only root can hand a file to another uid; CI and the qualification VM run this suite as root. + if (typeof process.getuid !== 'function' || process.getuid() !== 0) { + t.diagnostic('SKIPPED: needs uid 0 to chown the recording to another user'); + t.skip('needs uid 0'); return; + } + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + assert.equal(binding.status, 'session_bound'); + await s.recorder.close(); + fs.chownSync(s.file, 65534, 65534); + const info = fs.lstatSync(s.file); + assert.ok(info.isFile() && info.nlink === 1 && info.uid === 65534, 'only the owner differs'); + await endSession(s, { closeRecorder: false }); + assert.equal(binding.status, 'source_unavailable'); + await a.manager.flush(); assert.equal(a.requests.length, 0); assert.equal(a.uploaded.length, 0); + assert.ok(fs.existsSync(s.file), 'the binding never deletes the recording'); +}); + +for (const [name, mutate] of [ + ['report absent (no recorder)', s => { s.session._recorderIO = undefined; }], + ['recording replaced by a symlink', s => { const other = path.join(s.ctx.sessionDirectory, 'other.ogg'); fs.renameSync(s.file, other); fs.symlinkSync(other, s.file); }], + ['recording hard-linked elsewhere', s => { fs.linkSync(s.file, path.join(s.ctx.sessionDirectory, 'alias.ogg')); }], + ['recording deleted by the application', s => { fs.rmSync(s.file); }], + ['recorder path outside the session directory', s => { Object.defineProperty(s.session._recorderIO, 'outputPath', { value: '/tmp/elsewhere/audio.ogg' }); }], + ['another primary session', s => { s.ctx._primaryAgentSession = {}; }], +]) test(`${name}: source_unavailable, nothing uploaded`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + assert.equal(binding.status, 'session_bound'); + await s.recorder.close(); + mutate(s); + s.span.end(); + try { await s.ctx._onSessionEnd(); } catch { /* the SDK's own hook may reject on a broken session */ } + assert.equal(binding.status, 'source_unavailable'); + await a.manager.flush(); assert.equal(a.uploaded.length, 0); +}); + +test('T-LS-05: the file swapped after identify and before transfer fails closed', async t => { + const a = await attachmentFixture({ workers: 1 }); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + // Occupy the single worker so the owned-file job waits in the queue after identify. + const release = a.holdNext(); + const blocker = a.manager.attach(s.span.spanContext(), { kind: 'Bytes', data: new Uint8Array([1, 2, 3]) }, { mimeType: 'audio/wav', direction: 'output' }); + assert.equal(blocker.admission, 'queued'); + await s.recorder.close(); s.span.end(); + const ending = endSession(s, { closeRecorder: false }); + while (!binding.recording) await new Promise(r => setTimeout(r, 5)); + const original = fs.readFileSync(s.file); + const swap = path.join(s.ctx.sessionDirectory, 'swap.ogg'); + fs.writeFileSync(swap, Buffer.alloc(original.length, 7)); fs.renameSync(swap, s.file); + release(); + await ending; + assert.equal(binding.status, 'source_unavailable'); + assert.equal(a.uploaded.filter(b => b.length === original.length).length, 0, 'swapped bytes never uploaded'); +}); + +test('deadline: a stalled transfer ends incomplete within the bound and the SDK hook still runs', async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true, finalizationTimeoutMs: 300 }); + const release = a.holdNext(); t.after(release); + const started = Date.now(); + await endSession(s); + const elapsed = Date.now() - started; + assert.equal(binding.status, 'incomplete'); + assert.ok(elapsed < 3000, `finalization took ${elapsed} ms`); + assert.deepEqual(s.session._recordedEvents, []); +}); + +test('bind is idempotent for identical options and reports conflict otherwise; close() restores the hook', async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const options = { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }; + const first = bindLiveKitSessionRecording(s.session, s.ctx, options); + assert.equal(bindLiveKitSessionRecording(s.session, s.ctx, options), first); + assert.equal(bindLiveKitSessionRecording(s.session, s.ctx, { ...options, finalizationTimeoutMs: 1000 }).status, 'conflict'); + assert.ok(first instanceof LiveKitSessionRecording); + first.close(); + assert.equal(Object.prototype.hasOwnProperty.call(s.ctx, '_onSessionEnd'), false); + await endSession(s); assert.equal(a.requests.length, 0); +}); + +test('invalid options throw synchronously and never touch the session', () => { + assert.throws(() => bindLiveKitSessionRecording({}, {}, { manager: {}, agents: sdk.agents }), TypeError); + assert.throws(() => bindLiveKitSessionRecording({}, {}, { manager: {}, agents: sdk.agents, ownedRecordingFiles: true, finalizationTimeoutMs: 30001 }), RangeError); + assert.throws(() => bindLiveKitSessionRecording({}, {}, { manager: {}, agents: sdk.agents, ownedRecordingFiles: true, capture: { newRecording: true } }), TypeError); +}); + +test('a recording whose owner is not the worker uid is rejected (any uid; worker uid reported differently)', async t => { + // Runs without root too: the file keeps its real owner and the worker reports another uid. + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + assert.equal(binding.status, 'session_bound'); + await s.recorder.close(); s.span.end(); + const real = process.getuid; + process.getuid = () => real.call(process) + 1; + let status; + try { status = await binding.finalize(); } finally { process.getuid = real; } + assert.equal(status, 'source_unavailable'); + await a.manager.flush(); assert.equal(a.requests.length, 0); assert.equal(a.uploaded.length, 0); + assert.ok(fs.existsSync(s.file), 'the binding never deletes the recording'); + await s.ctx._onSessionEnd(); +}); diff --git a/typescript/packages/traceai_pipecat/README.md b/typescript/packages/traceai_pipecat/README.md index 814f914d9..87ee27678 100644 --- a/typescript/packages/traceai_pipecat/README.md +++ b/typescript/packages/traceai_pipecat/README.md @@ -45,6 +45,16 @@ If you already have an OpenTelemetry provider, reuse it instead of creating anot In Future AGI, open your configured project and look for `Pipecat Client Event: connected`, `botReady` and `disconnected`. `transportStateChanged`, `error` and `messageError` also produce client event spans. Each span describes a notification, not a server operation's duration. An optional `parentContext` associates events with an application-owned span without modifying transport headers or bot messages. +## Reference an existing server recording + +Pass `serverRecording` to `observePipecatClient` when your authenticated application server already provides the current conversation's recording occurrence. The function is reevaluated for each event; return `undefined` when the server session ends or changes, including during reconnect. + +Its return type, `PipecatServerRecording`, contains `schemaVersion: 1`, `parent: { traceId, spanId }`, `occurrenceId` and `conversationId`. Use the **original server trace/span** and UUID occurrence/conversation identifiers. Do not populate it from bot messages, participant names, media URLs or provider tokens. + +Valid identities add an OpenTelemetry span link to the server parent with the occurrence and conversation identifiers. The event's `pipecat.server_recording.status` is `reference_only`; missing, malformed or unsupported identities report `source_unavailable`. The observer does not copy the occurrence onto the client span, grant media access or resolve a playback URL. Server-side authorization is still required to view the original occurrence. + +This is correlation only: it does not record the microphone, camera or browser RTC stream, and requires no Future AGI credential in the browser. + ## Existing Node instrumentation `PipecatInstrumentation`, `manuallyInstrument()` and `isPatched()` remain available from the root entry point. Register the instrumentation with your existing Node tracer provider **before requiring** `@pipecat-ai/client-js`. For already loaded modules or ESM, pass the actual imported module to `manuallyInstrument()`. diff --git a/typescript/packages/traceai_pipecat/src/client.ts b/typescript/packages/traceai_pipecat/src/client.ts index 1509610d1..ce91430f5 100644 --- a/typescript/packages/traceai_pipecat/src/client.ts +++ b/typescript/packages/traceai_pipecat/src/client.ts @@ -8,6 +8,8 @@ import { Tracer, } from "@opentelemetry/api"; import { VERSION } from "./version"; +import { pipecatRecordingLink, PipecatServerRecording } from "./recording"; +export type { PipecatServerRecording } from "./recording"; const EVENTS = [ "connected", "disconnected", "botReady", "transportStateChanged", @@ -26,6 +28,11 @@ export interface PipecatClientTracingOptions { tracer?: Tracer; /** Optional application-owned parent; never sent to the bot or transport. */ parentContext?: Context; + /** Resolve only the current authenticated server session; reevaluated per event. + * Returning undefined on disconnect/reconnect prevents stale session linkage. + * Identity is a reference, never authorization to upload, relink or play media. + */ + serverRecording?: () => PipecatServerRecording | undefined; } const observers = new WeakMap void>(); @@ -47,13 +54,19 @@ export function observePipecatClient( const listener = (state: unknown) => { // Telemetry must not interrupt application callbacks. Do not log their data. try { + let recording; + try { recording = pipecatRecordingLink(options.serverRecording?.()); } catch { /* unavailable */ } const span = tracer.startSpan(`Pipecat Client Event: ${event}`, { kind: SpanKind.CLIENT, attributes: { "fi.span.kind": "CHAIN", "pipecat.telemetry_source": "client", "pipecat.event": event, + ...(options.serverRecording ? { + "pipecat.server_recording.status": recording ? "reference_only" : "source_unavailable", + } : {}), }, + links: recording ? [recording] : [], }, options.parentContext ?? context.active()); // Only inspect the fixed SDK error state; never serialize or coerce event data. const isError = event === "error" || event === "messageError" || diff --git a/typescript/packages/traceai_pipecat/src/index.ts b/typescript/packages/traceai_pipecat/src/index.ts index 6ec737d6e..13ea23417 100644 --- a/typescript/packages/traceai_pipecat/src/index.ts +++ b/typescript/packages/traceai_pipecat/src/index.ts @@ -8,3 +8,4 @@ export { PipecatInstrumentation, PipecatInstrumentationConfig } from "./instrume export { isPatched } from "./instrumentation"; export { VERSION } from "./version"; export { observePipecatClient, PipecatEventSource, PipecatClientTracingOptions } from "./client"; +export type { PipecatServerRecording } from "./recording"; diff --git a/typescript/packages/traceai_pipecat/src/recording.ts b/typescript/packages/traceai_pipecat/src/recording.ts new file mode 100644 index 000000000..00f906616 --- /dev/null +++ b/typescript/packages/traceai_pipecat/src/recording.ts @@ -0,0 +1,45 @@ +import type { Link } from "@opentelemetry/api"; + +/** + * An application-server-authorized identity, NOT a browser upload capability. + * Obtain it from your authenticated server, never from an RTC/bot event payload. + * The occurrence remains on its original server parent; a link grants no access. + */ +export interface PipecatServerRecording { + readonly schemaVersion: 1; + readonly parent: { readonly traceId: string; readonly spanId: string }; + readonly occurrenceId: string; + /** Opaque session UUID; no participant names or provider credentials. */ + readonly conversationId: string; +} + +const uuid = (value: unknown): value is string => typeof value === "string" && + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(value) && + value !== "00000000-0000-0000-0000-000000000000"; + +/** Content-free correlation only. It never calls an upload, relink or resolve API. */ +export function pipecatRecordingLink(value: PipecatServerRecording | undefined): Link | undefined { + try { + if (value?.schemaVersion !== 1) return; + // Preserve all trace bits; this reference-only browser leaf must not import + // the upload manager merely to normalize the two contract spellings. + const sourceTraceId = value.parent?.traceId; + const traceId = uuid(sourceTraceId) ? sourceTraceId.replace(/-/g, "") : sourceTraceId; + const spanId = value.parent?.spanId; + const occurrenceId = value.occurrenceId; + const conversationId = value.conversationId; + if (typeof traceId !== "string" || !/^[0-9a-f]{32}$/.test(traceId) || /^0+$/.test(traceId) || + typeof spanId !== "string" || !/^[0-9a-f]{16}$/.test(spanId) || + /^0+$/.test(spanId) || !uuid(occurrenceId) || !uuid(conversationId)) return; + return { + context: { traceId, spanId, traceFlags: 0, isRemote: true }, + attributes: { + "pipecat.server_recording.occurrence_id": occurrenceId, + "pipecat.server_recording.conversation_id": conversationId, + }, + }; + } catch { + // Malformed or unavailable server identity must not interrupt RTC callbacks. + return; + } +} diff --git a/typescript/packages/traceai_pipecat/test/recording.test.cjs b/typescript/packages/traceai_pipecat/test/recording.test.cjs new file mode 100644 index 000000000..dd6a3ead1 --- /dev/null +++ b/typescript/packages/traceai_pipecat/test/recording.test.cjs @@ -0,0 +1,78 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } = require('@opentelemetry/sdk-trace-base'); +const { observePipecatClient } = require('../dist/src/client.js'); +const { createClient } = require('./transport.cjs'); +const identity = () => ({ schemaVersion: 1, + parent: { traceId: '01234567-89ab-cdef-0123-456789abcdef', spanId: '0123456789abcdef' }, + occurrenceId: '10000000-0000-4000-8000-000000000001', conversationId: '10000000-0000-4000-8000-000000000002' }); + +for (const name of ['pipecat-floor', '@pipecat-ai/client-js']) { + test(`${name}: server recording stays on its parent without media or credentials`, async t => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + t.after(() => provider.shutdown()); + const { client, transport } = createClient(require(name)); + let current = identity(); + Object.defineProperty(current, 'sourceUrl', { get() { throw Error('synthetic-secret-must-not-read'); } }); + const stop = observePipecatClient(client, { tracer: provider.getTracer('recording-test'), serverRecording: () => current }); + t.after(stop); + await client.connect(); await client.disconnect(); + const first = exporter.getFinishedSpans(); assert.ok(first.length > 0); + for (const s of first) { + assert.equal(s.attributes['pipecat.server_recording.status'], 'reference_only'); + assert.equal(s.links.length, 1); + assert.equal(s.links[0].context.traceId, current.parent.traceId.replaceAll('-', '')); + assert.equal(s.links[0].context.spanId, current.parent.spanId); + assert.equal(s.links[0].attributes['pipecat.server_recording.occurrence_id'], current.occurrenceId); + assert.equal(s.attributes['fi.attachment.refs'], undefined); // not a relink + } + // The app clears identity when its server session ends. Reconnect never falls + // back to a prior session or to a bot-ready payload's untrusted identifiers. + current = undefined; + await client.connect(); await client.disconnect(); + for (const s of exporter.getFinishedSpans().slice(first.length)) { + assert.equal(s.attributes['pipecat.server_recording.status'], 'source_unavailable'); + assert.equal(s.links.length, 0); + } + const sent = JSON.stringify(transport.sent); + assert.ok(!sent.includes('10000000-0000')); assert.ok(!sent.includes('01234567')); + stop(); assert.equal(client.listenerCount('connected'), 0); + }); + + test(`${name}: a throwing identity callback never interrupts the client and reports source_unavailable`, async t => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + t.after(() => provider.shutdown()); + const { client } = createClient(require(name)); + let seen = 0; client.on('connected', () => { seen++; }); + const stop = observePipecatClient(client, { tracer: provider.getTracer('throwing-test'), + serverRecording: () => { throw new Error('synthetic-private-failure'); } }); + t.after(stop); await client.connect(); await client.disconnect(); + assert.equal(seen, 1); + const spans = exporter.getFinishedSpans(); assert.ok(spans.length > 0); + for (const s of spans) { + assert.equal(s.attributes['pipecat.server_recording.status'], 'source_unavailable'); + assert.deepEqual(s.links, []); assert.ok(!JSON.stringify({ a: s.attributes, e: s.events, st: s.status }).includes('synthetic-private-failure')); + } + }); + + for (const [caseName, supplied] of [ + ['missing', undefined], ['version', { ...identity(), schemaVersion: 2 }], + ['zero trace', { ...identity(), parent: { ...identity().parent, traceId: '0'.repeat(32) } }], + ['bad span', { ...identity(), parent: { ...identity().parent, spanId: 'secret=synthetic' } }], + ['URL occurrence', { ...identity(), occurrenceId: 'https://invalid.example/?key=synthetic' }], + ['participant name', { ...identity(), conversationId: 'synthetic-private-person' }], + ]) test(`${name}: ${caseName} identity has only source_unavailable status`, async t => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + t.after(() => provider.shutdown()); + const { client } = createClient(require(name)); + const stop = observePipecatClient(client, { tracer: provider.getTracer('invalid-test'), serverRecording: () => supplied }); + t.after(stop); await client.connect(); await client.disconnect(); + for (const s of exporter.getFinishedSpans()) { + assert.equal(s.attributes['pipecat.server_recording.status'], 'source_unavailable'); + assert.deepEqual(s.links, []); assert.ok(!JSON.stringify(s.attributes).includes('synthetic-private')); + } + }); +}