From be3d5f7548e1b455545aaf7db4ce53874b9f0fa4 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 11 Nov 2026 01:35:50 -0800 Subject: [PATCH 1/8] feat(voice): restore TS Pipecat recording link and ElevenLabs speech attachments Restores the ts-voice lane's saved work (refs/recovery/att-b-voice-1111) onto the current TS attachment core: - Pipecat client events link to an authenticated server recording occurrence (reference only, no browser capture). - ElevenLabs STT Blob input and complete timestamped MP3 output attach through the shared provider manager. Co-Authored-By: Claude Opus 5.5 --- .../packages/traceai_elevenlabs/README.md | 18 ++- .../traceai_elevenlabs/src/attachments.ts | 96 ++++++++++++++++ .../packages/traceai_elevenlabs/src/index.ts | 1 + .../traceai_elevenlabs/src/instrumentation.ts | 19 +++- .../traceai_elevenlabs/src/operation.ts | 10 +- .../traceai_elevenlabs/src/responses.ts | 5 +- .../test/attachments-fixture.cjs | 71 ++++++++++++ .../test/attachments.test.cjs | 105 ++++++++++++++++++ .../traceai_elevenlabs/test/fixture.cjs | 9 +- .../traceai_elevenlabs/test/fixtures/tone.mp3 | Bin 0 -> 2551 bytes typescript/packages/traceai_pipecat/README.md | 10 ++ .../packages/traceai_pipecat/src/client.ts | 13 +++ .../packages/traceai_pipecat/src/index.ts | 1 + .../packages/traceai_pipecat/src/recording.ts | 45 ++++++++ .../traceai_pipecat/test/recording.test.cjs | 61 ++++++++++ 15 files changed, 451 insertions(+), 13 deletions(-) create mode 100644 typescript/packages/traceai_elevenlabs/src/attachments.ts create mode 100644 typescript/packages/traceai_elevenlabs/test/attachments-fixture.cjs create mode 100644 typescript/packages/traceai_elevenlabs/test/attachments.test.cjs create mode 100644 typescript/packages/traceai_elevenlabs/test/fixtures/tone.mp3 create mode 100644 typescript/packages/traceai_pipecat/src/recording.ts create mode 100644 typescript/packages/traceai_pipecat/test/recording.test.cjs diff --git a/typescript/packages/traceai_elevenlabs/README.md b/typescript/packages/traceai_elevenlabs/README.md index 56772dc9a..084c2375c 100644 --- a/typescript/packages/traceai_elevenlabs/README.md +++ b/typescript/packages/traceai_elevenlabs/README.md @@ -107,9 +107,19 @@ The optional `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` and `ELEVENLABS_BASE_URL` over | `textToSpeech.streamWithTimestamps` | You finish or stop iterating timestamp chunks | | `speechToText.convert` | Synchronous transcription completes, including multi-channel output | -Use these methods on the wrapped client as usual. `withRawResponse()`, `then`, `catch`, `finally`, byte-stream readers (including BYOB when the upstream stream supports it), iteration and piping remain available. Timestamp responses retain audio and alignment data in your application, but that data is not exported. Requests that set `webhook: true`, realtime/WebSocket APIs and other SDK namespaces pass through without integration spans. +Use these methods on the wrapped client as usual. `withRawResponse()`, `then`, `catch`, `finally`, byte-stream readers (including BYOB when the upstream stream supports it), iteration and piping remain available. Timestamp responses retain audio and alignment data in your application. By default, neither is exported. Requests that set `webhook: true`, realtime/WebSocket APIs and other SDK namespaces pass through without integration spans. -Hosted ElevenLabs Agents conversations are a separate integration. This package does not export conversation webhooks, audio attachments, usage tokens or estimated costs. +Hosted ElevenLabs Agents conversations are a separate integration. This package does not export conversation webhooks, usage tokens or estimated costs. + +## Optional binary attachments + +Binary capture is off by default. If your destination supports the attachment service, explicitly enable your provider's attachment manager and pass that **same manager** as `attachments: provider.attachments` to `ElevenLabsInstrumentation`. Await `provider.attachments.ready()` at startup to avoid missing early calls; continue to use the provider's `shutdown()` for delivery. The adapter never creates another uploader or enables recording. + +- `textToSpeech.convertWithTimestamps` can attach a complete MP3 response up to 256 KiB. The output format must be the SDK default or a supported MP3 format. Generated audio is not proof of playback. +- `speechToText.convert` can attach an existing `Blob`/`File` when you also set `attachmentInputMimeType` to its exact MIME type: `audio/wav`, `audio/mpeg`, `audio/ogg`, `audio/mp4`, `audio/webm` or `audio/flac`. Filenames are not exported; mismatched or missing types are not sniffed. +- Streams, PCM/ulaw output, local paths and remote URLs are not captured. Alignment data stays in your SDK response, not in attachment metadata. Use the stream methods normally; tracing does not drain, tee or accumulate their bytes. + +Attachments obey the manager's default-off, server, sampling and PII policies in addition to the instrumentation's directional hide settings. Without a qualified speech sanitizer, `piiRedaction` denies capture. Existing raw audio may contain embedded metadata; a policy requiring metadata removal also denies it. A `queued` result on the span means admitted for transfer, **not** validated or playable. Content is out of band, never base64 in OTLP. ## Privacy @@ -123,7 +133,7 @@ export FI_HIDE_ERROR_DETAILS=true `FI_HIDE_INPUT_MESSAGES` / `FI_HIDE_INPUT_TEXT` and their output equivalents also hide the corresponding speech text. You can pass core `traceConfig` options to the instrumentation instead. These settings affect traces, not SDK request or response values. -The package never exports raw audio, base64, alignment text, upload paths, source URLs, request headers or provider error bodies. Errors record an error status, a fixed error type, and an HTTP status when available, not exception messages or stack traces—even if `FI_HIDE_ERROR_DETAILS` is false. Your application is responsible for content it adds to parent spans and context metadata. +Trace attributes never contain raw audio, base64, alignment text, upload paths, source URLs, request headers or provider error bodies. Binary audio leaves the process only through the optional attachment manager described above. Errors record an error status, a fixed error type, and an HTTP status when available, not exception messages or stack traces—even if `FI_HIDE_ERROR_DETAILS` is false. Your application is responsible for content it adds to parent spans and context metadata. ## Stream lifecycle @@ -139,5 +149,5 @@ For module-wide registration, call `instrumentation.manuallyInstrument(sdk)` wit - **No span after receiving audio:** consume or cancel the returned stream. Receiving headers is not successful completion. Use `shutdown()` before exiting a short script. - **Spans appear separately from your application trace:** initialize an async context manager once and call the wrapped client within the active application span. -- **Text is missing:** check the `FI_HIDE_*` variables and the instrumentation's `traceConfig`. Audio and provider error text are intentionally never captured. +- **Text is missing:** check the `FI_HIDE_*` variables and the instrumentation's `traceConfig`. Binary audio needs the separate attachment opt-in; provider error text is never captured. - **Transcription spans are missing:** `webhook: true` submits an asynchronous job; this package does not trace its later completion. diff --git a/typescript/packages/traceai_elevenlabs/src/attachments.ts b/typescript/packages/traceai_elevenlabs/src/attachments.ts new file mode 100644 index 000000000..c68e83c78 --- /dev/null +++ b/typescript/packages/traceai_elevenlabs/src/attachments.ts @@ -0,0 +1,96 @@ +import type { Span } from "@opentelemetry/api"; +import type { TraceConfig } from "@traceai/fi-core"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import { record, safely } from "./operation.js"; + +export type ElevenLabsInputAudioMimeType = + | "audio/wav" + | "audio/mpeg" + | "audio/ogg" + | "audio/mp4" + | "audio/webm" + | "audio/flac"; + +// This is a bounded complete-response path, NOT a recording/stream accumulator. +const MAX_DECODED_BYTES = 256 * 1024; +const MAX_ENCODED_CHARACTERS = Math.ceil(MAX_DECODED_BYTES / 3) * 4; +const INPUT_TYPES: readonly string[] = [ + "audio/wav", "audio/mpeg", "audio/ogg", "audio/mp4", "audio/webm", "audio/flac", +]; + +/** Adapters never own an uploader or change the provider manager's lifecycle. */ +export class SpeechAttachments { + constructor( + private readonly manager: AttachmentManager, + private readonly span: Span, + private readonly policy: TraceConfig, + private readonly outputFormat: string | undefined, + ) {} + + private allowed(direction: "input" | "output"): boolean { + // Adapter-local restrictions can only tighten the shared manager's policy. + // No qualified speech sanitizer is available here. + return !this.policy.piiRedaction && (direction === "input" + ? !(this.policy.hideInputs || this.policy.hideInputMessages || this.policy.hideInputText) + : !(this.policy.hideOutputs || this.policy.hideOutputMessages || this.policy.hideOutputText)); + } + + input(request: unknown, mimeType: ElevenLabsInputAudioMimeType): void { + safely(() => { + if (!INPUT_TYPES.includes(mimeType)) return; + const result = this.manager.attach(this.span, { + kind: "Blob", + // Core checks default-off, server policy, PII, scope and sampling BEFORE + // this getter. No file/stream property is inspected on a denied capture. + get data(): Blob { + const file = record(request).file; + if (!(file instanceof Blob) || file.type !== mimeType) { + throw new Error("unsupported_audio_source"); + } + return file; + }, + }, { + mimeType, + direction: "input", + purpose: "model_input", + capture: { captureContent: this.allowed("input") }, + }); + this.span.setAttribute("elevenlabs.attachment.input", result.admission); + }); + } + + timestamped(value: unknown): void { + safely(() => { + // Raw PCM/ulaw require sample-format metadata that the core does not yet + // carry. Do not label those bytes MP3. The SDK's default is mp3_44100_128. + if (this.outputFormat !== undefined && + !/^mp3_(22050_32|24000_48|44100_(32|64|96|128|192))$/.test(this.outputFormat)) { + this.span.setAttribute("elevenlabs.attachment.output", "source_unavailable"); + return; + } + const result = this.manager.attach(this.span, { + kind: "Bytes", + get data(): Uint8Array { + const encoded = record(value).audioBase64; + if (typeof encoded !== "string" || encoded.length === 0 || + encoded.length > MAX_ENCODED_CHARACTERS) return new Uint8Array(0); + // Buffer.from tolerates malformed base64; require canonical padding and + // alphabet first, then a round-trip before retaining decoded bytes. + if (encoded.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) { + throw new Error("unsupported_audio_source"); + } + const decoded = Buffer.from(encoded, "base64"); + if (decoded.length > MAX_DECODED_BYTES) return new Uint8Array(0); + if (decoded.toString("base64") !== encoded) throw new Error("unsupported_audio_source"); + return decoded; + }, + }, { + mimeType: "audio/mpeg", + direction: "output", + purpose: "model_output", + capture: { captureContent: this.allowed("output") }, + }); + this.span.setAttribute("elevenlabs.attachment.output", result.admission); + }); + } +} diff --git a/typescript/packages/traceai_elevenlabs/src/index.ts b/typescript/packages/traceai_elevenlabs/src/index.ts index 6c95040eb..3c116683c 100644 --- a/typescript/packages/traceai_elevenlabs/src/index.ts +++ b/typescript/packages/traceai_elevenlabs/src/index.ts @@ -1,3 +1,4 @@ export { ElevenLabsInstrumentation } from "./instrumentation.js"; export type { ElevenLabsInstrumentationConfig } from "./instrumentation.js"; export { VERSION } from "./version.js"; +export type { ElevenLabsInputAudioMimeType } from "./attachments.js"; diff --git a/typescript/packages/traceai_elevenlabs/src/instrumentation.ts b/typescript/packages/traceai_elevenlabs/src/instrumentation.ts index 116dbd1ba..140d3634b 100644 --- a/typescript/packages/traceai_elevenlabs/src/instrumentation.ts +++ b/typescript/packages/traceai_elevenlabs/src/instrumentation.ts @@ -28,6 +28,8 @@ import { safely, TEXT_LIMIT, } from "./operation.js"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import { SpeechAttachments, ElevenLabsInputAudioMimeType } from "./attachments.js"; import { observeResponse } from "./responses.js"; import { VERSION } from "./version.js"; @@ -68,6 +70,10 @@ export interface ElevenLabsInstrumentationConfig { instrumentationConfig?: InstrumentationConfig; tracerProvider?: TracerProvider; traceConfig?: TraceConfigOptions; + /** The configured provider's shared manager. Omitted means no binary capture. */ + attachments?: AttachmentManager; + /** Explicit MIME for STT Blob/File inputs; no sniffing or stream reads. */ + attachmentInputMimeType?: ElevenLabsInputAudioMimeType; /** Absolute span lifetime, including request and consumption. Default 300000 ms. */ maxSpanDurationMs?: number; /** Bound active telemetry, not SDK calls. Excess calls pass through. Default 1024. */ @@ -272,9 +278,8 @@ export class ElevenLabsInstrumentation extends InstrumentationBase { if (voice) attributes["elevenlabs.voice.id"] = voice; // Unlike JSON body overrides, the SDK skips nullish query values. const query = stt ? {} : record(options.queryParams); - const format = identifier( - bodyField(query, "output_format", undefined) ?? request.outputFormat, - ); + const formatValue = bodyField(query, "output_format", undefined) ?? request.outputFormat; + const format = identifier(formatValue); if (format) attributes["elevenlabs.output_format"] = format; if ( !stt && @@ -295,12 +300,20 @@ export class ElevenLabsInstrumentation extends InstrumentationBase { { kind: SpanKind.CLIENT, attributes }, callContext, ); + const attachments = this.options.attachments + ? new SpeechAttachments(this.options.attachments, span, policy, + formatValue === undefined ? undefined : format ?? "unsupported") + : undefined; operation = new Operation( span, policy, bounded(this.options.maxSpanDurationMs, 300_000, 86_400_000), (ended) => this.active.delete(ended), + attachments, ); + if (stt && this.options.attachmentInputMimeType && + !(options.abortSignal instanceof AbortSignal && options.abortSignal.aborted)) + attachments?.input(request, this.options.attachmentInputMimeType); this.active.add(operation); operation.watchAbort(record(args[stt ? 1 : 2]).abortSignal); callContext = trace.setSpan(callContext, span); diff --git a/typescript/packages/traceai_elevenlabs/src/operation.ts b/typescript/packages/traceai_elevenlabs/src/operation.ts index 38d29ded2..d6dce13d1 100644 --- a/typescript/packages/traceai_elevenlabs/src/operation.ts +++ b/typescript/packages/traceai_elevenlabs/src/operation.ts @@ -1,6 +1,7 @@ import { diag, Span, SpanStatusCode } from "@opentelemetry/api"; import { SemanticConventions as S } from "@traceai/fi-semantic-conventions"; import { TraceConfig } from "@traceai/fi-core"; +import type { SpeechAttachments } from "./attachments.js"; let diagnosed = false; @@ -58,6 +59,7 @@ export class Operation { private readonly policy: TraceConfig, timeoutMs: number, private readonly remove: (operation: Operation) => void, + private attachments?: SpeechAttachments, ) { this.span = span; this.timer = setTimeout(() => this.end("incomplete"), timeoutMs); @@ -74,9 +76,14 @@ export class Operation { } } + timestampedAttachment(value: unknown): void { + if (this.span) this.attachments?.timestamped(value); + } + audio(value: unknown, timestamps = false): void { safely(() => { - if (!this.span) return; + if (!this.span || this.policy.hideOutputs || this.policy.hideOutputMessages || + this.policy.hideOutputText || this.policy.piiRedaction) return; const size = timestamps ? typeof record(value).audioBase64 === "string" ? (record(value).audioBase64 as string).length @@ -152,6 +159,7 @@ export class Operation { const span = this.span; if (!span) return; this.span = undefined; + this.attachments = undefined; if (this.timer) clearTimeout(this.timer); this.timer = undefined; safely(() => this.detachAbort?.()); diff --git a/typescript/packages/traceai_elevenlabs/src/responses.ts b/typescript/packages/traceai_elevenlabs/src/responses.ts index 4af9cf43d..dc4ac662b 100644 --- a/typescript/packages/traceai_elevenlabs/src/responses.ts +++ b/typescript/packages/traceai_elevenlabs/src/responses.ts @@ -191,7 +191,10 @@ export function observeResponse( data = timestampStream(data as AsyncIterable, operation); } else { if (method === "speechToText.convert") operation.output(data); - else operation.audio(data, true); + else { + operation.timestampedAttachment(data); + operation.audio(data, true); + } operation.end("complete"); } }); diff --git a/typescript/packages/traceai_elevenlabs/test/attachments-fixture.cjs b/typescript/packages/traceai_elevenlabs/test/attachments-fixture.cjs new file mode 100644 index 000000000..085184ce8 --- /dev/null +++ b/typescript/packages/traceai_elevenlabs/test/attachments-fixture.cjs @@ -0,0 +1,71 @@ +const http = require('node:http'); +const { once } = require('node:events'); +const { randomUUID } = require('node:crypto'); +const { AttachmentManager, createAttachmentHttpTransport } = require('@traceai/fi-core/attachments'); +const ids = [1, 2, 3].map(n => `10000000-0000-4000-8000-00000000000${n}`); +const expiry = () => new Date(Date.now() + 60000).toISOString(); + +async function attachmentFixture(config = {}, serverPolicy = {}) { + const calls = [], uploaded = [], requests = [], uploads = new Map(); + const policy = { version: 'v1', enabled: true, allowed_sources: ['Bytes', 'Blob'], allowed_modalities: ['audio'], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, + record_audio: false, record_video: false, ...serverPolicy }; + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const chunk of req) chunks.push(chunk); + const bytes = Buffer.concat(chunks); + calls.push({ method: req.method, path: req.url }); + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('/attachment-sessions')) { + const body = JSON.parse(bytes); + res.end(JSON.stringify({ schema_version: 1, scope: { organization_id: ids[0], workspace_id: ids[1], project_id: ids[2] }, + capture_session_id: body.capture_session_id, policy, limits: { max_upload_bytes: 8388608, segment_bytes: 8388608, + segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, + active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, + initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, + playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }, + expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); + } else if (req.url.endsWith('/uploads')) { + const body = JSON.parse(bytes); requests.push(body); + const upload = { upload_id: randomUUID(), attachment_id: body.attachment_id, occurrence_id: body.occurrence_id, + state: 'pending_upload', reserved_bytes: body.declared_bytes, expires_at: expiry(), + transfer_mode: 'authenticated_stream', write_capability: 'synthetic-write-capability' }; + uploads.set(upload.upload_id, upload); res.statusCode = 201; res.end(JSON.stringify(upload)); + } else { + const id = req.url.split('/').at(-2); + if (!uploads.has(id) || req.headers.authorization !== 'Bearer synthetic-write-capability') { res.statusCode = 403; res.end('{}'); return; } + if (req.url.endsWith('/content')) { + uploaded.push(bytes); res.end(JSON.stringify({ upload_id: id, state: 'pending_upload', reason: 'queued' })); + } else { + res.statusCode = 202; res.end(JSON.stringify({ upload_id: id, state: 'validating', reason: 'binding_pending' })); + } + } + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const client = async req => { + const body = req.body instanceof Blob ? Buffer.from(await req.body.arrayBuffer()) : req.body; + return new Promise((resolve, reject) => { + const r = http.request(`http://127.0.0.1:${server.address().port}${new URL(req.url).pathname}`, + { method: req.method, headers: req.headers, signal: req.signal }, response => { + const chunks = []; response.on('data', c => chunks.push(c)); + response.on('end', () => resolve({ status: response.statusCode, body: Buffer.concat(chunks) })); + }); + r.on('error', reject); r.end(body); + }); + }; + const manager = new AttachmentManager({ enabled: true, projectName: 'synthetic-voice', projectType: 'observe', + transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic' }, client }), ...config }); + await manager.ready(); + return { manager, calls, uploaded, requests, + async close() { await manager.shutdown(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } }; +} + +function wav() { + const samples = 2400, out = Buffer.alloc(44 + samples * 2); + out.write('RIFF'); out.writeUInt32LE(out.length - 8, 4); out.write('WAVEfmt ', 8); + out.writeUInt32LE(16, 16); out.writeUInt16LE(1, 20); out.writeUInt16LE(1, 22); + out.writeUInt32LE(24000, 24); out.writeUInt32LE(48000, 28); out.writeUInt16LE(2, 32); out.writeUInt16LE(16, 34); + out.write('data', 36); out.writeUInt32LE(samples * 2, 40); + for (let i = 0; i < samples; i++) out.writeInt16LE(Math.round(Math.sin(i * 2 * Math.PI * 440 / 24000) * 2000), 44 + i * 2); + return out; +} +module.exports = { attachmentFixture, wav }; diff --git a/typescript/packages/traceai_elevenlabs/test/attachments.test.cjs b/typescript/packages/traceai_elevenlabs/test/attachments.test.cjs new file mode 100644 index 000000000..dc3ac56db --- /dev/null +++ b/typescript/packages/traceai_elevenlabs/test/attachments.test.cjs @@ -0,0 +1,105 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const { fixture, bytes } = require('./fixture.cjs'); +const { attachmentFixture, wav } = require('./attachments-fixture.cjs'); +const { SpeechAttachments } = require('../dist/src/attachments.js'); +const { generateTraceConfig } = require('@traceai/fi-core'); +const tone = readFileSync(require('node:path').join(__dirname, 'fixtures/tone.mp3')); + +for (const sdkName of ['@elevenlabs/sdk-floor', '@elevenlabs/elevenlabs-js']) { + const SDK = require(sdkName); + test(`${sdkName}: timestamp response uploads once; cached awaits retain data and parent`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager }, { base64: tone.toString('base64') }); t.after(() => f.close()); + const response = f.client.textToSpeech.convertWithTimestamps('synthetic-voice', { text: 'synthetic', outputFormat: 'mp3_44100_128' }); + const first = await response, second = await response.withRawResponse(); + assert.equal(first, second.data); + assert.deepEqual(Buffer.from(first.audioBase64, 'base64'), tone); + assert.equal(first.alignment.characterStartTimesSeconds[0], 0); + await a.manager.flush(); + assert.equal(a.uploaded.length, 1); assert.deepEqual(a.uploaded[0], tone); + const span = f.spans()[0]; + const refs = JSON.parse(span.attributes['fi.attachment.refs']); + assert.equal(refs.length, 1); assert.equal(refs[0].occurrence_id, a.requests[0].occurrence_id); + assert.equal(a.requests[0].association.parent.span_id, span.spanContext().spanId); + assert.equal(a.requests[0].association.parent.trace_id.replaceAll('-', ''), span.spanContext().traceId); + assert.equal(a.requests[0].mime_type, 'audio/mpeg'); + const wire = JSON.stringify(await f.flush()); + assert.ok(!wire.includes(tone.toString('base64'))); + assert.ok(!wire.includes('synthetic-write-capability')); + }); + + test(`${sdkName}: immutable typed STT Blob uploads independently without filename`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager, attachmentInputMimeType: 'audio/wav' }); t.after(() => f.close()); + const audio = wav(); const blob = new File([audio], 'synthetic-private-name.wav', { type: 'audio/wav' }); + const result = await f.client.speechToText.convert({ file: blob, modelId: 'scribe_v1' }); + assert.equal(result.text, 'synthetic-transcript-elevenlabs'); + await a.manager.flush(); assert.equal(a.uploaded.length, 1); assert.deepEqual(a.uploaded[0], audio); + assert.deepEqual(Buffer.from(await blob.arrayBuffer()), audio); + assert.equal(a.requests[0].direction, 'input'); assert.equal(a.requests[0].mime_type, 'audio/wav'); + assert.ok(!JSON.stringify(a.requests).includes('synthetic-private-name')); + assert.ok(!JSON.stringify(await f.flush()).includes('synthetic-private-name')); + }); + + for (const [name, managerConfig, localConfig, policy] of [ + ['disabled', { enabled: false }, {}], + ['core hidden input/output', { traceConfig: { hideInputs: true, hideOutputs: true } }, {}], + ['local hidden messages', {}, { hideInputMessages: true, hideOutputMessages: true }], + ['local hidden text', {}, { hideInputText: true, hideOutputText: true }], + ['local PII', {}, { piiRedaction: true }], + ['core PII', { traceConfig: { piiRedaction: true } }, {}], + ['server redaction', {}, {}, { require_media_redaction: true }], + ['environment denial', { environment: () => ({ FI_ATTACHMENTS_ENABLED: 'false' }) }, {}], + ]) test(`${sdkName}: ${name} prevents all attachment upload traffic`, async t => { + const a = await attachmentFixture(managerConfig, policy); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager, attachmentInputMimeType: 'audio/wav', traceConfig: localConfig }, + { base64: tone.toString('base64') }); t.after(() => f.close()); + await f.client.textToSpeech.convertWithTimestamps('voice', { text: 'synthetic' }); + await f.client.speechToText.convert({ file: new Blob([wav()], { type: 'audio/wav' }), modelId: 'scribe_v1' }); + await a.manager.flush(); + assert.equal(a.uploaded.length, 0); assert.equal(a.requests.length, 0); + assert.ok(a.calls.every(c => c.path.endsWith('/attachment-sessions'))); + assert.ok(f.spans().every(s => s.attributes['fi.attachment.refs'] === undefined)); + }); + + test(`${sdkName}: streams retain cancellation and never masquerade as complete audio`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager }); t.after(() => f.close()); + const stream = await f.client.textToSpeech.stream('voice', { text: 'synthetic' }); + await stream.cancel('synthetic-cancel'); + await bytes(await f.client.textToSpeech.convert('voice', { text: 'synthetic' })); + for await (const chunk of await f.client.textToSpeech.streamWithTimestamps('voice', { text: 'synthetic' })) { + assert.equal(typeof chunk.audioBase64, 'string'); break; + } + await a.manager.flush(); assert.equal(a.uploaded.length, 0); + assert.ok(f.spans().some(s => s.attributes['elevenlabs.completion'] === 'cancelled')); + }); + + for (const [name, base64, format] of [ + ['oversized complete response', Buffer.alloc(256 * 1024 + 1).toString('base64'), 'mp3_44100_128'], + ['malformed base64', 'a!==', 'mp3_44100_128'], + ['raw PCM is not MP3', tone.toString('base64'), 'pcm_24000'], + ['unrecognized format cannot become the default', tone.toString('base64'), 'bad/format'], + ]) test(`${sdkName}: ${name} fails closed without changing SDK response`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager }, { base64 }); t.after(() => f.close()); + const value = await f.client.textToSpeech.convertWithTimestamps('voice', { text: 'synthetic', outputFormat: 'mp3_44100_128' }, + { queryParams: { output_format: format } }); + assert.equal(value.audioBase64, base64); await a.manager.flush(); assert.equal(a.uploaded.length, 0); + }); +} + +for (const config of [{ enabled: false }, { traceConfig: { piiRedaction: true } }, { traceConfig: { hideInputs: true, hideOutputs: true } }]) { + test(`denied sources are never inspected: ${JSON.stringify(config)}`, async t => { + const a = await attachmentFixture(config); t.after(() => a.close()); + let reads = 0; + const span = { spanContext: () => ({ traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }), + isRecording: () => true, setAttribute() {} }; + const capture = new SpeechAttachments(a.manager, span, generateTraceConfig(), undefined); + capture.timestamped({ get audioBase64() { reads++; throw Error('synthetic-private'); } }); + capture.input({ get file() { reads++; throw Error('synthetic-private'); } }, 'audio/wav'); + await a.manager.flush(); assert.equal(reads, 0); assert.equal(a.uploaded.length, 0); + }); +} diff --git a/typescript/packages/traceai_elevenlabs/test/fixture.cjs b/typescript/packages/traceai_elevenlabs/test/fixture.cjs index 11b0e4bd6..f2b057374 100644 --- a/typescript/packages/traceai_elevenlabs/test/fixture.cjs +++ b/typescript/packages/traceai_elevenlabs/test/fixture.cjs @@ -44,7 +44,8 @@ const attributes = (span) => ]), ); -async function fixture(SDK, options = {}) { +async function fixture(SDK, options = {}, media = {}) { + const speech = { ...timestamp, audio_base64: media.base64 ?? BASE64 }; const requests = [], wire = []; const attempts = new Map(); @@ -122,14 +123,14 @@ async function fixture(SDK, options = {}) { if (req.url.includes("with-timestamps")) { res.setHeader("content-type", "application/json"); if (req.url.includes("/stream/")) { - res.write(JSON.stringify(timestamp) + "\n"); + res.write(JSON.stringify(speech) + "\n"); await delay(15); res.end( mode === "bad-stream" ? "{bad-json}\n" - : JSON.stringify(timestamp) + "\n", + : JSON.stringify(speech) + "\n", ); - } else res.end(JSON.stringify(timestamp)); + } else res.end(JSON.stringify(speech)); return; } res.setHeader("content-type", "audio/mpeg"); diff --git a/typescript/packages/traceai_elevenlabs/test/fixtures/tone.mp3 b/typescript/packages/traceai_elevenlabs/test/fixtures/tone.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..0e538fc9aff5eb635fdd6993fe675a962a4dda91 GIT binary patch literal 2551 zcmd^>X;70%8pmHk2%sPlgP?!|1VRQ0z9EDgG|i1_BJij{@q{eOl3U#wi9ZU8_9^9^X1 zIjaqA(Yj8~fKt&qZo17dl}T2@m%VN7qUaB;{ga{xqyLgymfBv(D_41ums6G+#!Ww! z4|WT8!VQQp9SgX@-Js3VsYCr))zkB-?>Tenyl3KlPjo-d%yg#ZQci!Y?S$_0-ar~^ zzyhG~R1Rki2iPP>XXUj23uh)Z_3IT>n=l`@VxW665iV+s#l48tLTvQEGHy=xh$HW> z&$7J%_$ZqmcA4^0#Te3^>1OM?6tq?=CH6GxDb|7oqLZF3case{bg?<~QQneF$ZW}I|F#PSb zUCrIeH7O&I*f==1d~K9@RD?yjb|*CW);x;jTc`SrA&~9}XiI~3G}Ldc5I62Ri9;1z zlGbS%$3V~!Xb6meXnFY=<^t-Z&x6}sTWo(gnrt0ih{k`)&<3H!;%+2&utgQnf6IYY z3k?*$sM~!w)&sIdeqe?x0xWX527%cd{|IbagBswFr;| z4=`L!^W=928JUEnl7j?Iju}^_O)m-FhLYy^6rh)hJJ{XM$Ei$>B|GoHfiEfgG0;ZQ zWpHMj;|DiJd0jx*Y(QXPV2MsYjZC;jyWBK;b0F%&&gT+Zf%^!>Xb;}8cp#Bz&u+-d zcrjk#ZbGVDeo|^cBcnI*2FyR^RvZC%+`uzUAw!!R6RVf zHJK8WWJ)6SC!aiX>*jW3i}AZ=i#p$pd|a59yAMuFGBtehR+L>VZ!0s#xBhWcH-k56 z5>XjKRU?d>VJ3gd$qtIt1%e}K=>d@yuoAjyMnF_85=?MhY>Ss6ouisF-mr~TSgr8b z22m{gY|I>+?zL%`(vvT4^kseRzRj-i0olm>QfB*JX1Uj;-RUxZcw_j0y(etCs5C{X z=gRHS)R!I|XYZsO_1m2HO#eLH?9F0U?n%9$5(kL5lC9I#9u|jb1c!(RpXeWxQ`ROo zR{d)3Pc?DW@YvlfOTF2Ou^T9uVpwO-Rq(H0!82N)L6dC?p>1s1#l2C=@rR-k7=d zXRr_pycZsZyNi^S;a%OxvF>=ID=*ETT8%gQIEvNvjC*#ja_{jMgHNEb6t7q?>K>g>lpxpy8@?u+Q?6X{C&vGyC1 z-(1aKxX$u#Z0ik_4dPhV5$yr2i7VwXh5A(XyExev=souUPq1rm(m1s@;h(b-GbxlS2Gl)`bKe3JiY?R4jujsO>76e&QdYm0T9z z_i{t|m7MBzI#uKosu`lZa7M=s&X^Ef{9PB%tE%$OQZ9?H+y;&E14e3$6yOF31P~c1 zrtr3XO8b04J70O|4`3a#vq|RoI*~RCLGxZ^`~8PwcsbE5qoFVfyuUQ%OD6?-OOugSFD%6xF|F7i=?ctcRwyb~nnxhbRg1))}L@b_~>3N);7^Wc| zIhC|!nd(u?041`0^3f3V34$7YQFB)lvb&#e)4i-ylP2%%W5tFzjy}t09e|+Wufx7p z%ege2_c%GV$C%_Caq3(N^J#*-q1t?!Kyb&r6H1PChwJg=Pgw%S!Lw%W2j^N_ z-`NlPWa-&`7A0`F62s;uVPW)!46CM PipecatServerRecording | undefined; } const observers = new WeakMap void>(); @@ -47,13 +54,19 @@ export function observePipecatClient( const listener = (state: unknown) => { // Telemetry must not interrupt application callbacks. Do not log their data. try { + let recording; + try { recording = pipecatRecordingLink(options.serverRecording?.()); } catch { /* unavailable */ } const span = tracer.startSpan(`Pipecat Client Event: ${event}`, { kind: SpanKind.CLIENT, attributes: { "fi.span.kind": "CHAIN", "pipecat.telemetry_source": "client", "pipecat.event": event, + ...(options.serverRecording ? { + "pipecat.server_recording.status": recording ? "reference_only" : "source_unavailable", + } : {}), }, + links: recording ? [recording] : [], }, options.parentContext ?? context.active()); // Only inspect the fixed SDK error state; never serialize or coerce event data. const isError = event === "error" || event === "messageError" || diff --git a/typescript/packages/traceai_pipecat/src/index.ts b/typescript/packages/traceai_pipecat/src/index.ts index 6ec737d6e..13ea23417 100644 --- a/typescript/packages/traceai_pipecat/src/index.ts +++ b/typescript/packages/traceai_pipecat/src/index.ts @@ -8,3 +8,4 @@ export { PipecatInstrumentation, PipecatInstrumentationConfig } from "./instrume export { isPatched } from "./instrumentation"; export { VERSION } from "./version"; export { observePipecatClient, PipecatEventSource, PipecatClientTracingOptions } from "./client"; +export type { PipecatServerRecording } from "./recording"; diff --git a/typescript/packages/traceai_pipecat/src/recording.ts b/typescript/packages/traceai_pipecat/src/recording.ts new file mode 100644 index 000000000..00f906616 --- /dev/null +++ b/typescript/packages/traceai_pipecat/src/recording.ts @@ -0,0 +1,45 @@ +import type { Link } from "@opentelemetry/api"; + +/** + * An application-server-authorized identity, NOT a browser upload capability. + * Obtain it from your authenticated server, never from an RTC/bot event payload. + * The occurrence remains on its original server parent; a link grants no access. + */ +export interface PipecatServerRecording { + readonly schemaVersion: 1; + readonly parent: { readonly traceId: string; readonly spanId: string }; + readonly occurrenceId: string; + /** Opaque session UUID; no participant names or provider credentials. */ + readonly conversationId: string; +} + +const uuid = (value: unknown): value is string => typeof value === "string" && + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(value) && + value !== "00000000-0000-0000-0000-000000000000"; + +/** Content-free correlation only. It never calls an upload, relink or resolve API. */ +export function pipecatRecordingLink(value: PipecatServerRecording | undefined): Link | undefined { + try { + if (value?.schemaVersion !== 1) return; + // Preserve all trace bits; this reference-only browser leaf must not import + // the upload manager merely to normalize the two contract spellings. + const sourceTraceId = value.parent?.traceId; + const traceId = uuid(sourceTraceId) ? sourceTraceId.replace(/-/g, "") : sourceTraceId; + const spanId = value.parent?.spanId; + const occurrenceId = value.occurrenceId; + const conversationId = value.conversationId; + if (typeof traceId !== "string" || !/^[0-9a-f]{32}$/.test(traceId) || /^0+$/.test(traceId) || + typeof spanId !== "string" || !/^[0-9a-f]{16}$/.test(spanId) || + /^0+$/.test(spanId) || !uuid(occurrenceId) || !uuid(conversationId)) return; + return { + context: { traceId, spanId, traceFlags: 0, isRemote: true }, + attributes: { + "pipecat.server_recording.occurrence_id": occurrenceId, + "pipecat.server_recording.conversation_id": conversationId, + }, + }; + } catch { + // Malformed or unavailable server identity must not interrupt RTC callbacks. + return; + } +} diff --git a/typescript/packages/traceai_pipecat/test/recording.test.cjs b/typescript/packages/traceai_pipecat/test/recording.test.cjs new file mode 100644 index 000000000..cca5996dc --- /dev/null +++ b/typescript/packages/traceai_pipecat/test/recording.test.cjs @@ -0,0 +1,61 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } = require('@opentelemetry/sdk-trace-base'); +const { observePipecatClient } = require('../dist/src/client.js'); +const { createClient } = require('./transport.cjs'); +const identity = () => ({ schemaVersion: 1, + parent: { traceId: '01234567-89ab-cdef-0123-456789abcdef', spanId: '0123456789abcdef' }, + occurrenceId: '10000000-0000-4000-8000-000000000001', conversationId: '10000000-0000-4000-8000-000000000002' }); + +for (const name of ['pipecat-floor', '@pipecat-ai/client-js']) { + test(`${name}: server recording stays on its parent without media or credentials`, async t => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + t.after(() => provider.shutdown()); + const { client, transport } = createClient(require(name)); + let current = identity(); + Object.defineProperty(current, 'sourceUrl', { get() { throw Error('synthetic-secret-must-not-read'); } }); + const stop = observePipecatClient(client, { tracer: provider.getTracer('recording-test'), serverRecording: () => current }); + t.after(stop); + await client.connect(); await client.disconnect(); + const first = exporter.getFinishedSpans(); assert.ok(first.length > 0); + for (const s of first) { + assert.equal(s.attributes['pipecat.server_recording.status'], 'reference_only'); + assert.equal(s.links.length, 1); + assert.equal(s.links[0].context.traceId, current.parent.traceId.replaceAll('-', '')); + assert.equal(s.links[0].context.spanId, current.parent.spanId); + assert.equal(s.links[0].attributes['pipecat.server_recording.occurrence_id'], current.occurrenceId); + assert.equal(s.attributes['fi.attachment.refs'], undefined); // not a relink + } + // The app clears identity when its server session ends. Reconnect never falls + // back to a prior session or to a bot-ready payload's untrusted identifiers. + current = undefined; + await client.connect(); await client.disconnect(); + for (const s of exporter.getFinishedSpans().slice(first.length)) { + assert.equal(s.attributes['pipecat.server_recording.status'], 'source_unavailable'); + assert.equal(s.links.length, 0); + } + const sent = JSON.stringify(transport.sent); + assert.ok(!sent.includes('10000000-0000')); assert.ok(!sent.includes('01234567')); + stop(); assert.equal(client.listenerCount('connected'), 0); + }); + + for (const [caseName, supplied] of [ + ['missing', undefined], ['version', { ...identity(), schemaVersion: 2 }], + ['zero trace', { ...identity(), parent: { ...identity().parent, traceId: '0'.repeat(32) } }], + ['bad span', { ...identity(), parent: { ...identity().parent, spanId: 'secret=synthetic' } }], + ['URL occurrence', { ...identity(), occurrenceId: 'https://invalid.example/?key=synthetic' }], + ['participant name', { ...identity(), conversationId: 'synthetic-private-person' }], + ]) test(`${name}: ${caseName} identity has only source_unavailable status`, async t => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + t.after(() => provider.shutdown()); + const { client } = createClient(require(name)); + const stop = observePipecatClient(client, { tracer: provider.getTracer('invalid-test'), serverRecording: () => supplied }); + t.after(stop); await client.connect(); await client.disconnect(); + for (const s of exporter.getFinishedSpans()) { + assert.equal(s.attributes['pipecat.server_recording.status'], 'source_unavailable'); + assert.deepEqual(s.links, []); assert.ok(!JSON.stringify(s.attributes).includes('synthetic-private')); + } + }); +} From f5748cf37d7480106fb96e94e9e345b54575ed50 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 11 Nov 2026 01:56:51 -0800 Subject: [PATCH 2/8] feat(elevenlabs): attach consumed TTS streams through the shared stream writer - convert, stream and streamWithTimestamps feed only chunks the application already read (instrumentation-owned bridge) into startAttachmentStream; nothing is read ahead, teed or drained. - The stream finishes before the span ends so its reference projects; cancel, error, timeout, early break or over 4 MiB uploads nothing. - Non-MP3 output formats fail closed with source_unavailable. - Bound calls carry the shared media-owner context marker so wire-level capture can skip the same bytes. - Size and latency metadata keep their previous behaviour; test doubles mint 1 h sessions. Co-Authored-By: Claude Opus 5.5 --- .../packages/traceai_elevenlabs/README.md | 5 +- .../traceai_elevenlabs/src/attachments.ts | 130 +++++++++++++++--- .../traceai_elevenlabs/src/instrumentation.ts | 7 + .../traceai_elevenlabs/src/operation.ts | 19 ++- .../traceai_elevenlabs/src/responses.ts | 12 +- .../test/attachments-fixture.cjs | 3 +- .../test/attachments.test.cjs | 111 ++++++++++++++- .../traceai_elevenlabs/test/fixture.cjs | 6 +- 8 files changed, 256 insertions(+), 37 deletions(-) diff --git a/typescript/packages/traceai_elevenlabs/README.md b/typescript/packages/traceai_elevenlabs/README.md index 084c2375c..a26db5015 100644 --- a/typescript/packages/traceai_elevenlabs/README.md +++ b/typescript/packages/traceai_elevenlabs/README.md @@ -117,9 +117,10 @@ Binary capture is off by default. If your destination supports the attachment se - `textToSpeech.convertWithTimestamps` can attach a complete MP3 response up to 256 KiB. The output format must be the SDK default or a supported MP3 format. Generated audio is not proof of playback. - `speechToText.convert` can attach an existing `Blob`/`File` when you also set `attachmentInputMimeType` to its exact MIME type: `audio/wav`, `audio/mpeg`, `audio/ogg`, `audio/mp4`, `audio/webm` or `audio/flac`. Filenames are not exported; mismatched or missing types are not sniffed. -- Streams, PCM/ulaw output, local paths and remote URLs are not captured. Alignment data stays in your SDK response, not in attachment metadata. Use the stream methods normally; tracing does not drain, tee or accumulate their bytes. +- `textToSpeech.convert`, `textToSpeech.stream` and `textToSpeech.streamWithTimestamps` can attach the MP3 your application **reads to the end**, up to 4 MiB per call (about 4 minutes at the default 128 kbit/s). Tracing copies only the chunks you already read; it never reads ahead, tees or drains the stream. If you cancel, stop iterating, hit an error or exceed 4 MiB, nothing is uploaded: a partial file is never stored as if complete. While a stream is being read, the manager reserves the full 4 MiB, so many concurrent long streams can report `queue_full` and pass through uncaptured. +- PCM/ulaw output, local paths and remote URLs are not captured. Alignment data stays in your SDK response, not in attachment metadata. -Attachments obey the manager's default-off, server, sampling and PII policies in addition to the instrumentation's directional hide settings. Without a qualified speech sanitizer, `piiRedaction` denies capture. Existing raw audio may contain embedded metadata; a policy requiring metadata removal also denies it. A `queued` result on the span means admitted for transfer, **not** validated or playable. Content is out of band, never base64 in OTLP. +Attachments obey the manager's default-off, server, sampling and PII policies in addition to the instrumentation's directional hide settings. Without a qualified speech sanitizer, `piiRedaction` denies capture. Existing raw audio may contain embedded metadata; a policy requiring metadata removal also denies it. The span's `elevenlabs.attachment.input` / `elevenlabs.attachment.output` value is the admission result; `queued` means admitted for transfer, **not** validated or playable. Content is out of band, never base64 in OTLP. A call made through this bound instrumentation carries the shared media-owner context marker (`traceai.auto.attachment_owner.v1`), so wire-level capture that honours the marker does not upload the same audio again. ## Privacy diff --git a/typescript/packages/traceai_elevenlabs/src/attachments.ts b/typescript/packages/traceai_elevenlabs/src/attachments.ts index c68e83c78..afcc8b8e8 100644 --- a/typescript/packages/traceai_elevenlabs/src/attachments.ts +++ b/typescript/packages/traceai_elevenlabs/src/attachments.ts @@ -1,6 +1,6 @@ import type { Span } from "@opentelemetry/api"; import type { TraceConfig } from "@traceai/fi-core"; -import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import type { AttachmentManager, AttachmentStream } from "@traceai/fi-core/attachments"; import { record, safely } from "./operation.js"; export type ElevenLabsInputAudioMimeType = @@ -11,15 +11,39 @@ export type ElevenLabsInputAudioMimeType = | "audio/webm" | "audio/flac"; -// This is a bounded complete-response path, NOT a recording/stream accumulator. +// A bounded complete-response path for convertWithTimestamps (one JSON value). const MAX_DECODED_BYTES = 256 * 1024; -const MAX_ENCODED_CHARACTERS = Math.ceil(MAX_DECODED_BYTES / 3) * 4; +/** + * Consumed-stream cap. The manager reserves the whole cap when a stream is admitted, so it + * also bounds per-call memory. About 4.4 minutes of the SDK's default 128 kbit/s MP3. A longer + * response uploads nothing (`too_large`); a truncated object is never uploaded as complete. + */ +export const ELEVENLABS_STREAM_MAX_BYTES = 4 * 1024 * 1024; const INPUT_TYPES: readonly string[] = [ "audio/wav", "audio/mpeg", "audio/ogg", "audio/mp4", "audio/webm", "audio/flac", ]; +const MP3_FORMAT = /^mp3_(22050_32|24000_48|44100_(32|64|96|128|192))$/; +const BASE64 = /^[A-Za-z0-9+/]*={0,2}$/; -/** Adapters never own an uploader or change the provider manager's lifecycle. */ +/** Canonical, bounded base64 only. Buffer.from tolerates malformed input, so check first. */ +function decode(encoded: unknown, maxBytes: number): Uint8Array | undefined { + if (typeof encoded !== "string" || encoded.length === 0 || + encoded.length > Math.ceil(maxBytes / 3) * 4) return undefined; + if (encoded.length % 4 !== 0 || !BASE64.test(encoded)) return undefined; + const decoded = Buffer.from(encoded, "base64"); + if (decoded.length === 0 || decoded.length > maxBytes) return undefined; + if (decoded.toString("base64") !== encoded) return undefined; + return decoded; +} + +/** + * Adapters never own an uploader or change the provider manager's lifecycle. Every capture + * goes through the shared manager's admission before any read, decode or copy. + */ export class SpeechAttachments { + private stream?: AttachmentStream; + private streamState: "idle" | "open" | "closed" = "idle"; + constructor( private readonly manager: AttachmentManager, private readonly span: Span, @@ -35,6 +59,11 @@ export class SpeechAttachments { : !(this.policy.hideOutputs || this.policy.hideOutputMessages || this.policy.hideOutputText)); } + /** Raw PCM/ulaw lack sample-format metadata in v1; never label those bytes MP3. */ + private mp3(): boolean { + return this.outputFormat === undefined || MP3_FORMAT.test(this.outputFormat); + } + input(request: unknown, mimeType: ElevenLabsInputAudioMimeType): void { safely(() => { if (!INPUT_TYPES.includes(mimeType)) return; @@ -61,27 +90,15 @@ export class SpeechAttachments { timestamped(value: unknown): void { safely(() => { - // Raw PCM/ulaw require sample-format metadata that the core does not yet - // carry. Do not label those bytes MP3. The SDK's default is mp3_44100_128. - if (this.outputFormat !== undefined && - !/^mp3_(22050_32|24000_48|44100_(32|64|96|128|192))$/.test(this.outputFormat)) { + if (!this.mp3()) { this.span.setAttribute("elevenlabs.attachment.output", "source_unavailable"); return; } const result = this.manager.attach(this.span, { kind: "Bytes", get data(): Uint8Array { - const encoded = record(value).audioBase64; - if (typeof encoded !== "string" || encoded.length === 0 || - encoded.length > MAX_ENCODED_CHARACTERS) return new Uint8Array(0); - // Buffer.from tolerates malformed base64; require canonical padding and - // alphabet first, then a round-trip before retaining decoded bytes. - if (encoded.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) { - throw new Error("unsupported_audio_source"); - } - const decoded = Buffer.from(encoded, "base64"); - if (decoded.length > MAX_DECODED_BYTES) return new Uint8Array(0); - if (decoded.toString("base64") !== encoded) throw new Error("unsupported_audio_source"); + const decoded = decode(record(value).audioBase64, MAX_DECODED_BYTES); + if (!decoded) throw new Error("unsupported_audio_source"); return decoded; }, }, { @@ -93,4 +110,79 @@ export class SpeechAttachments { this.span.setAttribute("elevenlabs.attachment.output", result.admission); }); } + + /** + * Admit the consumed stream once per call, before any byte is copied or decoded. A denied + * admission retains nothing and is never retried for this call. + */ + private open(): AttachmentStream | undefined { + if (this.streamState === "idle") { + this.streamState = "closed"; + if (!this.mp3()) { + this.span.setAttribute("elevenlabs.attachment.output", "source_unavailable"); + return undefined; + } + const stream = this.manager.startAttachmentStream(this.span, { + mimeType: "audio/mpeg", + direction: "output", + purpose: "model_output", + maxBytes: ELEVENLABS_STREAM_MAX_BYTES, + capture: { captureContent: this.allowed("output") }, + }); + if (stream.reason !== "queued") { + this.span.setAttribute("elevenlabs.attachment.output", stream.reason); + return undefined; + } + this.stream = stream; + this.streamState = "open"; + } + return this.streamState === "open" ? this.stream : undefined; + } + + /** + * Observe one chunk the APPLICATION already pulled through the instrumentation-owned bridge + * (design 6.4). tryWrite copies synchronously, so a BYOB view may be handed back right after. + */ + chunk(bytes: Uint8Array): void { + safely(() => { + if (this.streamState === "closed" || !(bytes instanceof Uint8Array) || bytes.byteLength === 0) return; + const stream = this.open(); + if (stream && !stream.tryWrite(bytes)) this.close("incomplete"); + }); + } + + /** One base64 chunk of a consumed timestamp stream; decoded only after admission. */ + timestampChunk(value: unknown): void { + safely(() => { + if (this.streamState === "closed") return; + const encoded = record(value).audioBase64; + if (typeof encoded !== "string" || encoded.length === 0) return; + const stream = this.open(); + if (!stream) return; + const decoded = decode(encoded, ELEVENLABS_STREAM_MAX_BYTES); + // A malformed or oversized chunk means the object would be incomplete: upload nothing. + if (!decoded || !stream.tryWrite(decoded)) this.close("incomplete"); + }); + } + + /** + * Called by the operation BEFORE its span ends, so a finished stream's reference is + * projected on a recording span. Anything but a complete read aborts: nothing uploads. + */ + close(completion: "complete" | "incomplete"): void { + safely(() => { + const stream = this.stream; + const wasOpen = this.streamState === "open"; + this.stream = undefined; + this.streamState = "closed"; + if (!stream || !wasOpen) return; + if (completion === "complete") { + const handle = stream.finish(); + this.span.setAttribute("elevenlabs.attachment.output", handle.admission); + } else { + stream.abort("incomplete"); + this.span.setAttribute("elevenlabs.attachment.output", stream.reason); + } + }); + } } diff --git a/typescript/packages/traceai_elevenlabs/src/instrumentation.ts b/typescript/packages/traceai_elevenlabs/src/instrumentation.ts index 140d3634b..c662cf840 100644 --- a/typescript/packages/traceai_elevenlabs/src/instrumentation.ts +++ b/typescript/packages/traceai_elevenlabs/src/instrumentation.ts @@ -1,6 +1,7 @@ import { Attributes, context, + createContextKey, SpanKind, trace, TracerProvider, @@ -30,6 +31,9 @@ import { } from "./operation.js"; import type { AttachmentManager } from "@traceai/fi-core/attachments"; import { SpeechAttachments, ElevenLabsInputAudioMimeType } from "./attachments.js"; + +/** Context marker read by `@traceai/auto` wire capture; `createContextKey` is `Symbol.for`. */ +const ATTACHMENT_OWNER = createContextKey("traceai.auto.attachment_owner.v1"); import { observeResponse } from "./responses.js"; import { VERSION } from "./version.js"; @@ -317,6 +321,9 @@ export class ElevenLabsInstrumentation extends InstrumentationBase { this.active.add(operation); operation.watchAbort(record(args[stt ? 1 : 2]).abortSignal); callContext = trace.setSpan(callContext, span); + // This adapter owns the call's media, so wire-level auto capture must not upload the + // same bytes again (shared key; no dependency on the auto package). + if (attachments) callContext = callContext.setValue(ATTACHMENT_OWNER, true); } } catch { diagnose(); diff --git a/typescript/packages/traceai_elevenlabs/src/operation.ts b/typescript/packages/traceai_elevenlabs/src/operation.ts index d6dce13d1..2ec233916 100644 --- a/typescript/packages/traceai_elevenlabs/src/operation.ts +++ b/typescript/packages/traceai_elevenlabs/src/operation.ts @@ -80,10 +80,19 @@ export class Operation { if (this.span) this.attachments?.timestamped(value); } + /** A chunk the application consumed from an instrumentation-owned stream. */ + audioChunk(value: Uint8Array): void { + if (this.span) this.attachments?.chunk(value); + } + + /** A timestamp chunk the application consumed; audio is decoded only after admission. */ + timestampChunk(value: unknown): void { + if (this.span) this.attachments?.timestampChunk(value); + } + audio(value: unknown, timestamps = false): void { safely(() => { - if (!this.span || this.policy.hideOutputs || this.policy.hideOutputMessages || - this.policy.hideOutputText || this.policy.piiRedaction) return; + if (!this.span) return; const size = timestamps ? typeof record(value).audioBase64 === "string" ? (record(value).audioBase64 as string).length @@ -158,8 +167,12 @@ export class Operation { end(completion: Completion): void { const span = this.span; if (!span) return; - this.span = undefined; + // Finish or abort a consumed stream while the span still records, so a complete + // object projects its reference and anything else uploads nothing. + const attachments = this.attachments; this.attachments = undefined; + safely(() => attachments?.close(completion === "complete" ? "complete" : "incomplete")); + this.span = undefined; if (this.timer) clearTimeout(this.timer); this.timer = undefined; safely(() => this.detachAbort?.()); diff --git a/typescript/packages/traceai_elevenlabs/src/responses.ts b/typescript/packages/traceai_elevenlabs/src/responses.ts index dc4ac662b..80e7eb369 100644 --- a/typescript/packages/traceai_elevenlabs/src/responses.ts +++ b/typescript/packages/traceai_elevenlabs/src/responses.ts @@ -55,6 +55,8 @@ function audioStream( reader.value.releaseLock(); } else { operation.audio(next.value); + // Copy before the view goes back to the consumer (BYOB) or downstream. + operation.audioChunk(next.value); if (request) request.respondWithNewView(next.value); else controller.enqueue(next.value); } @@ -94,11 +96,11 @@ function observeIterator( try { const next = await iterator.next(...args); if (next.done) operation.end("complete"); - else - operation.audio( - metadata ? record(next.value).data : next.value, - true, - ); + else { + const value = metadata ? record(next.value).data : next.value; + operation.audio(value, true); + operation.timestampChunk(value); + } return next; } catch (error) { operation.error(error); diff --git a/typescript/packages/traceai_elevenlabs/test/attachments-fixture.cjs b/typescript/packages/traceai_elevenlabs/test/attachments-fixture.cjs index 085184ce8..f690aba5a 100644 --- a/typescript/packages/traceai_elevenlabs/test/attachments-fixture.cjs +++ b/typescript/packages/traceai_elevenlabs/test/attachments-fixture.cjs @@ -3,7 +3,8 @@ const { once } = require('node:events'); const { randomUUID } = require('node:crypto'); const { AttachmentManager, createAttachmentHttpTransport } = require('@traceai/fi-core/attachments'); const ids = [1, 2, 3].map(n => `10000000-0000-4000-8000-00000000000${n}`); -const expiry = () => new Date(Date.now() + 60000).toISOString(); +// Sessions of at least 1 h, so the core's near-expiry renewal never adds a bootstrap here. +const expiry = () => new Date(Date.now() + 3600000).toISOString(); async function attachmentFixture(config = {}, serverPolicy = {}) { const calls = [], uploaded = [], requests = [], uploads = new Map(); diff --git a/typescript/packages/traceai_elevenlabs/test/attachments.test.cjs b/typescript/packages/traceai_elevenlabs/test/attachments.test.cjs index dc3ac56db..e2e8ea138 100644 --- a/typescript/packages/traceai_elevenlabs/test/attachments.test.cjs +++ b/typescript/packages/traceai_elevenlabs/test/attachments.test.cjs @@ -58,23 +58,109 @@ for (const sdkName of ['@elevenlabs/sdk-floor', '@elevenlabs/elevenlabs-js']) { { base64: tone.toString('base64') }); t.after(() => f.close()); await f.client.textToSpeech.convertWithTimestamps('voice', { text: 'synthetic' }); await f.client.speechToText.convert({ file: new Blob([wav()], { type: 'audio/wav' }), modelId: 'scribe_v1' }); + await bytes(await f.client.textToSpeech.convert('voice', { text: 'synthetic' })); + await bytes(await f.client.textToSpeech.stream('voice', { text: 'synthetic' })); + for await (const chunk of await f.client.textToSpeech.streamWithTimestamps('voice', { text: 'synthetic' })) assert.ok(chunk); await a.manager.flush(); assert.equal(a.uploaded.length, 0); assert.equal(a.requests.length, 0); assert.ok(a.calls.every(c => c.path.endsWith('/attachment-sessions'))); assert.ok(f.spans().every(s => s.attributes['fi.attachment.refs'] === undefined)); }); - test(`${sdkName}: streams retain cancellation and never masquerade as complete audio`, async t => { + for (const method of ['convert', 'stream']) { + test(`${sdkName}: fully consumed ${method} uploads the exact MP3 once, before the span ends`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager }, { audio: tone }); t.after(() => f.close()); + const received = await bytes(await f.client.textToSpeech[method]('voice', { text: 'synthetic' })); + assert.deepEqual(Buffer.from(received), tone); + await a.manager.flush(); + assert.equal(a.uploaded.length, 1); assert.deepEqual(a.uploaded[0], tone); + const span = f.spans()[0]; + assert.equal(span.attributes['elevenlabs.completion'], 'complete'); + assert.equal(span.attributes['elevenlabs.attachment.output'], 'queued'); + const refs = JSON.parse(span.attributes['fi.attachment.refs']); + assert.equal(refs.length, 1); assert.equal(refs[0].occurrence_id, a.requests[0].occurrence_id); + assert.equal(a.requests[0].mime_type, 'audio/mpeg'); assert.equal(a.requests[0].direction, 'output'); + assert.equal(a.requests[0].declared_bytes, tone.length); + assert.equal(a.requests[0].association.parent.span_id, span.spanContext().spanId); + }); + } + + test(`${sdkName}: BYOB reads keep consumer buffers and upload identical bytes`, async t => { const a = await attachmentFixture(); t.after(() => a.close()); - const f = await fixture(SDK, { attachments: a.manager }); t.after(() => f.close()); + const f = await fixture(SDK, { attachments: a.manager }, { audio: tone }); t.after(() => f.close()); const stream = await f.client.textToSpeech.stream('voice', { text: 'synthetic' }); - await stream.cancel('synthetic-cancel'); - await bytes(await f.client.textToSpeech.convert('voice', { text: 'synthetic' })); + let reader; + try { reader = stream.getReader({ mode: 'byob' }); } catch { t.skip('upstream stream is not a byte stream'); return; } + const out = []; let buffer = new ArrayBuffer(1024); + for (;;) { + const { done, value } = await reader.read(new Uint8Array(buffer, 0, 1024)); + if (done) break; + out.push(Buffer.from(value)); buffer = value.buffer; + } + assert.deepEqual(Buffer.concat(out), tone); + await a.manager.flush(); assert.equal(a.uploaded.length, 1); assert.deepEqual(a.uploaded[0], tone); + }); + + test(`${sdkName}: cancelled or broken streams upload nothing and keep cancellation`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager }, { audio: tone }); t.after(() => f.close()); + const stream = await f.client.textToSpeech.stream('voice', { text: 'synthetic' }); + const reader = stream.getReader(); + assert.equal((await reader.read()).done, false); + await reader.cancel('synthetic-cancel'); + const broken = await f.client.textToSpeech.convert('voice', { text: 'synthetic' }, { headers: { 'x-case': 'broken' } }); + await assert.rejects(bytes(broken)); for await (const chunk of await f.client.textToSpeech.streamWithTimestamps('voice', { text: 'synthetic' })) { assert.equal(typeof chunk.audioBase64, 'string'); break; } + await a.manager.flush(); assert.equal(a.uploaded.length, 0); assert.equal(a.requests.length, 0); + const spans = f.spans(); + assert.ok(spans.some(s => s.attributes['elevenlabs.completion'] === 'cancelled')); + assert.ok(spans.every(s => s.attributes['fi.attachment.refs'] === undefined)); + assert.equal(a.manager.status().openStreams, 0); + }); + + test(`${sdkName}: fully iterated timestamp stream uploads the decoded MP3 chunks as one object`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager }, { base64: tone.toString('base64') }); t.after(() => f.close()); + let chunks = 0; + for await (const chunk of await f.client.textToSpeech.streamWithTimestamps('voice', { text: 'synthetic' })) { + assert.equal(chunk.audioBase64, tone.toString('base64')); chunks++; + } + assert.equal(chunks, 2); + await a.manager.flush(); + assert.equal(a.uploaded.length, 1); assert.deepEqual(a.uploaded[0], Buffer.concat([tone, tone])); + const span = f.spans()[0]; + assert.equal(JSON.parse(span.attributes['fi.attachment.refs']).length, 1); + assert.ok(!JSON.stringify(await f.flush()).includes(tone.toString('base64'))); + }); + + test(`${sdkName}: raw PCM streams are never labelled MP3`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager }, { audio: tone }); t.after(() => f.close()); + await bytes(await f.client.textToSpeech.stream('voice', { text: 'synthetic', outputFormat: 'pcm_24000' })); await a.manager.flush(); assert.equal(a.uploaded.length, 0); - assert.ok(f.spans().some(s => s.attributes['elevenlabs.completion'] === 'cancelled')); + assert.equal(f.spans()[0].attributes['elevenlabs.attachment.output'], 'source_unavailable'); + }); + + test(`${sdkName}: the bound adapter marks its call context as the media owner`, async t => { + const { AsyncLocalStorageContextManager } = require('@opentelemetry/context-async-hooks'); + const { context } = require('@opentelemetry/api'); + const manager = new AsyncLocalStorageContextManager().enable(); + context.setGlobalContextManager(manager); + t.after(() => { context.disable(); }); + const key = Symbol.for('traceai.auto.attachment_owner.v1'); + const seen = []; + const original = globalThis.fetch; + globalThis.fetch = (...args) => { seen.push(context.active().getValue(key)); return original(...args); }; + t.after(() => { globalThis.fetch = original; }); + const a = await attachmentFixture(); t.after(() => a.close()); + const bound = await fixture(SDK, { attachments: a.manager }, { audio: tone }); t.after(() => bound.close()); + const plain = await fixture(SDK, {}, { audio: tone }); t.after(() => plain.close()); + await bytes(await bound.client.textToSpeech.convert('voice', { text: 'synthetic' })); + await bytes(await plain.client.textToSpeech.convert('voice', { text: 'synthetic' })); + assert.deepEqual(seen, [true, undefined]); }); for (const [name, base64, format] of [ @@ -103,3 +189,18 @@ for (const config of [{ enabled: false }, { traceConfig: { piiRedaction: true } await a.manager.flush(); assert.equal(reads, 0); assert.equal(a.uploaded.length, 0); }); } + +test('a stream above the cap uploads nothing (too_large) and returns its reservation', async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const attrs = {}; + const span = { spanContext: () => ({ traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }), + isRecording: () => true, setAttribute(k, v) { attrs[k] = v; } }; + const capture = new SpeechAttachments(a.manager, span, generateTraceConfig(), undefined); + const { ELEVENLABS_STREAM_MAX_BYTES } = require('../dist/src/attachments.js'); + capture.chunk(new Uint8Array(ELEVENLABS_STREAM_MAX_BYTES)); + capture.chunk(new Uint8Array(1)); + capture.close('complete'); + await a.manager.flush(); + assert.equal(a.uploaded.length, 0); assert.equal(attrs['elevenlabs.attachment.output'], 'too_large'); + assert.equal(a.manager.status().ownedBytes, 0); +}); diff --git a/typescript/packages/traceai_elevenlabs/test/fixture.cjs b/typescript/packages/traceai_elevenlabs/test/fixture.cjs index f2b057374..3230b2eb8 100644 --- a/typescript/packages/traceai_elevenlabs/test/fixture.cjs +++ b/typescript/packages/traceai_elevenlabs/test/fixture.cjs @@ -137,11 +137,13 @@ async function fixture(SDK, options = {}, media = {}) { res.flushHeaders(); await delay(mode === "slow" ? 120 : 10); if (res.destroyed) return; - res.write(AUDIO.subarray(0, 4)); + const audio = media.audio ?? AUDIO; + const split = Math.min(4, audio.length); + res.write(audio.subarray(0, split)); await delay(mode === "slow" ? 120 : 10); if (res.destroyed) return; if (mode === "broken") res.destroy(); - else res.end(AUDIO.subarray(4)); + else res.end(audio.subarray(split)); } catch { res.destroy(); } From 907c107a4d5cae99dfd325052206e5ca726d8758 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 11 Nov 2026 02:51:36 -0800 Subject: [PATCH 3/8] feat(livekit): upload an existing LiveKit Agents session recording New Node-only subpath @traceai/fi-instrumentation-livekit/agents: bindLiveKitSessionRecording(session, ctx, { manager, agents, ownedRecordingFiles, finalizationTimeoutMs, capture }). - Never enables, starts, stops or deletes a recording. Chains the SDK's own JobContext._onSessionEnd, which runs after AgentSession.close() closed the RecorderIO and after the application's onSessionEnd, and always runs the SDK step afterwards. - Only the job's primary session, its own closed recorder and the fixed audio.ogg in the job's session directory qualify; the file goes through the core's owned-recording lease (identify, no-follow open, identity bind, per-slice check) as tier single, direction mixed. - Deny wins: core policy via public preflight at bind and again at finalization, LiveKit project/session redaction, the LIVEKIT_TELEMETRY_ALLOW_PII kill switch and session audio off. - Exact supported versions 1.8.0, 1.8.1, 1.9.0-1.9.5; others report unsupported_version. No package peer, so the lockfile is unchanged. - CI job runs the real SDK (floor 1.8.0, latest 1.9.5) with its real RecorderIO writing Ogg/Opus; the workflow gains workflow_dispatch. - Pipecat: a throwing server-identity callback reports source_unavailable without interrupting client callbacks. Co-Authored-By: Claude Opus 5.5 --- .../workflows/livekit-typescript-tests.yml | 38 +++ typescript/packages/traceai_livekit/README.md | 29 ++ .../packages/traceai_livekit/package.json | 12 + .../packages/traceai_livekit/src/agents.ts | 266 ++++++++++++++++++ .../traceai_livekit/test/agents-fixture.cjs | 139 +++++++++ .../traceai_livekit/test/agents-recording.cjs | 198 +++++++++++++ .../traceai_pipecat/test/recording.test.cjs | 17 ++ 7 files changed, 699 insertions(+) create mode 100644 typescript/packages/traceai_livekit/src/agents.ts create mode 100644 typescript/packages/traceai_livekit/test/agents-fixture.cjs create mode 100644 typescript/packages/traceai_livekit/test/agents-recording.cjs diff --git a/.github/workflows/livekit-typescript-tests.yml b/.github/workflows/livekit-typescript-tests.yml index a546d66b9..a7b111fcc 100644 --- a/.github/workflows/livekit-typescript-tests.yml +++ b/.github/workflows/livekit-typescript-tests.yml @@ -9,6 +9,7 @@ on: - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/pnpm-lock.yaml' - '.github/workflows/livekit-typescript-tests.yml' + workflow_dispatch: permissions: contents: read @@ -85,3 +86,40 @@ jobs: env "LIVEKIT_COMPAT_DIR=$RUNNER_TEMP/livekit-sdk" \ "LIVEKIT_INSTRUMENTATION_DIR=$RUNNER_TEMP/livekit-sdk/node_modules/@traceai/fi-instrumentation-livekit" \ "$(command -v node)" --test packages/traceai_livekit/test/readme-shutdown.cjs + + agents-recording: + name: LiveKit Agents recording binding (${{ matrix.agents }}) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # Floor and latest of the exact versions the binding supports. + agents: ['1.8.0', '1.9.5'] + defaults: + run: + shell: bash + working-directory: typescript + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22.23.4' + - name: Install locked dependencies and the real Agents SDK + run: | + corepack enable + pnpm --filter @traceai/fi-instrumentation-livekit... install --frozen-lockfile --ignore-scripts + npm install --prefix "$RUNNER_TEMP/livekit-agents" --ignore-scripts --no-audit --no-fund \ + "@livekit/agents@${{ matrix.agents }}" "zod@^3.25.76" + - name: Build + run: | + pnpm run build:deps + pnpm --filter @traceai/fi-instrumentation-livekit build + - name: Real Agents session recording with loopback-only networking + run: | + sudo unshare --net -- sh -c 'ip link set lo up; exec "$@"' sh \ + env "LIVEKIT_AGENTS_DIR=$RUNNER_TEMP/livekit-agents" "$(command -v node)" \ + --test packages/traceai_livekit/test/agents-recording.cjs + git diff --exit-code -- pnpm-lock.yaml diff --git a/typescript/packages/traceai_livekit/README.md b/typescript/packages/traceai_livekit/README.md index 673f3f35c..902b90e0b 100644 --- a/typescript/packages/traceai_livekit/README.md +++ b/typescript/packages/traceai_livekit/README.md @@ -77,6 +77,35 @@ const room = new client.Room(); `livekit-client` implements room methods on each instance. If you already have a room, use `instrumentation.instrumentRoom(room)` instead. Repeated registration does not add duplicate spans. Existing instrumented rooms resume tracing when you call `enable()` after `disable()`. Manually registered SDK modules should also be registered again to restore their participant and media hooks. +## Upload an existing LiveKit Agents recording (optional) + +LiveKit Agents for Node can record a session to `audio.ogg` in the job's session directory when you start the session with recording on (`session.start({ ..., record: true })`, or the project's LiveKit Cloud setting). This package can upload **that existing file** to Future AGI at session end. It never turns recording on, never starts or stops the recorder, and never deletes the file. Attachments stay off unless you enable them on your provider. + +Supported `@livekit/agents` versions: 1.8.0, 1.8.1 and 1.9.0 to 1.9.5. Any other version reports `unsupported_version` and uploads nothing. Linux and macOS only. + +```javascript +const agents = require("@livekit/agents"); +const { bindLiveKitSessionRecording } = require("@traceai/fi-instrumentation-livekit/agents"); + +// Inside your agent entry, after `await session.start({ agent, room: ctx.room, record: true })`: +const recording = bindLiveKitSessionRecording(session, ctx, { + manager: provider.attachments, // the same manager your Future AGI provider created + agents, // the @livekit/agents module your application uses + ownedRecordingFiles: true, // explicit opt-in; anything else reports "disabled" + finalizationTimeoutMs: 30000, // upload deadline at session end, at most 30 s +}); +// recording.status is "session_bound" when it will run at session end. +``` + +How it works: + +- Bind right after `session.start(...)` returns. The session's `agent_session` span is captured then, and the upload is linked to it. Point LiveKit's telemetry at your Future AGI provider (`agents.telemetry.setTracerProvider(provider)`) so that span is exported; if the span is not recording, the binding reports `correlation_unavailable`. +- At session end, LiveKit closes the recorder, runs your `onSessionEnd`, then calls its own session-end step. The binding runs at the start of that step, uploads `audio.ogg` (mixed user and agent audio, up to the attachment service's single-upload limit), and then lets LiveKit continue. If your `onSessionEnd` moves or deletes the file, the binding reports `source_unavailable`. +- Only the job's primary session, its own recorder and the fixed `audio.ogg` in the job's session directory qualify. A symlink, a hard link, or a file swapped during upload fails closed. +- It uploads nothing when attachments are off, when your trace config hides inputs or outputs or redacts PII, when the server policy denies it, when LiveKit project or session redaction is on, when `LIVEKIT_TELEMETRY_ALLOW_PII` is false, or when session audio recording is off. +- Give your worker a shutdown grace longer than `finalizationTimeoutMs` plus your own session-end callbacks. A process killed before the upload finishes loses it. +- `recording.status` is content-free. `validating` means the file reached Future AGI and is being checked; it is not yet playable. `recording.close()` detaches the binding without touching the session. + ## Control sensitive data Tokens, URL credentials, URL paths, query strings, and raw audio/video are not captured as span attributes. Room names, participant identifiers, track names, publish options, and SDK error messages can still contain sensitive information. Use your collector's attribute filtering for identifying metadata, and configure content/error redaction when constructing the instrumentor: diff --git a/typescript/packages/traceai_livekit/package.json b/typescript/packages/traceai_livekit/package.json index feb18db86..a84d5e3e2 100644 --- a/typescript/packages/traceai_livekit/package.json +++ b/typescript/packages/traceai_livekit/package.json @@ -9,6 +9,18 @@ "types": "./dist/src/index.d.ts", "import": "./dist/src/index.js", "require": "./dist/src/index.js" + }, + "./agents": { + "types": "./dist/src/agents.d.ts", + "import": "./dist/src/agents.js", + "require": "./dist/src/agents.js" + } + }, + "typesVersions": { + "*": { + "agents": [ + "dist/src/agents.d.ts" + ] } }, "files": [ diff --git a/typescript/packages/traceai_livekit/src/agents.ts b/typescript/packages/traceai_livekit/src/agents.ts new file mode 100644 index 000000000..8ad84a603 --- /dev/null +++ b/typescript/packages/traceai_livekit/src/agents.ts @@ -0,0 +1,266 @@ +/** + * Opt-in upload of a bound LiveKit Agents (Node) session's EXISTING local recording. + * + * Node only; import it from `@traceai/fi-instrumentation-livekit/agents`, never from the root entry. + * This binding never enables, starts or stops a recording, and never deletes the file. It supports + * only the exact `@livekit/agents` versions the component suite qualifies; the private lifecycle + * seams it uses are confined to this file and fail closed on any other version. + */ +import { lstat } from "node:fs/promises"; +import * as path from "node:path"; +import { trace, type Context } from "@opentelemetry/api"; +import type { + AttachmentAdmission, + AttachmentCapturePolicy, + AttachmentCompletion, + AttachmentCorrelation, + AttachmentManager, + AttachmentOwnedRecordingHandle, + AttachmentRecordingFileScope, +} from "@traceai/fi-core/attachments"; +import { createRecordingFileScope } from "@traceai/fi-core/attachments/node"; + +/** `@livekit/agents` versions whose session-end ordering and recorder seams were qualified. */ +export const LIVEKIT_AGENTS_SUPPORTED_VERSIONS: readonly string[] = Object.freeze([ + "1.8.0", "1.8.1", "1.9.0", "1.9.1", "1.9.2", "1.9.3", "1.9.4", "1.9.5", +]); +const RECORDING_FILE = "audio.ogg"; +const MAX_TIMEOUT_MS = 30_000; +const PII_FALSY = new Set(["0", "false", "no", "off"]); + +/** Content-free outcome. `validating` means the file reached quarantine, NOT that it is playable. */ +export type LiveKitSessionRecordingStatus = + | "session_bound" + | "disabled" + | "unsupported_version" + | "conflict" + | AttachmentAdmission + | AttachmentCompletion["status"]; + +export interface LiveKitSessionRecordingOptions { + /** The provider's shared manager (`provider.attachments`). This binding never creates another. */ + manager: AttachmentManager; + /** The `@livekit/agents` module your application uses (`import * as agents from "@livekit/agents"`). */ + agents: { version?: unknown; JobContext?: unknown; voice?: { AgentSession?: unknown } }; + /** Must be exactly `true` to upload an existing recording. Anything else reports `disabled`. */ + ownedRecordingFiles: boolean; + /** Upload deadline at session end, at most 30 000 ms (default 30 000). */ + finalizationTimeoutMs?: number; + /** Additional restrictions only; a binding never relaxes the manager's policy. */ + capture?: AttachmentCapturePolicy; +} + +interface RecorderLike { readonly recording?: unknown; readonly outputPath?: unknown } +interface SessionLike { + rootSpanContext?: Context; + _recorderIO?: RecorderLike; + _redactionEnabled?: unknown; + sessionOptions?: { recordingOptions?: { audio?: unknown; redaction?: unknown } }; +} +interface JobLike { + _primaryAgentSession?: unknown; + readonly sessionDirectory?: unknown; + _redactionEnabled?: unknown; + readonly job?: { enableRedaction?: unknown }; + makeSessionReport?: (session?: unknown) => { audioRecordingPath?: unknown }; + _onSessionEnd?: () => Promise; +} + +const BINDING = Symbol.for("traceai.livekit.agents.recording_binding.v1"); +const isFn = (value: unknown): value is (...args: unknown[]) => unknown => typeof value === "function"; + +function nativeDenial(session: SessionLike, job: JobLike, env: NodeJS.ProcessEnv): LiveKitSessionRecordingStatus | undefined { + // Native integration restrictions come first, before any recording-path inspection. + const pii = env.LIVEKIT_TELEMETRY_ALLOW_PII; + if (pii !== undefined && PII_FALSY.has(pii.trim().toLowerCase())) return "policy_denied"; + const options = session.sessionOptions?.recordingOptions; + // Text redaction never sanitizes a binary recording; any redaction requirement denies it. + if (options?.redaction || session._redactionEnabled !== false || job._redactionEnabled !== false || job.job?.enableRedaction) + return "redaction_unavailable"; + if (options?.audio !== true) return "policy_denied"; + return undefined; +} + +/** + * The binding's state is content-free. `close()` only detaches this adapter: it never stops the + * application's session, shuts down the shared manager, or enables or deletes a recording. + */ +export class LiveKitSessionRecording { + status: LiveKitSessionRecordingStatus = "session_bound"; + private session?: SessionLike; + private job?: JobLike; + private correlation?: AttachmentCorrelation; + private root?: string; + private wrapper?: () => Promise; + private hadOwnHook = false; + private ownHook?: () => Promise; + private handle?: AttachmentOwnedRecordingHandle; + private finalizing?: Promise; + private closed = false; + /** Set when the deadline wins; a late step must not open, upload or keep anything. */ + private stopped = false; + + /** @internal */ + constructor( + private readonly manager: AttachmentManager, + private readonly timeoutMs: number, + private readonly capture: AttachmentCapturePolicy | undefined, + private readonly env: NodeJS.ProcessEnv, + ) {} + + /** @internal */ + attach(session: SessionLike, job: JobLike, correlation: AttachmentCorrelation, root: string): void { + this.session = session; this.job = job; this.correlation = correlation; this.root = root; + const original = job._onSessionEnd!; + this.hadOwnHook = Object.prototype.hasOwnProperty.call(job, "_onSessionEnd"); + this.ownHook = this.hadOwnHook ? original : undefined; + const wrapper = async () => { + // The SDK awaits this after AgentSession.close() (recorder closed) and the application's + // onSessionEnd, and before its own report upload. Its own work always still runs. + try { await this.finalize(); } finally { await original.call(job); } + }; + this.wrapper = wrapper; + job._onSessionEnd = wrapper; + (job as Record)[BINDING] = this; + } + + /** @internal Same arguments as a previous bind. */ + same(manager: AttachmentManager, session: unknown, timeoutMs: number, capture: AttachmentCapturePolicy | undefined): boolean { + return manager === this.manager && session === this.session && timeoutMs === this.timeoutMs && + JSON.stringify(capture ?? {}) === JSON.stringify(this.capture ?? {}); + } + + /** The bounded transfer handle, once a file was handed to the manager. */ + get recording(): AttachmentOwnedRecordingHandle | undefined { return this.handle; } + + /** At most once; bounded by the finalization deadline. Never rejects. */ + finalize(): Promise { + this.finalizing ??= this.run(); + return this.finalizing; + } + + private async run(): Promise { + if (this.closed || this.status !== "session_bound") return this.status; + let scope: AttachmentRecordingFileScope | undefined; + let timer: ReturnType | undefined; + const deadline = Date.now() + this.timeoutMs; + try { + const outcome = await Promise.race([ + this.capture_(s => { scope = s; }, deadline), + new Promise(resolve => { timer = setTimeout(() => resolve("incomplete"), this.timeoutMs); }), + ]); + this.status = outcome; + } catch { + // Never log the report, path or error text. + this.status = "source_unavailable"; + } finally { + this.stopped = true; + if (timer) clearTimeout(timer); + if (this.handle && this.status !== "validating") this.handle.cancel(); + if (scope) await scope.close().catch(() => undefined); + this.close(); + } + return this.status; + } + + private async capture_(keep: (scope: AttachmentRecordingFileScope) => void, deadline: number): Promise { + const session = this.session!, job = this.job!, root = this.root!; + const denial = nativeDenial(session, job, this.env); + if (denial) return denial; + // The report path is NOT authority. Only the job's primary session, its SDK recorder (closed) + // and the SDK's fixed filename under the bound job's session directory are eligible. + if (job._primaryAgentSession !== session || job.sessionDirectory !== root) return "source_unavailable"; + const recorder = session._recorderIO; + const expected = path.join(root, RECORDING_FILE); + if (!recorder || recorder.recording !== false || recorder.outputPath !== expected) return "source_unavailable"; + const report = job.makeSessionReport!(session); + if (report?.audioRecordingPath !== expected) return "source_unavailable"; + if (this.stopped) return "incomplete"; + // Policy is checked again before any stat or open of the session directory. + const ticket = this.manager.preflight(this.correlation, { mimeType: "audio/ogg", direction: "mixed", maxBytes: 1, + operation: "import_existing", sourceKind: "OwnedRecordingFile", purpose: "conversation_recording", capture: this.capture }); + ticket.release(); + if (!ticket.admitted) return ticket.reason; + const info = await lstat(expected); + if (this.stopped) return "incomplete"; + if (!info.isFile() || info.nlink !== 1 || (typeof process.getuid === "function" && info.uid !== process.getuid())) return "source_unavailable"; + const scope = await createRecordingFileScope(root, { enabled: true }); + if (this.stopped) { await scope.close(); return "incomplete"; } + keep(scope); + const identity = await scope.identify(RECORDING_FILE); + if (this.stopped) return "incomplete"; + const handle = this.manager.attachOwnedRecording(this.correlation, scope.source(RECORDING_FILE, identity), { + mimeType: "audio/ogg", direction: "mixed", purpose: "conversation_recording", capture: this.capture, + }); + this.handle = handle; + if (handle.reason !== "queued") return handle.reason; + const result = await handle.wait(Math.max(1, deadline - Date.now())); + return result.status; + } + + /** Detach. Restores the SDK hook only if nobody replaced this binding's wrapper since. */ + close(): void { + if (this.closed) return; + this.closed = true; + const job = this.job; + if (job && this.wrapper && job._onSessionEnd === this.wrapper) { + if (this.hadOwnHook) job._onSessionEnd = this.ownHook; + else delete job._onSessionEnd; + } + if (job && (job as Record)[BINDING] === this) delete (job as Record)[BINDING]; + this.session = undefined; this.job = undefined; this.root = undefined; this.wrapper = undefined; this.ownHook = undefined; + } +} + +/** + * Bind AFTER `await session.start(...)`. The session span is snapshotted now, before the SDK ends it. + * At session end the SDK's own `_onSessionEnd` is chained, not replaced: it runs after + * `AgentSession.close()` and your `onSessionEnd`, and before LiveKit's report upload. If your + * `onSessionEnd` deletes or replaces `audio.ogg`, this binding reports `source_unavailable`. + * + * Give the worker a shutdown grace longer than this deadline plus your own callbacks. Hard process + * termination loses the upload. + */ +export function bindLiveKitSessionRecording(session: unknown, jobContext: unknown, options: LiveKitSessionRecordingOptions): LiveKitSessionRecording { + if (!options || typeof options.ownedRecordingFiles !== "boolean") throw new TypeError("ownedRecordingFiles must be an explicit boolean"); + const timeoutMs = options.finalizationTimeoutMs ?? MAX_TIMEOUT_MS; + if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > MAX_TIMEOUT_MS) + throw new RangeError("finalizationTimeoutMs must be within (0, 30000]"); + if (options.capture?.newRecording) throw new TypeError("a session upload never creates a recording"); + const job = jobContext as JobLike | undefined; + const previous = job && typeof job === "object" ? (job as Record)[BINDING] : undefined; + const binding = new LiveKitSessionRecording(options.manager, timeoutMs, options.capture, process.env); + if (previous instanceof LiveKitSessionRecording) { + if (previous.same(options.manager, session, timeoutMs, options.capture) && options.ownedRecordingFiles) return previous; + binding.status = "conflict"; + return binding; + } + const fail = (status: LiveKitSessionRecordingStatus) => { binding.status = status; binding.close(); return binding; }; + if (options.ownedRecordingFiles !== true) return fail("disabled"); + try { + const agents = options.agents; + if (!agents || typeof agents.version !== "string" || !LIVEKIT_AGENTS_SUPPORTED_VERSIONS.includes(agents.version)) return fail("unsupported_version"); + const JobContext = agents.JobContext, AgentSession = agents.voice?.AgentSession; + if (!isFn(JobContext) || !isFn(AgentSession) || !(jobContext instanceof JobContext) || !(session instanceof AgentSession)) return fail("unsupported_version"); + const s = session as SessionLike; + if (!isFn(job!._onSessionEnd) || !isFn(job!.makeSessionReport)) return fail("unsupported_version"); + // Never fall back to an ambient span from an unrelated late callback. + const span = s.rootSpanContext ? trace.getSpan(s.rootSpanContext) : undefined; + if (!span || !span.isRecording()) return fail("correlation_unavailable"); + const correlation = options.manager.correlation(span); + // Public admission only; the file is neither stat'ed nor opened while bound. + const ticket = options.manager.preflight(correlation, { mimeType: "audio/ogg", direction: "mixed", maxBytes: 1, + operation: "import_existing", sourceKind: "OwnedRecordingFile", purpose: "conversation_recording", capture: options.capture }); + ticket.release(); + if (!ticket.admitted) return fail(ticket.reason); + if (job!._primaryAgentSession !== session) return fail("source_unavailable"); + const denial = nativeDenial(s, job!, process.env); + if (denial) return fail(denial); + const root = job!.sessionDirectory; + if (typeof root !== "string" || !path.isAbsolute(root)) return fail("source_unavailable"); + binding.attach(s, job!, correlation, root); + } catch { + return fail("source_unavailable"); + } + return binding; +} diff --git a/typescript/packages/traceai_livekit/test/agents-fixture.cjs b/typescript/packages/traceai_livekit/test/agents-fixture.cjs new file mode 100644 index 000000000..52416a6eb --- /dev/null +++ b/typescript/packages/traceai_livekit/test/agents-fixture.cjs @@ -0,0 +1,139 @@ +/* Loopback attachment control plane plus real LiveKit Agents (Node) objects. Synthetic data only. */ +const http = require('node:http'); +const fs = require('node:fs'); +const path = require('node:path'); +const { once } = require('node:events'); +const { randomUUID } = require('node:crypto'); +const { AttachmentManager, createAttachmentHttpTransport } = require('@traceai/fi-core/attachments'); + +const ids = [1, 2, 3].map(n => `10000000-0000-4000-8000-00000000000${n}`); +// Sessions of at least 1 h, so the core's near-expiry renewal never adds a bootstrap here. +const expiry = () => new Date(Date.now() + 3600000).toISOString(); +const SLICE = 256 * 1024; + +async function readBody(body) { + if (body === undefined || typeof body === 'string' || body instanceof Uint8Array) return body; + if (typeof Blob !== 'undefined' && body instanceof Blob) return Buffer.from(await body.arrayBuffer()); + // Ranged owned-file source: check before every slice and after the last, like the Node client. + const out = Buffer.alloc(body.size); + for (let offset = 0; offset < body.size; offset += SLICE) { + await body.check(); + const chunk = await body.read(offset, Math.min(SLICE, body.size - offset)); + Buffer.from(chunk).copy(out, offset); + } + await body.check(); + return out; +} + +async function attachmentFixture(config = {}, serverPolicy = {}) { + const calls = [], uploaded = [], requests = [], uploads = new Map(); + let hold; const held = []; + const policy = { version: 'v1', enabled: true, allowed_sources: ['Bytes', 'Blob', 'OwnedRecordingFile'], allowed_modalities: ['audio'], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, + record_audio: false, record_video: false, ...serverPolicy }; + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const chunk of req) chunks.push(chunk); + const bytes = Buffer.concat(chunks); + calls.push({ method: req.method, path: req.url }); + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('/attachment-sessions')) { + const body = JSON.parse(bytes); + res.end(JSON.stringify({ schema_version: 1, scope: { organization_id: ids[0], workspace_id: ids[1], project_id: ids[2] }, + capture_session_id: body.capture_session_id, policy, limits: { max_upload_bytes: 8388608, segment_bytes: 8388608, + segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, + active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, + initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, + playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }, + expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); + } else if (req.url.endsWith('/uploads')) { + const body = JSON.parse(bytes); requests.push(body); + const upload = { upload_id: randomUUID(), attachment_id: body.attachment_id, occurrence_id: body.occurrence_id, + state: 'pending_upload', reserved_bytes: body.declared_bytes, expires_at: expiry(), + transfer_mode: 'authenticated_stream', write_capability: 'synthetic-write-capability' }; + uploads.set(upload.upload_id, upload); res.statusCode = 201; res.end(JSON.stringify(upload)); + } else { + const id = req.url.split('/').at(-2); + if (!uploads.has(id) || req.headers.authorization !== 'Bearer synthetic-write-capability') { res.statusCode = 403; res.end('{}'); return; } + if (req.url.endsWith('/content')) { + if (hold) { const gate = hold; hold = undefined; held.push(id); await gate; } + uploaded.push(bytes); res.end(JSON.stringify({ upload_id: id, state: 'pending_upload', reason: 'queued' })); + } else { + res.statusCode = 202; res.end(JSON.stringify({ upload_id: id, state: 'validating', reason: 'binding_pending' })); + } + } + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const client = async req => { + const body = await readBody(req.body); + return new Promise((resolve, reject) => { + const r = http.request(`http://127.0.0.1:${server.address().port}${new URL(req.url).pathname}`, + { method: req.method, headers: req.headers, signal: req.signal }, response => { + const chunks = []; response.on('data', c => chunks.push(c)); + response.on('end', () => resolve({ status: response.statusCode, body: Buffer.concat(chunks) })); + }); + r.on('error', reject); r.end(body); + }); + }; + const manager = new AttachmentManager({ enabled: true, projectName: 'synthetic-voice', projectType: 'observe', + transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic' }, client }), ...config }); + await manager.ready(); + return { manager, calls, uploaded, requests, held, + /** Hold the next content PUT until the returned release() is called. */ + holdNext() { let release; hold = new Promise(r => { release = r; }); return () => release(); }, + async close() { await manager.shutdown(1000); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } }; +} + +/** Real `@livekit/agents` objects from an installed copy (LIVEKIT_AGENTS_DIR). */ +function loadAgents(dir) { + const main = require.resolve('@livekit/agents', { paths: [dir] }); + const agents = require(main); + try { agents.initializeLogger({ pretty: false, level: 'error' }); } catch { /* already initialized */ } + // RecorderIO is internal to the SDK; load the real class from the installed copy's dist. + const dist = main.slice(0, main.lastIndexOf(`${path.sep}dist${path.sep}`) + 6); + const { RecorderIO } = require(path.join(dist, 'voice', 'recorder_io', 'index.cjs')); + const rtc = require(require.resolve('@livekit/rtc-node', { paths: [path.dirname(main)] })); + return { agents, RecorderIO, AudioFrame: rtc.AudioFrame, rtc }; +} + +let jobs = 0; +/** + * A real JobContext and AgentSession in the state `AgentSession.start({ record: true })` leaves them: + * primary session, audio recording option on, session span context, and the SDK's own RecorderIO + * writing `${ctx.sessionDirectory}/audio.ogg`. + */ +async function startedSession(sdk, tracer, { redaction = false, audio = true, record = true } = {}) { + const { agents, RecorderIO } = sdk; + const { EventEmitter } = require('node:events'); + const room = new EventEmitter(); room.isConnected = false; + const jobId = `AJ_synthetic_${process.pid}_${++jobs}`; + const info = { job: { id: jobId, room: { name: 'synthetic-room', sid: 'RM_synthetic' }, enableRecording: false, enableRedaction: redaction, agentName: '' }, + url: 'ws://127.0.0.1:9', acceptArguments: {}, token: 'synthetic', workerId: 'synthetic' }; + const ctx = new agents.JobContext({ userData: {} }, info, room, () => {}, () => {}, {}); + const session = new agents.voice.AgentSession({}); + session.sessionOptions.recordingOptions = { ...session.sessionOptions.recordingOptions, audio, redaction: false }; + ctx._primaryAgentSession = session; + const span = tracer.startSpan('agent_session'); + const { trace, ROOT_CONTEXT } = require('@opentelemetry/api'); + session.rootSpanContext = trace.setSpan(ROOT_CONTEXT, span); + let recorder; + if (record) { + class In extends agents.voice.AudioInput {} + class Out extends agents.voice.AudioOutput { + constructor() { super(48000); } + flush() { super.flush(); setTimeout(() => this.onPlaybackFinished({ playbackPosition: 0.5, interrupted: false }), 20); } + clearBuffer() {} + } + recorder = new RecorderIO({ agentSession: session }); + recorder.recordInput(new In()); const out = recorder.recordOutput(new Out()); + await recorder.start(path.join(ctx.sessionDirectory, 'audio.ogg')); + session._recorderIO = recorder; + const n = 24000, data = new Int16Array(n); + for (let i = 0; i < n; i++) data[i] = Math.round(8000 * Math.sin(2 * Math.PI * 1000 * i / 48000)); + await out.captureFrame(new sdk.AudioFrame(data, 48000, 1, n)); out.flush(); + await new Promise(resolve => setTimeout(resolve, 700)); + } + return { ctx, session, span, recorder, file: path.join(ctx.sessionDirectory, 'audio.ogg'), + cleanup() { try { span.end(); } catch { /* ended */ } fs.rmSync(ctx.sessionDirectory, { recursive: true, force: true }); } }; +} + +module.exports = { attachmentFixture, loadAgents, startedSession }; diff --git a/typescript/packages/traceai_livekit/test/agents-recording.cjs b/typescript/packages/traceai_livekit/test/agents-recording.cjs new file mode 100644 index 000000000..cfa3a871e --- /dev/null +++ b/typescript/packages/traceai_livekit/test/agents-recording.cjs @@ -0,0 +1,198 @@ +/* + * V04: the session-recording binding against a REAL installed `@livekit/agents` (LIVEKIT_AGENTS_DIR): + * real JobContext._onSessionEnd and makeSessionReport, real AgentSession, and the SDK's own RecorderIO + * writing a real Ogg/Opus file. The control plane is a loopback HTTP double. Synthetic data only. + */ +const { test, after } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); +const { BasicTracerProvider } = require('@opentelemetry/sdk-trace-base'); +const { attachmentFixture, loadAgents, startedSession } = require('./agents-fixture.cjs'); + +const dir = process.env.LIVEKIT_AGENTS_DIR; +if (!dir) throw new Error('LIVEKIT_AGENTS_DIR must point at an install of @livekit/agents'); +const pkg = process.env.LIVEKIT_INSTRUMENTATION_DIR || path.join(__dirname, '..'); +const { bindLiveKitSessionRecording, LIVEKIT_AGENTS_SUPPORTED_VERSIONS, LiveKitSessionRecording } = + require(path.join(pkg, 'dist', 'src', 'agents.js')); +const sdk = loadAgents(dir); +const tracer = new BasicTracerProvider().getTracer('synthetic'); +// The RTC runtime the SDK loads keeps native threads alive until disposed. +after(async () => { await sdk.rtc.dispose?.(); }); +const ffprobe = (() => { try { execFileSync('ffprobe', ['-version'], { stdio: 'ignore' }); return true; } catch { return false; } })(); + +/** What the SDK's finalizeSession does: AgentSession.close() closes the recorder, then ctx._onSessionEnd(). */ +async function endSession(s, { closeRecorder = true } = {}) { + if (closeRecorder && s.recorder) await s.recorder.close(); + s.span.end(); + s.session._recordedEvents = ['synthetic-event']; + await s.ctx._onSessionEnd(); +} + +test(`real @livekit/agents ${sdk.agents.version} is a qualified version`, () => { + assert.ok(LIVEKIT_AGENTS_SUPPORTED_VERSIONS.includes(sdk.agents.version)); + assert.equal(typeof sdk.agents.JobContext.prototype._onSessionEnd, 'function'); + assert.equal(typeof sdk.agents.JobContext.prototype.makeSessionReport, 'function'); +}); + +test('uploads the SDK-written audio.ogg once, byte-identical, after the recorder closes; SDK hook still runs', async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + assert.equal(binding.status, 'session_bound'); + assert.equal(a.requests.length, 0, 'binding opens nothing'); + await endSession(s); + assert.equal(binding.status, 'validating'); + const file = fs.readFileSync(s.file); + assert.equal(a.uploaded.length, 1); assert.ok(a.uploaded[0].equals(file)); + const request = a.requests[0]; + assert.equal(request.mime_type, 'audio/ogg'); assert.equal(request.direction, 'mixed'); + assert.equal(request.source_kind, 'OwnedRecordingFile'); assert.equal(request.purpose, 'conversation_recording'); + assert.equal(request.declared_bytes, file.length); + assert.equal(request.association.parent.span_id, s.span.spanContext().spanId); + assert.equal(request.association.parent.trace_id.replaceAll('-', ''), s.span.spanContext().traceId); + assert.equal('channel' in request || 'role' in request || 'recording_id' in request, false); + assert.deepEqual(s.session._recordedEvents, [], 'the SDK _onSessionEnd ran after the binding'); + assert.equal(Object.prototype.hasOwnProperty.call(s.ctx, '_onSessionEnd'), false, 'hook restored'); + assert.ok(fs.existsSync(s.file), 'the binding never deletes the recording'); + if (ffprobe) { + const out = path.join(s.ctx.sessionDirectory, 'uploaded-copy.ogg'); fs.writeFileSync(out, a.uploaded[0]); + const probe = execFileSync('ffprobe', ['-v', 'error', '-show_entries', 'stream=codec_name,channels', '-of', 'csv=p=0', out]).toString().trim(); + assert.match(probe, /^opus,[12]$/); + } +}); + +test('a recorder still recording at finalization is source_unavailable; the SDK then closes it as before', async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + await endSession(s, { closeRecorder: false }); + assert.equal(binding.status, 'source_unavailable'); assert.equal(a.requests.length, 0); + assert.equal(s.recorder.recording, false, 'SDK _onSessionEnd closed the recorder'); +}); + +for (const [name, options, expected] of [ + ['ownedRecordingFiles false', { ownedRecordingFiles: false }, 'disabled'], + ['unsupported agents version', { agents: { ...sdk.agents, version: '1.7.0' } }, 'unsupported_version'], + ['foreign session object', { foreignSession: true }, 'unsupported_version'], +]) test(`${name} binds nothing (${expected})`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer, { record: false }); t.after(() => s.cleanup()); + const session = options.foreignSession ? {} : s.session; + const binding = bindLiveKitSessionRecording(session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true, ...options }); + assert.equal(binding.status, expected); + assert.equal(Object.prototype.hasOwnProperty.call(s.ctx, '_onSessionEnd'), false); +}); + +for (const [name, setup, expected, config, policy] of [ + ['attachments disabled', {}, 'disabled', { enabled: false }], + ['environment kill switch', {}, 'disabled', { environment: () => ({ FI_ATTACHMENTS_ENABLED: 'false' }) }], + ['core PII redaction', {}, 'redaction_unavailable', { traceConfig: { piiRedaction: true } }], + ['core hidden input', {}, 'policy_denied', { traceConfig: { hideInputs: true } }], + ['core hidden output', {}, 'policy_denied', { traceConfig: { hideOutputs: true } }], + ['server requires media redaction', {}, 'redaction_unavailable', {}, { require_media_redaction: true }], + ['server disallows owned files', {}, 'policy_denied', {}, { allowed_sources: ['Bytes'] }], + ['LiveKit project redaction', { redaction: true }, 'redaction_unavailable'], + ['session audio recording off', { audio: false }, 'policy_denied'], +]) test(`${name}: denied at bind (${expected}), no stat, no hook, no traffic`, async t => { + const a = await attachmentFixture(config, policy); t.after(() => a.close()); + const s = await startedSession(sdk, tracer, setup); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + assert.equal(binding.status, expected); + assert.equal(Object.prototype.hasOwnProperty.call(s.ctx, '_onSessionEnd'), false); + await endSession(s); + await a.manager.flush(); + assert.equal(a.requests.length, 0); assert.equal(a.uploaded.length, 0); +}); + +test('LIVEKIT_TELEMETRY_ALLOW_PII=0 denies the recording (deny wins)', async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const saved = process.env.LIVEKIT_TELEMETRY_ALLOW_PII; process.env.LIVEKIT_TELEMETRY_ALLOW_PII = '0'; + t.after(() => { if (saved === undefined) delete process.env.LIVEKIT_TELEMETRY_ALLOW_PII; else process.env.LIVEKIT_TELEMETRY_ALLOW_PII = saved; }); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + assert.equal(binding.status, 'policy_denied'); + await endSession(s); assert.equal(a.requests.length, 0); +}); + +test('policy tightened after bind: finalization re-checks before any stat or open', async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + s.ctx._redactionEnabled = true; // e.g. the project mandated redaction mid-session + await endSession(s); + assert.equal(binding.status, 'redaction_unavailable'); assert.equal(a.requests.length, 0); +}); + +for (const [name, mutate] of [ + ['report absent (no recorder)', s => { s.session._recorderIO = undefined; }], + ['recording replaced by a symlink', s => { const other = path.join(s.ctx.sessionDirectory, 'other.ogg'); fs.renameSync(s.file, other); fs.symlinkSync(other, s.file); }], + ['recording hard-linked elsewhere', s => { fs.linkSync(s.file, path.join(s.ctx.sessionDirectory, 'alias.ogg')); }], + ['recording deleted by the application', s => { fs.rmSync(s.file); }], + ['recorder path outside the session directory', s => { Object.defineProperty(s.session._recorderIO, 'outputPath', { value: '/tmp/elsewhere/audio.ogg' }); }], + ['another primary session', s => { s.ctx._primaryAgentSession = {}; }], +]) test(`${name}: source_unavailable, nothing uploaded`, async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + assert.equal(binding.status, 'session_bound'); + await s.recorder.close(); + mutate(s); + s.span.end(); + try { await s.ctx._onSessionEnd(); } catch { /* the SDK's own hook may reject on a broken session */ } + assert.equal(binding.status, 'source_unavailable'); + await a.manager.flush(); assert.equal(a.uploaded.length, 0); +}); + +test('T-LS-05: the file swapped after identify and before transfer fails closed', async t => { + const a = await attachmentFixture({ workers: 1 }); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + // Occupy the single worker so the owned-file job waits in the queue after identify. + const release = a.holdNext(); + const blocker = a.manager.attach(s.span.spanContext(), { kind: 'Bytes', data: new Uint8Array([1, 2, 3]) }, { mimeType: 'audio/wav', direction: 'output' }); + assert.equal(blocker.admission, 'queued'); + await s.recorder.close(); s.span.end(); + const ending = endSession(s, { closeRecorder: false }); + while (!binding.recording) await new Promise(r => setTimeout(r, 5)); + const original = fs.readFileSync(s.file); + const swap = path.join(s.ctx.sessionDirectory, 'swap.ogg'); + fs.writeFileSync(swap, Buffer.alloc(original.length, 7)); fs.renameSync(swap, s.file); + release(); + await ending; + assert.equal(binding.status, 'source_unavailable'); + assert.equal(a.uploaded.filter(b => b.length === original.length).length, 0, 'swapped bytes never uploaded'); +}); + +test('deadline: a stalled transfer ends incomplete within the bound and the SDK hook still runs', async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true, finalizationTimeoutMs: 300 }); + const release = a.holdNext(); t.after(release); + const started = Date.now(); + await endSession(s); + const elapsed = Date.now() - started; + assert.equal(binding.status, 'incomplete'); + assert.ok(elapsed < 3000, `finalization took ${elapsed} ms`); + assert.deepEqual(s.session._recordedEvents, []); +}); + +test('bind is idempotent for identical options and reports conflict otherwise; close() restores the hook', async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const options = { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }; + const first = bindLiveKitSessionRecording(s.session, s.ctx, options); + assert.equal(bindLiveKitSessionRecording(s.session, s.ctx, options), first); + assert.equal(bindLiveKitSessionRecording(s.session, s.ctx, { ...options, finalizationTimeoutMs: 1000 }).status, 'conflict'); + assert.ok(first instanceof LiveKitSessionRecording); + first.close(); + assert.equal(Object.prototype.hasOwnProperty.call(s.ctx, '_onSessionEnd'), false); + await endSession(s); assert.equal(a.requests.length, 0); +}); + +test('invalid options throw synchronously and never touch the session', () => { + assert.throws(() => bindLiveKitSessionRecording({}, {}, { manager: {}, agents: sdk.agents }), TypeError); + assert.throws(() => bindLiveKitSessionRecording({}, {}, { manager: {}, agents: sdk.agents, ownedRecordingFiles: true, finalizationTimeoutMs: 30001 }), RangeError); + assert.throws(() => bindLiveKitSessionRecording({}, {}, { manager: {}, agents: sdk.agents, ownedRecordingFiles: true, capture: { newRecording: true } }), TypeError); +}); diff --git a/typescript/packages/traceai_pipecat/test/recording.test.cjs b/typescript/packages/traceai_pipecat/test/recording.test.cjs index cca5996dc..dd6a3ead1 100644 --- a/typescript/packages/traceai_pipecat/test/recording.test.cjs +++ b/typescript/packages/traceai_pipecat/test/recording.test.cjs @@ -40,6 +40,23 @@ for (const name of ['pipecat-floor', '@pipecat-ai/client-js']) { stop(); assert.equal(client.listenerCount('connected'), 0); }); + test(`${name}: a throwing identity callback never interrupts the client and reports source_unavailable`, async t => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + t.after(() => provider.shutdown()); + const { client } = createClient(require(name)); + let seen = 0; client.on('connected', () => { seen++; }); + const stop = observePipecatClient(client, { tracer: provider.getTracer('throwing-test'), + serverRecording: () => { throw new Error('synthetic-private-failure'); } }); + t.after(stop); await client.connect(); await client.disconnect(); + assert.equal(seen, 1); + const spans = exporter.getFinishedSpans(); assert.ok(spans.length > 0); + for (const s of spans) { + assert.equal(s.attributes['pipecat.server_recording.status'], 'source_unavailable'); + assert.deepEqual(s.links, []); assert.ok(!JSON.stringify({ a: s.attributes, e: s.events, st: s.status }).includes('synthetic-private-failure')); + } + }); + for (const [caseName, supplied] of [ ['missing', undefined], ['version', { ...identity(), schemaVersion: 2 }], ['zero trace', { ...identity(), parent: { ...identity().parent, traceId: '0'.repeat(32) } }], From 50cb99562a6c0ea860dc15ccab45228447cf7b3e Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 11 Nov 2026 03:33:59 -0800 Subject: [PATCH 4/8] test(livekit): close a still-running recorder in cleanup after a failed assertion A failing assertion used to leave the SDK RecorderIO running, which kept the test process alive until the job timeout. Co-Authored-By: Claude Opus 5.5 --- .../packages/traceai_livekit/test/agents-fixture.cjs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/typescript/packages/traceai_livekit/test/agents-fixture.cjs b/typescript/packages/traceai_livekit/test/agents-fixture.cjs index 52416a6eb..f39116f90 100644 --- a/typescript/packages/traceai_livekit/test/agents-fixture.cjs +++ b/typescript/packages/traceai_livekit/test/agents-fixture.cjs @@ -133,7 +133,12 @@ async function startedSession(sdk, tracer, { redaction = false, audio = true, re await new Promise(resolve => setTimeout(resolve, 700)); } return { ctx, session, span, recorder, file: path.join(ctx.sessionDirectory, 'audio.ogg'), - cleanup() { try { span.end(); } catch { /* ended */ } fs.rmSync(ctx.sessionDirectory, { recursive: true, force: true }); } }; + // Runs even when an assertion failed, so a still-running recorder cannot keep the process alive. + async cleanup() { + try { if (recorder?.recording) await recorder.close(); } catch { /* best effort */ } + try { span.end(); } catch { /* ended */ } + fs.rmSync(ctx.sessionDirectory, { recursive: true, force: true }); + } }; } module.exports = { attachmentFixture, loadAgents, startedSession }; From 457a1553d804dd9af7fc2108355ec06d71283f80 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 11 Nov 2026 03:45:18 -0800 Subject: [PATCH 5/8] docs(livekit): state the Agents recording binding is qualified on Linux The owned-file scope requires a canonical root; macOS's default temp directory sits under the /var symlink, so it fails closed there. Co-Authored-By: Claude Opus 5.5 --- typescript/packages/traceai_livekit/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typescript/packages/traceai_livekit/README.md b/typescript/packages/traceai_livekit/README.md index 902b90e0b..395dda9fb 100644 --- a/typescript/packages/traceai_livekit/README.md +++ b/typescript/packages/traceai_livekit/README.md @@ -81,7 +81,7 @@ const room = new client.Room(); LiveKit Agents for Node can record a session to `audio.ogg` in the job's session directory when you start the session with recording on (`session.start({ ..., record: true })`, or the project's LiveKit Cloud setting). This package can upload **that existing file** to Future AGI at session end. It never turns recording on, never starts or stops the recorder, and never deletes the file. Attachments stay off unless you enable them on your provider. -Supported `@livekit/agents` versions: 1.8.0, 1.8.1 and 1.9.0 to 1.9.5. Any other version reports `unsupported_version` and uploads nothing. Linux and macOS only. +Supported `@livekit/agents` versions: 1.8.0, 1.8.1 and 1.9.0 to 1.9.5. Any other version reports `unsupported_version` and uploads nothing. Qualified on Linux. On macOS the default session directory is under `/var`, which is a symlink, so the binding reports `source_unavailable` there. Windows is not supported. ```javascript const agents = require("@livekit/agents"); From 1ee199012dee8f1b6b388a2efa4a8d44ac6156e0 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:37:36 -0800 Subject: [PATCH 6/8] feat(voice): adopt the clamped stream reservation and cover session-end deny order - ElevenLabs: consumed streams now rely on the core's min(4 MiB, server single-upload limit) reservation, so servers whose max_upload_bytes is below 4 MiB admit them. Tests cover a 1 MiB server (real SDK floor and current), the exact-limit / limit+1 boundary, and the owned-budget concurrency (7 open 4 MiB streams, the 8th queue_full). README states the real concurrency instead of "many". - LiveKit Agents binding: the session-end policy re-check now runs first, before the SDK report is built and before any stat or open. New tests watch the session directory's fs calls and assert none happen before a deny (five tightening paths, with and without the file), plus a control, and reject a recording owned by another user. Co-Authored-By: Claude Opus 5.5 --- .../packages/traceai_elevenlabs/README.md | 2 +- .../traceai_elevenlabs/src/attachments.ts | 8 +- .../test/attachments-fixture.cjs | 4 +- .../test/attachments.test.cjs | 63 ++++++++++++++ typescript/packages/traceai_livekit/README.md | 4 +- .../packages/traceai_livekit/src/agents.ts | 11 +-- .../traceai_livekit/test/agents-recording.cjs | 87 +++++++++++++++++++ 7 files changed, 166 insertions(+), 13 deletions(-) diff --git a/typescript/packages/traceai_elevenlabs/README.md b/typescript/packages/traceai_elevenlabs/README.md index a26db5015..20c920098 100644 --- a/typescript/packages/traceai_elevenlabs/README.md +++ b/typescript/packages/traceai_elevenlabs/README.md @@ -117,7 +117,7 @@ Binary capture is off by default. If your destination supports the attachment se - `textToSpeech.convertWithTimestamps` can attach a complete MP3 response up to 256 KiB. The output format must be the SDK default or a supported MP3 format. Generated audio is not proof of playback. - `speechToText.convert` can attach an existing `Blob`/`File` when you also set `attachmentInputMimeType` to its exact MIME type: `audio/wav`, `audio/mpeg`, `audio/ogg`, `audio/mp4`, `audio/webm` or `audio/flac`. Filenames are not exported; mismatched or missing types are not sniffed. -- `textToSpeech.convert`, `textToSpeech.stream` and `textToSpeech.streamWithTimestamps` can attach the MP3 your application **reads to the end**, up to 4 MiB per call (about 4 minutes at the default 128 kbit/s). Tracing copies only the chunks you already read; it never reads ahead, tees or drains the stream. If you cancel, stop iterating, hit an error or exceed 4 MiB, nothing is uploaded: a partial file is never stored as if complete. While a stream is being read, the manager reserves the full 4 MiB, so many concurrent long streams can report `queue_full` and pass through uncaptured. +- `textToSpeech.convert`, `textToSpeech.stream` and `textToSpeech.streamWithTimestamps` can attach the MP3 your application **reads to the end**, up to 4 MiB per call (about 4 minutes at the default 128 kbit/s), or up to the server's single-upload limit when that is smaller. Tracing copies only the chunks you already read; it never reads ahead, tees or drains the stream. If you cancel, stop iterating, hit an error or exceed that cap, nothing is uploaded: a partial file is never stored as if complete. While a stream is being read, the manager reserves the whole cap for it (4 MiB, or the smaller server limit) out of its shared in-memory budget of about 31 MiB. With 4 MiB reservations about 7 streams can be captured at once on an otherwise idle manager; further concurrent streams report `queue_full` and pass through uncaptured. - PCM/ulaw output, local paths and remote URLs are not captured. Alignment data stays in your SDK response, not in attachment metadata. Attachments obey the manager's default-off, server, sampling and PII policies in addition to the instrumentation's directional hide settings. Without a qualified speech sanitizer, `piiRedaction` denies capture. Existing raw audio may contain embedded metadata; a policy requiring metadata removal also denies it. The span's `elevenlabs.attachment.input` / `elevenlabs.attachment.output` value is the admission result; `queued` means admitted for transfer, **not** validated or playable. Content is out of band, never base64 in OTLP. A call made through this bound instrumentation carries the shared media-owner context marker (`traceai.auto.attachment_owner.v1`), so wire-level capture that honours the marker does not upload the same audio again. diff --git a/typescript/packages/traceai_elevenlabs/src/attachments.ts b/typescript/packages/traceai_elevenlabs/src/attachments.ts index afcc8b8e8..5e6b3aeaf 100644 --- a/typescript/packages/traceai_elevenlabs/src/attachments.ts +++ b/typescript/packages/traceai_elevenlabs/src/attachments.ts @@ -14,9 +14,11 @@ export type ElevenLabsInputAudioMimeType = // A bounded complete-response path for convertWithTimestamps (one JSON value). const MAX_DECODED_BYTES = 256 * 1024; /** - * Consumed-stream cap. The manager reserves the whole cap when a stream is admitted, so it - * also bounds per-call memory. About 4.4 minutes of the SDK's default 128 kbit/s MP3. A longer - * response uploads nothing (`too_large`); a truncated object is never uploaded as complete. + * Requested consumed-stream cap: about 4.4 minutes of the SDK's default 128 kbit/s MP3. The + * manager reserves min(this, the server's single-upload limit, its owned budget) when a stream is + * admitted, so a server whose `max_upload_bytes` is below 4 MiB still admits the stream with a + * smaller cap. The reservation bounds per-call memory. A response above the effective cap uploads + * nothing (`too_large`); a truncated object is never uploaded as complete. */ export const ELEVENLABS_STREAM_MAX_BYTES = 4 * 1024 * 1024; const INPUT_TYPES: readonly string[] = [ diff --git a/typescript/packages/traceai_elevenlabs/test/attachments-fixture.cjs b/typescript/packages/traceai_elevenlabs/test/attachments-fixture.cjs index f690aba5a..23a7f78e7 100644 --- a/typescript/packages/traceai_elevenlabs/test/attachments-fixture.cjs +++ b/typescript/packages/traceai_elevenlabs/test/attachments-fixture.cjs @@ -6,7 +6,7 @@ const ids = [1, 2, 3].map(n => `10000000-0000-4000-8000-00000000000${n}`); // Sessions of at least 1 h, so the core's near-expiry renewal never adds a bootstrap here. const expiry = () => new Date(Date.now() + 3600000).toISOString(); -async function attachmentFixture(config = {}, serverPolicy = {}) { +async function attachmentFixture(config = {}, serverPolicy = {}, serverLimits = {}) { const calls = [], uploaded = [], requests = [], uploads = new Map(); const policy = { version: 'v1', enabled: true, allowed_sources: ['Bytes', 'Blob'], allowed_modalities: ['audio'], require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, @@ -23,7 +23,7 @@ async function attachmentFixture(config = {}, serverPolicy = {}) { segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, - playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }, + playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5, ...serverLimits }, expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); } else if (req.url.endsWith('/uploads')) { const body = JSON.parse(bytes); requests.push(body); diff --git a/typescript/packages/traceai_elevenlabs/test/attachments.test.cjs b/typescript/packages/traceai_elevenlabs/test/attachments.test.cjs index e2e8ea138..91f0b3f11 100644 --- a/typescript/packages/traceai_elevenlabs/test/attachments.test.cjs +++ b/typescript/packages/traceai_elevenlabs/test/attachments.test.cjs @@ -6,6 +6,7 @@ const { attachmentFixture, wav } = require('./attachments-fixture.cjs'); const { SpeechAttachments } = require('../dist/src/attachments.js'); const { generateTraceConfig } = require('@traceai/fi-core'); const tone = readFileSync(require('node:path').join(__dirname, 'fixtures/tone.mp3')); +const MiB = 1024 * 1024; for (const sdkName of ['@elevenlabs/sdk-floor', '@elevenlabs/elevenlabs-js']) { const SDK = require(sdkName); @@ -86,6 +87,21 @@ for (const sdkName of ['@elevenlabs/sdk-floor', '@elevenlabs/elevenlabs-js']) { }); } + for (const method of ['convert', 'stream']) { + test(`${sdkName}: a server single-upload limit below 4 MiB still admits and uploads a consumed ${method}`, async t => { + // The stream reserves min(4 MiB, max_upload_bytes); before the core clamp this was denied too_large. + const a = await attachmentFixture({}, {}, { max_upload_bytes: MiB }); t.after(() => a.close()); + const f = await fixture(SDK, { attachments: a.manager }, { audio: tone }); t.after(() => f.close()); + const received = await bytes(await f.client.textToSpeech[method]('voice', { text: 'synthetic' })); + assert.deepEqual(Buffer.from(received), tone); + await a.manager.flush(); + assert.equal(f.spans()[0].attributes['elevenlabs.attachment.output'], 'queued'); + assert.equal(a.uploaded.length, 1); assert.deepEqual(a.uploaded[0], tone); + assert.equal(a.requests[0].declared_bytes, tone.length); + assert.equal(a.manager.status().ownedBytes, 0); + }); + } + test(`${sdkName}: BYOB reads keep consumer buffers and upload identical bytes`, async t => { const a = await attachmentFixture(); t.after(() => a.close()); const f = await fixture(SDK, { attachments: a.manager }, { audio: tone }); t.after(() => f.close()); @@ -204,3 +220,50 @@ test('a stream above the cap uploads nothing (too_large) and returns its reserva assert.equal(a.uploaded.length, 0); assert.equal(attrs['elevenlabs.attachment.output'], 'too_large'); assert.equal(a.manager.status().ownedBytes, 0); }); + +let spans = 0; +function syntheticSpan(attrs = {}) { + const spanId = (++spans).toString(16).padStart(16, '0'); + return { spanContext: () => ({ traceId: '0123456789abcdef0123456789abcdef', spanId, traceFlags: 1 }), + isRecording: () => true, setAttribute(k, v) { attrs[k] = v; } }; +} + +test('a server limit below 4 MiB caps the stream there: exactly the limit uploads, one byte more is too_large', async t => { + const a = await attachmentFixture({}, {}, { max_upload_bytes: MiB }); t.after(() => a.close()); + const fits = {}, over = {}; + const ok = new SpeechAttachments(a.manager, syntheticSpan(fits), generateTraceConfig(), undefined); + ok.chunk(new Uint8Array(MiB - 1)); ok.chunk(new Uint8Array(1)); ok.close('complete'); + const big = new SpeechAttachments(a.manager, syntheticSpan(over), generateTraceConfig(), undefined); + big.chunk(new Uint8Array(MiB)); big.chunk(new Uint8Array(1)); big.close('complete'); + await a.manager.flush(); + assert.equal(fits['elevenlabs.attachment.output'], 'queued'); + assert.equal(over['elevenlabs.attachment.output'], 'too_large'); + assert.equal(a.uploaded.length, 1); assert.equal(a.uploaded[0].length, MiB); + assert.equal(a.requests.length, 1); + assert.equal(a.manager.status().ownedBytes, 0); +}); + +test('concurrency: about 7 open 4 MiB streams fit the owned budget; the 8th is queue_full; a 1 MiB server limit fits more', async t => { + const { ELEVENLABS_STREAM_MAX_BYTES } = require('../dist/src/attachments.js'); + const open = async (serverLimits, n) => { + const a = await attachmentFixture({}, {}, serverLimits); t.after(() => a.close()); + const out = []; + for (let i = 0; i < n; i++) { + const attrs = {}; + const capture = new SpeechAttachments(a.manager, syntheticSpan(attrs), generateTraceConfig(), undefined); + capture.chunk(new Uint8Array(1)); // admission happens at the first observed chunk + out.push({ capture, attrs }); + } + return { a, out }; + }; + const wide = await open({}, 8); + assert.equal(ELEVENLABS_STREAM_MAX_BYTES, 4 * MiB); + assert.deepEqual(wide.out.map(s => s.attrs['elevenlabs.attachment.output'] ?? 'open'), [...Array(7).fill('open'), 'queue_full']); + for (const s of wide.out) s.capture.close('incomplete'); + assert.equal(wide.a.manager.status().ownedBytes, 0); + const narrow = await open({ max_upload_bytes: MiB }, 16); + assert.ok(narrow.out.every(s => s.attrs['elevenlabs.attachment.output'] === undefined), 'every 1 MiB reservation admitted'); + for (const s of narrow.out) s.capture.close('incomplete'); + await narrow.a.manager.flush(); + assert.equal(narrow.a.uploaded.length, 0); assert.equal(narrow.a.manager.status().ownedBytes, 0); +}); diff --git a/typescript/packages/traceai_livekit/README.md b/typescript/packages/traceai_livekit/README.md index 395dda9fb..94c6b82d2 100644 --- a/typescript/packages/traceai_livekit/README.md +++ b/typescript/packages/traceai_livekit/README.md @@ -101,8 +101,8 @@ How it works: - Bind right after `session.start(...)` returns. The session's `agent_session` span is captured then, and the upload is linked to it. Point LiveKit's telemetry at your Future AGI provider (`agents.telemetry.setTracerProvider(provider)`) so that span is exported; if the span is not recording, the binding reports `correlation_unavailable`. - At session end, LiveKit closes the recorder, runs your `onSessionEnd`, then calls its own session-end step. The binding runs at the start of that step, uploads `audio.ogg` (mixed user and agent audio, up to the attachment service's single-upload limit), and then lets LiveKit continue. If your `onSessionEnd` moves or deletes the file, the binding reports `source_unavailable`. -- Only the job's primary session, its own recorder and the fixed `audio.ogg` in the job's session directory qualify. A symlink, a hard link, or a file swapped during upload fails closed. -- It uploads nothing when attachments are off, when your trace config hides inputs or outputs or redacts PII, when the server policy denies it, when LiveKit project or session redaction is on, when `LIVEKIT_TELEMETRY_ALLOW_PII` is false, or when session audio recording is off. +- Only the job's primary session, its own recorder and the fixed `audio.ogg` in the job's session directory qualify. The file must be owned by the worker's own user. A symlink, a hard link, a file owned by another user, or a file swapped during upload fails closed. +- It uploads nothing when attachments are off, when your trace config hides inputs or outputs or redacts PII, when the server policy denies it, when LiveKit project or session redaction is on, when `LIVEKIT_TELEMETRY_ALLOW_PII` is false, or when session audio recording is off. These checks run again at session end, before the file is opened or even checked, so a policy tightened during the session still wins. - Give your worker a shutdown grace longer than `finalizationTimeoutMs` plus your own session-end callbacks. A process killed before the upload finishes loses it. - `recording.status` is content-free. `validating` means the file reached Future AGI and is being checked; it is not yet playable. `recording.close()` detaches the binding without touching the session. diff --git a/typescript/packages/traceai_livekit/src/agents.ts b/typescript/packages/traceai_livekit/src/agents.ts index 8ad84a603..453e7a6d2 100644 --- a/typescript/packages/traceai_livekit/src/agents.ts +++ b/typescript/packages/traceai_livekit/src/agents.ts @@ -167,6 +167,12 @@ export class LiveKitSessionRecording { const session = this.session!, job = this.job!, root = this.root!; const denial = nativeDenial(session, job, this.env); if (denial) return denial; + // The manager policy is checked again FIRST at session end, before the SDK report is built and + // before any stat or open of the session directory: a policy tightened mid-session wins. + const ticket = this.manager.preflight(this.correlation, { mimeType: "audio/ogg", direction: "mixed", maxBytes: 1, + operation: "import_existing", sourceKind: "OwnedRecordingFile", purpose: "conversation_recording", capture: this.capture }); + ticket.release(); + if (!ticket.admitted) return ticket.reason; // The report path is NOT authority. Only the job's primary session, its SDK recorder (closed) // and the SDK's fixed filename under the bound job's session directory are eligible. if (job._primaryAgentSession !== session || job.sessionDirectory !== root) return "source_unavailable"; @@ -176,11 +182,6 @@ export class LiveKitSessionRecording { const report = job.makeSessionReport!(session); if (report?.audioRecordingPath !== expected) return "source_unavailable"; if (this.stopped) return "incomplete"; - // Policy is checked again before any stat or open of the session directory. - const ticket = this.manager.preflight(this.correlation, { mimeType: "audio/ogg", direction: "mixed", maxBytes: 1, - operation: "import_existing", sourceKind: "OwnedRecordingFile", purpose: "conversation_recording", capture: this.capture }); - ticket.release(); - if (!ticket.admitted) return ticket.reason; const info = await lstat(expected); if (this.stopped) return "incomplete"; if (!info.isFile() || info.nlink !== 1 || (typeof process.getuid === "function" && info.uid !== process.getuid())) return "source_unavailable"; diff --git a/typescript/packages/traceai_livekit/test/agents-recording.cjs b/typescript/packages/traceai_livekit/test/agents-recording.cjs index cfa3a871e..79938c574 100644 --- a/typescript/packages/traceai_livekit/test/agents-recording.cjs +++ b/typescript/packages/traceai_livekit/test/agents-recording.cjs @@ -125,6 +125,93 @@ test('policy tightened after bind: finalization re-checks before any stat or ope assert.equal(binding.status, 'redaction_unavailable'); assert.equal(a.requests.length, 0); }); +/** + * Records every fs call on the session directory made while the binding is still finalizing + * (status `session_bound`). The binding assigns its outcome before the SDK's own `_onSessionEnd` + * runs, so the SDK's later work is excluded. Covers the binding's lstat and the core scope's + * realpath/open (both go through the shared `node:fs` promises object). + */ +function watchSessionFs(t, s, binding) { + const fsp = require('node:fs/promises'); + const seen = []; + const inDir = p => { try { return String(p).startsWith(s.ctx.sessionDirectory); } catch { return false; } }; + const wrap = (target, names) => { + for (const name of names) { + const original = target[name]; + if (typeof original !== 'function') continue; + target[name] = function (p, ...rest) { + if (binding.status === 'session_bound' && inDir(p)) seen.push(`${name}:${path.relative(s.ctx.sessionDirectory, String(p)) || '.'}`); + return original.call(this, p, ...rest); + }; + t.after(() => { target[name] = original; }); + } + }; + wrap(fsp, ['lstat', 'stat', 'open', 'realpath', 'readFile', 'readdir', 'opendir', 'access']); + wrap(fs, ['lstatSync', 'statSync', 'openSync', 'realpathSync', 'readFileSync', 'readdirSync', 'accessSync', 'createReadStream']); + return seen; +} + +test('control: an admitted session end does stat and open audio.ogg (the fs watch sees the binding)', async t => { + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + const seen = watchSessionFs(t, s, binding); + await endSession(s); + assert.equal(binding.status, 'validating'); + assert.ok(seen.includes('lstat:audio.ogg'), `seen: ${seen.join(',')}`); + assert.ok(seen.includes('open:audio.ogg'), `seen: ${seen.join(',')}`); + assert.equal(a.uploaded.length, 1); +}); + +for (const [name, tighten, expected] of [ + ['manager PII redaction enabled', a => a.manager.updatePolicy({ traceConfig: { piiRedaction: true } }), 'redaction_unavailable'], + ['manager output hidden', a => a.manager.updatePolicy({ traceConfig: { hideOutputs: true } }), 'policy_denied'], + ['manager disabled', a => a.manager.updatePolicy({ enabled: false }), 'disabled'], + ['environment kill switch flipped', (a, env) => { env.FI_ATTACHMENTS_ENABLED = 'false'; }, 'disabled'], + ['manager invalidated (revoked)', a => a.manager.invalidate('policy_denied'), 'policy_denied'], +]) { + for (const removeFile of [false, true]) { + test(`session-end deny before any stat or open: ${name}${removeFile ? ', recording removed' : ''} -> ${expected}`, async t => { + const env = {}; + const a = await attachmentFixture({ environment: () => env }); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + assert.equal(binding.status, 'session_bound', 'admitted at bind'); + const seen = watchSessionFs(t, s, binding); + await s.recorder.close(); + tighten(a, env); + // A removed file would make a stat-first order report source_unavailable instead of the policy reason. + if (removeFile) fs.rmSync(s.file); + await endSession(s, { closeRecorder: false }); + assert.equal(binding.status, expected); + assert.deepEqual(seen, [], 'no stat, open or realpath of the session directory before the deny'); + await a.manager.flush().catch(() => undefined); + assert.equal(a.requests.length, 0); assert.equal(a.uploaded.length, 0); + assert.deepEqual(s.session._recordedEvents, [], 'the SDK _onSessionEnd still ran'); + }); + } +} + +test('a recording file owned by another user is rejected (source_unavailable), nothing uploaded', async t => { + // Only root can hand a file to another uid; CI and the qualification VM run this suite as root. + if (typeof process.getuid !== 'function' || process.getuid() !== 0) { + t.diagnostic('SKIPPED: needs uid 0 to chown the recording to another user'); + t.skip('needs uid 0'); return; + } + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + assert.equal(binding.status, 'session_bound'); + await s.recorder.close(); + fs.chownSync(s.file, 65534, 65534); + const info = fs.lstatSync(s.file); + assert.ok(info.isFile() && info.nlink === 1 && info.uid === 65534, 'only the owner differs'); + await endSession(s, { closeRecorder: false }); + assert.equal(binding.status, 'source_unavailable'); + await a.manager.flush(); assert.equal(a.requests.length, 0); assert.equal(a.uploaded.length, 0); + assert.ok(fs.existsSync(s.file), 'the binding never deletes the recording'); +}); + for (const [name, mutate] of [ ['report absent (no recorder)', s => { s.session._recorderIO = undefined; }], ['recording replaced by a symlink', s => { const other = path.join(s.ctx.sessionDirectory, 'other.ogg'); fs.renameSync(s.file, other); fs.symlinkSync(other, s.file); }], From 31409b33ec704687de1a5a936d8420b0165fe0fd Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:16:30 -0800 Subject: [PATCH 7/8] test(livekit): reject a foreign-owned recording without root as well Co-Authored-By: Claude Opus 5.5 --- .../traceai_livekit/test/agents-recording.cjs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/typescript/packages/traceai_livekit/test/agents-recording.cjs b/typescript/packages/traceai_livekit/test/agents-recording.cjs index 79938c574..e95f55ed4 100644 --- a/typescript/packages/traceai_livekit/test/agents-recording.cjs +++ b/typescript/packages/traceai_livekit/test/agents-recording.cjs @@ -283,3 +283,20 @@ test('invalid options throw synchronously and never touch the session', () => { assert.throws(() => bindLiveKitSessionRecording({}, {}, { manager: {}, agents: sdk.agents, ownedRecordingFiles: true, finalizationTimeoutMs: 30001 }), RangeError); assert.throws(() => bindLiveKitSessionRecording({}, {}, { manager: {}, agents: sdk.agents, ownedRecordingFiles: true, capture: { newRecording: true } }), TypeError); }); + +test('a recording whose owner is not the worker uid is rejected (any uid; worker uid reported differently)', async t => { + // Runs without root too: the file keeps its real owner and the worker reports another uid. + const a = await attachmentFixture(); t.after(() => a.close()); + const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); + const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); + assert.equal(binding.status, 'session_bound'); + await s.recorder.close(); s.span.end(); + const real = process.getuid; + process.getuid = () => real.call(process) + 1; + let status; + try { status = await binding.finalize(); } finally { process.getuid = real; } + assert.equal(status, 'source_unavailable'); + await a.manager.flush(); assert.equal(a.requests.length, 0); assert.equal(a.uploaded.length, 0); + assert.ok(fs.existsSync(s.file), 'the binding never deletes the recording'); + await s.ctx._onSessionEnd(); +}); From fd7ffc41f8890958366bb3b4c2d1a70f02648502 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:53:44 -0800 Subject: [PATCH 8/8] test(livekit): keep the test's own rmSync outside the session-end fs watch rmSync lstats the file it removes, so watching from before the removal recorded the test's own call. The watch now starts right before the session end it observes. Co-Authored-By: Claude Opus 5.5 --- typescript/packages/traceai_livekit/test/agents-recording.cjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/typescript/packages/traceai_livekit/test/agents-recording.cjs b/typescript/packages/traceai_livekit/test/agents-recording.cjs index e95f55ed4..5954887ac 100644 --- a/typescript/packages/traceai_livekit/test/agents-recording.cjs +++ b/typescript/packages/traceai_livekit/test/agents-recording.cjs @@ -177,11 +177,12 @@ for (const [name, tighten, expected] of [ const s = await startedSession(sdk, tracer); t.after(() => s.cleanup()); const binding = bindLiveKitSessionRecording(s.session, s.ctx, { manager: a.manager, agents: sdk.agents, ownedRecordingFiles: true }); assert.equal(binding.status, 'session_bound', 'admitted at bind'); - const seen = watchSessionFs(t, s, binding); await s.recorder.close(); tighten(a, env); // A removed file would make a stat-first order report source_unavailable instead of the policy reason. if (removeFile) fs.rmSync(s.file); + // Watch only the session end; the test's own recorder close and rmSync (which lstats) stay outside. + const seen = watchSessionFs(t, s, binding); await endSession(s, { closeRecorder: false }); assert.equal(binding.status, expected); assert.deepEqual(seen, [], 'no stat, open or realpath of the session directory before the deny');