diff --git a/.github/workflows/fireworks-typescript-tests.yml b/.github/workflows/fireworks-typescript-tests.yml index ac7fdffff..0d4f11b43 100644 --- a/.github/workflows/fireworks-typescript-tests.yml +++ b/.github/workflows/fireworks-typescript-tests.yml @@ -7,6 +7,7 @@ on: - 'typescript/packages/traceai_fireworks/**' - 'typescript/packages/traceai_openai/**' - 'typescript/packages/fi-core/**' + - 'typescript/packages/provider-media/**' - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/pnpm-lock.yaml' - '.github/workflows/fireworks-typescript-tests.yml' @@ -50,9 +51,10 @@ jobs: pnpm --dir packages/fi-core pack --out "$RUNNER_TEMP/fi-core.tgz" pnpm --dir packages/traceai_openai pack --out "$RUNNER_TEMP/openai.tgz" pnpm --dir packages/traceai_fireworks pack --out "$RUNNER_TEMP/fireworks.tgz" + pnpm --dir packages/provider-media pack --out "$RUNNER_TEMP/provider-media.tgz" npm install --prefix "$RUNNER_TEMP/consumer" --ignore-scripts --no-audit --no-fund \ "$RUNNER_TEMP/openai.tgz" "$RUNNER_TEMP/fireworks.tgz" "openai@${{ matrix.sdk }}" \ - "$RUNNER_TEMP/fi-core.tgz" @traceai/fi-semantic-conventions@1.0.0 \ + "$RUNNER_TEMP/fi-core.tgz" "$RUNNER_TEMP/provider-media.tgz" @traceai/fi-semantic-conventions@1.0.0 \ @opentelemetry/sdk-trace-base@2.0.1 @opentelemetry/exporter-trace-otlp-http@0.202.0 \ typescript@5.8.3 @types/node@22.15.29 - name: Network-isolated automatic, manual and dual-instrumentor contracts diff --git a/.github/workflows/google-genai-sdk-compat.yml b/.github/workflows/google-genai-sdk-compat.yml index 9b9ae5dd2..455d71850 100644 --- a/.github/workflows/google-genai-sdk-compat.yml +++ b/.github/workflows/google-genai-sdk-compat.yml @@ -5,6 +5,7 @@ on: paths: - 'typescript/packages/traceai_google_genai_sdk/**' - 'typescript/packages/fi-core/**' + - 'typescript/packages/provider-media/**' - 'typescript/packages/fi-semantic-conventions/**' - '.github/workflows/google-genai-sdk-compat.yml' - '.github/scripts/loopback-network-diagnostics.cjs' @@ -39,11 +40,13 @@ jobs: run: | set -euo pipefail npm install --global pnpm@10.30.1 - pnpm --filter @traceai/fi-core... install --frozen-lockfile --ignore-scripts + pnpm --filter @traceai/provider-media... install --frozen-lockfile --ignore-scripts pnpm --filter @traceai/fi-semantic-conventions run build pnpm --filter @traceai/fi-core run build + pnpm --filter @traceai/provider-media run build pnpm --dir packages/fi-semantic-conventions pack --out "$RUNNER_TEMP/semconv.tgz" pnpm --dir packages/fi-core pack --out "$RUNNER_TEMP/core.tgz" + pnpm --dir packages/provider-media pack --out "$RUNNER_TEMP/provider-media.tgz" - name: Build adapter against published shared dependencies shell: bash run: | @@ -51,6 +54,8 @@ jobs: source="$GITHUB_WORKSPACE/typescript/packages/traceai_google_genai_sdk" cp -R "$source" "$RUNNER_TEMP/build" cd "$RUNNER_TEMP/build" + # Workspace-only shared packages are staged from this PR's tarballs. + npm pkg set "dependencies.@traceai/fi-core=file:$RUNNER_TEMP/core.tgz" "dependencies.@traceai/provider-media=file:$RUNNER_TEMP/provider-media.tgz" npm install --ignore-scripts sandbox-exec -f "$source/test/network.sb" npm run build npm pack --ignore-scripts --pack-destination "$RUNNER_TEMP" @@ -63,9 +68,9 @@ jobs: npm init -y # Exercise this PR's core fix, including the legacy coexistence adapter, # rather than silently resolving the already-published broken core. - npm pkg set 'overrides.@traceai/fi-core=$@traceai/fi-core' 'overrides.@traceai/fi-semantic-conventions=$@traceai/fi-semantic-conventions' + npm pkg set 'overrides.@traceai/fi-core=$@traceai/fi-core' 'overrides.@traceai/fi-semantic-conventions=$@traceai/fi-semantic-conventions' 'overrides.@traceai/provider-media=$@traceai/provider-media' tarball=$(find "$RUNNER_TEMP" -maxdepth 1 -name 'traceai-google-genai-sdk-*.tgz' -print) - npm install --ignore-scripts "$tarball" "$RUNNER_TEMP/core.tgz" "$RUNNER_TEMP/semconv.tgz" '@google/genai@${{ matrix.sdk }}' '@opentelemetry/sdk-trace-node@2.0.1' 'typescript@5.9.3' '@types/node@22' '@traceai/google-genai@0.1.0' '@google/generative-ai@0.24.1' + npm install --ignore-scripts "$tarball" "$RUNNER_TEMP/core.tgz" "$RUNNER_TEMP/semconv.tgz" "$RUNNER_TEMP/provider-media.tgz" '@google/genai@${{ matrix.sdk }}' '@opentelemetry/sdk-trace-node@2.0.1' 'typescript@5.9.3' '@types/node@22' '@traceai/google-genai@0.1.0' '@google/generative-ai@0.24.1' npm ls @traceai/fi-core @traceai/fi-semantic-conventions - name: Real SDK, OTLP, privacy and exact customer snippet without internet shell: bash diff --git a/.github/workflows/google-generative-ai-compat.yml b/.github/workflows/google-generative-ai-compat.yml index ad715104d..4fbdc2fe2 100644 --- a/.github/workflows/google-generative-ai-compat.yml +++ b/.github/workflows/google-generative-ai-compat.yml @@ -6,6 +6,7 @@ on: paths: - 'typescript/packages/traceai_google_genai/**' - 'typescript/packages/fi-core/**' + - 'typescript/packages/provider-media/**' - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/pnpm-lock.yaml' - 'typescript/tsconfig*.json' @@ -41,6 +42,7 @@ jobs: sandbox-exec -f "$policy" pnpm --dir typescript --filter '@traceai/google-genai...' run build sandbox-exec -f "$policy" pnpm --dir typescript --filter @traceai/google-genai test --runInBand sandbox-exec -f "$policy" pnpm --dir typescript/packages/fi-core pack --out "$RUNNER_TEMP/fi-core.tgz" + sandbox-exec -f "$policy" pnpm --dir typescript/packages/provider-media pack --out "$RUNNER_TEMP/provider-media.tgz" sandbox-exec -f "$policy" pnpm --dir typescript/packages/traceai_google_genai pack --pack-destination "$RUNNER_TEMP/tarballs" - name: Install packed adapter and staged core with real SDKs env: @@ -49,7 +51,7 @@ jobs: tarball=$(find "$RUNNER_TEMP/tarballs" -name '*.tgz' -type f) npm install --prefix "$RUNNER_TEMP/consumer" --ignore-scripts --no-audit --no-fund \ "$tarball" "@google/generative-ai@$LEGACY_SDK" @google/genai@2.32.0 \ - "$RUNNER_TEMP/fi-core.tgz" @opentelemetry/sdk-trace-node@2.0.1 \ + "$RUNNER_TEMP/fi-core.tgz" "$RUNNER_TEMP/provider-media.tgz" @opentelemetry/sdk-trace-node@2.0.1 \ @opentelemetry/sdk-trace-base@2.0.1 \ @opentelemetry/exporter-trace-otlp-http@0.202.0 \ @opentelemetry/exporter-trace-otlp-proto@0.202.0 \ diff --git a/.github/workflows/groq-typescript-tests.yml b/.github/workflows/groq-typescript-tests.yml index 52350ed6e..beda4bca0 100644 --- a/.github/workflows/groq-typescript-tests.yml +++ b/.github/workflows/groq-typescript-tests.yml @@ -6,6 +6,7 @@ on: paths: - 'typescript/packages/traceai_groq/**' - 'typescript/packages/fi-core/**' + - 'typescript/packages/provider-media/**' - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/pnpm-lock.yaml' - 'typescript/tsconfig*.json' @@ -15,6 +16,7 @@ on: paths: - 'typescript/packages/traceai_groq/**' - 'typescript/packages/fi-core/**' + - 'typescript/packages/provider-media/**' - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/pnpm-lock.yaml' - '.github/workflows/groq-typescript-tests.yml' diff --git a/.github/workflows/mistral-typescript-tests.yml b/.github/workflows/mistral-typescript-tests.yml index 111e29a6a..ced906c84 100644 --- a/.github/workflows/mistral-typescript-tests.yml +++ b/.github/workflows/mistral-typescript-tests.yml @@ -6,6 +6,7 @@ on: paths: - 'typescript/packages/traceai_mistral/**' - 'typescript/packages/fi-core/**' + - 'typescript/packages/provider-media/**' - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/tsconfig*.json' - 'typescript/pnpm-lock.yaml' diff --git a/.github/workflows/openai-sdk-contracts.yml b/.github/workflows/openai-sdk-contracts.yml index a58934f90..60060f48e 100644 --- a/.github/workflows/openai-sdk-contracts.yml +++ b/.github/workflows/openai-sdk-contracts.yml @@ -6,6 +6,7 @@ on: paths: - 'typescript/packages/traceai_openai/**' - 'typescript/packages/fi-core/**' + - 'typescript/packages/provider-media/**' - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/pnpm-lock.yaml' - '.github/workflows/openai-sdk-contracts.yml' @@ -47,9 +48,10 @@ jobs: pnpm --filter @traceai/openai run build pnpm --dir packages/traceai_openai pack --out "$RUNNER_TEMP/openai.tgz" pnpm --dir packages/fi-core pack --out "$RUNNER_TEMP/core.tgz" + pnpm --dir packages/provider-media pack --out "$RUNNER_TEMP/provider-media.tgz" npm install --prefix "$RUNNER_TEMP/consumer" --ignore-scripts --no-audit --no-fund \ "$RUNNER_TEMP/openai.tgz" "openai@${{ matrix.sdk }}" \ - "$RUNNER_TEMP/core.tgz" @traceai/fi-semantic-conventions@1.0.0 \ + "$RUNNER_TEMP/core.tgz" "$RUNNER_TEMP/provider-media.tgz" @traceai/fi-semantic-conventions@1.0.0 \ @opentelemetry/sdk-trace-base@2.0.1 @opentelemetry/context-async-hooks@2.0.1 \ @opentelemetry/exporter-trace-otlp-http@0.202.0 \ @opentelemetry/exporter-trace-otlp-proto@0.202.0 @opentelemetry/otlp-transformer@0.202.0 \ @@ -58,7 +60,7 @@ jobs: run: | mkdir -p "$RUNNER_TEMP/strict-consumer" cat > "$RUNNER_TEMP/strict-consumer/package.json" <; +const record = (value: unknown): RecordValue => value !== null && typeof value === "object" && !Array.isArray(value) ? value as RecordValue : {}; +const list = (value: unknown): unknown[] => Array.isArray(value) ? value : []; +// Bound synchronous base64 decoding independently of the manager queue. Raw bytes and +// Blobs are bounded by the manager alone (preflight too_large / queue_full). +const MAX_DECODED_BYTES = 1024 * 1024; +const MAX_ENCODED_CHARS = Math.ceil(MAX_DECODED_BYTES / 3) * 4; +const MAX_PARTS = 64; +// Consumed speech streams have no length up front. The core reserves min(this, the negotiated +// single-upload cap) while they are read, so servers whose max_upload_bytes is below 4 MiB still +// admit the stream; a body over the reservation is discarded whole. +const MAX_SPEECH_STREAM_BYTES = 4 * 1024 * 1024; +const DATA_URI_TYPES = ["image/png", "image/jpeg", "image/webp"] as const; +// Steps 11-12 of the core's preflight order run only after every privacy, identity and +// sampling check passed, so these two denials still allow sizing the source exactly. +const SIZE_DENIALS: ReadonlySet = new Set(["too_large", "queue_full"]); +const seen = new WeakMap>>(); + +type MediaFields = Omit; +type Owned = Extract; +/** A source sized from its shape only. `take` decodes or hands over the bytes, and runs only after admission. */ +type Described = { mimeType: string; size: number; take(maxBytes: number): Owned | undefined }; + +/** Exact decoded length of padded base64 from its shape only: no decode or allocation. 0 = unusable. */ +function base64Length(value: unknown): number { + if (typeof value !== "string" || !value.length || value.length > MAX_ENCODED_CHARS || value.length % 4 !== 0) return 0; + const length = (value.length / 4) * 3 - (value.endsWith("==") ? 2 : value.endsWith("=") ? 1 : 0); + return length > 0 && length <= MAX_DECODED_BYTES ? length : 0; +} +function base64Source(value: unknown, mimeType: string): Described | undefined { + const size = base64Length(value); + if (!size) return undefined; + return { mimeType, size, take(maxBytes) { + // Strict alphabet check and decode happen only after the ticket was admitted. + if (size > maxBytes || !/^[A-Za-z0-9+/]*={0,2}$/.test(value as string)) return undefined; + const decoded = Buffer.from(value as string, "base64"); + return decoded.byteLength === size ? { kind: "Bytes", data: decoded } : undefined; + } }; +} + +/** Typed extraction only. No recursion, URL fetch, provider file lookup, or filesystem access. */ +export class ProviderMediaCapture { + private readonly manager?: AttachmentManager; + private readonly correlation?: AttachmentCorrelation; + private readonly local: TraceConfigOptions; + private readonly seen: Set; + private attempted = 0; + + constructor(span: AttachmentSpan, options: ProviderMediaOptions = {}) { + // Honour an existing owner: exactly one adapter uploads a call's media. + this.manager = attachmentOwnedElsewhere(options.parentContext) ? undefined : options.attachments; + this.local = { ...options.traceConfig }; + this.seen = new Set(); + if (this.manager) { + // Late application-owned speech reads use this exact identity, never ambient context. + this.correlation = this.manager.correlation(span); + let spans = seen.get(this.manager); + if (!spans) { spans = new WeakMap(); seen.set(this.manager, spans); } + let positions = spans.get(span); + if (!positions) { positions = new Set(); spans.set(span, positions); } + this.seen = positions; + } + } + + /** Wrapper-local restrictions intersect the shared manager policy, never replace it. */ + private locallyDenied(mimeType: string, direction: AttachmentOptions["direction"]): boolean { + const config = generateTraceConfig(this.local); + const input = direction !== "output"; + return !!(config.piiRedaction || (input + ? config.hideInputs || config.hideInputMessages || config.hideInputText || (attachmentModality(mimeType) === "image" && config.hideInputImages) + : config.hideOutputs || config.hideOutputMessages || config.hideOutputText)); + } + + private ticket(mimeType: string, sourceKind: "Bytes" | "Blob", maxBytes: number, fields: MediaFields): AttachmentAdmissionTicket { + return this.manager!.preflight(this.correlation, { mimeType, direction: fields.direction, purpose: fields.purpose, capture: fields.capture, sourceKind, maxBytes }); + } + + /** + * SDK design section 1: preflight BEFORE the source is read, sized or decoded. A denied + * part is never read. The gate is sized by the adapter's cap (or the known Blob/ArrayBuffer + * length); after admission the exact size shrinks the reservation at commit. A gate denied + * only for size, a different exact MIME type, or a part larger than the gate is + * preflighted once more at its exact type and length. Never more than two preflights. + */ + private admit(key: string, gateMime: string, sourceKind: "Bytes" | "Blob", gateBytes: number, fields: MediaFields, + describe: () => Described | undefined): void { + const manager = this.manager; + // One budget slot per typed part. A manager without preflight (an older core) captures nothing. + if (!manager || !this.correlation || typeof manager.preflight !== "function" || this.attempted++ >= MAX_PARTS || this.seen.has(key)) return; + // Every ticket taken here is released on exit. release() is a no-op once a ticket was used. + const taken: AttachmentAdmissionTicket[] = []; + const preflight = (mimeType: string, maxBytes: number) => { + const ticket = this.ticket(mimeType, sourceKind, maxBytes, fields); + taken.push(ticket); + return ticket; + }; + try { + if (this.locallyDenied(gateMime, fields.direction) || !(gateBytes > 0)) return; + let ticket = preflight(gateMime, gateBytes); + if (!ticket.admitted && !SIZE_DENIALS.has(ticket.reason)) return; + const source = describe(); + if (!source || !(source.size > 0) || this.locallyDenied(source.mimeType, fields.direction)) return; + if (!ticket.admitted || source.mimeType !== gateMime || source.size > ticket.maxBytes) { + ticket.release(); + ticket = preflight(source.mimeType, source.size); + if (!ticket.admitted) return; + } + const owned = source.take(ticket.maxBytes); + if (!owned) return; + // Denied observations are not retained as per-part diagnostics or references. + if (ticket.attach(owned, { messageIndex: fields.messageIndex, partIndex: fields.partIndex, role: fields.role, channel: fields.channel, + conversationId: fields.conversationId, turnId: fields.turnId, toolCallId: fields.toolCallId }).admission === "queued") this.seen.add(key); + } catch { /* Never leak provider payloads or replace application behavior. */ } + finally { + // An admitted ticket that is not used returns its reservation at once. + for (const ticket of taken) { try { ticket.release(); } catch { /* bounded */ } } + } + } + + private base64(key: string, read: () => unknown, mimeType: string, fields: MediaFields): void { + this.admit(key, mimeType, "Bytes", MAX_DECODED_BYTES, fields, () => base64Source(read(), mimeType)); + } + + private dataUri(key: string, read: () => unknown, fields: MediaFields): void { + // Every candidate type is an image, so one image gate decides the whole part before + // the URL string is read. Never retain a remote/signed URL as fallback metadata. + this.admit(key, DATA_URI_TYPES[0], "Bytes", MAX_DECODED_BYTES, fields, () => { + const value = read(); + if (typeof value !== "string") return undefined; + for (const mimeType of DATA_URI_TYPES) { + const prefix = `data:${mimeType};base64,`; + if (value.startsWith(prefix)) return base64Source(value.slice(prefix.length), mimeType); + } + return undefined; + }); + } + + /** Gemini Content/Part shapes shared by all three maintained Google wrappers. */ + geminiContents(contents: unknown, direction: "input" | "output" = "input"): void { + this.safely(() => { + const values = Array.isArray(contents) ? contents : [contents]; + for (let m = 0; m < Math.min(values.length, MAX_PARTS); m++) { + const content = record(values[m]); + // Legacy SDK accepts Part[] directly. No arbitrary nested field walk. + const parts = content.parts === undefined && content.inlineData ? [content] : list(content.parts); + for (let p = 0; p < Math.min(parts.length, MAX_PARTS); p++) { + const inline = record(record(parts[p]).inlineData); + if (typeof inline.mimeType !== "string") continue; + this.base64(`gemini:${direction}:${m}:${p}`, () => inline.data, inline.mimeType, { + direction, purpose: direction === "input" ? "model_input" : "model_output", + messageIndex: m, partIndex: p, + }); + } + } + }); + } + + geminiResponse(response: unknown): void { + this.safely(() => { + const result = record(response); + const candidates = list((result.response ? record(result.response) : result).candidates); + this.geminiContents(candidates.slice(0, MAX_PARTS).map(c => record(c).content), "output"); + }); + } + + /** + * Compatible chat messages: inline image URL, plus typed input_audio for OpenAI-shaped + * routes that accept it. Groq chat is image-only, so its audio parts are not advertised or read. + */ + compatibleMessages(messages: unknown, dialect: "openai" | "groq" | "mistral" = "openai"): void { + this.safely(() => { + const values = list(messages); + for (let m = 0; m < Math.min(values.length, MAX_PARTS); m++) { + const parts = list(record(values[m]).content); + for (let p = 0; p < Math.min(parts.length, MAX_PARTS); p++) { + const part = record(parts[p]); + const options = { direction: "input" as const, purpose: "model_input" as const, messageIndex: m, partIndex: p }; + if (part.type === "image_url") { + this.dataUri(`chat:${m}:${p}`, () => { + const image = dialect === "mistral" ? part.imageUrl ?? part.image_url : part.image_url; + return typeof image === "string" ? image : record(image).url; + }, options); + } else if (part.type === "input_audio" && dialect === "openai") { + const audio = record(part.input_audio); + const mimeType = audio.format === "wav" ? "audio/wav" : audio.format === "mp3" ? "audio/mpeg" : undefined; + if (mimeType) this.base64(`chat:${m}:${p}`, () => audio.data, mimeType, options); + } + } + } + }); + } + + /** Converse only; unknown model families and InvokeModel JSON remain unsupported. */ + bedrockConverse(input: unknown): void { + this.safely(() => { + const body = record(input); + if (typeof body.modelId !== "string" || !/^(?:(?:us|eu|apac|global)\.)?(?:anthropic\.claude-(?:3|sonnet-4|opus-4|haiku-4)|amazon\.nova-(?:lite|pro|premier))/i.test(body.modelId)) return; + const messages = list(body.messages); + for (let m = 0; m < Math.min(messages.length, MAX_PARTS); m++) { + const parts = list(record(messages[m]).content); + for (let p = 0; p < Math.min(parts.length, MAX_PARTS); p++) { + const part = record(parts[p]); + this.bedrockBlock(`bedrock:${m}:${p}`, part, { direction: "input", purpose: "model_input", messageIndex: m, partIndex: p }); + // Typed tool results (for example computer-use screenshots) carry their own + // image/document blocks. Only these named blocks are read; no generic walk. + const tool = record(part.toolResult); + const toolCallId = typeof tool.toolUseId === "string" && /^[a-zA-Z0-9_-]{1,128}$/.test(tool.toolUseId) ? tool.toolUseId : undefined; + const results = list(tool.content); + for (let q = 0; q < Math.min(results.length, MAX_PARTS); q++) { + this.bedrockBlock(`bedrock:${m}:${p}:${q}`, record(results[q]), { + direction: "input", purpose: "tool_result", role: "tool", messageIndex: m, partIndex: p, ...(toolCallId ? { toolCallId } : {}), + }); + } + } + } + }); + } + + private bedrockBlock(key: string, part: RecordValue, options: MediaFields): void { + // Bedrock serializes Uint8Array bytes itself; S3 locations and other sources stay unsupported. + const media = record(part.image ?? part.document); + const mimeType = part.image ? ({ png: "image/png", jpeg: "image/jpeg", webp: "image/webp" } as Record)[String(media.format)] + : part.document && media.format === "pdf" ? "application/pdf" : undefined; + if (!mimeType) return; + this.admit(key, mimeType, "Bytes", MAX_DECODED_BYTES, options, () => { + const bytes = record(media.source).bytes; + // The core copies mutable bytes synchronously inside ticket.attach; no adapter-side copy. + return bytes instanceof Uint8Array && bytes.byteLength ? { mimeType, size: bytes.byteLength, take: max => bytes.byteLength <= max ? { kind: "Bytes", data: bytes } : undefined } : undefined; + }); + } + + /** Immutable File/Blob input only; never advance fs.ReadStream or fetch Response inputs. */ + speechInput(file: unknown): void { + this.safely(() => { + if (typeof Blob === "undefined" || !(file instanceof Blob) || attachmentModality(file.type) !== "audio") return; + // Blob type and size are metadata; the content is never read here. + const blob: Blob = file; const mimeType = blob.type; const size = blob.size; + this.admit("speech:input", mimeType, "Blob", size, { direction: "input", purpose: "model_input" }, + () => ({ mimeType, size, take: max => size <= max ? { kind: "Blob", data: blob } : undefined })); + }); + } + + /** + * Observe only bytes the caller consumed: whole-body arrayBuffer()/blob() reads, the chunks + * of a reader the caller obtained with response.body.getReader() (SDK design 6.4), and the + * chunks of an async iteration of response.body (`for await`, values()), up to the end of the + * stream. Unread, cancelled, partially read, released and broken-out-of bodies upload nothing, + * and so do a second reader or iterator on the same body, tee()/pipeTo()/pipeThrough()/clone() + * consumers, and any chunk-wise read of a Node-stream body (the node-fetch shim openai 4.x uses + * by default): only WHATWG ReadableStream bodies are observed chunk by chunk. On that transport + * a whole-body arrayBuffer() read is observed, but blob() is not (node-fetch resolves its own Blob + * class, not the global Blob). + */ + speechResponse(response: T): T { + this.safely(() => { + const target = response as unknown as Response; + if (!target || typeof target.headers?.get !== "function" || target.bodyUsed) return; + const mimeType = target.headers.get("content-type")?.split(";")[0].trim(); + if (!mimeType || attachmentModality(mimeType) !== "audio") return; + const capture = this; + for (const method of ["arrayBuffer", "blob"] as const) { + const original: (this: Response) => Promise = target[method]; + if (typeof original !== "function") continue; + Object.defineProperty(target, method, { configurable: true, writable: true, value: function (this: Response, ...args: []) { + // Borrowed methods must not attach another Response to this call's span. + if (this !== target) return original.apply(this, args); + // Preserve the native receiver/brand check and the application's error. + return original.apply(this, args).then(value => { + // Preflight at read time (not at hook install), so a late read is judged against the + // current policy and never holds an install-time reservation past its TTL. + capture.safely(() => capture.consumed(mimeType, value)); + return value; + }); + } }); + } + // Reading the body property neither locks nor reads the stream. + const body = target.body as ReadableStream | null; + // WHATWG streams only. A node-fetch body is a Node PassThrough: it has [Symbol.asyncIterator] + // but no getReader(), and its chunk-wise reads are deliberately left unobserved (the auto wire + // layer, which captures through fetch only, does not observe them either). + if (body && typeof (body as { getReader?: unknown }).getReader === "function") { + // Only the first reader or iterator is observed. releaseLock() (or an iterator's + // return() with preventCancel) does not end the stream, so a later consumer would see + // only the remaining bytes and must never upload them as whole. + let observed = false; + // `for await` and values() lock the stream with an internal reader, never the public + // getReader(), so each entry point is observed on this body instance. + const entries: Array<[PropertyKey, "read" | "next", readonly string[]]> = [ + ["getReader", "read", ["releaseLock", "cancel"]], + ["values", "next", ["return"]], + [Symbol.asyncIterator, "next", ["return"]], + ]; + for (const [name, pull, stops] of entries) { + const method = (body as unknown as Record)[name]; + if (typeof method !== "function") continue; + Object.defineProperty(body, name, { configurable: true, writable: true, value: function (this: ReadableStream, ...args: unknown[]) { + // A locked body throws here, unchanged, before anything is observed. + const consumer = (method as (...a: unknown[]) => unknown).apply(this, args); + if (this === body && !observed) { + observed = true; + capture.safely(() => capture.observeChunks(mimeType, consumer, pull, stops)); + } + return consumer; + } }); + } + } + }); + return response; + } + + /** A whole body the application itself read, sized from the value it already holds. */ + private consumed(mimeType: string, value: ArrayBuffer | Blob): void { + const fields: MediaFields = { direction: "output", purpose: "model_output" }; + if (value instanceof ArrayBuffer) { + const buffer: ArrayBuffer = value; const size = buffer.byteLength; + this.admit("speech:output", mimeType, "Bytes", size, fields, + () => ({ mimeType, size, take: max => size <= max ? { kind: "Bytes", data: new Uint8Array(buffer) } : undefined })); + } else if (typeof Blob !== "undefined" && value instanceof Blob) { + const blob: Blob = value; const size = blob.size; + this.admit("speech:output", mimeType, "Blob", size, fields, + () => ({ mimeType, size, take: max => size <= max ? { kind: "Blob", data: blob } : undefined })); + } + } + + /** + * Forward each chunk the caller's own reader read() or iterator next() resolved to a consumed + * stream (design 6). Never calls read, next, clone, tee or getReader itself. A whole stream + * within the cap uploads once at its end; a rejected pull, a non-byte chunk, an over-cap chunk + * or any stop method (releaseLock/cancel, or an iterator's return() on break) uploads nothing. + */ + private observeChunks(mimeType: string, consumer: unknown, pull: "read" | "next", stops: readonly string[]): void { + const manager = this.manager; + const target = consumer as Record | null; + const pullMethod = target?.[pull] as ((...a: unknown[]) => Promise<{ done?: boolean; value?: unknown }>) | undefined; + if (!manager || !this.correlation || typeof manager.startAttachmentStream !== "function" || !target || typeof pullMethod !== "function" || + this.seen.has("speech:output") || this.locallyDenied(mimeType, "output")) return; + // The whole budget is reserved at admission, so a write never fails mid-stream for budget reasons. + const stream = manager.startAttachmentStream(this.correlation, { mimeType, direction: "output", purpose: "model_output", maxBytes: MAX_SPEECH_STREAM_BYTES }); + if (stream.reason !== "queued") return; + let open = true; + const end = (complete: boolean) => { + if (!open) return; + open = false; + try { + if (!complete) stream.abort("incomplete"); + else if (stream.finish().admission === "queued") this.seen.add("speech:output"); + } catch { /* bounded */ } + }; + const define = (name: string, value: (...a: unknown[]) => unknown) => Object.defineProperty(target, name, { configurable: true, writable: true, value }); + // Defined before the consumer is returned: `for await` captures next() when it starts. + define(pull, function (this: unknown, ...args: unknown[]) { + const pending = pullMethod.apply(this, args); + if (this !== target || !open) return pending; + // The caller receives the same result or rejection; chunks are copied by tryWrite. + return pending.then(result => { + if (open) { + if (result?.done) end(true); + else if (!(result?.value instanceof Uint8Array) || !stream.tryWrite(result.value)) end(false); + } + return result; + }, error => { end(false); throw error; }); + }); + for (const name of stops) { + const stop = target[name]; + if (typeof stop === "function") define(name, function (this: unknown, ...args: unknown[]) { if (this === target) end(false); return stop.apply(this, args); }); + } + } + + /** + * Mark the context the provider SDK runs in, so wire-level capture (the auto module) and any nested + * adapter skip this call's media and only this capture uploads it. Called only for calls whose media + * this capture owns. An inert capture (no manager, or owned elsewhere) returns `ctx` unchanged. + * A local hide or PII restriction still claims the call: the restriction then holds for every layer. + */ + claim(ctx: C): C { + if (!this.manager) return ctx; + try { return ctx.setValue(ATTACHMENT_OWNER_KEY, true) as C; } catch { return ctx; } + } + + private safely(fn: () => void): void { if (this.manager) { try { fn(); } catch { /* Content-free fail-open. */ } } } +} + +function routeHost(resource: unknown, hosts: readonly string[]): boolean { + try { + const r = record(resource); + const client = record(r._client ?? r.client); + const url = new URL(String(client.baseURL)); + return url.protocol === "https:" && hosts.includes(url.hostname); + } catch { return false; } +} + +/** + * Explicit compatible chat host admission for OpenAI-SDK resources; unknown and + * text-only routes do not acquire media capture. The OpenAI chat hook itself is + * owned by the OpenAI adapter, which may consult this allowlist. + */ +export function compatibleMediaRoute(resource: unknown): boolean { + return routeHost(resource, ["api.openai.com", "api.fireworks.ai", "api.together.ai", "api.x.ai", "openrouter.ai", "api.groq.com", "api.mistral.ai"]); +} + +/** OpenAI-SDK audio resources: only hosts whose speech/transcription routes were qualified. */ +export function speechMediaRoute(resource: unknown): boolean { + return routeHost(resource, ["api.openai.com", "api.groq.com"]); +} + +/** True only while a configured manager has not been switched off. Tightening still wins later. */ +export function attachmentsActive(manager: AttachmentManager | undefined): manager is AttachmentManager { + try { return !!manager && manager.status().state !== "disabled"; } catch { return false; } +} + +export type ProviderMediaOperation = Readonly<{ + operation: string; + sources: readonly ("inline_base64" | "data_uri" | "bytes" | "blob" | "consumed_response" | "consumed_reader")[]; + modalities: readonly ("image" | "audio" | "document")[]; + directions: readonly ("input" | "output")[]; +}>; + +/** + * Typed per-package capability descriptors. Anything not listed (streams, URLs, + * provider file IDs, async operations, InvokeModel bodies, native OCR/speech + * routes) is unsupported and never captured. + */ +export const PROVIDER_MEDIA_CAPABILITIES: Readonly> = Object.freeze({ + "@traceai/google-genai-sdk": [{ operation: "models.generateContent", sources: ["inline_base64"], modalities: ["image", "audio", "document"], directions: ["input", "output"] }, + { operation: "models.generateContentStream", sources: ["inline_base64"], modalities: ["image", "audio", "document"], directions: ["input"] }], + "@traceai/google-genai": [{ operation: "GenerativeModel.generateContent", sources: ["inline_base64"], modalities: ["image", "audio", "document"], directions: ["input", "output"] }], + "@traceai/fi-instrumentation-vertexai": [{ operation: "GenerativeModel.generateContent", sources: ["inline_base64"], modalities: ["image", "audio", "document"], directions: ["input", "output"] }], + "@traceai/bedrock": [{ operation: "ConverseCommand", sources: ["bytes"], modalities: ["image", "document"], directions: ["input"] }], + "@traceai/groq": [{ operation: "chat.completions.create", sources: ["data_uri"], modalities: ["image"], directions: ["input"] }, + { operation: "audio.transcriptions.create", sources: ["blob"], modalities: ["audio"], directions: ["input"] }, + { operation: "audio.translations.create", sources: ["blob"], modalities: ["audio"], directions: ["input"] }, + { operation: "audio.speech.create", sources: ["consumed_response", "consumed_reader"], modalities: ["audio"], directions: ["output"] }], + "@traceai/mistral": [{ operation: "chat.complete", sources: ["data_uri"], modalities: ["image"], directions: ["input"] }], + "@traceai/portkey": [{ operation: "chat.completions.create", sources: ["data_uri", "inline_base64"], modalities: ["image", "audio"], directions: ["input"] }], + "@traceai/vllm": [{ operation: "chat.completions.create", sources: ["data_uri", "inline_base64"], modalities: ["image", "audio"], directions: ["input"] }], + "@traceai/openai": [{ operation: "audio.transcriptions.create", sources: ["blob"], modalities: ["audio"], directions: ["input"] }, + { operation: "audio.translations.create", sources: ["blob"], modalities: ["audio"], directions: ["input"] }, + { operation: "audio.speech.create", sources: ["consumed_response", "consumed_reader"], modalities: ["audio"], directions: ["output"] }], +}); diff --git a/typescript/packages/provider-media/test/fixtures.cjs b/typescript/packages/provider-media/test/fixtures.cjs new file mode 100644 index 000000000..595fd0f64 --- /dev/null +++ b/typescript/packages/provider-media/test/fixtures.cjs @@ -0,0 +1,43 @@ +const http = require('node:http'); +const { once } = require('node:events'); +const { randomUUID } = require('node:crypto'); +const { AttachmentManager, createAttachmentHttpTransport } = require('@traceai/fi-core/attachments'); +const { fake, identity } = require('../../fi-core/test/attachments-fixtures.cjs'); +const png = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jMZkAAAAASUVORK5CYII=', 'base64'); +function wav() { + const result = Buffer.alloc(46); result.write('RIFF'); result.writeUInt32LE(38, 4); result.write('WAVEfmt ', 8); + result.writeUInt32LE(16, 16); result.writeUInt16LE(1, 20); result.writeUInt16LE(1, 22); result.writeUInt32LE(16000, 24); + result.writeUInt32LE(32000, 28); result.writeUInt16LE(2, 32); result.writeUInt16LE(16, 34); result.write('data', 36); result.writeUInt32LE(2, 40); + return result; +} +async function fixture(options = {}) { + // sessionLimits overrides F01 session limits in the loopback bootstrap; everything else configures the manager. + const { sessionLimits, ...config } = options; + const uploads = []; const requests = []; const uploadsById = new Map(); + const server = http.createServer(async (req, res) => { + try { + const chunks = []; for await (const chunk of req) chunks.push(chunk); + const data = Buffer.concat(chunks); requests.push({ method: req.method, path: req.url }); + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('attachment-sessions')) { const session = await fake().transport.bootstrap(JSON.parse(data)); res.end(JSON.stringify(sessionLimits ? { ...session, limits: { ...session.limits, ...sessionLimits } } : session)); } + else if (req.url.endsWith('/uploads')) { + const input = JSON.parse(data); const upload = { ...await fake().transport.initialize(null, input), upload_id: randomUUID() }; + uploadsById.set(upload.upload_id, input); res.statusCode = 201; res.end(JSON.stringify(upload)); + } else { + const id = req.url.split('/').at(-2); + if (req.url.endsWith('/content')) uploads.push({ request: uploadsById.get(id), data }); + res.statusCode = req.url.endsWith('/complete') ? 202 : 200; + res.end(JSON.stringify({ upload_id: id, state: req.url.endsWith('/complete') ? 'validating' : 'pending_upload', reason: 'queued' })); + } + } catch { res.statusCode = 500; res.end('{}'); } + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const client = async req => { + const res = await fetch(`http://127.0.0.1:${server.address().port}${new URL(req.url).pathname}`, { method: req.method, headers: req.headers, body: req.body, signal: req.signal }); + return { status: res.status, body: new Uint8Array(await res.arrayBuffer()) }; + }; + const manager = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic' }, client }), ...config }); + await manager.ready(); + return { manager, uploads, requests, async close() { await manager.shutdown(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } }; +} +module.exports = { fixture, png, wav, identity }; diff --git a/typescript/packages/provider-media/test/install-sdks.cjs b/typescript/packages/provider-media/test/install-sdks.cjs new file mode 100644 index 000000000..f0acce534 --- /dev/null +++ b/typescript/packages/provider-media/test/install-sdks.cjs @@ -0,0 +1,29 @@ +/* Run only on an isolated qualification worker, never in the SDK or application. */ +const { execFileSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); +const root = process.argv[2]; +if (!root || !path.isAbsolute(root)) throw Error('Supply an absolute isolated SDK directory'); +fs.mkdirSync(root, { recursive: true }); +const entries = [ + ['openai', 'openai', '4.0.0', '7.55.0'], + // openai 4.x on its default node-fetch transport; audio.speech is first usable at 4.52.7 on this harness. + ['openai4', 'openai', '4.52.7', '4.104.0'], ['groq', 'groq-sdk', '0.3.0', '1.6.0'], + ['genai', '@google/genai', '1.33.0', '3.1.0'], ['gemini', '@google/generative-ai', '0.1.0', '0.24.1'], + ['vertex', '@google-cloud/vertexai', '1.0.0', '1.12.0'], ['bedrock', '@aws-sdk/client-bedrock-runtime', '3.693.0', '3.1166.0'], + // Typed Mistral image chunks start at 1.1.0; 0.x and 1.0.0 are legacy text clients and are not advertised. + ['mistral', '@mistralai/mistralai', '1.1.0', '3.5.0'], ['portkey', 'portkey-ai', '1.0.0', '3.1.0'], +]; +const dependencies = {}, versions = {}; +// Pinned latest releases keep CI reproducible. MEDIA_SDK_LATEST=resolve re-queries npm when re-qualifying. +const resolve = process.env.MEDIA_SDK_LATEST === 'resolve'; +for (const [name, pkg, floor, pinned] of entries) { + const latest = resolve || !pinned ? execFileSync('npm', ['view', pkg, 'version'], { encoding: 'utf8' }).trim() : pinned; + versions[name] = { package: pkg, floor, latest }; + dependencies[`sdk-${name}-floor`] = `npm:${pkg}@${floor}`; + dependencies[`sdk-${name}-latest`] = `npm:${pkg}@${latest}`; +} +fs.writeFileSync(path.join(root, 'package.json'), JSON.stringify({ private: true, dependencies }, null, 2)); +fs.writeFileSync(path.join(root, 'versions.json'), JSON.stringify(versions, null, 2)); +console.log(JSON.stringify(versions, null, 2)); +execFileSync('npm', ['install', '--prefix', root, '--ignore-scripts', '--no-audit', '--no-fund', '--legacy-peer-deps'], { stdio: 'inherit' }); diff --git a/typescript/packages/provider-media/test/layout.test.cjs b/typescript/packages/provider-media/test/layout.test.cjs new file mode 100644 index 000000000..e8826b5a0 --- /dev/null +++ b/typescript/packages/provider-media/test/layout.test.cjs @@ -0,0 +1,42 @@ +// Repository-layout guards for the shared helper. They need no build and no network. +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); + +const packageDir = path.resolve(__dirname, '..'); +const packagesDir = path.dirname(packageDir); +const repoRoot = path.resolve(packagesDir, '../..'); +const manifest = JSON.parse(fs.readFileSync(path.join(packageDir, 'package.json'), 'utf8')); +const releasePath = path.relative(repoRoot, packageDir).split(path.sep).join('/'); + +test('helper lives outside the traceai_* instrumentation directories', () => { + // `@traceai/auto` counts every packages/traceai_* directory (except auto and coding agents) + // as an instrumentation it must register. This helper is not an instrumentation. + assert.equal(path.basename(packagesDir), 'packages'); + assert.ok(!path.basename(packageDir).startsWith('traceai_'), `${path.basename(packageDir)} would be counted by the auto registry test`); + const dependents = fs.readdirSync(packagesDir) + .filter(dir => fs.existsSync(path.join(packagesDir, dir, 'package.json'))) + .filter(dir => manifest.name in (JSON.parse(fs.readFileSync(path.join(packagesDir, dir, 'package.json'), 'utf8')).dependencies ?? {})); + assert.ok(dependents.length > 0); + for (const dir of dependents) { + // Packages built with project references must point at the helper's real directory. + const tsconfig = fs.readFileSync(path.join(packagesDir, dir, 'tsconfig.json'), 'utf8'); + if (!tsconfig.includes('"references"')) continue; + assert.ok(tsconfig.includes(`"../${path.basename(packageDir)}"`), `${dir}/tsconfig.json must reference the helper by its directory`); + } +}); + +test('publishable helper has a release-please config and manifest entry', () => { + // release_plan.py inventories every non-private package with a name and version; a missing + // entry fails the Release workflow gates and blocks every dependent package's release. + assert.notEqual(manifest.private, true); + const config = JSON.parse(fs.readFileSync(path.join(repoRoot, 'release-please-config.json'), 'utf8')); + const versions = JSON.parse(fs.readFileSync(path.join(repoRoot, '.release-please-manifest.json'), 'utf8')); + const entry = config.packages[releasePath]; + assert.ok(entry, `release-please-config.json has no entry for ${releasePath}`); + assert.equal(entry['release-type'], 'node'); + assert.equal(entry['package-name'], manifest.name); + assert.equal(entry.component, `npm-${manifest.name.slice(1).replace('/', '-')}`); + assert.equal(versions[releasePath], manifest.version); +}); diff --git a/typescript/packages/provider-media/test/media.test.cjs b/typescript/packages/provider-media/test/media.test.cjs new file mode 100644 index 000000000..d03c5155c --- /dev/null +++ b/typescript/packages/provider-media/test/media.test.cjs @@ -0,0 +1,198 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { ProviderMediaCapture, compatibleMediaRoute } = require('../dist/src'); +const { fixture, png, wav, identity } = require('./fixtures.cjs'); +const inline = () => ({ inlineData: { mimeType: 'image/png', data: png.toString('base64') } }); +const messages = () => [{ role: 'user', content: [{ type: 'image_url', image_url: { url: `data:image/png;base64,${png.toString('base64')}` } }] }]; + +for (const kind of ['gemini', 'compatible', 'bedrock']) test(`${kind}: immutable bytes and exact message positions reach loopback attachment API`, async () => { + const f = await fixture(); + try { + const capture = new ProviderMediaCapture(identity, { attachments: f.manager }); + if (kind === 'gemini') capture.geminiContents([{ role: 'user', parts: [inline()] }]); + if (kind === 'compatible') capture.compatibleMessages(messages()); + if (kind === 'bedrock') capture.bedrockConverse({ modelId: 'anthropic.claude-3-5-sonnet-20241022-v2:0', messages: [{ role: 'user', content: [{ image: { format: 'png', source: { bytes: png } } }] }] }); + assert.deepEqual(await f.manager.flush(), { unfinished: 0, dropped: 0 }); + assert.equal(f.uploads.length, 1); assert.deepEqual(f.uploads[0].data, png); + assert.equal(f.uploads[0].request.message_index, 0); assert.equal(f.uploads[0].request.part_index, 0); + assert.equal(f.uploads[0].request.association.parent.trace_id, '01234567-89ab-cdef-0123-456789abcdef'); + } finally { await f.close(); } +}); + +for (const mode of ['disabled', 'hideInputs', 'hideInputMessages', 'hideInputImages', 'hideInputText', 'piiRedaction', 'localHide', 'localPii', 'unsampled']) { + test(`${mode}: no decoding, source access, upload or occurrence`, async () => { + const f = await fixture(mode === 'disabled' ? { enabled: false } : { traceConfig: { [mode]: true } }); + let reads = 0; + try { + const capture = new ProviderMediaCapture(mode === 'unsampled' ? { ...identity, traceFlags: 0 } : identity, { + attachments: f.manager, traceConfig: mode === 'localHide' ? { hideInputs: true } : mode === 'localPii' ? { piiRedaction: true } : {}, + }); + capture.geminiContents([{ parts: [{ inlineData: { mimeType: 'image/png', get data() { reads++; throw Error('synthetic-secret'); } } }] }]); + capture.compatibleMessages([{ content: [{ type: 'image_url', get image_url() { reads++; throw Error('synthetic-secret'); } }] }]); + capture.bedrockConverse({ modelId: 'anthropic.claude-3-5-sonnet-20241022-v2:0', messages: [{ content: [ + { image: { format: 'png', source: { get bytes() { reads++; throw Error('synthetic-secret'); } } } }, + { toolResult: { toolUseId: 'tool-1', content: [{ image: { format: 'png', source: { get bytes() { reads++; throw Error('synthetic-secret'); } } } }] } }] }] }); + capture.speechInput(new Blob([Buffer.from('synthetic-secret')], { type: 'audio/wav' })); + const before = f.requests.length; await f.manager.flush(); + // Hidden images do not hide audio: only the audio Blob may be admitted in that mode. + assert.equal(reads, 0); assert.equal(f.uploads.length, mode === 'hideInputImages' ? 1 : 0); + if (mode === 'hideInputImages') assert.equal(f.uploads[0].request.modality, 'audio'); else assert.equal(f.requests.length, before); + } finally { await f.close(); } + }); +} + +test('distinct repeated image positions survive; duplicate observation of the same span does not', async () => { + const f = await fixture(); + try { + const input = [{ parts: [inline(), inline()] }]; + new ProviderMediaCapture(identity, { attachments: f.manager }).geminiContents(input); + new ProviderMediaCapture(identity, { attachments: f.manager }).geminiContents(input); + await f.manager.flush(); assert.equal(f.uploads.length, 2); assert.notEqual(f.uploads[0].request.occurrence_id, f.uploads[1].request.occurrence_id); + } finally { await f.close(); } +}); + +test('unknown models, URLs, provider files and async operation values remain uncaptured', async () => { + const f = await fixture(); + try { + const c = new ProviderMediaCapture(identity, { attachments: f.manager }); + c.geminiContents([{ parts: [{ fileData: { mimeType: 'image/png', fileUri: 'https://private.invalid/secret' } }] }]); + c.geminiResponse({ name: 'operations/submitted', done: false, response: { generatedVideos: [{ video: { uri: 'https://private.invalid' } }] } }); + c.compatibleMessages([{ content: [{ type: 'image_url', image_url: { url: 'http://169.254.169.254/metadata' } }, { type: 'file', file: { file_id: 'private' } }] }]); + c.bedrockConverse({ modelId: 'unknown.text-only', messages: [{ content: [{ image: { format: 'png', source: { bytes: png } } }] }] }); + await f.manager.flush(); assert.equal(f.uploads.length, 0); + for (const baseURL of ['https://unknown.invalid', 'https://api.deepseek.com', 'https://api.cerebras.ai', 'http://api.openai.com', 'https://api.openai.com.evil.invalid']) assert.equal(compatibleMediaRoute({ _client: { baseURL } }), false); + } finally { await f.close(); } +}); + +test('generated Gemini Part uses output restrictions and stays distinct from input', async () => { + for (const traceConfig of [{}, { hideOutputs: true }, { hideOutputText: true }, { piiRedaction: true }]) { + const f = await fixture({ traceConfig }); + try { + new ProviderMediaCapture(identity, { attachments: f.manager }).geminiResponse({ candidates: [{ content: { parts: [inline()] } }] }); + await f.manager.flush(); assert.equal(f.uploads.length, Object.keys(traceConfig).length ? 0 : 1); + if (f.uploads.length) assert.equal(f.uploads[0].request.direction, 'output'); + } finally { await f.close(); } + } +}); + +for (const method of ['arrayBuffer', 'blob', 'unread', 'cancel', 'stream']) test(`speech ${method}: only bodies the application consumed to the end capture`, async () => { + const f = await fixture(); + try { + const response = new Response(wav(), { headers: { 'content-type': 'audio/wav' } }); + const c = new ProviderMediaCapture(identity, { attachments: f.manager }); + assert.equal(c.speechResponse(response), response); assert.equal(response.bodyUsed, false); + await f.manager.flush(); assert.equal(f.uploads.length, 0); + if (method === 'arrayBuffer') assert.deepEqual(Buffer.from(await response.arrayBuffer()), wav()); + if (method === 'blob') assert.deepEqual(Buffer.from(await (await response.blob()).arrayBuffer()), wav()); + if (method === 'cancel') await response.body.cancel(); + if (method === 'stream') { const reader = response.body.getReader(); while (!(await reader.read()).done) {} } + // A reader the application obtained and read to its end is a consumed stream (SDK design 6.4). + await f.manager.flush(); assert.equal(f.uploads.length, ['arrayBuffer', 'blob', 'stream'].includes(method) ? 1 : 0); + if (f.uploads.length) assert.deepEqual(f.uploads[0].data, wav()); + } finally { await f.close(); } +}); + +test('File/Blob input does not read opaque streams, names or paths', async () => { + const f = await fixture(); + try { + const c = new ProviderMediaCapture(identity, { attachments: f.manager }); + c.speechInput({ get path() { throw Error('must not read'); }, read() { throw Error('must not read'); } }); + c.speechInput(new Blob([wav()], { type: 'audio/wav' })); + await f.manager.flush(); assert.equal(f.uploads.length, 1); assert.deepEqual(f.uploads[0].data, wav()); + assert.equal(JSON.stringify(f.uploads[0].request).includes('filename'), false); + } finally { await f.close(); } +}); + +test('speech late read rechecks tightened policy and never mutates ended spans', async () => { + const f = await fixture(); let ended = false; let writes = 0; + const span = { spanContext: () => identity, isRecording: () => !ended, setAttribute() { if (ended) throw Error('ended mutation'); writes++; } }; + try { + const c = new ProviderMediaCapture(span, { attachments: f.manager }); + const response = c.speechResponse(new Response(wav(), { headers: { 'content-type': 'audio/wav' } })); + ended = true; const previous = writes; + f.manager.updatePolicy({ traceConfig: { hideOutputs: true } }); + await response.arrayBuffer(); await f.manager.flush(); + assert.equal(f.uploads.length, 0); assert.equal(writes, previous); + } finally { await f.close(); } +}); + +test('malformed and oversized base64 fail closed without changing provider input', async () => { + const f = await fixture(); + try { + for (const data of ['not-base64', 'A'.repeat(2 * 1024 * 1024)]) { + const part = Object.freeze({ inlineData: Object.freeze({ mimeType: 'image/png', data }) }); + new ProviderMediaCapture({ ...identity }, { attachments: f.manager }).geminiContents([{ parts: [part] }]); + } + await f.manager.flush(); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); + +test('data URI MIME descriptors consume one part budget and capture JPEG/WebP once with the exact type', async () => { + const f = await fixture(); + try { + const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0, 0x10, 0x4a, 0x46, 0x49, 0x46, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0xff, 0xd9]); + const parts = Array.from({ length: 70 }, () => ({ type: 'image_url', image_url: { url: `data:image/jpeg;base64,${jpeg.toString('base64')}` } })); + new ProviderMediaCapture(identity, { attachments: f.manager }).compatibleMessages([{ content: parts }]); + await f.manager.flush(); + assert.equal(f.uploads.length, 64); + assert.equal(new Set(f.uploads.map(u => u.request.mime_type)).size, 1); assert.equal(f.uploads[0].request.mime_type, 'image/jpeg'); + assert.deepEqual(f.uploads[0].data, jpeg); + // The manager runs two upload workers, so completion order is not part order: check the admitted set. + assert.deepEqual(f.uploads.map(u => u.request.part_index).sort((a, b) => a - b), Array.from({ length: 64 }, (_, i) => i)); + } finally { await f.close(); } +}); + +test('non-image data URIs, PDFs in image_url and unknown audio formats are not captured', async () => { + const f = await fixture(); + try { + new ProviderMediaCapture(identity, { attachments: f.manager }).compatibleMessages([{ content: [ + { type: 'image_url', image_url: { url: `data:application/pdf;base64,${png.toString('base64')}` } }, + { type: 'image_url', image_url: { url: `data:image/svg+xml;base64,${png.toString('base64')}` } }, + { type: 'input_audio', input_audio: { format: 'flac', data: wav().toString('base64') } }, + ] }]); + await f.manager.flush(); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); + +test('Bedrock typed tool-result screenshots keep tool identity; S3 locations and unsupported formats stay uncaptured', async () => { + const f = await fixture(); + try { + new ProviderMediaCapture(identity, { attachments: f.manager }).bedrockConverse({ modelId: 'us.anthropic.claude-sonnet-4-20250514-v1:0', messages: [ + { role: 'user', content: [{ toolResult: { toolUseId: 'tooluse_screen-1', content: [{ text: 'ok' }, { image: { format: 'png', source: { bytes: png } } }] } }] }, + { role: 'user', content: [{ image: { format: 'png', source: { s3Location: { uri: 's3://private/secret.png' } } } }, { image: { format: 'gif', source: { bytes: png } } }, + { document: { format: 'docx', name: 'private-name', source: { bytes: png } } }, { document: { format: 'pdf', name: 'private-name', source: { bytes: png } } }] }, + ] }); + await f.manager.flush(); + assert.equal(f.uploads.length, 2); + const tool = f.uploads.find(u => u.request.purpose === 'tool_result'); + assert.equal(tool.request.tool_call_id, 'tooluse_screen-1'); assert.equal(tool.request.message_index, 0); assert.deepEqual(tool.data, png); + const pdf = f.uploads.find(u => u.request.mime_type === 'application/pdf'); + assert.equal(pdf.request.message_index, 1); assert.equal(pdf.request.part_index, 3); + assert.equal(JSON.stringify(f.uploads.map(u => u.request)).includes('private'), false); + } finally { await f.close(); } +}); + +test('speech routes, manager activity and capability descriptors are explicit', async () => { + const { speechMediaRoute, attachmentsActive, PROVIDER_MEDIA_CAPABILITIES } = require('../dist/src'); + for (const [baseURL, allowed] of [['https://api.openai.com/v1', true], ['https://api.groq.com/openai/v1', true], ['https://api.fireworks.ai/inference/v1', false], ['http://api.openai.com/v1', false], ['https://api.openai.com.evil.invalid/v1', false]]) assert.equal(speechMediaRoute({ _client: { baseURL } }), allowed, baseURL); + assert.equal(attachmentsActive(undefined), false); + const off = await fixture({ enabled: false }); const on = await fixture(); + try { assert.equal(attachmentsActive(off.manager), false); assert.equal(attachmentsActive(on.manager), true); } finally { await off.close(); await on.close(); } + assert.ok(Object.isFrozen(PROVIDER_MEDIA_CAPABILITIES)); + for (const operations of Object.values(PROVIDER_MEDIA_CAPABILITIES)) for (const entry of operations) { + assert.equal(entry.sources.some(s => /url|file|stream|provider/i.test(s)), false); + } + assert.equal(PROVIDER_MEDIA_CAPABILITIES['@traceai/openai'].some(e => e.operation.startsWith('chat')), false); +}); + +test('input_audio is captured only for OpenAI-shaped routes that advertise it', async () => { + const f = await fixture(); + try { + const audio = [{ content: [{ type: 'input_audio', input_audio: { format: 'wav', data: wav().toString('base64') } }] }]; + new ProviderMediaCapture({ ...identity, spanId: '0000000000000001' }, { attachments: f.manager }).compatibleMessages(audio, 'groq'); + new ProviderMediaCapture({ ...identity, spanId: '0000000000000002' }, { attachments: f.manager }).compatibleMessages(audio, 'mistral'); + await f.manager.flush(); assert.equal(f.uploads.length, 0); + new ProviderMediaCapture(identity, { attachments: f.manager }).compatibleMessages(audio); + await f.manager.flush(); assert.equal(f.uploads.length, 1); assert.equal(f.uploads[0].request.mime_type, 'audio/wav'); assert.deepEqual(f.uploads[0].data, wav()); + } finally { await f.close(); } +}); diff --git a/typescript/packages/provider-media/test/owner.test.cjs b/typescript/packages/provider-media/test/owner.test.cjs new file mode 100644 index 000000000..bca1737ce --- /dev/null +++ b/typescript/packages/provider-media/test/owner.test.cjs @@ -0,0 +1,92 @@ +/* Shared media-owner marker: exactly one adapter uploads a call's media (auto wire layer, ElevenLabs, this helper). */ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { createRequire } = require('node:module'); +const { ProviderMediaCapture, ATTACHMENT_OWNER_KEY, attachmentOwnedElsewhere } = require('../dist/src'); +const { fixture, png, wav, identity } = require('./fixtures.cjs'); +const api = createRequire(path.resolve(__dirname, '../../fi-core/package.json'))('@opentelemetry/api'); + +const NAME = 'traceai.auto.attachment_owner.v1'; +const owned = () => api.ROOT_CONTEXT.setValue(Symbol.for(NAME), true); +/** Counts every admission call, so an owned call is shown to never reach the manager. */ +function counting(manager) { + const calls = { preflight: 0, stream: 0 }; + const preflight = manager.preflight.bind(manager), stream = manager.startAttachmentStream.bind(manager); + manager.preflight = (...a) => { calls.preflight++; return preflight(...a); }; + manager.startAttachmentStream = (...a) => { calls.stream++; return stream(...a); }; + return calls; +} +const idle = f => { const s = f.manager.status(); assert.equal(s.openStreams, 0); assert.equal(s.openTickets, 0); assert.equal(s.ownedBytes, 0); }; +const speech = () => new Response(wav(), { headers: { 'content-type': 'audio/wav' } }); +const drain = async reader => { const out = []; for (;;) { const { done, value } = await reader.read(); if (done) return Buffer.concat(out); out.push(value); } }; + +test('O-01 the marker is the OpenTelemetry context key the auto module and ElevenLabs use', () => { + assert.equal(ATTACHMENT_OWNER_KEY, Symbol.for(NAME)); + assert.equal(ATTACHMENT_OWNER_KEY, api.createContextKey(NAME)); + assert.equal(attachmentOwnedElsewhere(owned()), true); + for (const value of [undefined, false, 0, '']) assert.equal(attachmentOwnedElsewhere(api.ROOT_CONTEXT.setValue(ATTACHMENT_OWNER_KEY, value)), false); + assert.equal(attachmentOwnedElsewhere(undefined), false); + assert.equal(attachmentOwnedElsewhere({ getValue() { throw new Error('synthetic'); } }), false); +}); + +test('O-02 claim marks a new context only while this capture is active; the parent context is never mutated', async () => { + const f = await fixture(); + try { + const parent = api.ROOT_CONTEXT; + const claimed = new ProviderMediaCapture(identity, { attachments: f.manager, parentContext: parent }).claim(parent); + assert.notEqual(claimed, parent); assert.equal(claimed.getValue(ATTACHMENT_OWNER_KEY), true); + assert.equal(parent.getValue(ATTACHMENT_OWNER_KEY), undefined); + // Without a manager (unqualified route, attachments not configured) the call is not claimed. + assert.equal(new ProviderMediaCapture(identity, {}).claim(parent), parent); + // Already owned by another adapter: unchanged, and this capture stays inert. + const other = owned(); + assert.equal(new ProviderMediaCapture(identity, { attachments: f.manager, parentContext: other }).claim(other), other); + // A context whose setValue throws is returned unchanged; the application call is never affected. + const broken = { getValue: () => undefined, setValue() { throw new Error('synthetic'); } }; + assert.equal(new ProviderMediaCapture(identity, { attachments: f.manager }).claim(broken), broken); + } finally { await f.close(); } +}); + +test('O-03 a call owned by another adapter never reaches the manager and uploads nothing; the application keeps every byte', async () => { + const f = await fixture(); + try { + const calls = counting(f.manager); + const capture = new ProviderMediaCapture(identity, { attachments: f.manager, parentContext: owned() }); + capture.speechInput(new File([wav()], 'synthetic.wav', { type: 'audio/wav' })); + capture.geminiContents([{ parts: [{ inlineData: { mimeType: 'image/png', data: png.toString('base64') } }] }]); + capture.bedrockConverse({ modelId: 'amazon.nova-lite-v1:0', messages: [{ content: [{ image: { format: 'png', source: { bytes: png } } }] }] }); + capture.compatibleMessages([{ content: [{ type: 'image_url', image_url: { url: `data:image/png;base64,${png.toString('base64')}` } }] }]); + const whole = capture.speechResponse(speech()); + assert.deepEqual(Buffer.from(await whole.arrayBuffer()), wav()); + const streamed = capture.speechResponse(speech()); + assert.deepEqual(await drain(streamed.body.getReader()), wav()); + await f.manager.flush(); idle(f); + assert.deepEqual(calls, { preflight: 0, stream: 0 }); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); + +for (const value of [undefined, false]) test(`O-04 an unowned parent context (marker ${value}) captures exactly once`, async () => { + const f = await fixture(); + try { + const parentContext = api.ROOT_CONTEXT.setValue(ATTACHMENT_OWNER_KEY, value); + const capture = new ProviderMediaCapture(identity, { attachments: f.manager, parentContext }); + capture.speechInput(new File([wav()], 'synthetic.wav', { type: 'audio/wav' })); + const response = capture.speechResponse(speech()); + assert.deepEqual(await drain(response.body.getReader()), wav()); + await f.manager.flush(); idle(f); + assert.deepEqual(f.uploads.map(u => u.request.direction).sort(), ['input', 'output']); + for (const u of f.uploads) assert.deepEqual(u.data, wav()); + } finally { await f.close(); } +}); + +test('O-05 a locally hidden capture still claims the call, so no other layer uploads what the integration hides', async () => { + const f = await fixture(); + try { + const capture = new ProviderMediaCapture(identity, { attachments: f.manager, traceConfig: { hideOutputs: true, hideInputs: true }, parentContext: api.ROOT_CONTEXT }); + assert.equal(capture.claim(api.ROOT_CONTEXT).getValue(ATTACHMENT_OWNER_KEY), true); + capture.speechInput(new File([wav()], 'synthetic.wav', { type: 'audio/wav' })); + assert.deepEqual(Buffer.from(await capture.speechResponse(speech()).arrayBuffer()), wav()); + await f.manager.flush(); idle(f); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); diff --git a/typescript/packages/provider-media/test/preflight.test.cjs b/typescript/packages/provider-media/test/preflight.test.cjs new file mode 100644 index 000000000..040c4014d --- /dev/null +++ b/typescript/packages/provider-media/test/preflight.test.cjs @@ -0,0 +1,115 @@ +/* SDK design section 1: every provider part is preflighted before its source is read, sized or decoded. */ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { ProviderMediaCapture } = require('../dist/src'); +const { fixture, png, wav, identity } = require('./fixtures.cjs'); + +const warnings = []; +process.on('warning', warning => { if (/^FI_ATTACHMENTS_/.test(String(warning.code))) warnings.push(warning.code); }); +const b64 = png.toString('base64'); +const settled = async f => { assert.deepEqual(await f.manager.flush(), { unfinished: 0, dropped: 0 }); }; +/** Records preflight calls and source reads in one ordered log. */ +function spy(manager, log) { + const preflight = manager.preflight.bind(manager); + manager.preflight = (target, options) => { const ticket = preflight(target, options); log.push(`preflight:${options.mimeType}:${options.maxBytes}:${ticket.reason}`); return ticket; }; + return manager; +} + +test('P-01 the gate preflight runs before the first source read, and an admitted part is read once', async () => { + const f = await fixture(); const log = []; + try { + spy(f.manager, log); + const capture = new ProviderMediaCapture(identity, { attachments: f.manager }); + capture.geminiContents([{ parts: [{ inlineData: { mimeType: 'image/png', get data() { log.push('read'); return b64; } } }] }]); + await settled(f); + assert.deepEqual(log, [`preflight:image/png:${1024 * 1024}:queued`, 'read']); + assert.equal(f.uploads.length, 1); assert.deepEqual(f.uploads[0].data, png); + } finally { await f.close(); } +}); + +test('P-02 Bedrock bytes and data URIs are preflighted before their getters run', async () => { + const f = await fixture(); const log = []; + try { + spy(f.manager, log); + const capture = new ProviderMediaCapture(identity, { attachments: f.manager }); + capture.bedrockConverse({ modelId: 'amazon.nova-lite-v1:0', messages: [{ content: [{ image: { format: 'png', source: { get bytes() { log.push('read'); return png; } } } }] }] }); + capture.compatibleMessages([{ content: [{ type: 'image_url', get image_url() { log.push('read'); return { url: `data:image/png;base64,${b64}` }; } }] }]); + await settled(f); + assert.deepEqual(log.map(e => e.split(':')[0]), ['preflight', 'read', 'preflight', 'read']); + assert.equal(f.uploads.length, 2); + } finally { await f.close(); } +}); + +test('P-03 a manager without preflight (an older core) captures nothing and never reads', async () => { + let attaches = 0; let reads = 0; + const legacy = { correlation: () => ({ project() {}, snapshot: () => identity }), attach() { attaches++; return { admission: 'queued' }; }, status: () => ({ state: 'ready' }) }; + const capture = new ProviderMediaCapture(identity, { attachments: legacy }); + capture.geminiContents([{ parts: [{ inlineData: { mimeType: 'image/png', get data() { reads++; return b64; } } }] }]); + capture.speechInput(new Blob([wav()], { type: 'audio/wav' })); + assert.equal(attaches, 0); assert.equal(reads, 0); +}); + +test('P-04 a server upload cap below the adapter gate still admits a small part at its exact size', async () => { + const f = await fixture({ sessionLimits: { max_upload_bytes: 64 * 1024 } }); const log = []; + try { + spy(f.manager, log); + new ProviderMediaCapture(identity, { attachments: f.manager }).geminiContents([{ parts: [{ inlineData: { mimeType: 'image/png', data: b64 } }] }]); + await settled(f); + assert.deepEqual(log, [`preflight:image/png:${1024 * 1024}:too_large`, `preflight:image/png:${png.length}:queued`]); + assert.equal(f.uploads.length, 1); assert.deepEqual(f.uploads[0].data, png); + } finally { await f.close(); } +}); + +test('P-05 an owned budget below the adapter gate still admits a small part at its exact size', async () => { + const f = await fixture({ maxOwnedBytes: 256 * 1024 }); + try { + new ProviderMediaCapture(identity, { attachments: f.manager }).compatibleMessages([{ content: [{ type: 'image_url', image_url: { url: `data:image/png;base64,${b64}` } }] }]); + await settled(f); + assert.equal(f.uploads.length, 1); assert.deepEqual(f.uploads[0].data, png); + } finally { await f.close(); } +}); + +test('P-06 Bedrock bytes above the 1 MiB base64 gate are re-preflighted at their exact length and upload', async () => { + const f = await fixture(); + try { + const big = Buffer.alloc(1536 * 1024, 7); png.copy(big); + new ProviderMediaCapture(identity, { attachments: f.manager }).bedrockConverse({ modelId: 'anthropic.claude-3-5-sonnet-20241022-v2:0', messages: [{ content: [{ image: { format: 'png', source: { bytes: big } } }] }] }); + await settled(f); + assert.equal(f.uploads.length, 1); assert.equal(f.uploads[0].data.length, big.length); + } finally { await f.close(); } +}); + +test('P-07 an admitted ticket whose bytes fail strict decode is released, holding no budget or descriptor', async () => { + const f = await fixture(); + try { + new ProviderMediaCapture(identity, { attachments: f.manager }).geminiContents([{ parts: [{ inlineData: { mimeType: 'image/png', data: '!!!!'.repeat(8) } }] }]); + await settled(f); + const status = f.manager.status(); + assert.equal(f.uploads.length, 0); assert.equal(status.openTickets, 0); assert.equal(status.ownedBytes, 0); + } finally { await f.close(); } +}); + +test('P-08 speech bodies are preflighted when the application reads them, never when the hook is installed', async () => { + const f = await fixture(); const log = []; + try { + spy(f.manager, log); + const c = new ProviderMediaCapture(identity, { attachments: f.manager }); + const response = c.speechResponse(new Response(wav(), { headers: { 'content-type': 'audio/wav' } })); + assert.deepEqual(log, []); + await response.blob(); c.speechInput(new Blob([wav()], { type: 'audio/wav' })); + await settled(f); + assert.ok(log.length >= 2); assert.equal(f.uploads.length, 2); + for (const upload of f.uploads) assert.deepEqual(upload.data, wav()); + } finally { await f.close(); } +}); + +test('P-09 the loopback fixture speaks the V1 wire: no legacy-transport or wire-rejection warning fired', async () => { + const f = await fixture(); + try { + new ProviderMediaCapture(identity, { attachments: f.manager }).geminiContents([{ parts: [{ inlineData: { mimeType: 'image/png', data: b64 } }] }]); + await settled(f); + assert.equal(f.uploads.length, 1); + } finally { await f.close(); } + await new Promise(resolve => setImmediate(resolve)); + assert.deepEqual(warnings, []); +}); diff --git a/typescript/packages/provider-media/test/sdk.test.cjs b/typescript/packages/provider-media/test/sdk.test.cjs new file mode 100644 index 000000000..5a70c7446 --- /dev/null +++ b/typescript/packages/provider-media/test/sdk.test.cjs @@ -0,0 +1,363 @@ +/* Real SDKs, synthetic provider responses, real loopback attachment HTTP traffic. */ +const { test, before, after } = require('node:test'); +const assert = require('node:assert/strict'); +const http = require('node:http'); +const { once } = require('node:events'); +const { Readable } = require('node:stream'); +const { createRequire } = require('node:module'); +const { pathToFileURL } = require('node:url'); +const path = require('node:path'); +const { fixture, png, wav } = require('./fixtures.cjs'); +const coreLoad = createRequire(path.resolve(__dirname, '../../fi-core/package.json')); +const { NodeTracerProvider } = coreLoad('@opentelemetry/sdk-trace-node'); +const { SimpleSpanProcessor, InMemorySpanExporter } = coreLoad('@opentelemetry/sdk-trace-base'); +const api = coreLoad('@opentelemetry/api'); +const OWNER = Symbol.for('traceai.auto.attachment_owner.v1'); +const exporter = new InMemorySpanExporter(); +const provider = new NodeTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); +provider.register(); +const sdkLoad = createRequire(path.join(process.env.MEDIA_SDK_ROOT, 'package.json')); +const routes = []; +let server, base; +const originalFetch = globalThis.fetch; +const redirect = (input, init = {}) => { + const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url); + // Older SDK shims pass node-fetch-only options (agent, polyfilled signals, stream bodies). + const { agent, signal, ...rest } = init; + const body = rest.body; + const streamed = body != null && typeof body !== 'string' && !(body instanceof Uint8Array) && !(body instanceof ArrayBuffer) && + !(typeof FormData !== 'undefined' && body instanceof FormData) && !(typeof Blob !== 'undefined' && body instanceof Blob) && !(body instanceof URLSearchParams); + return originalFetch(`${base}${url.pathname}${url.search}`, { ...rest, ...(signal instanceof AbortSignal ? { signal } : {}), ...(streamed ? { duplex: 'half' } : {}) }); +}; +// The loopback fixture speaks HTTP/1.1; newer Bedrock clients default to HTTP/2. +const http1 = tier => { const { NodeHttpHandler } = createRequire(sdkLoad.resolve(`sdk-bedrock-${tier}`))('@smithy/node-http-handler'); return new NodeHttpHandler(); }; +const messages = [{ role: 'user', content: [{ type: 'image_url', image_url: { url: `data:image/png;base64,${png.toString('base64')}` } }] }]; +before(async () => { + server = http.createServer(async (req, res) => { + const chunks = []; for await (const chunk of req) chunks.push(chunk); + routes.push(req.url); + if (req.url.includes('/audio/speech')) { res.setHeader('content-type', 'audio/wav'); res.end(wav()); return; } + res.setHeader('content-type', 'application/json'); + if (req.url.includes('/audio/')) { res.end(JSON.stringify({ text: 'synthetic transcript' })); return; } + if (req.url.includes(':generateContent')) { res.end(JSON.stringify({ candidates: [{ content: { role: 'model', parts: [{ text: 'fixture' }, { inlineData: { mimeType: 'image/png', data: png.toString('base64') } }] }, finishReason: 'STOP' }], usageMetadata: { promptTokenCount: 1, candidatesTokenCount: 1, totalTokenCount: 2 } })); return; } + if (req.url.endsWith('/converse')) { res.end(JSON.stringify({ output: { message: { role: 'assistant', content: [{ text: 'fixture' }] } }, usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, stopReason: 'end_turn', metrics: { latencyMs: 1 } })); return; } + res.end(JSON.stringify({ id: 'synthetic-completion', object: 'chat.completion', created: 1, model: 'fixture', choices: [{ index: 0, message: { role: 'assistant', content: 'fixture' }, finish_reason: 'stop' }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })); + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); base = `http://127.0.0.1:${server.address().port}`; +}); +after(async () => { await provider.shutdown(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); }); + +// One tier per process: some instrumentations keep process-wide patch state. +// Built together with the OpenAI chat adapter (`captureOpenAIAttachments`, stacked lanes), that adapter captures the +// typed chat image once. This helper never adds a copy, so a chat call uploads exactly what that adapter uploads. +const openAIChatCaptures = () => typeof require('../../traceai_openai/dist/src').captureOpenAIAttachments === 'function' ? 1 : 0; +const tiers = process.env.MEDIA_SDK_TIER ? [process.env.MEDIA_SDK_TIER] : ['floor', 'latest']; +for (const tier of tiers) { + for (const privacy of ['enabled', 'disabled', 'hideInputs', 'piiRedaction']) test(`OpenAI ${tier}: speech ${privacy}; chat media stays with the OpenAI chat adapter`, async () => { + const sdk = sdkLoad(`sdk-openai-${tier}`); + const { OpenAIInstrumentation } = require('../../traceai_openai/dist/src'); + const f = await fixture(privacy === 'disabled' ? { enabled: false } : { traceConfig: privacy === 'enabled' ? {} : { [privacy]: true } }); + const inst = new OpenAIInstrumentation({ attachments: f.manager, instrumentationConfig: { enabled: false } }); + inst.setTracerProvider(provider); inst.enable(); inst.manuallyInstrument(sdk); + try { + const client = new sdk.OpenAI({ apiKey: 'synthetic', fetch: redirect }); + await client.chat.completions.create({ model: 'fixture', messages }); + // This lane does not own the chat hook; no duplicate chat capture is added here. + await f.manager.flush(); assert.equal(f.uploads.length, privacy === 'enabled' ? openAIChatCaptures() : 0); + if (client.audio.speech) { + const response = await client.audio.speech.create({ model: 'tts-1', input: 'fixture', voice: 'alloy', response_format: 'wav' }); + const before = f.uploads.length; await f.manager.flush(); assert.equal(f.uploads.length, before); + assert.deepEqual(Buffer.from(await response.arrayBuffer()), wav()); await f.manager.flush(); + assert.equal(f.uploads.length - before, ['enabled', 'hideInputs'].includes(privacy) ? 1 : 0); + if (f.uploads.length > before) assert.equal(f.uploads.at(-1).request.direction, 'output'); + const unread = await client.audio.speech.create({ model: 'tts-1', input: 'fixture', voice: 'alloy', response_format: 'wav' }); + await unread.body?.cancel?.(); const after = f.uploads.length; await f.manager.flush(); assert.equal(f.uploads.length, after); + } + const file = new File([wav()], 'synthetic-private-name.wav', { type: 'audio/wav' }); + for (const operation of ['transcriptions', 'translations']) { + const before = f.uploads.length; + const result = await client.audio[operation].create({ model: 'whisper-1', file }); assert.equal(result.text, 'synthetic transcript'); + await f.manager.flush(); assert.equal(f.uploads.length - before, privacy === 'enabled' ? 1 : 0); + if (f.uploads.length > before) assert.deepEqual(f.uploads.at(-1).data, wav()); + } + const spans = exporter.getFinishedSpans(); + assert.ok(spans.some(s => s.name === 'OpenAI Chat Completions')); + // Attachments OFF keeps audio calls span-free, exactly as before this change. + assert.equal(spans.some(s => s.name.startsWith('OpenAI Audio')), privacy !== 'disabled'); + const exported = JSON.stringify(spans.map(s => s.attributes)); + assert.equal(exported.includes(wav().toString('base64')), false); assert.equal(exported.includes('synthetic-private-name'), false); + assert.equal(JSON.stringify(f.uploads.map(u => u.request)).includes('synthetic-private-name'), false); + } finally { inst.disable(); await f.close(); exporter.reset(); } + }); + + test(`OpenAI ${tier}: speech capture only on qualified audio hosts`, async () => { + const sdk = sdkLoad(`sdk-openai-${tier}`); + const { OpenAIInstrumentation } = require('../../traceai_openai/dist/src'); + const f = await fixture(); const inst = new OpenAIInstrumentation({ attachments: f.manager, instrumentationConfig: { enabled: false } }); + inst.setTracerProvider(provider); inst.enable(); inst.manuallyInstrument(sdk); + try { + for (const baseURL of ['https://api.groq.com/openai/v1', 'https://unknown.invalid/v1', 'https://api.fireworks.ai/inference/v1']) { + const before = f.uploads.length; const client = new sdk.OpenAI({ apiKey: 'synthetic', baseURL, fetch: redirect }); + await client.audio.transcriptions.create({ model: 'whisper-1', file: new File([wav()], 'a.wav', { type: 'audio/wav' }) }); await f.manager.flush(); + assert.equal(f.uploads.length - before, baseURL.includes('api.groq.com') ? 1 : 0, baseURL); + } + } finally { inst.disable(); await f.close(); exporter.reset(); } + }); + + // openai 4.x's default Node transport is node-fetch 2: its body is a Node PassThrough, not a WHATWG stream. + // The SDK's own node-fetch is driven against the loopback fixture, so the Response and body are the shim's. + for (const privacy of ['enabled', 'disabled', 'hideInputs', 'piiRedaction']) test(`OpenAI 4.x ${tier} node-fetch shim: speech ${privacy}; chunk-wise and blob() reads upload nothing, arrayBuffer() is observed`, async () => { + const sdk = sdkLoad(`sdk-openai4-${tier}`); + const nodeFetch = createRequire(sdkLoad.resolve(`sdk-openai4-${tier}`))('node-fetch'); + const shim = (input, init = {}) => { + const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url); + // The SDK's keep-alive agent is for its https host; the loopback fixture is plain http. + const { agent, ...rest } = init; + return nodeFetch(`${base}${url.pathname}${url.search}`, rest); + }; + const { OpenAIInstrumentation } = require('../../traceai_openai/dist/src'); + const f = await fixture(privacy === 'disabled' ? { enabled: false } : { traceConfig: privacy === 'enabled' ? {} : { [privacy]: true } }); + const inst = new OpenAIInstrumentation({ attachments: f.manager, instrumentationConfig: { enabled: false } }); + inst.setTracerProvider(provider); inst.enable(); inst.manuallyInstrument(sdk); + try { + const client = new sdk.OpenAI({ apiKey: 'synthetic', fetch: shim }); + const speech = () => client.audio.speech.create({ model: 'tts-1', input: 'fixture', voice: 'alloy', response_format: 'wav' }); + for (const stopAfter of [Infinity, 2]) { + const response = await speech(); + assert.ok(response.body instanceof Readable, 'the row runs on the node-fetch transport'); + assert.equal(typeof response.body.getReader, 'undefined'); + const before = f.uploads.length; const out = []; let n = 0; + for await (const chunk of response.body) { out.push(chunk); if (++n >= stopAfter) break; } + if (stopAfter === Infinity) assert.deepEqual(Buffer.concat(out), wav()); + await f.manager.flush(); assert.equal(f.uploads.length, before, `for await (stop after ${stopAfter})`); + assert.equal(f.manager.status().openStreams, 0); + } + const response = await speech(); + const before = f.uploads.length; + assert.deepEqual(Buffer.from(await response.arrayBuffer()), wav()); await f.manager.flush(); + assert.equal(f.uploads.length - before, ['enabled', 'hideInputs'].includes(privacy) ? 1 : 0); + if (f.uploads.length > before) { assert.equal(f.uploads.at(-1).request.direction, 'output'); assert.deepEqual(f.uploads.at(-1).data, wav()); } + // node-fetch 2 resolves blob() to its own Blob class, not globalThis.Blob, so that read uploads nothing. + const blobbed = await (await speech()).blob(); + assert.equal(blobbed instanceof Blob, false); assert.equal(blobbed.size, wav().length); + const afterBlob = f.uploads.length; await f.manager.flush(); assert.equal(f.uploads.length, afterBlob, 'blob() on node-fetch'); + assert.equal(f.manager.status().openStreams, 0); + } finally { inst.disable(); await f.close(); exporter.reset(); } + }); + + // The context the SDK's own fetch runs in is the one wire-level capture (the auto module) inspects. + const marking = seen => (input, init) => { + const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url); + // Non-HTTP requests an SDK makes through the custom fetch (not provider calls) are recorded by scheme. + seen.push([url.protocol.startsWith('http') ? url.pathname.replace(/^.*\/(chat|audio)\//, '$1/') : url.protocol, api.context.active().getValue(OWNER)]); + return redirect(input, init); + }; + const ownerRows = async (client, model, voice, f, seen) => { + // Claimed: audio calls whose media this capture owns carry the marker; chat (owned by the OpenAI chat adapter) does not. + await client.chat.completions.create({ model: 'fixture', messages }); + if (client.audio?.speech) await (await client.audio.speech.create({ model: 'fixture', input: 'fixture', voice, response_format: 'wav' })).arrayBuffer(); + for (const operation of ['transcriptions', 'translations']) if (client.audio?.[operation]) await client.audio[operation].create({ model, file: new File([wav()], 'synthetic.wav', { type: 'audio/wav' }) }); + await f.manager.flush(); + const claimed = f.uploads.length; + // Honoured: an application or adapter that already owns the call's media keeps it; this capture uploads nothing. + await api.context.with(api.context.active().setValue(OWNER, true), async () => { + if (client.audio?.speech) assert.deepEqual(Buffer.from(await (await client.audio.speech.create({ model: 'fixture', input: 'fixture', voice, response_format: 'wav' })).arrayBuffer()), wav()); + for (const operation of ['transcriptions', 'translations']) if (client.audio?.[operation]) await client.audio[operation].create({ model, file: new File([wav()], 'synthetic.wav', { type: 'audio/wav' }) }); + }); + await f.manager.flush(); + return { claimed, honoured: f.uploads.length - claimed }; + }; + + for (const mode of ['enabled', 'disabled']) test(`OpenAI ${tier}: audio calls carry the shared owner marker (${mode}); an owned call uploads nothing`, async () => { + const sdk = sdkLoad(`sdk-openai-${tier}`); + const { OpenAIInstrumentation } = require('../../traceai_openai/dist/src'); + const f = await fixture(mode === 'disabled' ? { enabled: false } : {}); + const inst = new OpenAIInstrumentation({ attachments: f.manager, instrumentationConfig: { enabled: false } }); + inst.setTracerProvider(provider); inst.enable(); inst.manuallyInstrument(sdk); + try { + const seen = []; + const result = await ownerRows(new sdk.OpenAI({ apiKey: 'synthetic', fetch: marking(seen) }), 'whisper-1', 'alloy', f, seen); + assert.deepEqual(seen[0], ['chat/completions', undefined]); + const audio = seen.filter(([route]) => route.startsWith('audio/')); + const claimed = audio.slice(0, audio.length / 2); + assert.ok(claimed.length >= 2, JSON.stringify(seen)); + // Attachments OFF: audio calls are untouched, exactly as before (no span, no marker, no upload). + for (const [, value] of claimed) assert.equal(value, mode === 'enabled' ? true : undefined, JSON.stringify(seen)); + assert.deepEqual(result, { claimed: mode === 'enabled' ? claimed.length + openAIChatCaptures() : 0, honoured: 0 }); + } finally { inst.disable(); await f.close(); exporter.reset(); } + }); + + test(`OpenAI ${tier}: unqualified audio hosts are not claimed`, async () => { + const sdk = sdkLoad(`sdk-openai-${tier}`); + const { OpenAIInstrumentation } = require('../../traceai_openai/dist/src'); + const f = await fixture(); const inst = new OpenAIInstrumentation({ attachments: f.manager, instrumentationConfig: { enabled: false } }); + inst.setTracerProvider(provider); inst.enable(); inst.manuallyInstrument(sdk); + try { + const seen = []; + await new sdk.OpenAI({ apiKey: 'synthetic', baseURL: 'https://unknown.invalid/v1', fetch: marking(seen) }).audio.transcriptions.create({ model: 'whisper-1', file: new File([wav()], 'a.wav', { type: 'audio/wav' }) }); + await f.manager.flush(); + // Only the provider call is asserted by route; nothing the SDK fetches on this host carries the marker. + assert.deepEqual(seen.filter(([route]) => route.startsWith('audio/')), [['audio/transcriptions', undefined]], JSON.stringify(seen)); + for (const [, value] of seen) assert.equal(value, undefined, JSON.stringify(seen)); + assert.equal(f.uploads.length, 0); + } finally { inst.disable(); await f.close(); exporter.reset(); } + }); + + test(`Groq ${tier}: audio calls carry the shared owner marker; an owned call uploads nothing`, async () => { + const sdk = sdkLoad(`sdk-groq-${tier}`); + const { GroqInstrumentation } = require('../../traceai_groq/dist/src'); + const f = await fixture(); + const inst = new GroqInstrumentation({ attachments: f.manager, instrumentationConfig: { enabled: false } }); + inst.setTracerProvider(provider); inst.enable(); inst.manuallyInstrument(sdk); + try { + const Groq = sdk.Groq ?? sdk.default ?? sdk; + const seen = []; + const client = new Groq({ apiKey: 'synthetic', fetch: marking(seen) }); + const plain = [{ role: 'user', content: 'fixture' }]; + await client.chat.completions.create({ model: 'fixture', messages: plain }); + const result = await ownerRows(client, 'whisper-large-v3', 'fixture', f, seen); + const chat = seen.filter(([route]) => route === 'chat/completions'); + const audio = seen.filter(([route]) => route.startsWith('audio/')); + assert.equal(chat.length, 2); for (const [, value] of chat) assert.equal(value, undefined); + const owned = audio.length / 2; + for (const [, value] of audio.slice(0, owned)) assert.equal(value, true, JSON.stringify(seen)); + // Chat media (one typed image) is still captured; audio uploads once per claimed call and never under an owner. + assert.deepEqual(result, { claimed: 1 + owned, honoured: 0 }); + } finally { inst.disable(); await f.close(); exporter.reset(); } + }); + + for (const privacy of ['enabled', 'disabled', 'hideInputs', 'piiRedaction']) test(`Groq ${tier}: typed chat/speech ${privacy}`, async () => { + const sdk = sdkLoad(`sdk-groq-${tier}`); + const { GroqInstrumentation } = require('../../traceai_groq/dist/src'); + const f = await fixture(privacy === 'disabled' ? { enabled: false } : { traceConfig: privacy === 'enabled' ? {} : { [privacy]: true } }); + const inst = new GroqInstrumentation({ attachments: f.manager, instrumentationConfig: { enabled: false } }); + inst.setTracerProvider(provider); inst.enable(); inst.manuallyInstrument(sdk); + try { + const Groq = sdk.Groq ?? sdk.default ?? sdk; + const client = new Groq({ apiKey: 'synthetic', fetch: redirect }); + await client.chat.completions.create({ model: 'fixture', messages }); await f.manager.flush(); assert.equal(f.uploads.length, privacy === 'enabled' ? 1 : 0); + for (const operation of ['transcriptions', 'translations']) if (client.audio?.[operation]) { + const before = f.uploads.length; + await client.audio[operation].create({ model: 'whisper-large-v3', file: new File([wav()], 'synthetic.wav', { type: 'audio/wav' }) }); + await f.manager.flush(); assert.equal(f.uploads.length - before, privacy === 'enabled' ? 1 : 0); + } + if (client.audio?.speech) { + const response = await client.audio.speech.create({ model: 'fixture', input: 'fixture', voice: 'fixture', response_format: 'wav' }); + const before = f.uploads.length; await f.manager.flush(); assert.equal(f.uploads.length, before); + assert.deepEqual(Buffer.from(await response.arrayBuffer()), wav()); await f.manager.flush(); + assert.equal(f.uploads.length - before, ['enabled', 'hideInputs'].includes(privacy) ? 1 : 0); + } + } finally { inst.disable(); await f.close(); } + }); + + test(`Google GenAI ${tier}: original inline Parts in/out, no async submission media`, async () => { + const sdk = sdkLoad(`sdk-genai-${tier}`); + const { instrumentGoogleGenAI } = require('../../traceai_google_genai_sdk/dist'); + const f = await fixture(); const client = new sdk.GoogleGenAI({ apiKey: 'synthetic', httpOptions: { baseUrl: base } }); + const handle = instrumentGoogleGenAI(client, { attachments: f.manager, tracerProvider: provider }); + try { + await client.models.generateContent({ model: 'gemini-fixture', contents: [{ role: 'user', parts: [{ inlineData: { mimeType: 'image/png', data: png.toString('base64') } }] }] }); + await f.manager.flush(); assert.equal(f.uploads.length, 2); for (const u of f.uploads) assert.deepEqual(u.data, png); + } finally { handle.dispose(); await f.close(); } + }); + + test(`Legacy Gemini ${tier}: SDK normalizes typed Part array`, async () => { + const sdk = sdkLoad(`sdk-gemini-${tier}`); + const { GoogleGenAIInstrumentation } = require('../../traceai_google_genai/dist/src'); + const f = await fixture(); const inst = new GoogleGenAIInstrumentation({ attachments: f.manager, instrumentationConfig: { enabled: false } }); + inst.setTracerProvider(provider); inst.enable(); inst.manuallyInstrument(sdk); + globalThis.fetch = (input, init) => String(typeof input === 'object' ? input.url : input).startsWith('https://generativelanguage.googleapis.com') ? redirect(input, init) : originalFetch(input, init); + try { + await new sdk.GoogleGenerativeAI('synthetic').getGenerativeModel({ model: 'gemini-fixture' }).generateContent([{ inlineData: { mimeType: 'image/png', data: png.toString('base64') } }]); + await f.manager.flush(); assert.equal(f.uploads.length, 2); + } finally { globalThis.fetch = originalFetch; inst.disable(); await f.close(); } + }); + + test(`Bedrock ${tier}: real Converse serializer retains supported typed bytes`, async () => { + const sdk = sdkLoad(`sdk-bedrock-${tier}`); + const { BedrockInstrumentation } = require('../../traceai_bedrock/dist/src'); + const f = await fixture(); const inst = new BedrockInstrumentation({ attachments: f.manager, tracerProvider: provider, instrumentationConfig: { enabled: false } }); + inst.enable(); inst.manuallyInstrument(sdk); + const client = new sdk.BedrockRuntimeClient({ endpoint: base, region: 'us-east-1', credentials: { accessKeyId: 'synthetic', secretAccessKey: 'synthetic' }, requestHandler: http1(tier) }); + try { + await client.send(new sdk.ConverseCommand({ modelId: 'anthropic.claude-3-5-sonnet-20241022-v2:0', messages: [{ role: 'user', content: [{ image: { format: 'png', source: { bytes: png } } }] }] })); + await f.manager.flush(); assert.equal(f.uploads.length, 1); assert.deepEqual(f.uploads[0].data, png); + } finally { inst.disable(); client.destroy(); await f.close(); } + }); + + test(`Bedrock ${tier}: unsupported models keep span tracing without media capture`, async () => { + const sdk = sdkLoad(`sdk-bedrock-${tier}`); + const { BedrockInstrumentation } = require('../../traceai_bedrock/dist/src'); + const f = await fixture(); const inst = new BedrockInstrumentation({ attachments: f.manager, tracerProvider: provider, instrumentationConfig: { enabled: false } }); + inst.enable(); inst.manuallyInstrument(sdk); + const client = new sdk.BedrockRuntimeClient({ endpoint: base, region: 'us-east-1', credentials: { accessKeyId: 'synthetic', secretAccessKey: 'synthetic' }, requestHandler: http1(tier) }); + try { + await client.send(new sdk.ConverseCommand({ modelId: 'meta.llama3-70b-instruct-v1:0', messages: [{ role: 'user', content: [{ image: { format: 'png', source: { bytes: png } } }] }] })); + await f.manager.flush(); assert.equal(f.uploads.length, 0); + assert.ok(exporter.getFinishedSpans().some(s => s.name === 'bedrock.converse')); + } finally { inst.disable(); client.destroy(); await f.close(); exporter.reset(); } + }); + + test(`Vertex AI ${tier}: inline Parts in/out through the real SDK`, async () => { + const sdk = sdkLoad(`sdk-vertex-${tier}`); + const auth = createRequire(sdkLoad.resolve(`sdk-vertex-${tier}`))('google-auth-library'); + const restoreToken = auth.GoogleAuth.prototype.getAccessToken; + auth.GoogleAuth.prototype.getAccessToken = async () => 'synthetic-token'; + const { VertexAIInstrumentation } = require('../../traceai_vertexai/dist/src'); + const f = await fixture(); const inst = new VertexAIInstrumentation({ attachments: f.manager, instrumentationConfig: { enabled: false } }); + inst.setTracerProvider(provider); inst.enable(); inst.manuallyInstrument(sdk); + globalThis.fetch = (input, init) => String(typeof input === 'object' ? input.url : input).startsWith('https://vertex.fixture.invalid') ? redirect(input, init) : originalFetch(input, init); + try { + const model = new sdk.VertexAI({ project: 'synthetic-project', location: 'us-central1', apiEndpoint: 'vertex.fixture.invalid' }).getGenerativeModel({ model: 'gemini-fixture' }); + await model.generateContent({ contents: [{ role: 'user', parts: [{ inlineData: { mimeType: 'image/png', data: png.toString('base64') } }] }] }); + await f.manager.flush(); assert.equal(f.uploads.length, 2); + assert.deepEqual(f.uploads.map(u => u.request.direction).sort(), ['input', 'output']); for (const u of f.uploads) assert.deepEqual(u.data, png); + } finally { globalThis.fetch = originalFetch; auth.GoogleAuth.prototype.getAccessToken = restoreToken; inst.disable(); await f.close(); } + }); + + for (const privacy of ['enabled', 'hideInputImages', 'piiRedaction']) test(`Mistral ${tier}: typed imageUrl chunk ${privacy}`, async () => { + const imported = await import(pathToFileURL(sdkLoad.resolve(`sdk-mistral-${tier}`)).href); + const sdk = imported.Mistral ? imported : imported.default; + const { MistralInstrumentation } = require('../../traceai_mistral/dist/src'); + const f = await fixture({ traceConfig: privacy === 'enabled' ? {} : { [privacy]: true } }); + const inst = new MistralInstrumentation({ attachments: f.manager, instrumentationConfig: { enabled: false } }); + inst.setTracerProvider(provider); inst.enable(); inst.manuallyInstrument(sdk); + try { + const sent = []; + const client = new sdk.Mistral({ apiKey: 'synthetic', serverURL: base, httpClient: { request: async req => { sent.push(await req.clone().text()); return redirect(req.url, { method: req.method, headers: req.headers, body: await req.text() }); } } }); + await client.chat.complete({ model: 'pixtral-fixture', messages: [{ role: 'user', content: [{ type: 'text', text: 'describe' }, { type: 'image_url', imageUrl: `data:image/png;base64,${png.toString('base64')}` }] }] }); + assert.ok(sent.some(body => body.includes(png.toString('base64'))), 'SDK sent the typed image chunk'); + await f.manager.flush(); assert.equal(f.uploads.length, privacy === 'enabled' ? 1 : 0); + if (f.uploads.length) { assert.deepEqual(f.uploads[0].data, png); assert.equal(f.uploads[0].request.part_index, 1); } + } finally { inst.disable(); await f.close(); } + }); + + test(`Portkey ${tier}: OpenAI-shaped chat image parts`, async () => { + const sdk = sdkLoad(`sdk-portkey-${tier}`); + const { PortkeyInstrumentation } = require('../../traceai_portkey/dist/src'); + const f = await fixture(); const inst = new PortkeyInstrumentation({ attachments: f.manager, instrumentationConfig: { enabled: false } }); + inst.setTracerProvider(provider); inst.enable(); inst.manuallyInstrument(sdk); + try { + const Portkey = sdk.Portkey ?? sdk.default; + const client = new Portkey({ apiKey: 'synthetic', baseURL: `${base}/v1`, provider: 'openai' }); + inst.manuallyInstrument(client); + await client.chat.completions.create({ model: 'fixture', messages }); + await f.manager.flush(); assert.equal(f.uploads.length, 1); assert.deepEqual(f.uploads[0].data, png); + } finally { inst.disable(); await f.close(); } + }); + + test(`vLLM ${tier}: matched self-hosted route captures; unmatched route does not`, async () => { + const sdk = sdkLoad(`sdk-openai-${tier}`); + const { VLLMInstrumentation } = require('../../traceai_vllm/dist/cjs'); + const f = await fixture(); const inst = new VLLMInstrumentation({ attachments: f.manager, baseUrlPattern: '127.0.0.1', instrumentationConfig: { enabled: false } }); + inst.setTracerProvider(provider); inst.enable(); inst.manuallyInstrument(sdk); + try { + await new sdk.OpenAI({ apiKey: 'synthetic', baseURL: `${base}/v1` }).chat.completions.create({ model: 'fixture', messages }); + await f.manager.flush(); assert.equal(f.uploads.length, 1); + await new sdk.OpenAI({ apiKey: 'synthetic', baseURL: 'https://other.invalid/v1', fetch: redirect }).chat.completions.create({ model: 'fixture', messages }); + await f.manager.flush(); assert.equal(f.uploads.length, 1); + } finally { inst.disable(); await f.close(); } + }); +} + +test('provider requests reached only the loopback server', () => { assert.ok(routes.length >= 20 * tiers.length); }); diff --git a/typescript/packages/provider-media/test/speech-stream.test.cjs b/typescript/packages/provider-media/test/speech-stream.test.cjs new file mode 100644 index 000000000..839273d50 --- /dev/null +++ b/typescript/packages/provider-media/test/speech-stream.test.cjs @@ -0,0 +1,275 @@ +/* SDK design 6.4: a reader or async iterator the application obtained is observed chunk by chunk; nothing is pulled by the adapter. */ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { ProviderMediaCapture } = require('../dist/src'); +const { fixture, wav, identity } = require('./fixtures.cjs'); + +const audio = (bytes = wav(), chunk = 16) => { + let offset = 0; let pulls = 0; + const body = new ReadableStream({ pull(controller) { + pulls++; + if (offset >= bytes.length) { controller.close(); return; } + controller.enqueue(new Uint8Array(bytes.subarray(offset, offset + chunk))); offset += chunk; + } }, { highWaterMark: 0 }); + return { response: new Response(body, { headers: { 'content-type': 'audio/wav' } }), pulls: () => pulls }; +}; +const drain = async reader => { const out = []; for (;;) { const { done, value } = await reader.read(); if (done) return Buffer.concat(out); out.push(value); } }; +const idle = async f => { const s = f.manager.status(); assert.equal(s.openStreams, 0); assert.equal(s.openTickets, 0); assert.equal(s.ownedBytes, 0); }; + +test('S-01 a reader read to its end uploads exactly the chunks the application received', async () => { + const f = await fixture(); + try { + const { response, pulls } = audio(); + new ProviderMediaCapture(identity, { attachments: f.manager }).speechResponse(response); + assert.equal(pulls(), 0); + const received = await drain(response.body.getReader()); + assert.deepEqual(received, wav()); + await f.manager.flush(); await idle(f); + assert.equal(f.uploads.length, 1); assert.deepEqual(f.uploads[0].data, wav()); + assert.equal(f.uploads[0].request.direction, 'output'); assert.equal(f.uploads[0].request.mime_type, 'audio/wav'); + } finally { await f.close(); } +}); + +for (const stop of ['releaseLock', 'cancel', 'abandon']) test(`S-02 a partial read then ${stop} uploads nothing and returns its budget`, async () => { + const f = await fixture(); + try { + const { response } = audio(); + new ProviderMediaCapture(identity, { attachments: f.manager }).speechResponse(response); + const reader = response.body.getReader(); + await reader.read(); await reader.read(); + if (stop === 'releaseLock') reader.releaseLock(); + if (stop === 'cancel') await reader.cancel(); + await f.manager.flush(); + assert.equal(f.uploads.length, 0); + if (stop !== 'abandon') await idle(f); + else assert.equal(f.manager.status().openStreams, 1); + } finally { await f.close(); } +}); + +test('S-03 a stream over the 4 MiB cap uploads nothing; the application still reads every byte', async () => { + const f = await fixture(); + try { + const big = Buffer.alloc(4 * 1024 * 1024 + 64, 3); + const { response } = audio(big, 256 * 1024); + new ProviderMediaCapture(identity, { attachments: f.manager }).speechResponse(response); + const received = await drain(response.body.getReader()); + assert.equal(received.length, big.length); + await f.manager.flush(); await idle(f); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); + +for (const mode of ['hideOutputs', 'piiRedaction', 'localHide', 'disabled']) test(`S-04 ${mode}: no stream is opened and nothing uploads`, async () => { + const f = await fixture(mode === 'disabled' ? { enabled: false } : mode === 'localHide' ? {} : { traceConfig: { [mode]: true } }); + try { + const { response } = audio(); + new ProviderMediaCapture(identity, { attachments: f.manager, traceConfig: mode === 'localHide' ? { hideOutputs: true } : {} }).speechResponse(response); + const reader = response.body.getReader(); + await reader.read(); + assert.equal(f.manager.status().openStreams, 0); + assert.ok((await drain(reader)).length > 0); + await f.manager.flush(); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); + +test('S-05 a policy tightened mid-stream discards the stream', async () => { + const f = await fixture(); + try { + const { response } = audio(); + new ProviderMediaCapture(identity, { attachments: f.manager }).speechResponse(response); + const reader = response.body.getReader(); + await reader.read(); + f.manager.updatePolicy({ traceConfig: { hideOutputs: true } }); + await drain(reader); + await f.manager.flush(); await idle(f); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); + +test('S-06 a read error reaches the application unchanged and uploads nothing', async () => { + const f = await fixture(); + try { + let sent = false; + const body = new ReadableStream({ pull(controller) { if (!sent) { sent = true; controller.enqueue(new Uint8Array(wav().subarray(0, 8))); } else controller.error(new Error('synthetic-network')); } }, { highWaterMark: 0 }); + const response = new Response(body, { headers: { 'content-type': 'audio/wav' } }); + new ProviderMediaCapture(identity, { attachments: f.manager }).speechResponse(response); + const reader = response.body.getReader(); + await reader.read(); + await assert.rejects(reader.read(), /synthetic-network/); + await f.manager.flush(); await idle(f); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); + +test('S-07 the same span is captured once when the application reads through arrayBuffer()', async () => { + const f = await fixture(); + try { + const { response } = audio(); + new ProviderMediaCapture(identity, { attachments: f.manager }).speechResponse(response); + assert.deepEqual(Buffer.from(await response.arrayBuffer()), wav()); + await f.manager.flush(); await idle(f); + assert.equal(f.uploads.length, 1); assert.deepEqual(f.uploads[0].data, wav()); + } finally { await f.close(); } +}); + +test('S-08 a second reader after releaseLock sees only the rest of the body, so nothing uploads', async () => { + const f = await fixture(); + try { + const { response } = audio(); + new ProviderMediaCapture(identity, { attachments: f.manager }).speechResponse(response); + const first = response.body.getReader(); + const head = [(await first.read()).value, (await first.read()).value]; + first.releaseLock(); + const rest = await drain(response.body.getReader()); + assert.deepEqual(Buffer.concat([...head, rest]), wav()); + await f.manager.flush(); await idle(f); + assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); + +/* The core reserves min(4 MiB, the negotiated max_upload_bytes) for a consumed stream (ts-core 01b3327a). */ +const SERVER_CAP = 32 * 1024; +const pcm = size => { const out = Buffer.alloc(size); wav().copy(out); out.writeUInt32LE(size - 8, 4); out.writeUInt32LE(size - 44, 40); return out; }; + +test('S-09 a server max_upload_bytes below 4 MiB still admits the reader stream, reserving only the server cap', async () => { + const f = await fixture({ sessionLimits: { max_upload_bytes: SERVER_CAP } }); + try { + const body = pcm(20 * 1024); + const { response } = audio(body, 4096); + new ProviderMediaCapture(identity, { attachments: f.manager }).speechResponse(response); + const reader = response.body.getReader(); + const first = await reader.read(); + assert.equal(f.manager.status().openStreams, 1); + assert.equal(f.manager.status().ownedBytes, SERVER_CAP, 'reservation = min(4 MiB, max_upload_bytes)'); + const rest = await drain(reader); + assert.deepEqual(Buffer.concat([first.value, rest]), body); + await f.manager.flush(); await idle(f); + assert.equal(f.uploads.length, 1); assert.deepEqual(f.uploads[0].data, body); + } finally { await f.close(); } +}); + +test('S-10 a body over the server cap (but under 4 MiB) is discarded whole; the application still reads every byte', async () => { + const f = await fixture({ sessionLimits: { max_upload_bytes: SERVER_CAP } }); + try { + const body = pcm(SERVER_CAP + 4096); + const { response } = audio(body, 4096); + new ProviderMediaCapture(identity, { attachments: f.manager }).speechResponse(response); + assert.deepEqual(await drain(response.body.getReader()), body); + await f.manager.flush(); await idle(f); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); + +/* `for await` and values() lock the body with an internal reader, never the public getReader(). */ +const iterate = async (iterable, stopAfter = Infinity) => { + const out = []; let n = 0; + for await (const chunk of iterable) { out.push(chunk); if (++n >= stopAfter) break; } + return Buffer.concat(out); +}; + +for (const entry of ['for await', 'values()']) test(`S-11 ${entry} to the end uploads exactly the chunks the application received, pulling nothing itself`, async () => { + const f = await fixture(); + try { + const { response, pulls } = audio(); + new ProviderMediaCapture(identity, { attachments: f.manager }).speechResponse(response); + const iterator = entry === 'values()' ? response.body.values() : response.body[Symbol.asyncIterator](); + assert.equal(pulls(), 0, 'obtaining the iterator reads nothing'); + assert.equal(f.manager.status().openStreams, 1); + const received = await iterate({ [Symbol.asyncIterator]: () => iterator }); + assert.deepEqual(received, wav()); + await f.manager.flush(); await idle(f); + assert.equal(f.uploads.length, 1); assert.deepEqual(f.uploads[0].data, wav()); + assert.equal(f.uploads[0].request.direction, 'output'); assert.equal(f.uploads[0].request.mime_type, 'audio/wav'); + } finally { await f.close(); } +}); + +test('S-12 a for-await loop that breaks early uploads nothing and returns its budget', async () => { + const f = await fixture(); + try { + const { response } = audio(); + new ProviderMediaCapture(identity, { attachments: f.manager }).speechResponse(response); + assert.equal((await iterate(response.body, 2)).length, 32); + await f.manager.flush(); await idle(f); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); + +test('S-13 values({ preventCancel }) then a later reader sees only the rest of the body, so nothing uploads', async () => { + const f = await fixture(); + try { + const { response } = audio(); + new ProviderMediaCapture(identity, { attachments: f.manager }).speechResponse(response); + const head = await iterate({ [Symbol.asyncIterator]: () => response.body.values({ preventCancel: true }) }, 2); + const rest = await drain(response.body.getReader()); + assert.deepEqual(Buffer.concat([head, rest]), wav()); + await f.manager.flush(); await idle(f); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); + +test('S-14 an iterated stream over the 4 MiB cap uploads nothing; the application still reads every byte', async () => { + const f = await fixture(); + try { + const big = Buffer.alloc(4 * 1024 * 1024 + 64, 3); + const { response } = audio(big, 256 * 1024); + new ProviderMediaCapture(identity, { attachments: f.manager }).speechResponse(response); + assert.equal((await iterate(response.body)).length, big.length); + await f.manager.flush(); await idle(f); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); + +test('S-15 an error during iteration reaches the application unchanged and uploads nothing', async () => { + const f = await fixture(); + try { + let sent = false; + const body = new ReadableStream({ pull(controller) { if (!sent) { sent = true; controller.enqueue(new Uint8Array(wav().subarray(0, 8))); } else controller.error(new Error('synthetic-network')); } }, { highWaterMark: 0 }); + const response = new Response(body, { headers: { 'content-type': 'audio/wav' } }); + new ProviderMediaCapture(identity, { attachments: f.manager }).speechResponse(response); + await assert.rejects(iterate(response.body), /synthetic-network/); + await f.manager.flush(); await idle(f); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); + +for (const mode of ['hideOutputs', 'piiRedaction', 'localHide', 'disabled']) test(`S-16 ${mode}: iterating opens no stream and nothing uploads`, async () => { + const f = await fixture(mode === 'disabled' ? { enabled: false } : mode === 'localHide' ? {} : { traceConfig: { [mode]: true } }); + try { + const { response } = audio(); + new ProviderMediaCapture(identity, { attachments: f.manager, traceConfig: mode === 'localHide' ? { hideOutputs: true } : {} }).speechResponse(response); + const iterator = response.body[Symbol.asyncIterator](); + await iterator.next(); + assert.equal(f.manager.status().openStreams, 0); + assert.ok((await iterate({ [Symbol.asyncIterator]: () => iterator })).length > 0); + await f.manager.flush(); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); + +test('S-17 iterating a body already locked by a reader throws the host error unchanged; the reader still uploads once', async () => { + const f = await fixture(); + try { + const { response } = audio(); + new ProviderMediaCapture(identity, { attachments: f.manager }).speechResponse(response); + const reader = response.body.getReader(); + assert.throws(() => response.body[Symbol.asyncIterator](), TypeError); + assert.throws(() => response.body.values(), TypeError); + assert.deepEqual(await drain(reader), wav()); + await f.manager.flush(); await idle(f); + assert.equal(f.uploads.length, 1); assert.deepEqual(f.uploads[0].data, wav()); + } finally { await f.close(); } +}); + +/* openai 4.x's default Node transport is node-fetch 2, whose body is a Node PassThrough. */ +const nodeStreamResponse = (bytes = wav(), chunk = 16) => { + const { PassThrough } = require('node:stream'); + const body = new PassThrough(); + for (let offset = 0; offset < bytes.length; offset += chunk) body.write(bytes.subarray(offset, offset + chunk)); + body.end(); + return { headers: new Headers({ 'content-type': 'audio/wav' }), bodyUsed: false, body }; +}; + +for (const stop of ['end', 'break']) test(`S-18 iterating a Node-stream body (node-fetch shim) to its ${stop} installs no hook and uploads nothing`, async () => { + const f = await fixture(); + try { + const response = nodeStreamResponse(); + new ProviderMediaCapture(identity, { attachments: f.manager }).speechResponse(response); + assert.equal(Object.getOwnPropertyDescriptor(response.body, Symbol.asyncIterator), undefined); + assert.equal(Object.getOwnPropertyDescriptor(response.body, 'values'), undefined); + const received = await iterate(response.body, stop === 'break' ? 2 : Infinity); + if (stop === 'end') assert.deepEqual(received, wav()); + assert.equal(f.manager.status().openStreams, 0); + await f.manager.flush(); await idle(f); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); diff --git a/typescript/packages/provider-media/tsconfig.esm.json b/typescript/packages/provider-media/tsconfig.esm.json new file mode 100644 index 000000000..c36c0cfd5 --- /dev/null +++ b/typescript/packages/provider-media/tsconfig.esm.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.base.esm.json", + "compilerOptions": { + "outDir": "dist/esm", + "rootDir": "src", + "tsBuildInfoFile": "dist/esm/tsconfig.esm.tsbuildinfo", + "baseUrl": "./src" + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "src/**/*.test.ts", + "src/**/__tests__/**" + ], + "references": [] +} diff --git a/typescript/packages/provider-media/tsconfig.esnext.json b/typescript/packages/provider-media/tsconfig.esnext.json new file mode 100644 index 000000000..43bc7e90f --- /dev/null +++ b/typescript/packages/provider-media/tsconfig.esnext.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.base.esnext.json", + "compilerOptions": { + "outDir": "dist/esnext", + "rootDir": "src", + "tsBuildInfoFile": "dist/esnext/tsconfig.esnext.tsbuildinfo", + "baseUrl": "./src" + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "src/**/*.test.ts", + "src/**/__tests__/**" + ], + "references": [] +} diff --git a/typescript/packages/provider-media/tsconfig.json b/typescript/packages/provider-media/tsconfig.json new file mode 100644 index 000000000..b810db2cf --- /dev/null +++ b/typescript/packages/provider-media/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": ".", + "baseUrl": "./src" + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "src/**/*.e2e.test.ts", + "src/**/e2e.test.ts" + ], + "references": [ + { + "path": "../fi-core" + } + ] +} diff --git a/typescript/packages/traceai_bedrock/package.json b/typescript/packages/traceai_bedrock/package.json index cca0c9460..ba68ac948 100644 --- a/typescript/packages/traceai_bedrock/package.json +++ b/typescript/packages/traceai_bedrock/package.json @@ -40,7 +40,8 @@ "@opentelemetry/api": "^1.9.0", "@opentelemetry/instrumentation": "^0.202.0", "@traceai/fi-core": "workspace:^", - "@traceai/fi-semantic-conventions": "workspace:^" + "@traceai/fi-semantic-conventions": "workspace:^", + "@traceai/provider-media": "workspace:^" }, "peerDependencies": { "@aws-sdk/client-bedrock-runtime": "^3.693.0" diff --git a/typescript/packages/traceai_bedrock/src/instrumentation.ts b/typescript/packages/traceai_bedrock/src/instrumentation.ts index 75e0b0261..b1c6d3337 100644 --- a/typescript/packages/traceai_bedrock/src/instrumentation.ts +++ b/typescript/packages/traceai_bedrock/src/instrumentation.ts @@ -1,3 +1,4 @@ +import { ProviderMediaCapture, type ProviderMediaOptions } from "@traceai/provider-media"; import { InstrumentationBase, InstrumentationConfig, @@ -97,10 +98,12 @@ export class BedrockInstrumentation extends InstrumentationBase { private fiTracer: FITracer; private tracerProvider?: TracerProvider; private traceConfig?: TraceConfigOptions; + private attachments?: ProviderMediaOptions["attachments"]; constructor({ instrumentationConfig, traceConfig, + attachments, tracerProvider, }: { /** @@ -113,6 +116,7 @@ export class BedrockInstrumentation extends InstrumentationBase { * @see {@link TraceConfigOptions} */ traceConfig?: TraceConfigOptions; + attachments?: ProviderMediaOptions["attachments"]; /** * An optional custom trace provider to be used for tracing. If not provided, a tracer will be created using the global tracer provider. * This is useful if you want to use a non-global tracer provider. @@ -128,6 +132,7 @@ export class BedrockInstrumentation extends InstrumentationBase { ); this.tracerProvider = tracerProvider; this.traceConfig = traceConfig; + this.attachments = attachments; this.fiTracer = new FITracer({ tracer: this.tracerProvider?.getTracer(INSTRUMENTATION_NAME) ?? this.tracer, @@ -433,6 +438,7 @@ export class BedrockInstrumentation extends InstrumentationBase { const contextAttributes = getAttributesFromContext(context.active()); span.setAttributes(contextAttributes); + new ProviderMediaCapture(span, { attachments: this.attachments, traceConfig: this.traceConfig }).bedrockConverse(command.input); extractConverseRequestAttributes({ span, command }); try { diff --git a/typescript/packages/traceai_bedrock/tsconfig.json b/typescript/packages/traceai_bedrock/tsconfig.json index 0e2ed2808..8bf8fdd37 100644 --- a/typescript/packages/traceai_bedrock/tsconfig.json +++ b/typescript/packages/traceai_bedrock/tsconfig.json @@ -8,6 +8,7 @@ "include": ["src/**/*.ts", "test/**/*.ts"], "exclude": ["src/**/*.test.ts", "src/**/__tests__/**", "test/**/*.ts"], "references": [ + { "path": "../provider-media" }, { "path": "../fi-core" }, { "path": "../fi-semantic-conventions" } ] diff --git a/typescript/packages/traceai_google_genai/package.json b/typescript/packages/traceai_google_genai/package.json index 3b3ccbf15..aa5f304b8 100644 --- a/typescript/packages/traceai_google_genai/package.json +++ b/typescript/packages/traceai_google_genai/package.json @@ -41,7 +41,8 @@ "@opentelemetry/instrumentation": "^0.202.0", "@opentelemetry/core": "^2.0.1", "@traceai/fi-core": "workspace:^", - "@traceai/fi-semantic-conventions": "workspace:^" + "@traceai/fi-semantic-conventions": "workspace:^", + "@traceai/provider-media": "workspace:^" }, "peerDependencies": { "@google/generative-ai": ">=0.1.0" diff --git a/typescript/packages/traceai_google_genai/src/instrumentation.ts b/typescript/packages/traceai_google_genai/src/instrumentation.ts index d2f24f187..acba503da 100644 --- a/typescript/packages/traceai_google_genai/src/instrumentation.ts +++ b/typescript/packages/traceai_google_genai/src/instrumentation.ts @@ -1,3 +1,4 @@ +import { ProviderMediaCapture, type ProviderMediaOptions } from "@traceai/provider-media"; import { InstrumentationBase, InstrumentationConfig, @@ -148,14 +149,17 @@ interface BatchEmbedContentsResponse { export class GoogleGenAIInstrumentation extends InstrumentationBase { private fiTracer!: FITracer; private _traceConfig?: TraceConfigOptions; + private attachments?: ProviderMediaOptions["attachments"]; public readonly instrumentationName = "@traceai/google-genai"; constructor({ instrumentationConfig, traceConfig, + attachments, }: { instrumentationConfig?: InstrumentationConfig; traceConfig?: TraceConfigOptions; + attachments?: ProviderMediaOptions["attachments"]; } = {}) { super( "@traceai/google-genai", @@ -163,6 +167,7 @@ export class GoogleGenAIInstrumentation extends InstrumentationBase { Object.assign({}, instrumentationConfig), ); this._traceConfig = traceConfig; + this.attachments = attachments; // InstrumentationBase calls enable() before our traceConfig is assigned. // Rebuild even when the globally registered tracer has not changed, so the // first manually or automatically instrumented call honors privacy options. @@ -223,7 +228,7 @@ export class GoogleGenAIInstrumentation extends InstrumentationBase { if (typeof original !== "function") return original; return function (this: GenerationModelMetadata, ...args: unknown[]) { return instrumentation.traceCall( - method, this, () => original.apply(this, args), + method, this, args[0], () => original.apply(this, args), () => { // Normalize only for capture. The SDK always receives the original arguments. const request = args[0]; @@ -325,6 +330,7 @@ export class GoogleGenAIInstrumentation extends InstrumentationBase { private traceCall( method: string, model: { model?: string }, + request: unknown, invoke: () => Promise, input: () => Attributes, output: (result: T) => Attributes, @@ -343,6 +349,8 @@ export class GoogleGenAIInstrumentation extends InstrumentationBase { }, }); } catch { return invoke(); } + const media = new ProviderMediaCapture(span, { attachments: this.attachments, traceConfig: this._traceConfig }); + if (method === "generateContent") capture(() => media.geminiContents(normalizeGeneration(request).contents)); capture(() => span.setAttributes(input())); const finish = (failed: boolean) => { // Provider errors may echo credentials (including malformed header values). @@ -361,6 +369,7 @@ export class GoogleGenAIInstrumentation extends InstrumentationBase { throw error; } return Promise.resolve(promise).then(result => { + if (method === "generateContent") media.geminiResponse(result); capture(() => span.setAttributes(output(result))); finish(false); return result; diff --git a/typescript/packages/traceai_google_genai/tsconfig.json b/typescript/packages/traceai_google_genai/tsconfig.json index 45fa28b90..01cbdb9d0 100644 --- a/typescript/packages/traceai_google_genai/tsconfig.json +++ b/typescript/packages/traceai_google_genai/tsconfig.json @@ -8,6 +8,7 @@ "include": ["src/**/*.ts", "test/**/*.ts"], "exclude": ["src/**/*.e2e.test.ts", "src/**/e2e.test.ts"], "references": [ + { "path": "../provider-media" }, { "path": "../fi-core" }, { "path": "../fi-semantic-conventions" } ] diff --git a/typescript/packages/traceai_google_genai_sdk/package.json b/typescript/packages/traceai_google_genai_sdk/package.json index becaa61d5..f2180c1a7 100644 --- a/typescript/packages/traceai_google_genai_sdk/package.json +++ b/typescript/packages/traceai_google_genai_sdk/package.json @@ -27,8 +27,9 @@ "dependencies": { "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^2.0.1", - "@traceai/fi-core": "^1.1.0", - "@traceai/fi-semantic-conventions": "^1.0.0" + "@traceai/fi-core": "workspace:^", + "@traceai/fi-semantic-conventions": "^1.0.0", + "@traceai/provider-media": "workspace:^" }, "peerDependencies": { "@google/genai": ">=1.33.0 <3.0.0" diff --git a/typescript/packages/traceai_google_genai_sdk/src/index.ts b/typescript/packages/traceai_google_genai_sdk/src/index.ts index 48748e85b..f429a1866 100644 --- a/typescript/packages/traceai_google_genai_sdk/src/index.ts +++ b/typescript/packages/traceai_google_genai_sdk/src/index.ts @@ -1,3 +1,4 @@ +import { ProviderMediaCapture, type ProviderMediaOptions } from "@traceai/provider-media"; import { context, trace, @@ -21,7 +22,7 @@ import { VERSION } from "./version.js"; import { wrapAPIPromise } from "./promise.js"; export { VERSION } from "./version.js"; -export interface GoogleGenAITracingOptions { +export interface GoogleGenAITracingOptions extends ProviderMediaOptions { tracerProvider?: TracerProvider; traceConfig?: TraceConfigOptions; /** Total captured content budget per call, in characters. Default 16,384; maximum 65,536. */ @@ -326,6 +327,7 @@ export function instrumentGoogleGenAI( activeCalls++; let span: Span | undefined; let capture: Capture | undefined; + let media: ProviderMediaCapture | undefined; let child = parent; safely(() => { span = tracer.startSpan( @@ -350,6 +352,10 @@ export function instrumentGoogleGenAI( child = trace.setSpan(parent, span); capture = new Capture(span, config, limit); const params = record(args[0]); + if (resource === "models" && (name === "generateContent" || name === "generateContentStream")) { + media = new ProviderMediaCapture(span, options); + media.geminiContents(params.contents); + } capture.input(params, name); if (resource === "interactions" && name !== "create") capture.string("google.genai.interaction_id", args[0]); @@ -431,6 +437,7 @@ export function instrumentGoogleGenAI( const observe = (value: unknown) => { if (ended) return; safely(() => { + if (name === "generateContent") media?.geminiResponse(value); capture?.response(value); const status = record(record(value).interaction).status ?? diff --git a/typescript/packages/traceai_groq/package.json b/typescript/packages/traceai_groq/package.json index 84bff269a..aff652f21 100644 --- a/typescript/packages/traceai_groq/package.json +++ b/typescript/packages/traceai_groq/package.json @@ -41,7 +41,8 @@ "@opentelemetry/instrumentation": "^0.202.0", "@opentelemetry/core": "^2.0.1", "@traceai/fi-core": "workspace:^", - "@traceai/fi-semantic-conventions": "workspace:^" + "@traceai/fi-semantic-conventions": "workspace:^", + "@traceai/provider-media": "workspace:^" }, "peerDependencies": { "groq-sdk": ">=0.3.0" diff --git a/typescript/packages/traceai_groq/src/instrumentation.ts b/typescript/packages/traceai_groq/src/instrumentation.ts index 22ef5debc..e30312212 100644 --- a/typescript/packages/traceai_groq/src/instrumentation.ts +++ b/typescript/packages/traceai_groq/src/instrumentation.ts @@ -1,3 +1,4 @@ +import { ProviderMediaCapture, type ProviderMediaOptions } from "@traceai/provider-media"; import { InstrumentationBase, InstrumentationConfig, @@ -131,16 +132,19 @@ function audioInput(body: Record): Record { export class GroqInstrumentation extends InstrumentationBase { private fiTracer!: FITracer; private _traceConfig?: TraceConfigOptions; + private attachments?: ProviderMediaOptions["attachments"]; // Base construction can call enable()/patch() before our constructor runs. private manualModules?: Set>; private patches?: Map; - constructor({ instrumentationConfig, traceConfig }: { + constructor({ instrumentationConfig, traceConfig, attachments }: { instrumentationConfig?: InstrumentationConfig; traceConfig?: TraceConfigOptions; + attachments?: ProviderMediaOptions["attachments"]; } = {}) { super("@traceai/groq", VERSION, { ...instrumentationConfig }); this._traceConfig = traceConfig; + this.attachments = attachments; this.refreshTracer(); } @@ -238,7 +242,14 @@ export class GroqInstrumentation extends InstrumentationBase { }, }, ); - const execContext = trace.setSpan(context.active(), span); + const parent = context.active(); + const media = new ProviderMediaCapture(span, { attachments: instrumentation.attachments, traceConfig: instrumentation._traceConfig, parentContext: parent }); + if (operation === "chat") media.compatibleMessages(body.messages, "groq"); + if (operation === "transcription" || operation === "translation") media.speechInput(body.file); + // Audio calls whose media this capture owns carry the shared owner marker, so wire-level + // capture (auto) never uploads a second copy. Chat media ownership is unchanged. + const spanContext = trace.setSpan(parent, span); + const execContext = operation === "chat" ? spanContext : media.claim(spanContext); const execPromise = safeExecuteInTheMiddle( () => context.with(execContext, () => original.apply(this, args)), error => { @@ -251,7 +262,7 @@ export class GroqInstrumentation extends InstrumentationBase { ); // Speech resolves to a binary Response owned by the caller. Record completion // at response availability, without reading/cloning the body or logging audio. - const output = operation === "chat" ? undefined : operation === "speech" ? () => ({}) : audioResponseAttributes; + const output = operation === "chat" ? undefined : operation === "speech" ? (result: unknown) => { media.speechResponse(result); return {}; } : audioResponseAttributes; return observeResponse(execPromise, span, execContext, output); }; } diff --git a/typescript/packages/traceai_groq/tsconfig.json b/typescript/packages/traceai_groq/tsconfig.json index 45fa28b90..01cbdb9d0 100644 --- a/typescript/packages/traceai_groq/tsconfig.json +++ b/typescript/packages/traceai_groq/tsconfig.json @@ -8,6 +8,7 @@ "include": ["src/**/*.ts", "test/**/*.ts"], "exclude": ["src/**/*.e2e.test.ts", "src/**/e2e.test.ts"], "references": [ + { "path": "../provider-media" }, { "path": "../fi-core" }, { "path": "../fi-semantic-conventions" } ] diff --git a/typescript/packages/traceai_mistral/package.json b/typescript/packages/traceai_mistral/package.json index 28579b65f..fec262cb3 100644 --- a/typescript/packages/traceai_mistral/package.json +++ b/typescript/packages/traceai_mistral/package.json @@ -43,7 +43,8 @@ "@opentelemetry/instrumentation": "^0.202.0", "@opentelemetry/core": "^2.0.1", "@traceai/fi-core": "workspace:^", - "@traceai/fi-semantic-conventions": "workspace:^" + "@traceai/fi-semantic-conventions": "workspace:^", + "@traceai/provider-media": "workspace:^" }, "peerDependencies": { "@mistralai/mistralai": ">=0.1.0" diff --git a/typescript/packages/traceai_mistral/src/instrumentation.ts b/typescript/packages/traceai_mistral/src/instrumentation.ts index 66cdc62d0..193612d3d 100644 --- a/typescript/packages/traceai_mistral/src/instrumentation.ts +++ b/typescript/packages/traceai_mistral/src/instrumentation.ts @@ -1,3 +1,4 @@ +import { ProviderMediaCapture, type ProviderMediaOptions } from "@traceai/provider-media"; import { InstrumentationBase, InstrumentationConfig, @@ -64,11 +65,15 @@ export class MistralInstrumentation extends InstrumentationBase { private registrations?: WeakMap>; public readonly instrumentationName = "@traceai/mistral"; - constructor({ instrumentationConfig, traceConfig }: { + private attachments?: ProviderMediaOptions["attachments"]; + + constructor({ instrumentationConfig, traceConfig, attachments }: { instrumentationConfig?: InstrumentationConfig; traceConfig?: TraceConfigOptions; + attachments?: ProviderMediaOptions["attachments"]; } = {}) { super("@traceai/mistral", VERSION, { ...instrumentationConfig }); + this.attachments = attachments; this.traceConfig = traceConfig; this.refreshTracer(); } @@ -174,6 +179,7 @@ export class MistralInstrumentation extends InstrumentationBase { attributes: inputAttributes(input, operation, instrumentation.resolvedConfig), })); if (!span) return original.apply(this, args); + if (operation === "chat") new ProviderMediaCapture(span, { attachments: instrumentation.attachments, traceConfig: instrumentation.resolvedConfig }).compatibleMessages(record(input).messages, "mistral"); const spanContext = trace.setSpan(context.active(), span); let ended = false; const finish = (error?: unknown) => { diff --git a/typescript/packages/traceai_mistral/tsconfig.json b/typescript/packages/traceai_mistral/tsconfig.json index 45fa28b90..01cbdb9d0 100644 --- a/typescript/packages/traceai_mistral/tsconfig.json +++ b/typescript/packages/traceai_mistral/tsconfig.json @@ -8,6 +8,7 @@ "include": ["src/**/*.ts", "test/**/*.ts"], "exclude": ["src/**/*.e2e.test.ts", "src/**/e2e.test.ts"], "references": [ + { "path": "../provider-media" }, { "path": "../fi-core" }, { "path": "../fi-semantic-conventions" } ] diff --git a/typescript/packages/traceai_openai/package.json b/typescript/packages/traceai_openai/package.json index 384a59239..1e8b6015d 100644 --- a/typescript/packages/traceai_openai/package.json +++ b/typescript/packages/traceai_openai/package.json @@ -46,7 +46,8 @@ "@opentelemetry/core": "^2.0.1", "@opentelemetry/instrumentation": "^0.202.0", "@traceai/fi-core": "workspace:^", - "@traceai/fi-semantic-conventions": "workspace:^" + "@traceai/fi-semantic-conventions": "workspace:^", + "@traceai/provider-media": "workspace:^" }, "peerDependencies": { "openai": "^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" diff --git a/typescript/packages/traceai_openai/src/instrumentation.ts b/typescript/packages/traceai_openai/src/instrumentation.ts index a04bc72d0..e75d25dbf 100644 --- a/typescript/packages/traceai_openai/src/instrumentation.ts +++ b/typescript/packages/traceai_openai/src/instrumentation.ts @@ -1,3 +1,5 @@ +import { patchSpeech, unpatchSpeech } from "./speech"; +import type { ProviderMediaOptions } from "@traceai/provider-media"; import openai from "openai"; import { InstrumentationBase, @@ -107,6 +109,7 @@ function getExecContext(span: Span) { export class OpenAIInstrumentation extends InstrumentationBase{ private fiTracer!: FITracer; private _traceConfig?: TraceConfigOptions; + private attachments?: ProviderMediaOptions["attachments"]; // Lazily initialized: InstrumentationBase can patch during super(). // A Set is needed so disable() also undoes every manually patched ESM SDK. private patchedModules?: Set; @@ -114,6 +117,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ constructor({ instrumentationConfig, traceConfig, + attachments, }: { /** * The config for the instrumentation @@ -125,6 +129,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ * @see {@link TraceConfigOptions} */ traceConfig?: TraceConfigOptions; + attachments?: ProviderMediaOptions["attachments"]; } = {}) { super( "@traceai/fi-instrumentation-openai", @@ -132,6 +137,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ Object.assign({}, instrumentationConfig), ); this._traceConfig = traceConfig; + this.attachments = attachments; // The base constructor can call enable() before our trace options exist. this.fiTracer = new FITracer({ tracer: this.tracer, traceConfig: this._traceConfig }); } @@ -628,6 +634,10 @@ export class OpenAIInstrumentation extends InstrumentationBase{ ); } + // Audio calls gain spans only while an explicitly supplied manager is active (checked per call, + // because the base constructor can patch before this.attachments is assigned). + patchSpeech(sdk, this, () => this.fiTracer, () => ({ attachments: this.attachments, traceConfig: this._traceConfig }), () => this.isEnabled()); + // Keep the legacy diagnostic property, but never use it as the guard. // Immutable class exports still work via the identity map. try { (sdk as typeof openai & { fiPatched?: boolean }).fiPatched = true; } catch { /* immutable SDK */ } @@ -643,6 +653,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ private unpatch(moduleExports: typeof openai, moduleVersion?: string) { const sdk = moduleExports?.OpenAI; if (!sdk || patchRegistry.owners.get(sdk) !== this) return; + unpatchSpeech(sdk, this); this._unwrap(sdk.Chat.Completions.prototype, "create"); this._unwrap(sdk.Completions.prototype, "create"); this._unwrap(sdk.Embeddings.prototype, "create"); diff --git a/typescript/packages/traceai_openai/src/speech.ts b/typescript/packages/traceai_openai/src/speech.ts new file mode 100644 index 000000000..d074d9b47 --- /dev/null +++ b/typescript/packages/traceai_openai/src/speech.ts @@ -0,0 +1,71 @@ +import { context, trace, SpanKind, SpanStatusCode } from "@opentelemetry/api"; +import { isTracingSuppressed } from "@opentelemetry/core"; +import { type FITracer } from "@traceai/fi-core"; +import { ProviderMediaCapture, attachmentsActive, speechMediaRoute, type ProviderMediaOptions } from "@traceai/provider-media"; +import { observeAPIPromise } from "./apiPromise"; + +type Create = (this: unknown, ...args: unknown[]) => Promise; +type Resource = { create: Create }; +type SDK = { Audio?: { Transcriptions?: { prototype: Resource }; Translations?: { prototype: Resource }; Speech?: { prototype: Resource } } }; +const owners = new WeakMap(); +function resources(sdk: object): Array<[Resource | undefined, string]> { + const audio = (sdk as SDK).Audio; + return [[audio?.Transcriptions?.prototype, "transcription"], [audio?.Translations?.prototype, "translation"], [audio?.Speech?.prototype, "speech"]]; +} + +/** Speech owns no new recorder. It observes input Blobs and explicitly consumed response bytes. */ +export function patchSpeech(sdk: object, owner: object, tracer: () => FITracer, options: () => ProviderMediaOptions, enabled: () => boolean): void { + for (const [resource, operation] of resources(sdk)) { + if (!resource || typeof resource.create !== "function" || owners.has(resource)) continue; + const original = resource.create; + const wrapped: Create = function (...args) { + const mediaOptions = options(); + const parent = context.active(); + // Attachments OFF (the default) leaves audio calls exactly as before: no span, no wrapper work. + if (!enabled() || isTracingSuppressed(parent) || !attachmentsActive(mediaOptions.attachments)) return original.apply(this, args); + const span = tracer().startSpan(`OpenAI Audio ${operation}`, { kind: SpanKind.CLIENT, attributes: { + "fi.span.kind": "LLM", "gen_ai.operation.name": operation, + } }); + // Provider bytes and filenames are not span attributes. The existing provider + // manager supplies policy, transport credentials and lifecycle to this hook. + // An adapter that already owns this call's media (shared owner marker) keeps it; otherwise this + // capture claims the call, so wire-level capture (auto) never uploads a second copy. + const media = new ProviderMediaCapture(span, speechMediaRoute(this) ? { ...mediaOptions, parentContext: parent } : {}); + if (operation !== "speech") { + try { media.speechInput((args[0] as { file?: unknown } | undefined)?.file); } catch { /* No payload diagnostics. */ } + } + let ended = false; + const finish = (failed: boolean) => { + if (ended) return; + ended = true; + if (failed) span.setAttribute("error.type", "provider_error"); + span.setStatus({ code: failed ? SpanStatusCode.ERROR : SpanStatusCode.OK }); + span.end(); + }; + const observe = (result: T): T => { + if (operation === "speech") media.speechResponse(result); + finish(false); + return result; + }; + try { + const result = context.with(media.claim(trace.setSpan(parent, span)), () => original.apply(this, args)); + return observeAPIPromise(result, observe, () => finish(true), response => { + if (response.status < 400) observe(response); + else finish(true); + }); + } catch (error) { finish(true); throw error; } + }; + resource.create = wrapped; + owners.set(resource, { owner, original, wrapped }); + } +} + +export function unpatchSpeech(sdk: object, owner: object): void { + for (const [resource] of resources(sdk)) { + if (!resource) continue; + const state = owners.get(resource); + if (state?.owner !== owner || resource.create !== state.wrapped) continue; + resource.create = state.original; + owners.delete(resource); + } +} diff --git a/typescript/packages/traceai_openai/tsconfig.json b/typescript/packages/traceai_openai/tsconfig.json index 7811a4e17..614463d46 100644 --- a/typescript/packages/traceai_openai/tsconfig.json +++ b/typescript/packages/traceai_openai/tsconfig.json @@ -8,6 +8,7 @@ "include": ["src/**/*.ts"], "exclude": ["src/**/*.test.ts", "src/**/__tests__/**", "test/**/*.ts"], "references": [ + { "path": "../provider-media" }, { "path": "../fi-core" }, { "path": "../fi-semantic-conventions" } ] diff --git a/typescript/packages/traceai_portkey/package.json b/typescript/packages/traceai_portkey/package.json index 286fdd942..a62530e9d 100644 --- a/typescript/packages/traceai_portkey/package.json +++ b/typescript/packages/traceai_portkey/package.json @@ -36,7 +36,8 @@ "@opentelemetry/core": "^2.0.1", "@opentelemetry/instrumentation": "^0.202.0", "@traceai/fi-core": "workspace:^", - "@traceai/fi-semantic-conventions": "workspace:^" + "@traceai/fi-semantic-conventions": "workspace:^", + "@traceai/provider-media": "workspace:^" }, "devDependencies": { "@opentelemetry/context-async-hooks": "^2.0.1", diff --git a/typescript/packages/traceai_portkey/src/instrumentation.ts b/typescript/packages/traceai_portkey/src/instrumentation.ts index 23573b1d3..ae50e2f2d 100644 --- a/typescript/packages/traceai_portkey/src/instrumentation.ts +++ b/typescript/packages/traceai_portkey/src/instrumentation.ts @@ -1,3 +1,4 @@ +import { ProviderMediaCapture, type ProviderMediaOptions } from "@traceai/provider-media"; import { InstrumentationBase, InstrumentationNodeModuleDefinition } from '@opentelemetry/instrumentation'; import { context, trace, Context, Span, SpanKind, SpanStatusCode } from '@opentelemetry/api'; import { isTracingSuppressed, suppressTracing } from '@opentelemetry/core'; @@ -5,7 +6,7 @@ import { FITracer, generateTraceConfig, TraceConfigOptions, safelyJSONStringify import { SemanticConventions as S, FISpanKind, MimeType } from '@traceai/fi-semantic-conventions'; import { VERSION } from './version'; -export interface PortkeyInstrumentationConfig { +export interface PortkeyInstrumentationConfig extends ProviderMediaOptions { instrumentationConfig?: Record; traceConfig?: TraceConfigOptions; } @@ -21,10 +22,12 @@ export class PortkeyInstrumentation extends InstrumentationBase { // Lazily initialized: InstrumentationBase calls enable() during super(). private patches?: Patch[]; private traceConfig?: TraceConfigOptions; + private attachments?: ProviderMediaOptions["attachments"]; constructor(config: PortkeyInstrumentationConfig = {}) { super('@traceai/fi-instrumentation-portkey', VERSION, config.instrumentationConfig || {}); this.traceConfig = config.traceConfig; + this.attachments = config.attachments; } protected init() { @@ -146,6 +149,7 @@ export class PortkeyInstrumentation extends InstrumentationBase { ); // Portkey Responses delegates to OpenAI internally. Suppress a second provider // span, while preserving the Portkey parent across request and stream work. + if (operation === "chat") new ProviderMediaCapture(span, { attachments: this.attachments, traceConfig: this.traceConfig }).compatibleMessages(params.messages); const exec = suppressTracing(trace.setSpan(context.active(), span)); const finish = (result: any) => { if (result && typeof result[Symbol.asyncIterator] === 'function') { diff --git a/typescript/packages/traceai_portkey/tsconfig.json b/typescript/packages/traceai_portkey/tsconfig.json index 087f63ee4..16ffd05fb 100644 --- a/typescript/packages/traceai_portkey/tsconfig.json +++ b/typescript/packages/traceai_portkey/tsconfig.json @@ -8,6 +8,7 @@ "include": ["src/**/*.ts"], "exclude": ["src/**/*.test.ts", "src/**/__tests__/**", "test/**/*.ts"], "references": [ + { "path": "../provider-media" }, { "path": "../fi-core" }, { "path": "../fi-semantic-conventions" } ] diff --git a/typescript/packages/traceai_vertexai/package.json b/typescript/packages/traceai_vertexai/package.json index 3f08496b9..e569273d1 100644 --- a/typescript/packages/traceai_vertexai/package.json +++ b/typescript/packages/traceai_vertexai/package.json @@ -49,7 +49,8 @@ "@opentelemetry/api": "^1.7.0", "@opentelemetry/instrumentation": "^0.46.0", "@traceai/fi-core": "workspace:^", - "@traceai/fi-semantic-conventions": "workspace:^" + "@traceai/fi-semantic-conventions": "workspace:^", + "@traceai/provider-media": "workspace:^" }, "devDependencies": { "@google-cloud/vertexai": "^1.0.0", diff --git a/typescript/packages/traceai_vertexai/src/instrumentation.ts b/typescript/packages/traceai_vertexai/src/instrumentation.ts index 6d70fbeb8..acf206380 100644 --- a/typescript/packages/traceai_vertexai/src/instrumentation.ts +++ b/typescript/packages/traceai_vertexai/src/instrumentation.ts @@ -1,3 +1,4 @@ +import { ProviderMediaCapture, type ProviderMediaOptions } from "@traceai/provider-media"; /** * OpenTelemetry instrumentation for Google Cloud Vertex AI. * @@ -25,7 +26,7 @@ import { } from "@traceai/fi-semantic-conventions"; import { VERSION } from "./version"; -export interface VertexAIInstrumentationConfig { +export interface VertexAIInstrumentationConfig extends ProviderMediaOptions { instrumentationConfig?: Record; traceConfig?: TraceConfigOptions; } @@ -38,6 +39,7 @@ export interface VertexAIInstrumentationConfig { export class VertexAIInstrumentation extends InstrumentationBase { private fiTracer!: FITracer; private _traceConfig?: TraceConfigOptions; + private attachments?: ProviderMediaOptions["attachments"]; constructor(config: VertexAIInstrumentationConfig = {}) { super( @@ -46,6 +48,7 @@ export class VertexAIInstrumentation extends InstrumentationBase { config.instrumentationConfig || {} ); this._traceConfig = config.traceConfig; + this.attachments = config.attachments; } override enable(): void { @@ -211,6 +214,8 @@ export class VertexAIInstrumentation extends InstrumentationBase { }, }); + const media = new ProviderMediaCapture(span, { attachments: this.attachments, traceConfig: this._traceConfig }); + media.geminiContents(request.contents); const execContext = trace.setSpan(context.active(), span); try { @@ -218,6 +223,7 @@ export class VertexAIInstrumentation extends InstrumentationBase { return original.apply(instance, args); }); + media.geminiResponse(result); this.setResponseAttributes(span, result); span.setStatus({ code: SpanStatusCode.OK }); span.end(); diff --git a/typescript/packages/traceai_vertexai/tsconfig.json b/typescript/packages/traceai_vertexai/tsconfig.json index 087f63ee4..16ffd05fb 100644 --- a/typescript/packages/traceai_vertexai/tsconfig.json +++ b/typescript/packages/traceai_vertexai/tsconfig.json @@ -8,6 +8,7 @@ "include": ["src/**/*.ts"], "exclude": ["src/**/*.test.ts", "src/**/__tests__/**", "test/**/*.ts"], "references": [ + { "path": "../provider-media" }, { "path": "../fi-core" }, { "path": "../fi-semantic-conventions" } ] diff --git a/typescript/packages/traceai_vllm/package.json b/typescript/packages/traceai_vllm/package.json index 5cca2a4f7..0798ed9e5 100644 --- a/typescript/packages/traceai_vllm/package.json +++ b/typescript/packages/traceai_vllm/package.json @@ -46,7 +46,8 @@ "@opentelemetry/core": "^1.29.0", "@opentelemetry/instrumentation": "^0.56.0", "@traceai/fi-core": "workspace:^", - "@traceai/fi-semantic-conventions": "workspace:^" + "@traceai/fi-semantic-conventions": "workspace:^", + "@traceai/provider-media": "workspace:^" }, "devDependencies": { "@opentelemetry/sdk-trace-base": "^1.29.0", diff --git a/typescript/packages/traceai_vllm/src/instrumentation.ts b/typescript/packages/traceai_vllm/src/instrumentation.ts index 8f0808acf..c26f3a1ba 100644 --- a/typescript/packages/traceai_vllm/src/instrumentation.ts +++ b/typescript/packages/traceai_vllm/src/instrumentation.ts @@ -1,3 +1,4 @@ +import { ProviderMediaCapture, type ProviderMediaOptions } from "@traceai/provider-media"; import { InstrumentationBase, InstrumentationConfig, @@ -161,7 +162,7 @@ interface StreamResponse { /** * Configuration for vLLM instrumentation */ -export interface VLLMInstrumentationConfig { +export interface VLLMInstrumentationConfig extends ProviderMediaOptions { instrumentationConfig?: InstrumentationConfig; traceConfig?: TraceConfigOptions; /** Base URL pattern to identify vLLM requests (optional) */ @@ -174,6 +175,7 @@ export interface VLLMInstrumentationConfig { export class VLLMInstrumentation extends InstrumentationBase { private fiTracer!: FITracer; private _traceConfig?: TraceConfigOptions; + private attachments?: ProviderMediaOptions["attachments"]; private _baseUrlPattern?: string | RegExp; private readonly modules = new Set(); private readonly patches = new Map(); @@ -187,6 +189,7 @@ export class VLLMInstrumentation extends InstrumentationBase { enabled: false, }); this._traceConfig = config.traceConfig; + this.attachments = config.attachments; this._baseUrlPattern = config.baseUrlPattern; this.setConfig(config.instrumentationConfig ?? {}); this.refreshTracer(); @@ -374,6 +377,7 @@ export class VLLMInstrumentation extends InstrumentationBase { }, }, ); + if (chat) new ProviderMediaCapture(span, { attachments: instrumentation.attachments, traceConfig: instrumentation._traceConfig }).compatibleMessages(body.messages); const finish = spanEnder(span); let result: Promise; try { diff --git a/typescript/packages/traceai_vllm/tsconfig.json b/typescript/packages/traceai_vllm/tsconfig.json index 68ed46255..6d1c424c1 100644 --- a/typescript/packages/traceai_vllm/tsconfig.json +++ b/typescript/packages/traceai_vllm/tsconfig.json @@ -17,4 +17,6 @@ }, "include": ["src/**/*.ts"], "exclude": ["src/**/*.test.ts", "src/**/__tests__/**", "node_modules", "dist", "examples"] +, + "references": [{ "path": "../provider-media" }] } diff --git a/typescript/pnpm-lock.yaml b/typescript/pnpm-lock.yaml index 114266c1c..6119a54fc 100644 --- a/typescript/pnpm-lock.yaml +++ b/typescript/pnpm-lock.yaml @@ -268,6 +268,16 @@ importers: specifier: ^5.8.3 version: 5.8.3 + packages/provider-media: + dependencies: + '@traceai/fi-core': + specifier: workspace:^ + version: link:../fi-core + devDependencies: + tsc-alias: + specifier: ^1.8.16 + version: 1.8.16 + packages/traceai_agent_core: dependencies: '@opentelemetry/api': @@ -561,6 +571,9 @@ importers: '@traceai/fi-semantic-conventions': specifier: workspace:^ version: link:../fi-semantic-conventions + '@traceai/provider-media': + specifier: workspace:^ + version: link:../provider-media devDependencies: '@aws-sdk/client-bedrock-runtime': specifier: ^3.693.0 @@ -1306,6 +1319,9 @@ importers: '@traceai/fi-semantic-conventions': specifier: workspace:^ version: link:../fi-semantic-conventions + '@traceai/provider-media': + specifier: workspace:^ + version: link:../provider-media devDependencies: '@google/generative-ai': specifier: ^0.21.0 @@ -1350,11 +1366,14 @@ importers: specifier: ^2.0.1 version: 2.7.1(@opentelemetry/api@1.9.1) '@traceai/fi-core': - specifier: ^1.1.0 - version: 1.1.0 + specifier: workspace:^ + version: link:../fi-core '@traceai/fi-semantic-conventions': specifier: ^1.0.0 version: 1.0.0 + '@traceai/provider-media': + specifier: workspace:^ + version: link:../provider-media devDependencies: '@google/genai': specifier: 2.32.0 @@ -1424,6 +1443,9 @@ importers: '@traceai/fi-semantic-conventions': specifier: workspace:^ version: link:../fi-semantic-conventions + '@traceai/provider-media': + specifier: workspace:^ + version: link:../provider-media devDependencies: '@opentelemetry/exporter-trace-otlp-proto': specifier: ^0.201.1 @@ -2044,6 +2066,9 @@ importers: '@traceai/fi-semantic-conventions': specifier: workspace:^ version: link:../fi-semantic-conventions + '@traceai/provider-media': + specifier: workspace:^ + version: link:../provider-media devDependencies: '@mistralai/mistralai': specifier: ^1.0.0 @@ -2281,6 +2306,9 @@ importers: '@traceai/fi-semantic-conventions': specifier: workspace:^ version: link:../fi-semantic-conventions + '@traceai/provider-media': + specifier: workspace:^ + version: link:../provider-media devDependencies: '@opentelemetry/exporter-trace-otlp-http': specifier: ^0.202.0 @@ -2679,6 +2707,9 @@ importers: '@traceai/fi-semantic-conventions': specifier: workspace:^ version: link:../fi-semantic-conventions + '@traceai/provider-media': + specifier: workspace:^ + version: link:../provider-media devDependencies: '@opentelemetry/context-async-hooks': specifier: ^2.0.1 @@ -2961,6 +2992,9 @@ importers: '@traceai/fi-semantic-conventions': specifier: workspace:^ version: link:../fi-semantic-conventions + '@traceai/provider-media': + specifier: workspace:^ + version: link:../provider-media devDependencies: '@google-cloud/vertexai': specifier: ^1.0.0 @@ -3041,6 +3075,9 @@ importers: '@traceai/fi-semantic-conventions': specifier: workspace:^ version: link:../fi-semantic-conventions + '@traceai/provider-media': + specifier: workspace:^ + version: link:../provider-media devDependencies: '@opentelemetry/sdk-trace-base': specifier: ^1.29.0