From e02c24ab005c826763687a8b003f9e470bbeed67 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 19:59:31 -0800 Subject: [PATCH 01/20] feat(openai,anthropic): capture typed inline media through the shared attachment manager OpenAI chat/responses input images, files and computer_call_output screenshots, Responses image_generation_call and Images generate b64 outputs; Anthropic Messages image/document blocks and nested tool_result media with computer-tool screenshot classification. Decode is lazy behind manager admission, inline only and capped at 256 KiB per part. Telemetry payloads are projected so media bytes and filenames never reach OTLP. Co-Authored-By: Claude Opus 5.5 --- .../traceai_anthropic/src/attachments.ts | 119 ++++++++ .../packages/traceai_anthropic/src/index.ts | 3 +- .../traceai_anthropic/src/instrumentation.ts | 18 +- .../traceai_openai/src/attachments.ts | 159 +++++++++++ .../packages/traceai_openai/src/index.ts | 3 +- .../traceai_openai/src/instrumentation.ts | 82 ++++-- .../test/attachments-providers.test.cjs | 256 ++++++++++++++++++ 7 files changed, 610 insertions(+), 30 deletions(-) create mode 100644 typescript/packages/traceai_anthropic/src/attachments.ts create mode 100644 typescript/packages/traceai_openai/src/attachments.ts create mode 100644 typescript/packages/traceai_openai/test/attachments-providers.test.cjs diff --git a/typescript/packages/traceai_anthropic/src/attachments.ts b/typescript/packages/traceai_anthropic/src/attachments.ts new file mode 100644 index 000000000..4c3d1e00e --- /dev/null +++ b/typescript/packages/traceai_anthropic/src/attachments.ts @@ -0,0 +1,119 @@ +import type { TraceConfigOptions } from "@traceai/fi-core"; +import type { AttachmentManager, AttachmentSpan, AttachmentCorrelation, AttachmentCapturePolicy, AttachmentOptions } from "@traceai/fi-core/attachments"; + +/** Native SDK span ownership is deliberately not overridden by this adapter. */ +export const anthropicAttachmentCapabilities = Object.freeze({ + schemaVersion: 1, + operations: ["messages.create.input", "tool_result.input"] as const, + sources: ["inline_base64"] as const, + unsupported: ["remote_url", "provider_file", "native_span_hookup", "generated_image", "beta_messages_hookup"] as const, + maxInlineBytes: 256 * 1024, +}); +type RecordValue = Record; +const record = (value: unknown): RecordValue => value !== null && typeof value === "object" ? value as RecordValue : {}; +const registryKey = Symbol.for("@traceai/anthropic/attachment-observations/v1"); +type Registry = WeakMap>>; +const host = globalThis as typeof globalThis & { [registryKey]?: Registry }; +const observations = (host[registryKey] ??= new WeakMap()); + +/** Observe only Messages' typed image/document and one-level tool_result blocks. + * The manager owns policy, upload, lifecycle and references. A native integration + * must supply its actual emitted span/correlation, never an ambient guessed span. + */ +export function captureAnthropicAttachments(manager: AttachmentManager | undefined, + span: AttachmentSpan | AttachmentCorrelation, payload: unknown, + options: { traceConfig?: TraceConfigOptions; capture?: AttachmentCapturePolicy } = {}): void { + if (!manager) return; + try { + const correlation = "snapshot" in span ? span : manager.correlation(span); + const identity = correlation.snapshot(); + if (!identity?.sampled) return; + const messages = record(payload).messages; + if (!Array.isArray(messages)) return; + let spans = observations.get(manager); + if (!spans) { spans = new Map(); observations.set(manager, spans); } + const spanKey = `${identity.traceId}/${identity.spanId}`; + const config = options.traceConfig ?? {}; + const hidden = config.hideInputs || config.hideInputMessages || config.hideInputText || config.piiRedaction; + const computerNames = new Set(); + const tools = record(payload).tools; + if (Array.isArray(tools)) for (const value of tools.slice(0, 256)) { + const tool = record(value); + if (typeof tool.type === "string" && /^computer_\d{8}$/.test(tool.type) && typeof tool.name === "string") computerNames.add(tool.name); + } + const computerCalls = new Set(); + for (const value of messages.slice(0, 256)) { + const message = record(value); + if (message.role !== "assistant" || !Array.isArray(message.content)) continue; + for (const value of message.content.slice(0, 256)) { + const block = record(value); + if (block.type === "tool_use" && typeof block.id === "string" && typeof block.name === "string" && computerNames.has(block.name)) computerCalls.add(block.id); + } + } + let visited = 0; + messages.slice(0, 256).forEach((value, messageIndex) => { + const message = record(value); + if (!Array.isArray(message.content)) return; + // Nested positions are relative to their tool result, identified by toolCallId. + const visit = (value: unknown, index: number, position: string, toolCallId?: string, errorDetail = false) => { + if (++visited > 256) return; + const part = record(value); + if (part.type !== "image" && part.type !== "document") return; + const source = record(part.source); + if (source.type !== "base64") return; + const mime = source.media_type; + if (typeof mime !== "string" || !(part.type === "image" ? ["image/png", "image/jpeg", "image/webp"] : ["application/pdf"]).includes(mime)) return; + const key = `input/${messageIndex}/${position}`; + if (spans!.get(spanKey)?.has(key)) return; + const role: AttachmentOptions["role"] = toolCallId ? "tool" : message.role === "assistant" ? "assistant" : "user"; + const handle = manager.attach(correlation, { kind: "Bytes", get data() { + const encoded = source.data; + if (typeof encoded !== "string" || !encoded.length || encoded.length % 4 !== 0 || + encoded.length > 4 * Math.ceil(anthropicAttachmentCapabilities.maxInlineBytes / 3) || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); + const bytes = Buffer.from(encoded, "base64"); + return bytes.length <= anthropicAttachmentCapabilities.maxInlineBytes && bytes.toString("base64") === encoded ? bytes : new Uint8Array(); + } }, { mimeType: mime, direction: "input", messageIndex, partIndex: index, role, + purpose: part.type === "document" ? "document" : toolCallId ? computerCalls.has(toolCallId) ? "computer_screenshot" : "tool_result" : "model_input", + ...(toolCallId ? { toolCallId } : {}), + capture: { ...options.capture, modelInput: true, errorDetail: errorDetail || options.capture?.errorDetail, + ...(hidden || (part.type === "image" && config.hideInputImages) || (errorDetail && config.hideErrorDetails) ? { captureContent: false } : {}) }, + }); + if (handle.admission === "queued") { + let positions = spans!.get(spanKey); + if (!positions) { + if (spans!.size >= 1024) spans!.delete(spans!.keys().next().value!); + positions = new Set(); spans!.set(spanKey, positions); + } + if (positions.size < 256) positions.add(key); + } + }; + message.content.slice(0, 256).forEach((value, index) => { + const block = record(value); + if (block.type === "tool_result" && Array.isArray(block.content)) { + // Invalid tool IDs fail closed instead of losing causal identity. + if (typeof block.tool_use_id !== "string" || !/^[a-zA-Z0-9_-]{1,128}$/.test(block.tool_use_id)) return; + block.content.slice(0, 256).forEach((child, childIndex) => visit(child, childIndex, `${index}/${childIndex}`, block.tool_use_id as string, block.is_error === true)); + } else visit(value, index, String(index)); + }); + }); + } catch { /* No source-bearing exceptions or changes to the application's result. */ } +} + +/** A telemetry-only projection; never mutates the request passed to the SDK. */ +export function projectAnthropicMedia(payload: unknown): any { + const body = record(payload); + if (!Array.isArray(body.messages)) return payload; + const project = (value: unknown, nested = false): unknown => { + const block = record(value); + if (block.type === "image" || block.type === "document") return { type: block.type }; + if (block.type === "tool_result" && Array.isArray(block.content)) return { + type: block.type, tool_use_id: block.tool_use_id, is_error: block.is_error, + content: nested ? [] : block.content.map(child => project(child, true)), + }; + return block; + }; + return { ...body, messages: body.messages.map(value => { + const message = record(value); + return Array.isArray(message.content) ? { ...message, content: message.content.map(value => project(value)) } : message; + }) }; +} diff --git a/typescript/packages/traceai_anthropic/src/index.ts b/typescript/packages/traceai_anthropic/src/index.ts index 378f38976..bc14a48fb 100644 --- a/typescript/packages/traceai_anthropic/src/index.ts +++ b/typescript/packages/traceai_anthropic/src/index.ts @@ -1,2 +1,3 @@ export * from './version'; -export * from './instrumentation'; \ No newline at end of file +export * from './instrumentation'; +export * from './attachments'; diff --git a/typescript/packages/traceai_anthropic/src/instrumentation.ts b/typescript/packages/traceai_anthropic/src/instrumentation.ts index 77d132430..7df2e6d6f 100644 --- a/typescript/packages/traceai_anthropic/src/instrumentation.ts +++ b/typescript/packages/traceai_anthropic/src/instrumentation.ts @@ -1,4 +1,6 @@ import Anthropic from "@anthropic-ai/sdk"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import { captureAnthropicAttachments, projectAnthropicMedia } from "./attachments"; import { InstrumentationBase, InstrumentationConfig, @@ -334,16 +336,22 @@ function finishInstrumentationStream( export class AnthropicInstrumentation extends InstrumentationBase{ private fiTracer!: FITracer; private _traceConfig?: TraceConfigOptions; + private attachments?: AttachmentManager; constructor({ instrumentationConfig, traceConfig, + attachments, }: { instrumentationConfig?: InstrumentationConfig; traceConfig?: TraceConfigOptions; + /** The provider-owned shared attachment manager. */ + attachments?: AttachmentManager; } = {}) { super("@traceai/instrumentation-anthropic", VERSION,Object.assign({}, instrumentationConfig)); this._traceConfig = traceConfig; + this.attachments = attachments; + this.fiTracer = new FITracer({ tracer: this.tracer, traceConfig: this._traceConfig }); } public override enable(): void { @@ -404,12 +412,13 @@ export class AnthropicInstrumentation extends InstrumentationBase{ // Not async: the SDK returns an APIPromise whose withResponse(), asResponse() // and lazy body parsing callers (including messages.stream()) depend on. return function patchedMessagesCreate(this: Anthropic.Messages, ...args: Parameters) { - const params = args[0] as Anthropic.MessageCreateParams; + const originalParams = args[0] as Anthropic.MessageCreateParams; + const params = projectAnthropicMedia(originalParams) as Anthropic.MessageCreateParams; const { messages: _messages, system: _system, tools: _tools, ...invocationParameters } = params; - const span = instrumentation.fiTracer.startSpan("Anthropic Messages Create", { - kind: SpanKind.INTERNAL, - attributes: { + const span = instrumentation.fiTracer.startSpan("Anthropic Messages Create", { kind: SpanKind.INTERNAL }); + captureAnthropicAttachments(instrumentation.attachments, span, originalParams, { traceConfig: instrumentation._traceConfig }); + span.setAttributes({ [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, [SemanticConventions.LLM_MODEL_NAME]: params.model, [SemanticConventions.LLM_PROVIDER]: LLMProvider.ANTHROPIC, @@ -418,7 +427,6 @@ export class AnthropicInstrumentation extends InstrumentationBase{ ...getAnthropicInputMessagesAttributes(params), ...getAnthropicToolsAttributes(params), [SemanticConventions.RAW_INPUT]: safelyJSONStringify(params) ?? "", - }, }); const execContext = getExecContext(span); diff --git a/typescript/packages/traceai_openai/src/attachments.ts b/typescript/packages/traceai_openai/src/attachments.ts new file mode 100644 index 000000000..d47e00ce0 --- /dev/null +++ b/typescript/packages/traceai_openai/src/attachments.ts @@ -0,0 +1,159 @@ +import type { TraceConfigOptions } from "@traceai/fi-core"; +import type { AttachmentManager, AttachmentOptions, AttachmentSpan, AttachmentCorrelation, AttachmentCapturePolicy } from "@traceai/fi-core/attachments"; + +/** Declarative handoff to auto/wire owners; this is not a native/wire hookup. */ +export const openAIAttachmentCapabilities = Object.freeze({ + schemaVersion: 1, + operations: ["chat.completions.create.input", "responses.create.input", "responses.create.output", "images.generate.output"] as const, + sources: ["inline_base64"] as const, + unsupported: ["remote_url", "provider_file", "image_edit_stream", "speech", "partial_image"] as const, + maxInlineBytes: 256 * 1024, +}); + +type RecordValue = Record; +const record = (value: unknown): RecordValue => value !== null && typeof value === "object" ? value as RecordValue : {}; +const list = (value: unknown): unknown[] => Array.isArray(value) ? value : []; +const MAX_PARTS = 256; +// Shared across CJS/ESM copies. Retain only admitted span IDs and positional keys, never sources. +const registryKey = Symbol.for("@traceai/openai/attachment-observations/v1"); +type Registry = WeakMap>>; +const host = globalThis as typeof globalThis & { [registryKey]?: Registry }; +const observations = (host[registryKey] ??= new WeakMap()); + +export interface OpenAIAttachmentCaptureOptions { + operation: "chat" | "responses" | "images"; + direction: "input" | "output"; + traceConfig?: TraceConfigOptions; + capture?: AttachmentCapturePolicy; + /** The Images request's typed output_format; never inferred from filenames. */ + imageFormat?: "png" | "jpeg" | "webp"; +} + +function restrictions(options: OpenAIAttachmentCaptureOptions): AttachmentCapturePolicy { + const config = options.traceConfig ?? {}; + const input = options.direction === "input"; + const hidden = input + ? config.hideInputs || config.hideInputMessages || config.hideInputText + : config.hideOutputs || config.hideOutputMessages || config.hideOutputText; + return { ...options.capture, + ...(hidden || config.piiRedaction ? { captureContent: false } : {}), + ...(input ? { modelInput: true } : {}), + }; +} + +/** Typed, synchronous observation only. No provider downloads or background stream consumption. + * Pass the SAME emitted identity to SDK and wire observations to deduplicate positions. + * Denied sources are never read: the shared manager calls the getter only after admission. + */ +export function captureOpenAIAttachments(manager: AttachmentManager | undefined, + span: AttachmentSpan | AttachmentCorrelation, payload: unknown, options: OpenAIAttachmentCaptureOptions): void { + if (!manager) return; + try { + const correlation = "snapshot" in span ? span : manager.correlation(span); + const identity = correlation.snapshot(); + if (!identity?.sampled) return; + let spans = observations.get(manager); + if (!spans) { spans = new Map(); observations.set(manager, spans); } + const spanKey = `${identity.traceId}/${identity.spanId}`; + const capture = restrictions(options); + let visited = 0; + const attach = (read: () => unknown, mimeTypes: string[], messageIndex: number, partIndex: number, + purpose: AttachmentOptions["purpose"], toolCallId?: string, raw = false, role: AttachmentOptions["role"] = "user") => { + if (++visited > MAX_PARTS) return; + const key = `${options.operation}/${options.direction}/${messageIndex}/${partIndex}`; + if (spans!.get(spanKey)?.has(key)) return; + for (const mimeType of mimeTypes) { + const handle = manager.attach(correlation, { kind: "Bytes", get data() { + const value = read(); + if (typeof value !== "string") return new Uint8Array(); + const prefix = `data:${mimeType};base64,`; + if (!raw && !value.startsWith(prefix)) return new Uint8Array(); + // Bound allocation BEFORE slicing/decoding. Padding and alphabet are strict. + const start = raw ? 0 : prefix.length; + const length = value.length - start; + if (length <= 0 || length > 4 * Math.ceil(openAIAttachmentCapabilities.maxInlineBytes / 3) || length % 4 !== 0) return new Uint8Array(); + const encoded = value.slice(start); + if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); + const bytes = Buffer.from(encoded, "base64"); + if (bytes.length > openAIAttachmentCapabilities.maxInlineBytes || bytes.toString("base64") !== encoded) return new Uint8Array(); + return bytes; + } }, { mimeType, direction: options.direction, purpose, messageIndex, partIndex, role, + ...(toolCallId ? { toolCallId } : {}), capture: { + ...capture, + ...(options.direction === "input" && mimeType.startsWith("image/") && options.traceConfig?.hideInputImages ? { captureContent: false } : {}), + } }); + if (handle.admission === "queued") { + let positions = spans!.get(spanKey); + if (!positions) { + if (spans!.size >= 1024) spans!.delete(spans!.keys().next().value!); + positions = new Set(); spans!.set(spanKey, positions); + } + if (positions.size < MAX_PARTS) positions.add(key); + return; + } + if (handle.admission !== "too_large" && handle.admission !== "source_unavailable") return; + } + }; + const images = ["image/png", "image/jpeg", "image/webp"]; + const parts = (content: unknown, messageIndex: number, role: AttachmentOptions["role"]) => { + list(content).slice(0, MAX_PARTS).forEach((value, partIndex) => { + const part = record(value); + if (part.type === "image_url") attach(() => record(part.image_url).url, images, messageIndex, partIndex, "model_input", undefined, false, role); + else if (part.type === "input_image") attach(() => part.image_url, images, messageIndex, partIndex, "model_input", undefined, false, role); + else if (part.type === "input_file" || part.type === "file") attach(() => part.type === "file" ? record(part.file).file_data : part.file_data, + ["application/pdf", "application/octet-stream"], messageIndex, partIndex, "document", undefined, false, role); + }); + }; + const body = record(payload); + if (options.direction === "output" && body.error) return; + if (options.direction === "input") { + const messages = options.operation === "chat" ? body.messages : body.input; + list(messages).slice(0, MAX_PARTS).forEach((value, messageIndex) => { + const message = record(value); + if (message.type === "computer_call_output") { + const output = record(message.output); + if (output.type === "computer_screenshot") attach(() => output.image_url, images, messageIndex, 0, "computer_screenshot", + typeof message.call_id === "string" ? message.call_id : undefined, false, "tool"); + } else if (!message.type || message.type === "message") { + const role = ["user", "assistant", "tool", "system"].includes(String(message.role)) ? message.role as AttachmentOptions["role"] : "unknown"; + parts(message.content, messageIndex, role); + } + }); + } else if (options.operation === "images") { + list(body.data).slice(0, MAX_PARTS).forEach((value, index) => { + const item = record(value); + // The Images API defaults to PNG. Explicit response format wins where provided. + const format = body.output_format ?? options.imageFormat; + const mime = format === "jpeg" ? "image/jpeg" : format === "webp" ? "image/webp" : "image/png"; + attach(() => item.b64_json, [mime], index, 0, "model_output", undefined, true, "assistant"); + }); + } else if (options.operation === "responses") { + list(body.output).slice(0, MAX_PARTS).forEach((value, index) => { + const item = record(value); + if (item.type === "image_generation_call" && item.status === "completed") { + const mime = item.output_format === "jpeg" ? "image/jpeg" : item.output_format === "webp" ? "image/webp" : "image/png"; + attach(() => item.result, [mime], index, 0, "model_output", undefined, true, "assistant"); + } + }); + } + } catch { /* Instrumentation must not throw or log source-bearing errors. */ } +} + +/** Telemetry-only typed projection. Never send media bytes, filenames or source URLs to OTLP. */ +export function projectOpenAIMedia(payload: unknown, operation: "chat" | "responses" | "images", direction: "input" | "output", redactImages = true): any { + const body = record(payload); + const projectPart = (value: unknown): unknown => { + const part = record(value); + return (["input_file", "file"].includes(String(part.type)) || redactImages && ["image_url", "input_image"].includes(String(part.type))) ? { type: part.type } : value; + }; + if (operation === "images") return { created: body.created }; + const field = direction === "input" ? operation === "chat" ? "messages" : "input" : "output"; + if (!Array.isArray(body[field])) return payload; + return { ...body, [field]: list(body[field]).map(value => { + const message = record(value); + if (message.type === "computer_call_output") return { type: message.type, call_id: message.call_id, output: { type: "computer_screenshot" } }; + if (message.type === "image_generation_call") return { type: message.type, id: message.id, status: message.status }; + if (Array.isArray(message.content)) return { ...message, content: message.content.map(projectPart) }; + return value; + }) }; +} diff --git a/typescript/packages/traceai_openai/src/index.ts b/typescript/packages/traceai_openai/src/index.ts index a38448910..e886a4c31 100644 --- a/typescript/packages/traceai_openai/src/index.ts +++ b/typescript/packages/traceai_openai/src/index.ts @@ -1,2 +1,3 @@ export * from './instrumentation'; -export * from './version'; \ No newline at end of file +export * from './version'; +export * from './attachments'; diff --git a/typescript/packages/traceai_openai/src/instrumentation.ts b/typescript/packages/traceai_openai/src/instrumentation.ts index a04bc72d0..1951022ea 100644 --- a/typescript/packages/traceai_openai/src/instrumentation.ts +++ b/typescript/packages/traceai_openai/src/instrumentation.ts @@ -1,4 +1,6 @@ import openai from "openai"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import { captureOpenAIAttachments, projectOpenAIMedia } from "./attachments"; import { InstrumentationBase, InstrumentationConfig, @@ -107,6 +109,7 @@ function getExecContext(span: Span) { export class OpenAIInstrumentation extends InstrumentationBase{ private fiTracer!: FITracer; private _traceConfig?: TraceConfigOptions; + private attachments?: AttachmentManager; // Lazily initialized: InstrumentationBase can patch during super(). // A Set is needed so disable() also undoes every manually patched ESM SDK. private patchedModules?: Set; @@ -114,6 +117,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ constructor({ instrumentationConfig, traceConfig, + attachments, }: { /** * The config for the instrumentation @@ -125,6 +129,8 @@ export class OpenAIInstrumentation extends InstrumentationBase{ * @see {@link TraceConfigOptions} */ traceConfig?: TraceConfigOptions; + /** The shared manager from register(...).attachments; never a per-adapter uploader. */ + attachments?: AttachmentManager; } = {}) { super( "@traceai/fi-instrumentation-openai", @@ -132,6 +138,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ Object.assign({}, instrumentationConfig), ); this._traceConfig = traceConfig; + this.attachments = attachments; // The base constructor can call enable() before our trace options exist. this.fiTracer = new FITracer({ tracer: this.tracer, traceConfig: this._traceConfig }); } @@ -223,7 +230,8 @@ export class OpenAIInstrumentation extends InstrumentationBase{ this: unknown, ...args: Parameters ) { - const body = args[0]; + const originalBody = args[0]; + const body = projectOpenAIMedia(originalBody, "chat", "input", !!instrumentation.attachments); // Prediction text is input content, captured only in maskable payloads. const { messages: _messages, prediction: _prediction, ...invocationParameters } = body; @@ -237,11 +245,11 @@ export class OpenAIInstrumentation extends InstrumentationBase{ // diag.debug(`@traceai/openai: this.fiTracer type: ${instrumentation.fiTracer?.constructor?.name}`); // --- END ADDED LOGS --- - const span = instrumentation.fiTracer.startSpan( - `OpenAI Chat Completions`, - { - kind: SpanKind.INTERNAL, - attributes: { + const span = instrumentation.fiTracer.startSpan("OpenAI Chat Completions", { kind: SpanKind.INTERNAL }); + captureOpenAIAttachments(instrumentation.attachments, span, originalBody, { + operation: "chat", direction: "input", traceConfig: instrumentation._traceConfig, + }); + span.setAttributes({ [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, [SemanticConventions.LLM_MODEL_NAME]: body.model, @@ -254,9 +262,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ ...getLLMInputMessagesAttributes(body), ...getLLMToolsJSONSchema(body), [SemanticConventions.RAW_INPUT]: safelyJSONStringify(body) ?? "", - }, - }, - ); + }); // --- ADD LOG --- // diag.debug(`@traceai/openai: ChatCompletion span STARTED: ${span.spanContext().spanId}`); @@ -519,21 +525,22 @@ export class OpenAIInstrumentation extends InstrumentationBase{ }; // Telemetry-only copy: MCP credentials must never reach any exported // duplicate, but the original SDK request must remain untouched. + const projectedBody = projectOpenAIMedia(body, "responses", "input", !!instrumentation.attachments); const telemetryBody = { - ...body, + ...projectedBody, ...(body.tools && { tools: sanitizeToolDefinitions(body.tools) }), - ...(body.input && { input: sanitizeResponsesInput(body.input) }), + ...(projectedBody.input && { input: sanitizeResponsesInput(projectedBody.input) }), }; const { input: _messages, instructions: _instructions, prompt, ...parameters } = telemetryBody; const invocationParameters = { ...parameters, ...(prompt && { prompt: { id: prompt.id, version: prompt.version } }), }; - const span = instrumentation.fiTracer.startSpan( - `OpenAI Responses`, - { - kind: SpanKind.INTERNAL, - attributes: { + const span = instrumentation.fiTracer.startSpan("OpenAI Responses", { kind: SpanKind.INTERNAL }); + captureOpenAIAttachments(instrumentation.attachments, span, body, { + operation: "responses", direction: "input", traceConfig: instrumentation._traceConfig, + }); + span.setAttributes({ [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, [SemanticConventions.LLM_MODEL_NAME]: body.model, @@ -544,12 +551,10 @@ export class OpenAIInstrumentation extends InstrumentationBase{ [SemanticConventions.PROMPT_TEMPLATE_VARIABLES]: safelyJSONStringify(prompt?.variables) ?? undefined, [SemanticConventions.LLM_PROVIDER]: providerForResource(this), [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", - ...getResponsesInputMessagesAttributes(body), + ...getResponsesInputMessagesAttributes(telemetryBody), ...getLLMToolsJSONSchema(body), [SemanticConventions.RAW_INPUT]: safelyJSONStringify(telemetryBody) ?? "", - }, - }, - ); + }); const contextSpan = span.getContextSpan(); const execContext = getExecContext(contextSpan); const requestStartedAt = performance.now(); @@ -581,16 +586,20 @@ export class OpenAIInstrumentation extends InstrumentationBase{ if (endSpan) span.end(); return; } + captureOpenAIAttachments(instrumentation.attachments, span, result, { + operation: "responses", direction: "output", traceConfig: instrumentation._traceConfig, + }); + const projected = projectOpenAIMedia(result, "responses", "output"); const output = { id: result.id, object: result.object, status: result.status, - model: result.model, output: result.output, usage: result.usage, + model: result.model, output: projected.output, usage: result.usage, }; span.setAttributes({ [SemanticConventions.OUTPUT_VALUE]: JSON.stringify(output), [SemanticConventions.OUTPUT_MIME_TYPE]: MimeType.JSON, [SemanticConventions.GEN_AI_RESPONSE_MODEL]: result.model, [SemanticConventions.GEN_AI_RESPONSE_ID]: result.id, - ...getResponsesOutputMessagesAttributes(result), + ...getResponsesOutputMessagesAttributes(projected), ...getResponsesUsageAttributes(result), [SemanticConventions.RAW_OUTPUT]: safelyJSONStringify(output) ?? "", }); @@ -628,6 +637,32 @@ export class OpenAIInstrumentation extends InstrumentationBase{ ); } + if (typeof module.OpenAI.Images?.prototype?.generate === "function") { + this._wrap(module.OpenAI.Images.prototype, "generate", (original: typeof module.OpenAI.Images.prototype.generate): any => { + return function(this: unknown, ...args: Parameters) { + const span = instrumentation.fiTracer.startSpan("OpenAI Images", { kind: SpanKind.INTERNAL, attributes: { + [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, + [SemanticConventions.LLM_PROVIDER]: providerForResource(this), + [SemanticConventions.GEN_AI_OPERATION_NAME]: "image_generation", + } }); + try { + const execContext = trace.setSpan(context.active(), span); + const promise = context.with(execContext, () => original.apply(this, args)); + return observeAPIPromise(promise, result => { + if (span.isRecording() && !isStream(result)) { + captureOpenAIAttachments(instrumentation.attachments, span, result, { + operation: "images", direction: "output", traceConfig: instrumentation._traceConfig, + imageFormat: (args[0] as { output_format?: "png" | "jpeg" | "webp" }).output_format, + }); + } + span.end(); + return result; + }, error => endSpanWithError(span, error), response => endRawResponseSpan(span, response)); + } catch (error) { endSpanWithError(span, error); throw error; } + }; + }); + } + // Keep the legacy diagnostic property, but never use it as the guard. // Immutable class exports still work via the identity map. try { (sdk as typeof openai & { fiPatched?: boolean }).fiPatched = true; } catch { /* immutable SDK */ } @@ -647,6 +682,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ this._unwrap(sdk.Completions.prototype, "create"); this._unwrap(sdk.Embeddings.prototype, "create"); if (sdk.Responses) this._unwrap(sdk.Responses.prototype, "create"); + if (sdk.Images) this._unwrap(sdk.Images.prototype, "generate"); try { (sdk as typeof openai & { fiPatched?: boolean }).fiPatched = false; } catch { /* immutable SDK */ } patchRegistry.owners.delete(sdk); // An automatic require hook may use a different namespace for this class. @@ -983,7 +1019,7 @@ function serializeChatCompletionInputMessage( if (part.type === "text") { return { type: "text", text: part.text }; } else if (part.type === "image_url") { - return { type: "image_url", image_url: { url: part.image_url.url } }; + return part.image_url ? { type: "image_url", image_url: { url: part.image_url.url } } : { type: "image_url" }; } return { type: part.type }; }); diff --git a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs new file mode 100644 index 000000000..dcba8d149 --- /dev/null +++ b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs @@ -0,0 +1,256 @@ +/* Cloud component qualification: real SDKs, synthetic media, loopback-only APIs. */ +const { test, after } = require('node:test'); +const assert = require('node:assert/strict'); +const http = require('node:http'); +const { once } = require('node:events'); +const { randomUUID } = require('node:crypto'); +const { pathToFileURL } = require('node:url'); +const path = require('node:path'); +const { AttachmentManager, createAttachmentHttpTransport } = require('@traceai/fi-core/attachments'); +const { BasicTracerProvider, SimpleSpanProcessor, InMemorySpanExporter } = require('@opentelemetry/sdk-trace-base'); +const { OpenAIInstrumentation } = require('../dist/src/instrumentation.js'); +const { AnthropicInstrumentation } = require('../../traceai_anthropic/dist/src/instrumentation.js'); +const oa = require('../dist/src/attachments.js'); +const an = require('../../traceai_anthropic/dist/src/attachments.js'); +const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aD1sAAAAASUVORK5CYII='; +const PDF = Buffer.from('%PDF-1.7\n% synthetic document\n%%EOF').toString('base64'); +const image = () => ({ type: 'image', source: { type: 'base64', media_type: 'image/png', data: PNG } }); +const document = () => ({ type: 'document', title: 'synthetic-private-filename', source: { type: 'base64', media_type: 'application/pdf', data: PDF } }); +const identity = () => ({ traceId: randomUUID().replace(/-/g, ''), spanId: randomUUID().replace(/-/g, '').slice(0, 16), traceFlags: 1 }); +const keepalive = setInterval(() => {}, 1000); +after(() => clearInterval(keepalive)); + +async function fixture(config = {}, serverPolicy = {}) { + const calls = [], initialized = [], uploaded = []; + const scope = { organization_id: randomUUID(), workspace_id: randomUUID(), project_id: randomUUID() }; + const expiry = () => new Date(Date.now() + 60000).toISOString(); + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const c of req) chunks.push(c); + const bytes = Buffer.concat(chunks); calls.push(req.url); + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('attachment-sessions')) { + const body = JSON.parse(bytes); + res.end(JSON.stringify({ schema_version: 1, scope, capture_session_id: body.capture_session_id, + policy: { version: 'fixture-v1', enabled: true, allowed_sources: ['Bytes'], allowed_modalities: ['image', 'document', 'file'], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, + record_audio: false, record_video: false, ...serverPolicy }, + limits: { max_upload_bytes: 1024 * 1024, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, + recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, + initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, + transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }, expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); + } else if (req.url.endsWith('/uploads')) { + const body = JSON.parse(bytes); initialized.push(body); + res.statusCode = 201; + res.end(JSON.stringify({ upload_id: body.attachment_id, attachment_id: body.attachment_id, occurrence_id: body.occurrence_id, + state: 'pending_upload', reserved_bytes: body.declared_bytes, expires_at: expiry(), transfer_mode: 'authenticated_stream', write_capability: 'synthetic-upload-only' })); + } else if (req.url.endsWith('/content')) { + assert.equal(req.headers.authorization, 'Bearer synthetic-upload-only'); uploaded.push(bytes); + res.end(JSON.stringify({ upload_id: req.url.split('/').at(-2), state: 'pending_upload', reason: 'queued' })); + } else { + res.statusCode = 202; res.end(JSON.stringify({ upload_id: req.url.split('/').at(-2), state: 'validating', reason: 'binding_pending' })); + } + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const client = req => new Promise((resolve, reject) => { + const request = http.request(`http://127.0.0.1:${server.address().port}${new URL(req.url).pathname}`, { method: req.method, headers: req.headers, signal: req.signal }, response => { + const chunks = []; response.on('data', c => chunks.push(c)); response.on('end', () => resolve({ status: response.statusCode, body: Buffer.concat(chunks) })); + }); request.on('error', reject); request.end(req.body); + }); + const manager = new AttachmentManager({ enabled: true, projectName: 'synthetic', projectType: 'observe', + transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic' }, client }), ...config }); + assert.equal(await manager.ready(), config.enabled !== false); + return { manager, calls, initialized, uploaded, async close() { await manager.shutdown(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } }; +} +const oaInput = () => ({ input: [ + { role: 'user', content: [{ type: 'input_image', image_url: `data:image/png;base64,${PNG}` }, { type: 'input_file', filename: 'synthetic-private-filename', file_data: `data:application/pdf;base64,${PDF}` }] }, + { type: 'computer_call_output', call_id: 'call_computer', output: { type: 'computer_screenshot', image_url: `data:image/png;base64,${PNG}` } }, +] }); +const anInput = () => ({ tools: [{ type: 'computer_20250124', name: 'computer', display_width_px: 800, display_height_px: 600 }], messages: [ + { role: 'assistant', content: [{ type: 'tool_use', id: 'tool_computer', name: 'computer', input: { action: 'screenshot' } }] }, + { role: 'user', content: [image(), document(), { type: 'tool_result', tool_use_id: 'tool_computer', content: [image(), image()] }] }, +] }); + +for (const provider of ['openai', 'anthropic']) { + const capture = (m, s, p, extra = {}) => provider === 'openai' + ? oa.captureOpenAIAttachments(m, s, p, { operation: 'responses', direction: 'input', ...extra }) + : an.captureAnthropicAttachments(m, s, p, extra); + const input = provider === 'openai' ? oaInput : anInput; + test(`${provider}: typed image/document/screenshots preserve bytes, causal positions and distinct repeated frames`, async () => { + const f = await fixture(); + try { + const id = identity(), body = input(), saved = JSON.stringify(body); + capture(f.manager, id, body); capture(f.manager, { ...id }, body); + await f.manager.flush(); + assert.equal(f.initialized.length, provider === 'openai' ? 3 : 4); + assert.equal(f.uploaded.length, f.initialized.length); + assert.deepEqual(f.uploaded.map(b => b.toString('base64')).sort(), (provider === 'openai' ? [PNG, PDF, PNG] : [PNG, PDF, PNG, PNG]).sort()); + assert.equal(new Set(f.initialized.map(r => r.occurrence_id)).size, f.initialized.length); + assert.ok(f.initialized.every(r => r.direction === 'input' && r.association.parent.span_id === id.spanId)); + assert.equal(f.initialized.filter(r => r.purpose === 'computer_screenshot').length, provider === 'openai' ? 1 : 2); + assert.ok(!JSON.stringify(f.initialized).includes('synthetic-private-filename')); + assert.equal(JSON.stringify(body), saved); + } finally { await f.close(); } + }); + for (const key of ['disabled', 'hideInputs', 'hideInputMessages', 'hideInputText', 'piiRedaction', 'hideInputImages', 'serverRedaction', 'serverMetadata']) { + test(`${provider}: ${key} denies protected sources without reading or uploading`, async () => { + const f = await fixture(key === 'disabled' ? { enabled: false } : { traceConfig: { [key]: true } }, + key === 'serverRedaction' ? { require_media_redaction: true } : key === 'serverMetadata' ? { require_metadata_removal: true } : {}); + try { + let read = false; + const source = provider === 'openai' ? { input: [{ type: 'computer_call_output', call_id: 'call_computer', output: { type: 'computer_screenshot', get image_url() { read = true; throw Error('synthetic-secret'); } } }] } + : { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_computer', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', get data() { read = true; throw Error('synthetic-secret'); } } }] }] }] }; + const before = f.calls.length; capture(f.manager, identity(), source); + await f.manager.flush(); assert.equal(read, false); assert.equal(f.calls.length, before); + if (key === 'disabled') assert.equal(f.calls.length, 0); + } finally { await f.close(); } + }); + } + for (const capturePolicy of [{ captureContent: false }, { recordInputs: false }, { muted: true }, { temporaryChat: true }]) { + test(`${provider}: integration restriction ${Object.keys(capturePolicy)[0]} wins`, async () => { + const f = await fixture(); try { capture(f.manager, identity(), input(), { capture: capturePolicy }); await f.manager.flush(); assert.equal(f.initialized.length, 0); } finally { await f.close(); } + }); + } + test(`${provider}: malformed and oversized inline data is bounded; foreign URLs/files are not retrieved`, async () => { + const f = await fixture(); + try { + for (const data of ['!', 'a===', 'AB==', 'AAAA'.repeat(100000), 'https://127.0.0.1:1/?synthetic-secret']) { + const body = provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_image', image_url: data }] }] } + : { messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data } }] }] }; + capture(f.manager, identity(), body); + } + capture(f.manager, identity(), provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_file', file_id: 'file_secret' }] }] } + : { messages: [{ role: 'user', content: [{ type: 'document', source: { type: 'file', file_id: 'file_secret' } }, null, { type: 'tool_result', tool_use_id: {}, content: [image()] }] }] }); + await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } + }); + test(`${provider}: unsampled identity and queued policy tightening cannot upload`, async () => { + const f = await fixture(); try { + capture(f.manager, { ...identity(), traceFlags: 0 }, input()); assert.equal(f.manager.status().ownedBytes, 0); + capture(f.manager, identity(), input()); f.manager.updatePolicy({ traceConfig: { hideInputs: true } }); await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } + }); +} + +test('OpenAI generated images obey output and PII restrictions', async () => { + for (const traceConfig of [{}, { hideOutputs: true }, { hideOutputMessages: true }, { hideOutputText: true }, { piiRedaction: true }]) { + const f = await fixture({ traceConfig }); try { + oa.captureOpenAIAttachments(f.manager, identity(), { data: [{ b64_json: PNG }] }, { operation: 'images', direction: 'output' }); + await f.manager.flush(); assert.equal(f.uploaded.length, Object.keys(traceConfig).length ? 0 : 1); + } finally { await f.close(); } + } +}); + +test('Telemetry projections strip typed bytes, filenames and signed source locations without mutating requests', () => { + for (const [body, project] of [[oaInput(), x => oa.projectOpenAIMedia(x, 'responses', 'input')], [anInput(), an.projectAnthropicMedia]]) { + const before = JSON.stringify(body), projected = JSON.stringify(project(body)); + assert.equal(JSON.stringify(body), before); + for (const secret of [PNG, PDF, 'synthetic-private-filename']) assert.ok(!projected.includes(secret)); + } +}); + +test('CJS/ESM observations share deduplication without creating a second capture', async () => { + const f = await fixture(); try { + const esm = await import(pathToFileURL(path.resolve(__dirname, '../dist/esm/attachments.js'))); + const s = identity(); oa.captureOpenAIAttachments(f.manager, s, oaInput(), { operation: 'responses', direction: 'input' }); + esm.captureOpenAIAttachments(f.manager, { ...s }, oaInput(), { operation: 'responses', direction: 'input' }); + const anthropicESM = await import(pathToFileURL(path.resolve(__dirname, '../../traceai_anthropic/dist/esm/attachments.js'))); + const a = identity(); an.captureAnthropicAttachments(f.manager, a, anInput()); anthropicESM.captureAnthropicAttachments(f.manager, { ...a }, anInput()); + await f.manager.flush(); assert.equal(f.uploaded.length, 7); + } finally { await f.close(); } +}); + +test('Anthropic tool-result images are not labelled computer screenshots without a typed computer tool', async () => { + const f = await fixture(); try { + an.captureAnthropicAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_photo', content: [image()] }] }] }); + await f.manager.flush(); assert.equal(f.initialized[0].purpose, 'tool_result'); assert.equal(f.initialized[0].part_index, 0); + } finally { await f.close(); } +}); + +test('Anthropic error-tool media respects hideErrorDetails before source access', async () => { + const f = await fixture({ traceConfig: { hideErrorDetails: true } }); try { + let read = false; + an.captureAnthropicAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_error', is_error: true, + content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', get data() { read = true; return PNG; } } }] }] }] }); + await f.manager.flush(); assert.equal(read, false); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } +}); + +test('OpenAI image error bodies are not treated as generated media', async () => { + const f = await fixture(); try { + oa.captureOpenAIAttachments(f.manager, identity(), { error: { message: 'synthetic' }, data: [{ b64_json: PNG }] }, { operation: 'images', direction: 'output' }); + await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } +}); + +test('Local instrumentor privacy restrictions cannot be loosened by a permissive shared manager', async () => { + const f = await fixture(); try { + oa.captureOpenAIAttachments(f.manager, identity(), oaInput(), { operation: 'responses', direction: 'input', traceConfig: { hideInputs: true } }); + an.captureAnthropicAttachments(f.manager, identity(), anInput(), { traceConfig: { piiRedaction: true } }); + await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } +}); + +async function providerServer() { + const requests = []; + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const c of req) chunks.push(c); + const body = JSON.parse(Buffer.concat(chunks)); requests.push({ path: req.url, body }); + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('/chat/completions')) res.end(JSON.stringify({ id: 'chat_fixture', object: 'chat.completion', model: body.model, choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })); + else if (req.url.endsWith('/responses')) res.end(JSON.stringify({ id: 'resp_fixture', object: 'response', status: 'completed', model: body.model, output: [{ type: 'computer_call', id: 'computer_fixture', call_id: 'call_computer', status: 'completed', action: { type: 'screenshot' } }, { type: 'image_generation_call', id: 'img_fixture', status: 'completed', result: PNG }], usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } })); + else if (req.url.endsWith('/images/generations')) res.end(JSON.stringify({ created: 1, data: [{ b64_json: PNG }] })); + else res.end(JSON.stringify({ id: 'msg_fixture', type: 'message', role: 'assistant', model: body.model, content: [{ type: 'text', text: 'ok' }], stop_reason: 'end_turn', stop_sequence: null, usage: { input_tokens: 1, output_tokens: 1 } })); + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + return { requests, url: `http://127.0.0.1:${server.address().port}/v1`, async close() { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } }; +} + +test('Real SDKs preserve APIPromise/request/response behavior and upload only through the shared manager', async () => { + const sdkModule = require(process.env.SDK_OPENAI || 'openai'); + const anthropicModule = require(process.env.SDK_ANTHROPIC || '../../traceai_anthropic/node_modules/@anthropic-ai/sdk'); + const OpenAI = sdkModule.OpenAI || sdkModule.default || sdkModule; + const Anthropic = anthropicModule.Anthropic || anthropicModule.default || anthropicModule; + const native = 'openTelemetry' in Anthropic.prototype; + const f = await fixture(), p = await providerServer(); + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + const oi = new OpenAIInstrumentation({ instrumentationConfig: { enabled: false }, attachments: f.manager }); + const ai = new AnthropicInstrumentation({ instrumentationConfig: { enabled: false }, attachments: f.manager }); + oi.setTracerProvider(provider); ai.setTracerProvider(provider); + oi.manuallyInstrument(sdkModule); ai.manuallyInstrument(anthropicModule); + try { + const client = new OpenAI({ apiKey: 'synthetic-provider-key', baseURL: p.url, maxRetries: 0 }); + const body = { model: 'synthetic-model', messages: [{ role: 'user', content: [{ type: 'image_url', image_url: { url: `data:image/png;base64,${PNG}` } }] }] }; + const pending = client.chat.completions.create(body); assert.equal(typeof pending.withResponse, 'function'); + assert.equal((await pending.withResponse()).data.choices[0].message.content, 'ok'); + let expected = 1; + if (client.responses) { + const result = await client.responses.create({ model: 'synthetic-model', ...oaInput() }); + assert.equal(result.output[0].call_id, 'call_computer'); expected += 4; + } + const generated = await client.images.generate({ prompt: 'synthetic', model: 'synthetic-model', response_format: 'b64_json' }); + assert.equal(generated.data[0].b64_json, PNG); expected++; + const ac = new Anthropic({ apiKey: 'synthetic-provider-key', baseURL: p.url, maxRetries: 0 }); + const request = ac.messages.create({ model: 'synthetic-model', max_tokens: 10, ...anInput() }); + assert.equal(typeof request.withResponse, 'function'); assert.equal((await request.withResponse()).data.content[0].text, 'ok'); + if (!native) expected += 4; + await f.manager.flush(); await provider.forceFlush(); + assert.equal(f.uploaded.length, expected); + assert.ok(p.requests.some(r => JSON.stringify(r.body).includes(PNG))); + assert.ok(!JSON.stringify(exporter.getFinishedSpans().map(s => s.attributes)).includes(PNG)); + assert.ok(exporter.getFinishedSpans().some(s => s.attributes['fi.attachment.refs'])); + // A raw-response caller owns the body; instrumentation must not parse for media. + const raw = await client.images.generate({ prompt: 'synthetic', response_format: 'b64_json' }).asResponse(); + assert.equal(raw.bodyUsed, false); await raw.arrayBuffer(); + await f.manager.flush(); assert.equal(f.uploaded.length, expected); + // Requests still reach the provider unchanged after privacy tightening; media egress stops. + f.manager.updatePolicy({ traceConfig: { hideInputs: true, piiRedaction: true } }); + await client.chat.completions.create(body); + await ac.messages.create({ model: 'synthetic-model', max_tokens: 10, ...anInput() }); + await client.images.generate({ prompt: 'synthetic', response_format: 'b64_json' }); + await f.manager.flush(); assert.equal(f.uploaded.length, expected); + console.log(JSON.stringify({ nativeAnthropic: native, responsesAvailable: !!client.responses, uploads: expected, spans: exporter.getFinishedSpans().length })); + } finally { + oi.disable(); ai.disable(); await provider.shutdown(); await f.close(); await p.close(); + } +}); From 52b95b0ff500fc4a50d69195a21ec5c5e6dbabd5 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 20:11:55 -0800 Subject: [PATCH 02/20] fix(openai,anthropic): project only admitted media and keep existing telemetry otherwise Capture now runs before the telemetry projection and returns the admitted positions. Only those parts are replaced by their fi-attachment occurrence URI; denied, disabled or absent capture leaves the span payload exactly as before. A second observer of the same span position reuses the first occurrence. Streamed Responses output media and the Images edit and variation endpoints are explicitly unsupported. Adds a network-isolated CI workflow against real SDK floor and latest releases. Co-Authored-By: Claude Opus 5.5 --- .../openai-anthropic-attachments-tests.yml | 70 ++++++++ .../traceai_anthropic/src/attachments.ts | 124 ++++++++----- .../traceai_anthropic/src/instrumentation.ts | 17 +- .../traceai_openai/src/attachments.ts | 167 +++++++++++------- .../traceai_openai/src/instrumentation.ts | 121 +++++++------ .../test/attachments-providers.test.cjs | 119 ++++++++++++- 6 files changed, 440 insertions(+), 178 deletions(-) create mode 100644 .github/workflows/openai-anthropic-attachments-tests.yml diff --git a/.github/workflows/openai-anthropic-attachments-tests.yml b/.github/workflows/openai-anthropic-attachments-tests.yml new file mode 100644 index 000000000..a8335241c --- /dev/null +++ b/.github/workflows/openai-anthropic-attachments-tests.yml @@ -0,0 +1,70 @@ +name: OpenAI and Anthropic TypeScript attachments +on: + pull_request: + paths: + - 'typescript/packages/traceai_openai/**' + - 'typescript/packages/traceai_anthropic/**' + - 'typescript/packages/fi-core/**' + - 'typescript/packages/fi-semantic-conventions/**' + - 'typescript/pnpm-lock.yaml' + - '.github/workflows/openai-anthropic-attachments-tests.yml' + workflow_dispatch: +permissions: + contents: read +jobs: + attachments: + name: OpenAI ${{ matrix.openai }} / Anthropic ${{ matrix.anthropic }} + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - { openai: '4.0.0', anthropic: '0.20.0' } + - { openai: '5.23.2', anthropic: '0.27.3' } + - { openai: '6.0.0', anthropic: '0.133.0' } + - { openai: '7.47.0', anthropic: '0.134.0' } + - { openai: 'latest', anthropic: 'latest' } + defaults: + run: + shell: bash + working-directory: typescript + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22.23.4' + - name: Install pinned workspace dependencies and real SDK fixtures + run: | + mkdir -p "$RUNNER_TEMP/corepack-bin" + corepack enable --install-directory "$RUNNER_TEMP/corepack-bin" pnpm + echo "$RUNNER_TEMP/corepack-bin" >> "$GITHUB_PATH" + corepack pnpm --filter @traceai/openai... --filter @traceai/anthropic... install --frozen-lockfile --ignore-scripts + npm install --prefix "$RUNNER_TEMP/sdks" --ignore-scripts --no-audit --no-fund \ + "openai-under-test@npm:openai@${{ matrix.openai }}" \ + "anthropic-under-test@npm:@anthropic-ai/sdk@${{ matrix.anthropic }}" + - name: Build CJS and ESM + run: | + pnpm run build:deps + pnpm --filter @traceai/openai run build + pnpm --filter @traceai/anthropic run build + - name: Network-isolated attachment adapters against real SDKs and loopback mocks + run: | + sudo unshare --net -- bash -c 'ip link set lo up; exec "$@"' bash \ + env PATH="$PATH" \ + SDK_OPENAI="$RUNNER_TEMP/sdks/node_modules/openai-under-test" \ + SDK_ANTHROPIC="$RUNNER_TEMP/sdks/node_modules/anthropic-under-test" \ + node --test packages/traceai_openai/test/attachments-providers.test.cjs + - name: Network-isolated package unit suites + if: matrix.openai == 'latest' + run: | + sudo unshare --net -- bash -c 'ip link set lo up; exec "$@"' bash \ + env PATH="$PATH" node packages/traceai_openai/node_modules/jest/bin/jest.js \ + --config packages/traceai_openai/jest.config.js --runInBand + sudo unshare --net -- bash -c 'ip link set lo up; exec "$@"' bash \ + env PATH="$PATH" node node_modules/jest/bin/jest.js \ + --config packages/traceai_anthropic/jest.config.js --runInBand + git diff --exit-code -- pnpm-lock.yaml diff --git a/typescript/packages/traceai_anthropic/src/attachments.ts b/typescript/packages/traceai_anthropic/src/attachments.ts index 4c3d1e00e..876ac52ae 100644 --- a/typescript/packages/traceai_anthropic/src/attachments.ts +++ b/typescript/packages/traceai_anthropic/src/attachments.ts @@ -6,114 +6,142 @@ export const anthropicAttachmentCapabilities = Object.freeze({ schemaVersion: 1, operations: ["messages.create.input", "tool_result.input"] as const, sources: ["inline_base64"] as const, - unsupported: ["remote_url", "provider_file", "native_span_hookup", "generated_image", "beta_messages_hookup"] as const, + /** Fail closed: no capture is attempted for these. */ + unsupported: ["remote_url", "provider_file", "native_span_hookup", "generated_image", "beta_messages_hookup", "streamed_output"] as const, maxInlineBytes: 256 * 1024, }); + +/** Admitted media positions for one observation: position key -> occurrence. */ +export type AnthropicCapturedMedia = ReadonlyMap>; +type Captured = Map>; type RecordValue = Record; const record = (value: unknown): RecordValue => value !== null && typeof value === "object" ? value as RecordValue : {}; +const MAX_PARTS = 256; +const MAX_SPANS = 1024; +const EMPTY: AnthropicCapturedMedia = new Map(); const registryKey = Symbol.for("@traceai/anthropic/attachment-observations/v1"); -type Registry = WeakMap>>; +type Registry = WeakMap>; const host = globalThis as typeof globalThis & { [registryKey]?: Registry }; const observations = (host[registryKey] ??= new WeakMap()); +const TOOL_ID = /^[a-zA-Z0-9_-]{1,128}$/; + +function decode(encoded: unknown): Uint8Array { + const max = anthropicAttachmentCapabilities.maxInlineBytes; + // Bound allocation BEFORE decoding. Padding and alphabet are strict. + if (typeof encoded !== "string" || !encoded.length || encoded.length % 4 !== 0 || + encoded.length > 4 * Math.ceil(max / 3) || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); + const bytes = Buffer.from(encoded, "base64"); + return bytes.length <= max && bytes.toString("base64") === encoded ? bytes : new Uint8Array(); +} -/** Observe only Messages' typed image/document and one-level tool_result blocks. +/** Observe only Messages' typed image/document blocks and one level of tool_result content. * The manager owns policy, upload, lifecycle and references. A native integration * must supply its actual emitted span/correlation, never an ambient guessed span. + * Returns the admitted positions so the caller can project them out of span telemetry. */ export function captureAnthropicAttachments(manager: AttachmentManager | undefined, span: AttachmentSpan | AttachmentCorrelation, payload: unknown, - options: { traceConfig?: TraceConfigOptions; capture?: AttachmentCapturePolicy } = {}): void { - if (!manager) return; + options: { traceConfig?: TraceConfigOptions; capture?: AttachmentCapturePolicy } = {}): AnthropicCapturedMedia { + if (!manager) return EMPTY; + const captured: Captured = new Map(); try { const correlation = "snapshot" in span ? span : manager.correlation(span); const identity = correlation.snapshot(); - if (!identity?.sampled) return; + if (!identity?.sampled) return EMPTY; const messages = record(payload).messages; - if (!Array.isArray(messages)) return; + if (!Array.isArray(messages)) return EMPTY; let spans = observations.get(manager); if (!spans) { spans = new Map(); observations.set(manager, spans); } const spanKey = `${identity.traceId}/${identity.spanId}`; const config = options.traceConfig ?? {}; const hidden = config.hideInputs || config.hideInputMessages || config.hideInputText || config.piiRedaction; + // Screenshot classification requires a typed computer tool AND an assistant tool_use naming it. const computerNames = new Set(); const tools = record(payload).tools; - if (Array.isArray(tools)) for (const value of tools.slice(0, 256)) { + if (Array.isArray(tools)) for (const value of tools.slice(0, MAX_PARTS)) { const tool = record(value); if (typeof tool.type === "string" && /^computer_\d{8}$/.test(tool.type) && typeof tool.name === "string") computerNames.add(tool.name); } const computerCalls = new Set(); - for (const value of messages.slice(0, 256)) { + for (const value of messages.slice(0, MAX_PARTS)) { const message = record(value); if (message.role !== "assistant" || !Array.isArray(message.content)) continue; - for (const value of message.content.slice(0, 256)) { + for (const value of message.content.slice(0, MAX_PARTS)) { const block = record(value); if (block.type === "tool_use" && typeof block.id === "string" && typeof block.name === "string" && computerNames.has(block.name)) computerCalls.add(block.id); } } let visited = 0; - messages.slice(0, 256).forEach((value, messageIndex) => { + messages.slice(0, MAX_PARTS).forEach((value, messageIndex) => { const message = record(value); if (!Array.isArray(message.content)) return; // Nested positions are relative to their tool result, identified by toolCallId. const visit = (value: unknown, index: number, position: string, toolCallId?: string, errorDetail = false) => { - if (++visited > 256) return; + if (++visited > MAX_PARTS) return; const part = record(value); if (part.type !== "image" && part.type !== "document") return; const source = record(part.source); if (source.type !== "base64") return; const mime = source.media_type; if (typeof mime !== "string" || !(part.type === "image" ? ["image/png", "image/jpeg", "image/webp"] : ["application/pdf"]).includes(mime)) return; - const key = `input/${messageIndex}/${position}`; - if (spans!.get(spanKey)?.has(key)) return; + const positionKey = `input/${messageIndex}/${position}`; + const previous = spans!.get(spanKey)?.get(positionKey); + if (previous) { captured.set(positionKey, previous); return; } const role: AttachmentOptions["role"] = toolCallId ? "tool" : message.role === "assistant" ? "assistant" : "user"; - const handle = manager.attach(correlation, { kind: "Bytes", get data() { - const encoded = source.data; - if (typeof encoded !== "string" || !encoded.length || encoded.length % 4 !== 0 || - encoded.length > 4 * Math.ceil(anthropicAttachmentCapabilities.maxInlineBytes / 3) || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); - const bytes = Buffer.from(encoded, "base64"); - return bytes.length <= anthropicAttachmentCapabilities.maxInlineBytes && bytes.toString("base64") === encoded ? bytes : new Uint8Array(); - } }, { mimeType: mime, direction: "input", messageIndex, partIndex: index, role, - purpose: part.type === "document" ? "document" : toolCallId ? computerCalls.has(toolCallId) ? "computer_screenshot" : "tool_result" : "model_input", - ...(toolCallId ? { toolCallId } : {}), - capture: { ...options.capture, modelInput: true, errorDetail: errorDetail || options.capture?.errorDetail, - ...(hidden || (part.type === "image" && config.hideInputImages) || (errorDetail && config.hideErrorDetails) ? { captureContent: false } : {}) }, - }); + const handle = manager.attach(correlation, { kind: "Bytes", get data() { return decode(source.data); } }, + { mimeType: mime, direction: "input", messageIndex, partIndex: index, role, + purpose: part.type === "document" ? "document" : toolCallId ? computerCalls.has(toolCallId) ? "computer_screenshot" : "tool_result" : "model_input", + ...(toolCallId ? { toolCallId } : {}), + capture: { ...options.capture, modelInput: true, errorDetail: errorDetail || options.capture?.errorDetail, + ...(hidden || (part.type === "image" && config.hideInputImages) || (errorDetail && config.hideErrorDetails) ? { captureContent: false } : {}) }, + }); if (handle.admission === "queued") { + const occurrence = Object.freeze({ occurrenceId: handle.occurrenceId, mimeType: mime }); let positions = spans!.get(spanKey); if (!positions) { - if (spans!.size >= 1024) spans!.delete(spans!.keys().next().value!); - positions = new Set(); spans!.set(spanKey, positions); + if (spans!.size >= MAX_SPANS) spans!.delete(spans!.keys().next().value!); + positions = new Map(); spans!.set(spanKey, positions); } - if (positions.size < 256) positions.add(key); + if (positions.size < MAX_PARTS) positions.set(positionKey, occurrence); + captured.set(positionKey, occurrence); } }; - message.content.slice(0, 256).forEach((value, index) => { + message.content.slice(0, MAX_PARTS).forEach((value, index) => { const block = record(value); if (block.type === "tool_result" && Array.isArray(block.content)) { // Invalid tool IDs fail closed instead of losing causal identity. - if (typeof block.tool_use_id !== "string" || !/^[a-zA-Z0-9_-]{1,128}$/.test(block.tool_use_id)) return; - block.content.slice(0, 256).forEach((child, childIndex) => visit(child, childIndex, `${index}/${childIndex}`, block.tool_use_id as string, block.is_error === true)); + if (typeof block.tool_use_id !== "string" || !TOOL_ID.test(block.tool_use_id)) return; + block.content.slice(0, MAX_PARTS).forEach((child, childIndex) => visit(child, childIndex, `${index}/${childIndex}`, block.tool_use_id as string, block.is_error === true)); } else visit(value, index, String(index)); }); }); } catch { /* No source-bearing exceptions or changes to the application's result. */ } + return captured; } -/** A telemetry-only projection; never mutates the request passed to the SDK. */ -export function projectAnthropicMedia(payload: unknown): any { - const body = record(payload); - if (!Array.isArray(body.messages)) return payload; - const project = (value: unknown, nested = false): unknown => { - const block = record(value); - if (block.type === "image" || block.type === "document") return { type: block.type }; - if (block.type === "tool_result" && Array.isArray(block.content)) return { - type: block.type, tool_use_id: block.tool_use_id, is_error: block.is_error, - content: nested ? [] : block.content.map(child => project(child, true)), +/** Telemetry-only projection. ONLY admitted positions are replaced with a `fi-attachment:` occurrence URI + * source, so uploaded bytes are not exported twice. Everything else is returned exactly as before, and the + * request passed to the SDK is never mutated. + */ +export function projectAnthropicMedia(payload: unknown, captured: AnthropicCapturedMedia): any { + if (!captured.size) return payload; + try { + const body = record(payload); + if (!Array.isArray(body.messages)) return payload; + const replace = (value: unknown, position: string): unknown => { + const occurrence = captured.get(position); + return occurrence ? { ...record(value), source: { type: "url", url: `fi-attachment:${occurrence.occurrenceId}` } } : value; }; - return block; - }; - return { ...body, messages: body.messages.map(value => { - const message = record(value); - return Array.isArray(message.content) ? { ...message, content: message.content.map(value => project(value)) } : message; - }) }; + return { ...body, messages: body.messages.map((value, messageIndex) => { + const message = record(value); + if (!Array.isArray(message.content)) return value; + return { ...message, content: message.content.map((value, index) => { + const block = record(value); + if (block.type === "tool_result" && Array.isArray(block.content)) { + return { ...block, content: block.content.map((child, childIndex) => replace(child, `input/${messageIndex}/${index}/${childIndex}`)) }; + } + return replace(value, `input/${messageIndex}/${index}`); + }) }; + }) }; + } catch { return payload; } } diff --git a/typescript/packages/traceai_anthropic/src/instrumentation.ts b/typescript/packages/traceai_anthropic/src/instrumentation.ts index 7df2e6d6f..91561e08c 100644 --- a/typescript/packages/traceai_anthropic/src/instrumentation.ts +++ b/typescript/packages/traceai_anthropic/src/instrumentation.ts @@ -413,16 +413,21 @@ export class AnthropicInstrumentation extends InstrumentationBase{ // and lazy body parsing callers (including messages.stream()) depend on. return function patchedMessagesCreate(this: Anthropic.Messages, ...args: Parameters) { const originalParams = args[0] as Anthropic.MessageCreateParams; - const params = projectAnthropicMedia(originalParams) as Anthropic.MessageCreateParams; - const { messages: _messages, system: _system, tools: _tools, ...invocationParameters } = params; + const { messages: _messages, system: _system, tools: _tools, ...invocationParameters } = originalParams; - const span = instrumentation.fiTracer.startSpan("Anthropic Messages Create", { kind: SpanKind.INTERNAL }); - captureAnthropicAttachments(instrumentation.attachments, span, originalParams, { traceConfig: instrumentation._traceConfig }); - span.setAttributes({ + // Content attributes are set after capture so admitted media can be projected out first. + const span = instrumentation.fiTracer.startSpan("Anthropic Messages Create", { + kind: SpanKind.INTERNAL, + attributes: { [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, - [SemanticConventions.LLM_MODEL_NAME]: params.model, + [SemanticConventions.LLM_MODEL_NAME]: originalParams.model, [SemanticConventions.LLM_PROVIDER]: LLMProvider.ANTHROPIC, [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", + }, + }); + const captured = captureAnthropicAttachments(instrumentation.attachments, span, originalParams, { traceConfig: instrumentation._traceConfig }); + const params = projectAnthropicMedia(originalParams, captured) as Anthropic.MessageCreateParams; + span.setAttributes({ [SemanticConventions.LLM_INVOCATION_PARAMETERS]: safelyJSONStringify(invocationParameters) ?? "", ...getAnthropicInputMessagesAttributes(params), ...getAnthropicToolsAttributes(params), diff --git a/typescript/packages/traceai_openai/src/attachments.ts b/typescript/packages/traceai_openai/src/attachments.ts index d47e00ce0..319628c03 100644 --- a/typescript/packages/traceai_openai/src/attachments.ts +++ b/typescript/packages/traceai_openai/src/attachments.ts @@ -6,17 +6,25 @@ export const openAIAttachmentCapabilities = Object.freeze({ schemaVersion: 1, operations: ["chat.completions.create.input", "responses.create.input", "responses.create.output", "images.generate.output"] as const, sources: ["inline_base64"] as const, - unsupported: ["remote_url", "provider_file", "image_edit_stream", "speech", "partial_image"] as const, + /** Fail closed: no capture is attempted for these. */ + unsupported: ["remote_url", "provider_file", "streamed_output", "partial_image", "images.edit", "images.createVariation", "speech"] as const, maxInlineBytes: 256 * 1024, }); +/** Admitted media positions for one observation: position key -> occurrence. */ +export type CapturedMedia = ReadonlyMap>; +type Captured = Map>; type RecordValue = Record; const record = (value: unknown): RecordValue => value !== null && typeof value === "object" ? value as RecordValue : {}; const list = (value: unknown): unknown[] => Array.isArray(value) ? value : []; const MAX_PARTS = 256; -// Shared across CJS/ESM copies. Retain only admitted span IDs and positional keys, never sources. +const MAX_SPANS = 1024; +const EMPTY: CapturedMedia = new Map(); +const IMAGES = ["image/png", "image/jpeg", "image/webp"]; +const FILES = ["application/pdf", "application/octet-stream"]; +// Shared across CJS/ESM copies. Retains only admitted span IDs, positions and occurrence IDs, never sources. const registryKey = Symbol.for("@traceai/openai/attachment-observations/v1"); -type Registry = WeakMap>>; +type Registry = WeakMap>; const host = globalThis as typeof globalThis & { [registryKey]?: Registry }; const observations = (host[registryKey] ??= new WeakMap()); @@ -29,6 +37,7 @@ export interface OpenAIAttachmentCaptureOptions { imageFormat?: "png" | "jpeg" | "webp"; } +/** The local instrumentor's restrictions; the shared manager applies its own policy as well (deny wins). */ function restrictions(options: OpenAIAttachmentCaptureOptions): AttachmentCapturePolicy { const config = options.traceConfig ?? {}; const input = options.direction === "input"; @@ -41,17 +50,38 @@ function restrictions(options: OpenAIAttachmentCaptureOptions): AttachmentCaptur }; } -/** Typed, synchronous observation only. No provider downloads or background stream consumption. - * Pass the SAME emitted identity to SDK and wire observations to deduplicate positions. +function decode(value: unknown, mimeType: string, raw: boolean): Uint8Array { + if (typeof value !== "string") return new Uint8Array(); + const prefix = `data:${mimeType};base64,`; + if (!raw && !value.startsWith(prefix)) return new Uint8Array(); + // Bound allocation BEFORE slicing/decoding. Padding and alphabet are strict. + const start = raw ? 0 : prefix.length; + const length = value.length - start; + if (length <= 0 || length > 4 * Math.ceil(openAIAttachmentCapabilities.maxInlineBytes / 3) || length % 4 !== 0) return new Uint8Array(); + const encoded = value.slice(start); + if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); + const bytes = Buffer.from(encoded, "base64"); + if (bytes.length > openAIAttachmentCapabilities.maxInlineBytes || bytes.toString("base64") !== encoded) return new Uint8Array(); + return bytes; +} + +const key = (options: Pick, messageIndex: number, partIndex: number) => + `${options.operation}/${options.direction}/${messageIndex}/${partIndex}`; + +/** Typed, synchronous observation only. No provider downloads or stream consumption. + * Pass the SAME emitted identity to SDK and wire observations: a second observation of a position + * returns the first occurrence instead of uploading again. * Denied sources are never read: the shared manager calls the getter only after admission. + * Returns the admitted positions so the caller can project them out of span telemetry. */ export function captureOpenAIAttachments(manager: AttachmentManager | undefined, - span: AttachmentSpan | AttachmentCorrelation, payload: unknown, options: OpenAIAttachmentCaptureOptions): void { - if (!manager) return; + span: AttachmentSpan | AttachmentCorrelation, payload: unknown, options: OpenAIAttachmentCaptureOptions): CapturedMedia { + if (!manager) return EMPTY; + const captured: Captured = new Map(); try { const correlation = "snapshot" in span ? span : manager.correlation(span); const identity = correlation.snapshot(); - if (!identity?.sampled) return; + if (!identity?.sampled) return EMPTY; let spans = observations.get(manager); if (!spans) { spans = new Map(); observations.set(manager, spans); } const spanKey = `${identity.traceId}/${identity.spanId}`; @@ -60,71 +90,63 @@ export function captureOpenAIAttachments(manager: AttachmentManager | undefined, const attach = (read: () => unknown, mimeTypes: string[], messageIndex: number, partIndex: number, purpose: AttachmentOptions["purpose"], toolCallId?: string, raw = false, role: AttachmentOptions["role"] = "user") => { if (++visited > MAX_PARTS) return; - const key = `${options.operation}/${options.direction}/${messageIndex}/${partIndex}`; - if (spans!.get(spanKey)?.has(key)) return; + const position = key(options, messageIndex, partIndex); + const previous = spans!.get(spanKey)?.get(position); + if (previous) { captured.set(position, previous); return; } for (const mimeType of mimeTypes) { - const handle = manager.attach(correlation, { kind: "Bytes", get data() { - const value = read(); - if (typeof value !== "string") return new Uint8Array(); - const prefix = `data:${mimeType};base64,`; - if (!raw && !value.startsWith(prefix)) return new Uint8Array(); - // Bound allocation BEFORE slicing/decoding. Padding and alphabet are strict. - const start = raw ? 0 : prefix.length; - const length = value.length - start; - if (length <= 0 || length > 4 * Math.ceil(openAIAttachmentCapabilities.maxInlineBytes / 3) || length % 4 !== 0) return new Uint8Array(); - const encoded = value.slice(start); - if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); - const bytes = Buffer.from(encoded, "base64"); - if (bytes.length > openAIAttachmentCapabilities.maxInlineBytes || bytes.toString("base64") !== encoded) return new Uint8Array(); - return bytes; - } }, { mimeType, direction: options.direction, purpose, messageIndex, partIndex, role, - ...(toolCallId ? { toolCallId } : {}), capture: { - ...capture, - ...(options.direction === "input" && mimeType.startsWith("image/") && options.traceConfig?.hideInputImages ? { captureContent: false } : {}), - } }); + const handle = manager.attach(correlation, { kind: "Bytes", get data() { return decode(read(), mimeType, raw); } }, + { mimeType, direction: options.direction, purpose, messageIndex, partIndex, role, + ...(toolCallId ? { toolCallId } : {}), capture: { + ...capture, + ...(options.direction === "input" && mimeType.startsWith("image/") && options.traceConfig?.hideInputImages ? { captureContent: false } : {}), + } }); if (handle.admission === "queued") { + const occurrence = Object.freeze({ occurrenceId: handle.occurrenceId, mimeType }); let positions = spans!.get(spanKey); if (!positions) { - if (spans!.size >= 1024) spans!.delete(spans!.keys().next().value!); - positions = new Set(); spans!.set(spanKey, positions); + if (spans!.size >= MAX_SPANS) spans!.delete(spans!.keys().next().value!); + positions = new Map(); spans!.set(spanKey, positions); } - if (positions.size < MAX_PARTS) positions.add(key); + if (positions.size < MAX_PARTS) positions.set(position, occurrence); + captured.set(position, occurrence); return; } - if (handle.admission !== "too_large" && handle.admission !== "source_unavailable") return; + // Only a non-matching data-URI type (decoded as empty) tries the next candidate. Denials stop. + if (handle.admission !== "too_large") return; } }; - const images = ["image/png", "image/jpeg", "image/webp"]; + const toolCallId = (value: unknown) => typeof value === "string" && /^[a-zA-Z0-9_-]{1,128}$/.test(value) ? value : undefined; const parts = (content: unknown, messageIndex: number, role: AttachmentOptions["role"]) => { list(content).slice(0, MAX_PARTS).forEach((value, partIndex) => { const part = record(value); - if (part.type === "image_url") attach(() => record(part.image_url).url, images, messageIndex, partIndex, "model_input", undefined, false, role); - else if (part.type === "input_image") attach(() => part.image_url, images, messageIndex, partIndex, "model_input", undefined, false, role); - else if (part.type === "input_file" || part.type === "file") attach(() => part.type === "file" ? record(part.file).file_data : part.file_data, - ["application/pdf", "application/octet-stream"], messageIndex, partIndex, "document", undefined, false, role); + if (part.type === "image_url") attach(() => record(part.image_url).url, IMAGES, messageIndex, partIndex, "model_input", undefined, false, role); + else if (part.type === "input_image") attach(() => part.image_url, IMAGES, messageIndex, partIndex, "model_input", undefined, false, role); + else if (part.type === "input_file") attach(() => part.file_data, FILES, messageIndex, partIndex, "document", undefined, false, role); + else if (part.type === "file") attach(() => record(part.file).file_data, FILES, messageIndex, partIndex, "document", undefined, false, role); }); }; const body = record(payload); - if (options.direction === "output" && body.error) return; + if (options.direction === "output" && body.error) return EMPTY; if (options.direction === "input") { - const messages = options.operation === "chat" ? body.messages : body.input; + const messages = options.operation === "chat" ? body.messages : options.operation === "responses" ? body.input : undefined; list(messages).slice(0, MAX_PARTS).forEach((value, messageIndex) => { const message = record(value); if (message.type === "computer_call_output") { const output = record(message.output); - if (output.type === "computer_screenshot") attach(() => output.image_url, images, messageIndex, 0, "computer_screenshot", - typeof message.call_id === "string" ? message.call_id : undefined, false, "tool"); + // A screenshot is model-bound input. Its call id keeps the causal link to the computer_call. + if (output.type === "computer_screenshot") attach(() => output.image_url, IMAGES, messageIndex, 0, "computer_screenshot", + toolCallId(message.call_id), false, "tool"); } else if (!message.type || message.type === "message") { const role = ["user", "assistant", "tool", "system"].includes(String(message.role)) ? message.role as AttachmentOptions["role"] : "unknown"; parts(message.content, messageIndex, role); } }); } else if (options.operation === "images") { + // The Images API defaults to PNG. The typed request/response format wins where provided. + const format = body.output_format ?? options.imageFormat; + const mime = format === "jpeg" ? "image/jpeg" : format === "webp" ? "image/webp" : "image/png"; list(body.data).slice(0, MAX_PARTS).forEach((value, index) => { const item = record(value); - // The Images API defaults to PNG. Explicit response format wins where provided. - const format = body.output_format ?? options.imageFormat; - const mime = format === "jpeg" ? "image/jpeg" : format === "webp" ? "image/webp" : "image/png"; attach(() => item.b64_json, [mime], index, 0, "model_output", undefined, true, "assistant"); }); } else if (options.operation === "responses") { @@ -137,23 +159,44 @@ export function captureOpenAIAttachments(manager: AttachmentManager | undefined, }); } } catch { /* Instrumentation must not throw or log source-bearing errors. */ } + return captured; } -/** Telemetry-only typed projection. Never send media bytes, filenames or source URLs to OTLP. */ -export function projectOpenAIMedia(payload: unknown, operation: "chat" | "responses" | "images", direction: "input" | "output", redactImages = true): any { - const body = record(payload); - const projectPart = (value: unknown): unknown => { - const part = record(value); - return (["input_file", "file"].includes(String(part.type)) || redactImages && ["image_url", "input_image"].includes(String(part.type))) ? { type: part.type } : value; - }; - if (operation === "images") return { created: body.created }; - const field = direction === "input" ? operation === "chat" ? "messages" : "input" : "output"; - if (!Array.isArray(body[field])) return payload; - return { ...body, [field]: list(body[field]).map(value => { - const message = record(value); - if (message.type === "computer_call_output") return { type: message.type, call_id: message.call_id, output: { type: "computer_screenshot" } }; - if (message.type === "image_generation_call") return { type: message.type, id: message.id, status: message.status }; - if (Array.isArray(message.content)) return { ...message, content: message.content.map(projectPart) }; - return value; - }) }; +const uri = (occurrence: { occurrenceId: string }) => `fi-attachment:${occurrence.occurrenceId}`; + +/** Telemetry-only projection. ONLY admitted positions are replaced with their `fi-attachment:` occurrence + * URI, so uploaded bytes are not exported twice. Everything else is returned exactly as before, and the + * request passed to the SDK is never mutated. + */ +export function projectOpenAIMedia(payload: unknown, captured: CapturedMedia, + operation: "chat" | "responses", direction: "input" | "output"): any { + if (!captured.size) return payload; + try { + const body = record(payload); + const field = direction === "input" ? operation === "chat" ? "messages" : "input" : "output"; + if (!Array.isArray(body[field])) return payload; + const at = (messageIndex: number, partIndex: number) => captured.get(key({ operation, direction }, messageIndex, partIndex)); + return { ...body, [field]: list(body[field]).map((value, messageIndex) => { + const message = record(value); + if (message.type === "computer_call_output") { + const occurrence = at(messageIndex, 0); + return occurrence ? { ...message, output: { ...record(message.output), image_url: uri(occurrence) } } : value; + } + if (message.type === "image_generation_call") { + const occurrence = at(messageIndex, 0); + return occurrence ? { ...message, result: uri(occurrence) } : value; + } + if (!Array.isArray(message.content)) return value; + return { ...message, content: message.content.map((value, partIndex) => { + const occurrence = at(messageIndex, partIndex); + if (!occurrence) return value; + const part = record(value); + if (part.type === "image_url") return { ...part, image_url: { ...record(part.image_url), url: uri(occurrence) } }; + if (part.type === "input_image") return { ...part, image_url: uri(occurrence) }; + if (part.type === "input_file") return { ...part, file_data: uri(occurrence) }; + if (part.type === "file") return { ...part, file: { ...record(part.file), file_data: uri(occurrence) } }; + return value; + }) }; + }) }; + } catch { return payload; } } diff --git a/typescript/packages/traceai_openai/src/instrumentation.ts b/typescript/packages/traceai_openai/src/instrumentation.ts index 1951022ea..855577d88 100644 --- a/typescript/packages/traceai_openai/src/instrumentation.ts +++ b/typescript/packages/traceai_openai/src/instrumentation.ts @@ -231,34 +231,29 @@ export class OpenAIInstrumentation extends InstrumentationBase{ ...args: Parameters ) { const originalBody = args[0]; - const body = projectOpenAIMedia(originalBody, "chat", "input", !!instrumentation.attachments); - // Prediction text is input content, captured only in maskable payloads. - const { messages: _messages, prediction: _prediction, ...invocationParameters } = body; - // diag.debug("@traceai/openai: ChatCompletion patch CALLED. Starting span..."); - - // --- ADD LOGS FOR TRACER AND CONTEXT --- - // const activeContextForSuppressionCheck = context.active(); - // const isSuppressed = isTracingSuppressed(activeContextForSuppressionCheck); - // diag.debug(`@traceai/openai: Is tracing suppressed? ${isSuppressed}`); - // diag.debug(`@traceai/openai: this.tracer type: ${instrumentation.tracer?.constructor?.name}`); // Accessing the base tracer - // diag.debug(`@traceai/openai: this.fiTracer type: ${instrumentation.fiTracer?.constructor?.name}`); - // --- END ADDED LOGS --- - const span = instrumentation.fiTracer.startSpan("OpenAI Chat Completions", { kind: SpanKind.INTERNAL }); - captureOpenAIAttachments(instrumentation.attachments, span, originalBody, { + // Content attributes are set after capture so admitted media can be projected out first. + const span = instrumentation.fiTracer.startSpan("OpenAI Chat Completions", { + kind: SpanKind.INTERNAL, + attributes: { + [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, + [SemanticConventions.LLM_MODEL_NAME]: originalBody.model, + [SemanticConventions.LLM_PROVIDER]: providerForResource(this), + [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", + }, + }); + const captured = captureOpenAIAttachments(instrumentation.attachments, span, originalBody, { operation: "chat", direction: "input", traceConfig: instrumentation._traceConfig, }); + const body = projectOpenAIMedia(originalBody, captured, "chat", "input") as typeof originalBody; + // Prediction text is input content, captured only in maskable payloads. + const { messages: _messages, prediction: _prediction, ...invocationParameters } = body; span.setAttributes({ - [SemanticConventions.FI_SPAN_KIND]: - FISpanKind.LLM, - [SemanticConventions.LLM_MODEL_NAME]: body.model, [SemanticConventions.INPUT_VALUE]: JSON.stringify(body), [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON, [SemanticConventions.LLM_INVOCATION_PARAMETERS]: JSON.stringify(invocationParameters), - [SemanticConventions.LLM_PROVIDER]: providerForResource(this), - [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", ...getLLMInputMessagesAttributes(body), ...getLLMToolsJSONSchema(body), [SemanticConventions.RAW_INPUT]: safelyJSONStringify(body) ?? "", @@ -523,9 +518,22 @@ export class OpenAIInstrumentation extends InstrumentationBase{ const body = args[0] as ResponseCreateParamsBase & { prompt?: { id: string; version?: string; variables?: Record }; }; + // Content attributes are set after capture so admitted media can be projected out first. + const span = instrumentation.fiTracer.startSpan("OpenAI Responses", { + kind: SpanKind.INTERNAL, + attributes: { + [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, + [SemanticConventions.LLM_MODEL_NAME]: body.model, + [SemanticConventions.LLM_PROVIDER]: providerForResource(this), + [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", + }, + }); + const captured = captureOpenAIAttachments(instrumentation.attachments, span, body, { + operation: "responses", direction: "input", traceConfig: instrumentation._traceConfig, + }); // Telemetry-only copy: MCP credentials must never reach any exported // duplicate, but the original SDK request must remain untouched. - const projectedBody = projectOpenAIMedia(body, "responses", "input", !!instrumentation.attachments); + const projectedBody = projectOpenAIMedia(body, captured, "responses", "input") as typeof body; const telemetryBody = { ...projectedBody, ...(body.tools && { tools: sanitizeToolDefinitions(body.tools) }), @@ -536,21 +544,12 @@ export class OpenAIInstrumentation extends InstrumentationBase{ ...parameters, ...(prompt && { prompt: { id: prompt.id, version: prompt.version } }), }; - const span = instrumentation.fiTracer.startSpan("OpenAI Responses", { kind: SpanKind.INTERNAL }); - captureOpenAIAttachments(instrumentation.attachments, span, body, { - operation: "responses", direction: "input", traceConfig: instrumentation._traceConfig, - }); span.setAttributes({ - [SemanticConventions.FI_SPAN_KIND]: - FISpanKind.LLM, - [SemanticConventions.LLM_MODEL_NAME]: body.model, [SemanticConventions.INPUT_VALUE]: JSON.stringify(telemetryBody), [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON, [SemanticConventions.LLM_INVOCATION_PARAMETERS]: JSON.stringify(invocationParameters), [SemanticConventions.PROMPT_TEMPLATE_VARIABLES]: safelyJSONStringify(prompt?.variables) ?? undefined, - [SemanticConventions.LLM_PROVIDER]: providerForResource(this), - [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", ...getResponsesInputMessagesAttributes(telemetryBody), ...getLLMToolsJSONSchema(body), [SemanticConventions.RAW_INPUT]: safelyJSONStringify(telemetryBody) ?? "", @@ -586,10 +585,12 @@ export class OpenAIInstrumentation extends InstrumentationBase{ if (endSpan) span.end(); return; } - captureOpenAIAttachments(instrumentation.attachments, span, result, { + // endSpan is false only on the streamed path. Streamed output media is unsupported + // until the core has a consumed-stream observation API. + const outputMedia = endSpan ? captureOpenAIAttachments(instrumentation.attachments, span, result, { operation: "responses", direction: "output", traceConfig: instrumentation._traceConfig, - }); - const projected = projectOpenAIMedia(result, "responses", "output"); + }) : new Map(); + const projected = projectOpenAIMedia(result, outputMedia, "responses", "output") as ResponseType; const output = { id: result.id, object: result.object, status: result.status, model: result.model, output: projected.output, usage: result.usage, @@ -637,28 +638,42 @@ export class OpenAIInstrumentation extends InstrumentationBase{ ); } + // images.generate: generated b64 outputs. Streamed partial images, edits and variations stay unsupported. if (typeof module.OpenAI.Images?.prototype?.generate === "function") { - this._wrap(module.OpenAI.Images.prototype, "generate", (original: typeof module.OpenAI.Images.prototype.generate): any => { - return function(this: unknown, ...args: Parameters) { - const span = instrumentation.fiTracer.startSpan("OpenAI Images", { kind: SpanKind.INTERNAL, attributes: { - [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, - [SemanticConventions.LLM_PROVIDER]: providerForResource(this), - [SemanticConventions.GEN_AI_OPERATION_NAME]: "image_generation", - } }); - try { - const execContext = trace.setSpan(context.active(), span); - const promise = context.with(execContext, () => original.apply(this, args)); - return observeAPIPromise(promise, result => { - if (span.isRecording() && !isStream(result)) { - captureOpenAIAttachments(instrumentation.attachments, span, result, { - operation: "images", direction: "output", traceConfig: instrumentation._traceConfig, - imageFormat: (args[0] as { output_format?: "png" | "jpeg" | "webp" }).output_format, - }); - } - span.end(); - return result; - }, error => endSpanWithError(span, error), response => endRawResponseSpan(span, response)); - } catch (error) { endSpanWithError(span, error); throw error; } + type ImagesGenerateType = typeof module.OpenAI.Images.prototype.generate; + this._wrap(module.OpenAI.Images.prototype, "generate", (original: ImagesGenerateType): any => { + return function patchedImagesGenerate(this: unknown, ...args: Parameters) { + const request = (args[0] ?? {}) as { model?: string; output_format?: "png" | "jpeg" | "webp" }; + const span = instrumentation.fiTracer.startSpan("OpenAI Images", { + kind: SpanKind.INTERNAL, + attributes: { + [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, + [SemanticConventions.LLM_PROVIDER]: providerForResource(this), + [SemanticConventions.GEN_AI_OPERATION_NAME]: "image_generation", + ...(typeof request.model === "string" ? { [SemanticConventions.LLM_MODEL_NAME]: request.model } : {}), + [SemanticConventions.INPUT_VALUE]: safelyJSONStringify(args[0]) ?? "", + [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON, + }, + }); + const contextSpan = span.getContextSpan(); + const execContext = getExecContext(contextSpan); + const execPromise = safeExecuteInTheMiddle>( + () => context.with(trace.setSpan(execContext, contextSpan), () => original.apply(this, args)), + (error) => { if (error) endSpanWithError(span, error); }, + ); + const wrappedPromise = observeAPIPromise(execPromise, (result) => { + if (!span.isRecording()) return result; + if (!isStream(result)) { + captureOpenAIAttachments(instrumentation.attachments, span, result, { + operation: "images", direction: "output", traceConfig: instrumentation._traceConfig, + imageFormat: request.output_format, + }); + } + span.setStatus({ code: SpanStatusCode.OK }); + span.end(); + return result; + }, (error) => endSpanWithError(span, error), (response) => endRawResponseSpan(span, response)); + return context.bind(execContext, wrappedPromise); }; }); } diff --git a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs index dcba8d149..5a0ceab82 100644 --- a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs +++ b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs @@ -140,21 +140,70 @@ test('OpenAI generated images obey output and PII restrictions', async () => { } }); -test('Telemetry projections strip typed bytes, filenames and signed source locations without mutating requests', () => { - for (const [body, project] of [[oaInput(), x => oa.projectOpenAIMedia(x, 'responses', 'input')], [anInput(), an.projectAnthropicMedia]]) { - const before = JSON.stringify(body), projected = JSON.stringify(project(body)); - assert.equal(JSON.stringify(body), before); - for (const secret of [PNG, PDF, 'synthetic-private-filename']) assert.ok(!projected.includes(secret)); - } +test('Telemetry projections replace only admitted media with occurrence URIs and never mutate requests', async () => { + const f = await fixture(); try { + const cases = [ + [oaInput(), (m, s, x) => oa.captureOpenAIAttachments(m, s, x, { operation: 'responses', direction: 'input' }), (x, c) => oa.projectOpenAIMedia(x, c, 'responses', 'input'), 3], + [anInput(), (m, s, x) => an.captureAnthropicAttachments(m, s, x), (x, c) => an.projectAnthropicMedia(x, c), 4], + ]; + for (const [body, capture, project, count] of cases) { + const before = JSON.stringify(body); + // Nothing admitted: the exact same payload object is returned, so existing telemetry is unchanged. + assert.equal(project(body, new Map()), body); + const captured = capture(f.manager, identity(), body); + assert.equal(captured.size, count); + const projected = JSON.stringify(project(body, captured)); + assert.equal(JSON.stringify(body), before); + for (const secret of [PNG, PDF]) assert.ok(!projected.includes(secret)); + for (const { occurrenceId } of captured.values()) assert.ok(projected.includes(`fi-attachment:${occurrenceId}`)); + } + await f.manager.flush(); + assert.equal(f.uploaded.length, 7); + } finally { await f.close(); } +}); + +test('Denied media keeps the existing telemetry payload untouched', async () => { + const f = await fixture({ traceConfig: { hideInputImages: true } }); try { + const body = oaInput(), captured = oa.captureOpenAIAttachments(f.manager, identity(), body, { operation: 'responses', direction: 'input' }); + // Only the PDF is admitted; both images stay exactly as the application sent them. + assert.deepEqual([...captured.values()].map(v => v.mimeType), ['application/pdf']); + const projected = oa.projectOpenAIMedia(body, captured, 'responses', 'input'); + assert.equal(projected.input[0].content[0], body.input[0].content[0]); + assert.equal(projected.input[1], body.input[1]); + assert.ok(projected.input[0].content[1].file_data.startsWith('fi-attachment:')); + } finally { await f.close(); } +}); + +test('Hostile payloads fail capture without throwing into the application', async () => { + const f = await fixture(); try { + const hostile = new Proxy({}, { get() { throw Error('synthetic-secret'); }, has() { throw Error('synthetic-secret'); } }); + for (const payload of [hostile, { input: [hostile] }, { messages: [hostile] }, { messages: [{ role: 'user', content: [hostile] }] }, null, 7, 'x']) { + assert.equal(oa.captureOpenAIAttachments(f.manager, identity(), payload, { operation: 'responses', direction: 'input' }).size, 0); + assert.equal(oa.captureOpenAIAttachments(f.manager, identity(), payload, { operation: 'chat', direction: 'input' }).size, 0); + assert.equal(an.captureAnthropicAttachments(f.manager, identity(), payload).size, 0); + assert.doesNotThrow(() => an.projectAnthropicMedia(payload, new Map([['input/0/0', { occurrenceId: 'x', mimeType: 'image/png' }]]))); + assert.doesNotThrow(() => oa.projectOpenAIMedia(payload, new Map([['responses/input/0/0', { occurrenceId: 'x', mimeType: 'image/png' }]]), 'responses', 'input')); + } + await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } +}); + +test('Capability descriptors name every unsupported source explicitly', () => { + for (const name of ['remote_url', 'provider_file', 'streamed_output', 'partial_image', 'images.edit', 'speech']) assert.ok(oa.openAIAttachmentCapabilities.unsupported.includes(name)); + for (const name of ['remote_url', 'provider_file', 'native_span_hookup', 'generated_image', 'streamed_output']) assert.ok(an.anthropicAttachmentCapabilities.unsupported.includes(name)); + assert.ok(Object.isFrozen(oa.openAIAttachmentCapabilities) && Object.isFrozen(an.anthropicAttachmentCapabilities)); }); test('CJS/ESM observations share deduplication without creating a second capture', async () => { const f = await fixture(); try { const esm = await import(pathToFileURL(path.resolve(__dirname, '../dist/esm/attachments.js'))); - const s = identity(); oa.captureOpenAIAttachments(f.manager, s, oaInput(), { operation: 'responses', direction: 'input' }); - esm.captureOpenAIAttachments(f.manager, { ...s }, oaInput(), { operation: 'responses', direction: 'input' }); + const s = identity(), first = oa.captureOpenAIAttachments(f.manager, s, oaInput(), { operation: 'responses', direction: 'input' }); + const second = esm.captureOpenAIAttachments(f.manager, { ...s }, oaInput(), { operation: 'responses', direction: 'input' }); + // The second observer reuses the first occurrence, so both projections point at one upload. + assert.deepEqual([...second.entries()], [...first.entries()]); const anthropicESM = await import(pathToFileURL(path.resolve(__dirname, '../../traceai_anthropic/dist/esm/attachments.js'))); - const a = identity(); an.captureAnthropicAttachments(f.manager, a, anInput()); anthropicESM.captureAnthropicAttachments(f.manager, { ...a }, anInput()); + const a = identity(), cjs = an.captureAnthropicAttachments(f.manager, a, anInput()), mjs = anthropicESM.captureAnthropicAttachments(f.manager, { ...a }, anInput()); + assert.deepEqual([...mjs.entries()], [...cjs.entries()]); await f.manager.flush(); assert.equal(f.uploaded.length, 7); } finally { await f.close(); } }); @@ -195,6 +244,14 @@ async function providerServer() { const server = http.createServer(async (req, res) => { const chunks = []; for await (const c of req) chunks.push(c); const body = JSON.parse(Buffer.concat(chunks)); requests.push({ path: req.url, body }); + if (req.url.endsWith('/responses') && body.stream) { + // A streamed Responses call whose completed response carries a generated image. + const response = { id: 'resp_stream', object: 'response', status: 'completed', model: body.model, output: [{ type: 'image_generation_call', id: 'img_stream', status: 'completed', result: PNG }], usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } }; + res.setHeader('content-type', 'text/event-stream'); + res.write(`event: response.created\ndata: ${JSON.stringify({ type: 'response.created', sequence_number: 0, response: { ...response, status: 'in_progress', output: [] } })}\n\n`); + res.end(`event: response.completed\ndata: ${JSON.stringify({ type: 'response.completed', sequence_number: 1, response })}\n\n`); + return; + } res.setHeader('content-type', 'application/json'); if (req.url.endsWith('/chat/completions')) res.end(JSON.stringify({ id: 'chat_fixture', object: 'chat.completion', model: body.model, choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })); else if (req.url.endsWith('/responses')) res.end(JSON.stringify({ id: 'resp_fixture', object: 'response', status: 'completed', model: body.model, output: [{ type: 'computer_call', id: 'computer_fixture', call_id: 'call_computer', status: 'completed', action: { type: 'screenshot' } }, { type: 'image_generation_call', id: 'img_fixture', status: 'completed', result: PNG }], usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } })); @@ -239,6 +296,18 @@ test('Real SDKs preserve APIPromise/request/response behavior and upload only th assert.ok(p.requests.some(r => JSON.stringify(r.body).includes(PNG))); assert.ok(!JSON.stringify(exporter.getFinishedSpans().map(s => s.attributes)).includes(PNG)); assert.ok(exporter.getFinishedSpans().some(s => s.attributes['fi.attachment.refs'])); + // Each admitted reference appears as an occurrence URI in its own span's telemetry. + for (const span of exporter.getFinishedSpans()) { + const refs = JSON.parse(span.attributes['fi.attachment.refs'] ?? '[]'), text = JSON.stringify(span.attributes); + for (const ref of refs.filter(r => r.direction === 'input')) assert.ok(text.includes(`fi-attachment:${ref.occurrence_id}`), span.name); + } + if (client.responses) { + // Streamed output media is unsupported: the stream is delivered unchanged and nothing uploads. + const stream = await client.responses.create({ model: 'synthetic-model', input: 'synthetic', stream: true }); + const events = []; for await (const event of stream) events.push(event.type); + assert.deepEqual(events, ['response.created', 'response.completed']); + await f.manager.flush(); assert.equal(f.uploaded.length, expected); + } // A raw-response caller owns the body; instrumentation must not parse for media. const raw = await client.images.generate({ prompt: 'synthetic', response_format: 'b64_json' }).asResponse(); assert.equal(raw.bodyUsed, false); await raw.arrayBuffer(); @@ -254,3 +323,35 @@ test('Real SDKs preserve APIPromise/request/response behavior and upload only th oi.disable(); ai.disable(); await provider.shutdown(); await f.close(); await p.close(); } }); + +test('A configured but disabled manager leaves existing span telemetry byte-identical', async () => { + const sdkModule = require(process.env.SDK_OPENAI || 'openai'); + const anthropicModule = require(process.env.SDK_ANTHROPIC || '../../traceai_anthropic/node_modules/@anthropic-ai/sdk'); + const OpenAI = sdkModule.OpenAI || sdkModule.default || sdkModule; + const Anthropic = anthropicModule.Anthropic || anthropicModule.default || anthropicModule; + const native = 'openTelemetry' in Anthropic.prototype; + const p = await providerServer(), disabled = await fixture({ enabled: false }); + const run = async attachments => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + const oi = new OpenAIInstrumentation({ instrumentationConfig: { enabled: false }, attachments }); + const ai = new AnthropicInstrumentation({ instrumentationConfig: { enabled: false }, attachments }); + oi.setTracerProvider(provider); ai.setTracerProvider(provider); + oi.manuallyInstrument(sdkModule); ai.manuallyInstrument(anthropicModule); + try { + const client = new OpenAI({ apiKey: 'synthetic-provider-key', baseURL: p.url, maxRetries: 0 }); + await client.chat.completions.create({ model: 'synthetic-model', messages: [{ role: 'user', content: [{ type: 'image_url', image_url: { url: `data:image/png;base64,${PNG}` } }] }] }); + if (client.responses) await client.responses.create({ model: 'synthetic-model', ...oaInput() }); + if (!native) await new Anthropic({ apiKey: 'synthetic-provider-key', baseURL: p.url, maxRetries: 0 }).messages.create({ model: 'synthetic-model', max_tokens: 10, ...anInput() }); + await provider.forceFlush(); + const strip = ({ name, attributes }) => ({ name, attributes: Object.fromEntries(Object.entries(attributes).filter(([k]) => !/latency|duration|time/i.test(k))) }); + return exporter.getFinishedSpans().map(strip); + } finally { oi.disable(); ai.disable(); await provider.shutdown(); } + }; + try { + const baseline = await run(undefined), off = await run(disabled.manager); + assert.ok(baseline.length >= 1); + assert.deepEqual(off, baseline); + assert.equal(disabled.calls.length, 0); + } finally { await disabled.close(); await p.close(); } +}); From 92662aecc430a07ffaffcc3ef6d0d023b229b063 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 20:13:48 -0800 Subject: [PATCH 03/20] test(openai,anthropic): cover local-only privacy restrictions independently of the manager Co-Authored-By: Claude Opus 5.5 --- .../test/attachments-providers.test.cjs | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs index 5a0ceab82..5008ace23 100644 --- a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs +++ b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs @@ -232,9 +232,31 @@ test('OpenAI image error bodies are not treated as generated media', async () => }); test('Local instrumentor privacy restrictions cannot be loosened by a permissive shared manager', async () => { + for (const traceConfig of [{ hideInputs: true }, { hideInputMessages: true }, { hideInputText: true }, { piiRedaction: true }]) { + const f = await fixture(); try { + assert.equal(oa.captureOpenAIAttachments(f.manager, identity(), oaInput(), { operation: 'responses', direction: 'input', traceConfig }).size, 0); + assert.equal(an.captureAnthropicAttachments(f.manager, identity(), anInput(), { traceConfig }).size, 0); + await f.manager.flush(); assert.equal(f.initialized.length, 0, JSON.stringify(traceConfig)); + } finally { await f.close(); } + } + for (const traceConfig of [{ hideOutputs: true }, { piiRedaction: true }]) { + const f = await fixture(); try { + oa.captureOpenAIAttachments(f.manager, identity(), { data: [{ b64_json: PNG }] }, { operation: 'images', direction: 'output', traceConfig }); + await f.manager.flush(); assert.equal(f.initialized.length, 0, JSON.stringify(traceConfig)); + } finally { await f.close(); } + } + // Image-only restriction: the PDF still uploads, no image (including screenshots) does. + const f = await fixture(); try { + oa.captureOpenAIAttachments(f.manager, identity(), oaInput(), { operation: 'responses', direction: 'input', traceConfig: { hideInputImages: true } }); + an.captureAnthropicAttachments(f.manager, identity(), anInput(), { traceConfig: { hideInputImages: true } }); + await f.manager.flush(); assert.deepEqual(f.initialized.map(r => r.mime_type), ['application/pdf', 'application/pdf']); + } finally { await f.close(); } +}); + +test('Local hideErrorDetails denies Anthropic error tool media even when the manager allows it', async () => { const f = await fixture(); try { - oa.captureOpenAIAttachments(f.manager, identity(), oaInput(), { operation: 'responses', direction: 'input', traceConfig: { hideInputs: true } }); - an.captureAnthropicAttachments(f.manager, identity(), anInput(), { traceConfig: { piiRedaction: true } }); + an.captureAnthropicAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_error', is_error: true, content: [image()] }] }] }, + { traceConfig: { hideErrorDetails: true } }); await f.manager.flush(); assert.equal(f.initialized.length, 0); } finally { await f.close(); } }); From 059e0b91516d2b53901d39fd01fdc5b66660aad4 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 20:19:33 -0800 Subject: [PATCH 04/20] fix(openai): keep the Responses telemetry copy loosely typed for older SDK typings Co-Authored-By: Claude Opus 5.5 --- typescript/packages/traceai_openai/src/instrumentation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typescript/packages/traceai_openai/src/instrumentation.ts b/typescript/packages/traceai_openai/src/instrumentation.ts index 855577d88..c457aaf20 100644 --- a/typescript/packages/traceai_openai/src/instrumentation.ts +++ b/typescript/packages/traceai_openai/src/instrumentation.ts @@ -533,7 +533,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ }); // Telemetry-only copy: MCP credentials must never reach any exported // duplicate, but the original SDK request must remain untouched. - const projectedBody = projectOpenAIMedia(body, captured, "responses", "input") as typeof body; + const projectedBody = projectOpenAIMedia(body, captured, "responses", "input"); const telemetryBody = { ...projectedBody, ...(body.tools && { tools: sanitizeToolDefinitions(body.tools) }), From 4b52c19ca5e6f7c47827ca93ebde41cf4bc73b77 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 20:39:13 -0800 Subject: [PATCH 05/20] test(openai,anthropic): pin the 256 KiB inline decode bound with valid data URIs The cloud mutation run showed the OpenAI cap was not exercised: the oversized case was not a data URI, so it failed the prefix check first. Co-Authored-By: Claude Opus 5.5 --- .../test/attachments-providers.test.cjs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs index 5008ace23..c9a9be53a 100644 --- a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs +++ b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs @@ -131,6 +131,19 @@ for (const provider of ['openai', 'anthropic']) { }); } +test('Inline parts are capped at exactly 256 KiB of decoded bytes for both providers', async () => { + const f = await fixture(); try { + const at = Buffer.alloc(256 * 1024, 7).toString('base64'), over = Buffer.alloc(256 * 1024 + 3, 7).toString('base64'); + for (const data of [at, over]) { + oa.captureOpenAIAttachments(f.manager, identity(), { input: [{ role: 'user', content: [{ type: 'input_image', image_url: `data:image/png;base64,${data}` }] }] }, { operation: 'responses', direction: 'input' }); + oa.captureOpenAIAttachments(f.manager, identity(), { data: [{ b64_json: data }] }, { operation: 'images', direction: 'output' }); + an.captureAnthropicAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data } }] }] }); + } + await f.manager.flush(); + assert.deepEqual(f.uploaded.map(b => b.length), [262144, 262144, 262144]); + } finally { await f.close(); } +}); + test('OpenAI generated images obey output and PII restrictions', async () => { for (const traceConfig of [{}, { hideOutputs: true }, { hideOutputMessages: true }, { hideOutputText: true }, { piiRedaction: true }]) { const f = await fixture({ traceConfig }); try { From 912c4fb831ba1608796530be482967ef7d4514ab Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:44:36 -0800 Subject: [PATCH 06/20] feat(openai,anthropic): preflight every media part; capture streamed Responses output Adopt the core preflight API (sdk-streaming-design section 1) in place of lazy `get data()` getters: - Each inline part is preflighted (deny-wins, budget reserved) before its value is read or decoded. Denied parts are never read. - An admitted ticket that is not used (malformed base64, MIME outside the allowlist, decode failure) is released at once, not left to the TTL sweep. - OpenAI data URIs naming a sibling MIME of the same modality are released and re-admitted under the exact MIME before decoding. - When the server negotiates an upload limit below the 256 KiB inline cap, preflight returns too_large only after every privacy step passed, so the part is re-preflighted with its exact decoded length. Small media keeps capturing as before. Streamed Responses: the generated image in the terminal response.completed event that the application consumed from its own iterator is now captured and projected out of the span output, before the span ends. Partial (preview) images and streamed images.generate stay unsupported and are named in the capability descriptor. Test fixtures mint one-hour sessions (core renewal window), assert no FI_ATTACHMENTS_* degradation warnings and no leaked tickets or bytes. Co-Authored-By: Claude Opus 5.5 --- .../traceai_anthropic/src/attachments.ts | 50 ++++++--- .../traceai_openai/src/attachments.ts | 100 ++++++++++++------ .../traceai_openai/src/instrumentation.ts | 8 +- .../test/attachments-providers.test.cjs | 71 +++++++++++-- 4 files changed, 172 insertions(+), 57 deletions(-) diff --git a/typescript/packages/traceai_anthropic/src/attachments.ts b/typescript/packages/traceai_anthropic/src/attachments.ts index 876ac52ae..ce45cd139 100644 --- a/typescript/packages/traceai_anthropic/src/attachments.ts +++ b/typescript/packages/traceai_anthropic/src/attachments.ts @@ -25,17 +25,26 @@ const host = globalThis as typeof globalThis & { [registryKey]?: Registry }; const observations = (host[registryKey] ??= new WeakMap()); const TOOL_ID = /^[a-zA-Z0-9_-]{1,128}$/; -function decode(encoded: unknown): Uint8Array { - const max = anthropicAttachmentCapabilities.maxInlineBytes; - // Bound allocation BEFORE decoding. Padding and alphabet are strict. - if (typeof encoded !== "string" || !encoded.length || encoded.length % 4 !== 0 || - encoded.length > 4 * Math.ceil(max / 3) || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); - const bytes = Buffer.from(encoded, "base64"); - return bytes.length <= max && bytes.toString("base64") === encoded ? bytes : new Uint8Array(); +const MAX_ENCODED = 4 * Math.ceil(anthropicAttachmentCapabilities.maxInlineBytes / 3); + +/** Exact decoded length from the encoded string's shape only (no decode); 0 when out of bounds. */ +function bound(encoded: unknown): number { + if (typeof encoded !== "string" || !encoded.length || encoded.length % 4 !== 0 || encoded.length > MAX_ENCODED) return 0; + return (encoded.length / 4) * 3 - (encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0); +} + +/** Strict decode after admission. The ticket's budget is a hard bound; padding and alphabet are strict. */ +function decode(encoded: unknown, maxBytes: number): Uint8Array | undefined { + // Bound allocation BEFORE decoding. + if (!bound(encoded) || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded as string)) return undefined; + const bytes = Buffer.from(encoded as string, "base64"); + return bytes.length && bytes.length <= maxBytes && bytes.length <= anthropicAttachmentCapabilities.maxInlineBytes && + bytes.toString("base64") === encoded ? bytes : undefined; } /** Observe only Messages' typed image/document blocks and one level of tool_result content. - * The manager owns policy, upload, lifecycle and references. A native integration + * Every part is preflighted (deny-wins, budget reserved) before its base64 is read or decoded; an admitted + * ticket that is not used is released at once. The manager owns policy, upload, lifecycle and references. A native integration * must supply its actual emitted span/correlation, never an ambient guessed span. * Returns the admitted positions so the caller can project them out of span telemetry. */ @@ -88,13 +97,24 @@ export function captureAnthropicAttachments(manager: AttachmentManager | undefin const previous = spans!.get(spanKey)?.get(positionKey); if (previous) { captured.set(positionKey, previous); return; } const role: AttachmentOptions["role"] = toolCallId ? "tool" : message.role === "assistant" ? "assistant" : "user"; - const handle = manager.attach(correlation, { kind: "Bytes", get data() { return decode(source.data); } }, - { mimeType: mime, direction: "input", messageIndex, partIndex: index, role, - purpose: part.type === "document" ? "document" : toolCallId ? computerCalls.has(toolCallId) ? "computer_screenshot" : "tool_result" : "model_input", - ...(toolCallId ? { toolCallId } : {}), - capture: { ...options.capture, modelInput: true, errorDetail: errorDetail || options.capture?.errorDetail, - ...(hidden || (part.type === "image" && config.hideInputImages) || (errorDetail && config.hideErrorDetails) ? { captureContent: false } : {}) }, - }); + const request = { mimeType: mime, direction: "input" as const, + purpose: (part.type === "document" ? "document" : toolCallId ? computerCalls.has(toolCallId) ? "computer_screenshot" : "tool_result" : "model_input") as AttachmentOptions["purpose"], + capture: { ...options.capture, modelInput: true, errorDetail: errorDetail || options.capture?.errorDetail, + ...(hidden || (part.type === "image" && config.hideInputImages) || (errorDetail && config.hideErrorDetails) ? { captureContent: false } : {}) } }; + // Preflight (deny-wins, budget reserved) before the base64 source is read or decoded (design section 1). + // The source is read at most once. + let data: unknown, dataRead = false; + const read = () => { if (!dataRead) { dataRead = true; data = source.data; } return data; }; + let ticket = manager.preflight(correlation, { ...request, maxBytes: anthropicAttachmentCapabilities.maxInlineBytes }); + // too_large is evaluated after every privacy, correlation and sampling step, so sizing the source exactly is safe here. + if (ticket.reason === "too_large") { + const exact = bound(read()); + if (exact > 0) ticket = manager.preflight(correlation, { ...request, maxBytes: exact }); + } + if (!ticket.admitted) return; + const bytes = decode(read(), ticket.maxBytes); + if (!bytes) { ticket.release(); return; } + const handle = ticket.attach({ kind: "Bytes", data: bytes }, { messageIndex, partIndex: index, role, ...(toolCallId ? { toolCallId } : {}) }); if (handle.admission === "queued") { const occurrence = Object.freeze({ occurrenceId: handle.occurrenceId, mimeType: mime }); let positions = spans!.get(spanKey); diff --git a/typescript/packages/traceai_openai/src/attachments.ts b/typescript/packages/traceai_openai/src/attachments.ts index 319628c03..cf0cac04c 100644 --- a/typescript/packages/traceai_openai/src/attachments.ts +++ b/typescript/packages/traceai_openai/src/attachments.ts @@ -4,10 +4,11 @@ import type { AttachmentManager, AttachmentOptions, AttachmentSpan, AttachmentCo /** Declarative handoff to auto/wire owners; this is not a native/wire hookup. */ export const openAIAttachmentCapabilities = Object.freeze({ schemaVersion: 1, - operations: ["chat.completions.create.input", "responses.create.input", "responses.create.output", "images.generate.output"] as const, + operations: ["chat.completions.create.input", "responses.create.input", "responses.create.output", "responses.stream.output", "images.generate.output"] as const, sources: ["inline_base64"] as const, - /** Fail closed: no capture is attempted for these. */ - unsupported: ["remote_url", "provider_file", "streamed_output", "partial_image", "images.edit", "images.createVariation", "speech"] as const, + /** Fail closed: no capture is attempted for these. Streamed Responses output is captured only from the + * terminal `response.completed` event the application consumed; partial (preview) images never are. */ + unsupported: ["remote_url", "provider_file", "partial_image", "images.generate.stream", "images.edit", "images.createVariation", "speech"] as const, maxInlineBytes: 256 * 1024, }); @@ -50,18 +51,35 @@ function restrictions(options: OpenAIAttachmentCaptureOptions): AttachmentCaptur }; } -function decode(value: unknown, mimeType: string, raw: boolean): Uint8Array { - if (typeof value !== "string") return new Uint8Array(); - const prefix = `data:${mimeType};base64,`; - if (!raw && !value.startsWith(prefix)) return new Uint8Array(); - // Bound allocation BEFORE slicing/decoding. Padding and alphabet are strict. - const start = raw ? 0 : prefix.length; +type Parsed = { mimeType: string; encoded: string; bound: number }; +const MAX_ENCODED = 4 * Math.ceil(openAIAttachmentCapabilities.maxInlineBytes / 3); + +/** Structural parse only: no decode or allocation beyond the string the application already holds. + * Returns the candidate MIME named by a data URI (or the typed MIME for raw base64) and the exact decoded length. + */ +function parse(value: unknown, mimeTypes: string[], raw: boolean): Parsed | undefined { + if (typeof value !== "string") return undefined; + let mimeType = mimeTypes[0], start = 0; + if (!raw) { + const comma = value.indexOf(","); + const header = comma > 5 && comma < 128 ? value.slice(0, comma) : ""; + const match = /^data:([a-z]+\/[a-z0-9.+-]+);base64$/.exec(header); + if (!match || !mimeTypes.includes(match[1])) return undefined; + mimeType = match[1]; start = comma + 1; + } const length = value.length - start; - if (length <= 0 || length > 4 * Math.ceil(openAIAttachmentCapabilities.maxInlineBytes / 3) || length % 4 !== 0) return new Uint8Array(); + // Bound allocation BEFORE slicing/decoding. + if (length <= 0 || length > MAX_ENCODED || length % 4 !== 0) return undefined; const encoded = value.slice(start); - if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); + const padding = encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0; + return { mimeType, encoded, bound: (length / 4) * 3 - padding }; +} + +/** Strict decode after admission. Padding and alphabet are strict; the ticket's budget is a hard bound. */ +function decode(encoded: string, maxBytes: number): Uint8Array | undefined { + if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return undefined; const bytes = Buffer.from(encoded, "base64"); - if (bytes.length > openAIAttachmentCapabilities.maxInlineBytes || bytes.toString("base64") !== encoded) return new Uint8Array(); + if (!bytes.length || bytes.length > maxBytes || bytes.length > openAIAttachmentCapabilities.maxInlineBytes || bytes.toString("base64") !== encoded) return undefined; return bytes; } @@ -71,7 +89,8 @@ const key = (options: Pick= MAX_SPANS) spans!.delete(spans!.keys().next().value!); - positions = new Map(); spans!.set(spanKey, positions); - } - if (positions.size < MAX_PARTS) positions.set(position, occurrence); - captured.set(position, occurrence); - return; + // The source is read at most once, and only after a preflight admitted this modality (design section 1). + let source: Parsed | undefined, sourceRead = false; + const parsed = () => { if (!sourceRead) { sourceRead = true; source = parse(read(), mimeTypes, raw); } return source; }; + const preflight = (mimeType: string) => { + const request = { mimeType, direction: options.direction, purpose, capture: { + ...capture, + ...(options.direction === "input" && mimeType.startsWith("image/") && options.traceConfig?.hideInputImages ? { captureContent: false } : {}), + } }; + let ticket = manager.preflight(correlation, { ...request, maxBytes: openAIAttachmentCapabilities.maxInlineBytes }); + // too_large is evaluated after every privacy, correlation and sampling step, so sizing the source exactly is safe here. + if (ticket.reason === "too_large") { + const bound = parsed()?.bound ?? 0; + if (bound > 0) ticket = manager.preflight(correlation, { ...request, maxBytes: bound }); } - // Only a non-matching data-URI type (decoded as empty) tries the next candidate. Denials stop. - if (handle.admission !== "too_large") return; + return ticket; + }; + let ticket = preflight(mimeTypes[0]); + if (!ticket.admitted) return; + const found = parsed(); + if (!found) { ticket.release(); return; } + // A data URI names a sibling MIME of the same modality: re-admit under the exact type before decoding. + if (found.mimeType !== mimeTypes[0]) { + ticket.release(); + ticket = preflight(found.mimeType); + if (!ticket.admitted) return; + } + const bytes = decode(found.encoded, ticket.maxBytes); + if (!bytes) { ticket.release(); return; } + const handle = ticket.attach({ kind: "Bytes", data: bytes }, { messageIndex, partIndex, role, ...(toolCallId ? { toolCallId } : {}) }); + if (handle.admission !== "queued") return; + const occurrence = Object.freeze({ occurrenceId: handle.occurrenceId, mimeType: found.mimeType }); + let positions = spans!.get(spanKey); + if (!positions) { + if (spans!.size >= MAX_SPANS) spans!.delete(spans!.keys().next().value!); + positions = new Map(); spans!.set(spanKey, positions); } + if (positions.size < MAX_PARTS) positions.set(position, occurrence); + captured.set(position, occurrence); }; const toolCallId = (value: unknown) => typeof value === "string" && /^[a-zA-Z0-9_-]{1,128}$/.test(value) ? value : undefined; const parts = (content: unknown, messageIndex: number, role: AttachmentOptions["role"]) => { diff --git a/typescript/packages/traceai_openai/src/instrumentation.ts b/typescript/packages/traceai_openai/src/instrumentation.ts index c457aaf20..d2d05e7a4 100644 --- a/typescript/packages/traceai_openai/src/instrumentation.ts +++ b/typescript/packages/traceai_openai/src/instrumentation.ts @@ -585,11 +585,11 @@ export class OpenAIInstrumentation extends InstrumentationBase{ if (endSpan) span.end(); return; } - // endSpan is false only on the streamed path. Streamed output media is unsupported - // until the core has a consumed-stream observation API. - const outputMedia = endSpan ? captureOpenAIAttachments(instrumentation.attachments, span, result, { + // On the streamed path (endSpan false) `result` is the terminal response the application consumed + // from its own iterator, observed before the span ends. Partial (preview) images are never captured. + const outputMedia = captureOpenAIAttachments(instrumentation.attachments, span, result, { operation: "responses", direction: "output", traceConfig: instrumentation._traceConfig, - }) : new Map(); + }); const projected = projectOpenAIMedia(result, outputMedia, "responses", "output") as ResponseType; const output = { id: result.id, object: result.object, status: result.status, diff --git a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs index c9a9be53a..368cf49fd 100644 --- a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs +++ b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs @@ -19,11 +19,19 @@ const document = () => ({ type: 'document', title: 'synthetic-private-filename', const identity = () => ({ traceId: randomUUID().replace(/-/g, ''), spanId: randomUUID().replace(/-/g, '').slice(0, 16), traceFlags: 1 }); const keepalive = setInterval(() => {}, 1000); after(() => clearInterval(keepalive)); +// The core reports a silently degraded fixture (legacy transport, rejected F01 shape) as a process warning. +const coreWarnings = []; +process.on('warning', warning => { if (/^FI_ATTACHMENTS_/.test(warning.code ?? '')) coreWarnings.push(warning.code); }); +// Preflight tickets that are not used must be released at once, never left to the TTL sweep. +const assertNoOpenTickets = async manager => { + assert.equal(manager.status().openTickets, 0, 'open tickets'); await manager.flush(); assert.equal(manager.status().ownedBytes, 0, 'owned bytes'); +}; -async function fixture(config = {}, serverPolicy = {}) { +async function fixture(config = {}, serverPolicy = {}, serverLimits = {}) { const calls = [], initialized = [], uploaded = []; const scope = { organization_id: randomUUID(), workspace_id: randomUUID(), project_id: randomUUID() }; - const expiry = () => new Date(Date.now() + 60000).toISOString(); + // Sessions of one hour or more: the core renews within max(60 s, 20% of lifetime) of expiry. + const expiry = () => new Date(Date.now() + 3600000).toISOString(); const server = http.createServer(async (req, res) => { const chunks = []; for await (const c of req) chunks.push(c); const bytes = Buffer.concat(chunks); calls.push(req.url); @@ -37,7 +45,7 @@ async function fixture(config = {}, serverPolicy = {}) { limits: { max_upload_bytes: 1024 * 1024, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, - transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }, expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); + transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5, ...serverLimits }, expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); } else if (req.url.endsWith('/uploads')) { const body = JSON.parse(bytes); initialized.push(body); res.statusCode = 201; @@ -121,6 +129,23 @@ for (const provider of ['openai', 'anthropic']) { capture(f.manager, identity(), provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_file', file_id: 'file_secret' }] }] } : { messages: [{ role: 'user', content: [{ type: 'document', source: { type: 'file', file_id: 'file_secret' } }, null, { type: 'tool_result', tool_use_id: {}, content: [image()] }] }] }); await f.manager.flush(); assert.equal(f.initialized.length, 0); + await assertNoOpenTickets(f.manager); + } finally { await f.close(); } + }); + test(`${provider}: a negotiated upload limit below the inline cap still captures small media exactly`, async () => { + const f = await fixture({}, {}, { max_upload_bytes: 100 * 1024 }); + try { + const id = identity(), captured = capture(f.manager, id, input()); + await f.manager.flush(); + assert.equal(captured.size, provider === 'openai' ? 3 : 4); + assert.equal(f.uploaded.length, captured.size); + assert.ok(f.initialized.every(r => r.declared_bytes <= 100 * 1024)); + // Above the negotiated limit nothing is decoded or uploaded, and the reservation is returned. + const big = Buffer.alloc(150 * 1024, 7).toString('base64'); + capture(f.manager, identity(), provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_image', image_url: `data:image/png;base64,${big}` }] }] } + : { messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: big } }] }] }); + await f.manager.flush(); assert.equal(f.uploaded.length, captured.size); + await assertNoOpenTickets(f.manager); } finally { await f.close(); } }); test(`${provider}: unsampled identity and queued policy tightening cannot upload`, async () => { @@ -202,7 +227,10 @@ test('Hostile payloads fail capture without throwing into the application', asyn }); test('Capability descriptors name every unsupported source explicitly', () => { - for (const name of ['remote_url', 'provider_file', 'streamed_output', 'partial_image', 'images.edit', 'speech']) assert.ok(oa.openAIAttachmentCapabilities.unsupported.includes(name)); + for (const name of ['remote_url', 'provider_file', 'partial_image', 'images.generate.stream', 'images.edit', 'speech']) assert.ok(oa.openAIAttachmentCapabilities.unsupported.includes(name)); + // Streamed Responses output is captured from the consumed terminal event; partial images never are. + assert.ok(oa.openAIAttachmentCapabilities.operations.includes('responses.stream.output')); + assert.ok(!oa.openAIAttachmentCapabilities.unsupported.includes('streamed_output')); for (const name of ['remote_url', 'provider_file', 'native_span_hookup', 'generated_image', 'streamed_output']) assert.ok(an.anthropicAttachmentCapabilities.unsupported.includes(name)); assert.ok(Object.isFrozen(oa.openAIAttachmentCapabilities) && Object.isFrozen(an.anthropicAttachmentCapabilities)); }); @@ -337,10 +365,22 @@ test('Real SDKs preserve APIPromise/request/response behavior and upload only th for (const ref of refs.filter(r => r.direction === 'input')) assert.ok(text.includes(`fi-attachment:${ref.occurrence_id}`), span.name); } if (client.responses) { - // Streamed output media is unsupported: the stream is delivered unchanged and nothing uploads. + // Streamed Responses: the stream is delivered unchanged; the generated image in the terminal event the + // application consumed uploads once and is projected out of that span's output telemetry. const stream = await client.responses.create({ model: 'synthetic-model', input: 'synthetic', stream: true }); - const events = []; for await (const event of stream) events.push(event.type); + const events = []; for await (const event of stream) { events.push(event.type); if (event.type === 'response.completed') assert.equal(event.response.output[0].result, PNG); } assert.deepEqual(events, ['response.created', 'response.completed']); + expected++; + await f.manager.flush(); await provider.forceFlush(); assert.equal(f.uploaded.length, expected); + assert.equal(f.uploaded.at(-1).toString('base64'), PNG); + const streamed = exporter.getFinishedSpans().filter(s => s.name === 'OpenAI Responses').at(-1); + const refs = JSON.parse(streamed.attributes['fi.attachment.refs'] ?? '[]'); + assert.equal(refs.length, 1); assert.equal(refs[0].direction, 'output'); assert.equal(refs[0].purpose, 'model_output'); + assert.ok(String(streamed.attributes['output.value']).includes(`fi-attachment:${refs[0].occurrence_id}`)); + assert.ok(!JSON.stringify(streamed.attributes).includes(PNG)); + // A consumer that stops early never reaches the terminal event: nothing is captured. + const early = await client.responses.create({ model: 'synthetic-model', input: 'synthetic', stream: true }); + for await (const event of early) { if (event.type === 'response.created') break; } await f.manager.flush(); assert.equal(f.uploaded.length, expected); } // A raw-response caller owns the body; instrumentation must not parse for media. @@ -390,3 +430,22 @@ test('A configured but disabled manager leaves existing span telemetry byte-iden assert.equal(disabled.calls.length, 0); } finally { await disabled.close(); await p.close(); } }); + +test('OpenAI data URIs of a sibling image type are admitted under their exact MIME', async () => { + const f = await fixture(); try { + const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 1, 2, 3, 0xff, 0xd9]).toString('base64'); + const captured = oa.captureOpenAIAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [ + { type: 'image_url', image_url: { url: `data:image/jpeg;base64,${jpeg}` } }, + { type: 'image_url', image_url: { url: `data:image/gif;base64,${jpeg}` } }, + ] }] }, { operation: 'chat', direction: 'input' }); + await f.manager.flush(); + assert.deepEqual([...captured.values()].map(v => v.mimeType), ['image/jpeg']); + assert.deepEqual(f.initialized.map(r => r.mime_type), ['image/jpeg']); + assert.equal(f.uploaded[0].toString('base64'), jpeg); + await assertNoOpenTickets(f.manager); + } finally { await f.close(); } +}); + +test('No core degradation warning was raised by any fixture', () => { + assert.deepEqual(coreWarnings, []); +}); From 88a4c233baa27a56e87fb359398bd0baec63f9e5 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 03:07:34 -0800 Subject: [PATCH 07/20] fix(openai,anthropic): retry a queue_full preflight at the part's exact size A cap-sized (256 KiB) preflight can return queue_full when the owned-byte budget is nearly used, where the previous attach() reserved only the actual length and admitted small media. queue_full, like too_large, is evaluated after every privacy, correlation and sampling step, so the part is sized exactly and preflighted once more. New test row for both providers. Co-Authored-By: Claude Opus 5.5 --- .../packages/traceai_anthropic/src/attachments.ts | 5 +++-- .../packages/traceai_openai/src/attachments.ts | 5 +++-- .../test/attachments-providers.test.cjs | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/typescript/packages/traceai_anthropic/src/attachments.ts b/typescript/packages/traceai_anthropic/src/attachments.ts index ce45cd139..cb72aee72 100644 --- a/typescript/packages/traceai_anthropic/src/attachments.ts +++ b/typescript/packages/traceai_anthropic/src/attachments.ts @@ -106,8 +106,9 @@ export function captureAnthropicAttachments(manager: AttachmentManager | undefin let data: unknown, dataRead = false; const read = () => { if (!dataRead) { dataRead = true; data = source.data; } return data; }; let ticket = manager.preflight(correlation, { ...request, maxBytes: anthropicAttachmentCapabilities.maxInlineBytes }); - // too_large is evaluated after every privacy, correlation and sampling step, so sizing the source exactly is safe here. - if (ticket.reason === "too_large") { + // too_large (negotiated limit below the cap) and queue_full (owned budget nearly used) are evaluated after every + // privacy, correlation and sampling step, so sizing the source exactly and retrying once is safe here. + if (ticket.reason === "too_large" || ticket.reason === "queue_full") { const exact = bound(read()); if (exact > 0) ticket = manager.preflight(correlation, { ...request, maxBytes: exact }); } diff --git a/typescript/packages/traceai_openai/src/attachments.ts b/typescript/packages/traceai_openai/src/attachments.ts index cf0cac04c..b185f6294 100644 --- a/typescript/packages/traceai_openai/src/attachments.ts +++ b/typescript/packages/traceai_openai/src/attachments.ts @@ -121,8 +121,9 @@ export function captureOpenAIAttachments(manager: AttachmentManager | undefined, ...(options.direction === "input" && mimeType.startsWith("image/") && options.traceConfig?.hideInputImages ? { captureContent: false } : {}), } }; let ticket = manager.preflight(correlation, { ...request, maxBytes: openAIAttachmentCapabilities.maxInlineBytes }); - // too_large is evaluated after every privacy, correlation and sampling step, so sizing the source exactly is safe here. - if (ticket.reason === "too_large") { + // too_large (negotiated limit below the cap) and queue_full (owned budget nearly used) are evaluated after every + // privacy, correlation and sampling step, so sizing the source exactly and retrying once is safe here. + if (ticket.reason === "too_large" || ticket.reason === "queue_full") { const bound = parsed()?.bound ?? 0; if (bound > 0) ticket = manager.preflight(correlation, { ...request, maxBytes: bound }); } diff --git a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs index 368cf49fd..fbb05607c 100644 --- a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs +++ b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs @@ -148,6 +148,20 @@ for (const provider of ['openai', 'anthropic']) { await assertNoOpenTickets(f.manager); } finally { await f.close(); } }); + test(`${provider}: a nearly used owned-byte budget still admits small media at its exact size`, async () => { + const f = await fixture({ maxOwnedBytes: 300 * 1024 }); + try { + const big = Buffer.alloc(200 * 1024, 7).toString('base64'); + const one = data => provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_image', image_url: `data:image/png;base64,${data}` }] }] } + : { messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data } }] }] }; + // Same tick: the 200 KiB part still owns its bytes, so a cap-sized reservation would not fit. + const first = capture(f.manager, identity(), one(big)), second = capture(f.manager, identity(), one(PNG)); + assert.equal(first.size, 1); assert.equal(second.size, 1); + await f.manager.flush(); + assert.deepEqual(f.uploaded.map(b => b.length).sort((a, b) => a - b), [Buffer.from(PNG, 'base64').length, 200 * 1024]); + await assertNoOpenTickets(f.manager); + } finally { await f.close(); } + }); test(`${provider}: unsampled identity and queued policy tightening cannot upload`, async () => { const f = await fixture(); try { capture(f.manager, { ...identity(), traceFlags: 0 }, input()); assert.equal(f.manager.status().ownedBytes, 0); From 944f2f1a29accb3c95024da0dfcc3f19a3c4034a Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 04:41:10 -0800 Subject: [PATCH 08/20] feat(auto): restore attachment lane work saved before the drive loss Recovered from refs/recovery/att-b-auto-1111 (base 7145f783), applied on feat/attachments-ts-core f0a56b37. Not yet migrated to the preflight API. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/auto-typescript-tests.yml | 2 +- .../packages/traceai_auto/src/attachments.ts | 41 ++++ .../packages/traceai_auto/src/config.ts | 7 + typescript/packages/traceai_auto/src/index.ts | 33 ++- .../packages/traceai_auto/src/libraries.ts | 8 +- .../traceai_auto/src/wire/attachments.ts | 127 ++++++++++ .../traceai_auto/src/wire/fetch-runtime.ts | 29 ++- .../packages/traceai_auto/src/wire/fetch.ts | 9 +- .../test/attachment-lifecycle.test.ts | 44 ++++ .../traceai_auto/test/attachments.test.ts | 223 ++++++++++++++++++ 10 files changed, 502 insertions(+), 21 deletions(-) create mode 100644 typescript/packages/traceai_auto/src/attachments.ts create mode 100644 typescript/packages/traceai_auto/src/wire/attachments.ts create mode 100644 typescript/packages/traceai_auto/test/attachment-lifecycle.test.ts create mode 100644 typescript/packages/traceai_auto/test/attachments.test.ts diff --git a/.github/workflows/auto-typescript-tests.yml b/.github/workflows/auto-typescript-tests.yml index ed29b9b42..bc932fc24 100644 --- a/.github/workflows/auto-typescript-tests.yml +++ b/.github/workflows/auto-typescript-tests.yml @@ -4,7 +4,7 @@ name: Zero-code agent (@traceai/auto) tests # integration tests load the built instrumentors, so the whole workspace is built. on: pull_request: - branches: [dev, main] + branches: [dev, main, feat/attachments-ts-core] paths: - 'typescript/packages/traceai_auto/**' - 'typescript/packages/fi-core/**' diff --git a/typescript/packages/traceai_auto/src/attachments.ts b/typescript/packages/traceai_auto/src/attachments.ts new file mode 100644 index 000000000..af19bed06 --- /dev/null +++ b/typescript/packages/traceai_auto/src/attachments.ts @@ -0,0 +1,41 @@ +import { createNodeAttachmentManager, type NodeAttachmentManagerOptions } from "@traceai/fi-core/attachments/node"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import type { Config, Options } from "./config"; + +export function attachmentOptions(config: Config, options: Options): NodeAttachmentManagerOptions { + const explicit = options.attachments ?? {}; + const restrictive = Object.fromEntries(Object.entries(explicit.traceConfig ?? {}).filter(([key, value]) => + (key.startsWith("hide") || key === "piiRedaction") && value === true)); + return { ...explicit, environment: () => process.env, projectName: config.projectName, projectType: config.projectType, + // Only resolved FI credentials, never model/provider credentials. A neutral + // exporter does not imply an attachment destination or credential authority. + headers: explicit.headers ?? (config.apiKey && config.secretKey ? { "x-api-key": config.apiKey, "x-secret-key": config.secretKey } : {}), + traceConfig: { ...explicit.traceConfig, ...config.traceConfig, ...restrictive } }; +} +interface AttachmentProvider { attachments?: AttachmentManager; getDelegate?(): AttachmentProvider } +export function selectAttachmentManager(provider: AttachmentProvider, config: Config, options: Options, ownsProvider: boolean): { manager: AttachmentManager; owned: boolean } { + const existing = options.attachmentManager ?? provider?.attachments ?? provider?.getDelegate?.()?.attachments; + if (existing) { + // Borrowing must not loosen application policy or turn capture on implicitly. + const effective = attachmentOptions(config, options); + const restrictions = Object.fromEntries(Object.entries(effective.traceConfig ?? {}).filter(([key, value]) => + (key.startsWith("hide") || key === "piiRedaction") && value === true)); + const environmentDenies = process.env.FI_ATTACHMENTS_ENABLED !== undefined && !/^(true|1|yes)$/i.test(process.env.FI_ATTACHMENTS_ENABLED.trim()); + existing.updatePolicy({ ...(options.attachments?.enabled === false || environmentDenies ? { enabled: false } : {}), traceConfig: restrictions }); + return { manager: existing, owned: ownsProvider && !options.attachmentManager }; + } + return { manager: createNodeAttachmentManager(attachmentOptions(config, options)), owned: true }; +} +export interface AttachmentReport { + state: ReturnType["state"] | "not_initialized"; + transport: "shared_manager"; integration: "component_only"; sources: string[]; unsupported: string[]; +} +export function attachmentReport(manager?: AttachmentManager): AttachmentReport { + return { + state: manager?.status().state ?? "not_initialized", + transport: "shared_manager", + integration: "component_only", + sources: ["explicit-policy native-span FormData Blob", "consumed response arrayBuffer/blob", "consumed PNG JSON output <=256KiB"], + unsupported: ["unqualified native span policy", "JSON inputs", "wire-only FormData", "incremental/encoded audio", "RemoteUrl/ProviderFile", "WebSocket/RTC", "new recordings", "product playback qualification"], + }; +} diff --git a/typescript/packages/traceai_auto/src/config.ts b/typescript/packages/traceai_auto/src/config.ts index cd216b331..59e475358 100644 --- a/typescript/packages/traceai_auto/src/config.ts +++ b/typescript/packages/traceai_auto/src/config.ts @@ -6,7 +6,14 @@ import type { TracerProvider } from "@opentelemetry/api"; import type { PrivacyOptions as TraceConfigOptions } from "./export-privacy"; import { compileOutgoingPolicy, type OutgoingPolicy, type OutgoingPropagationMode } from "./outgoing-policy"; +import type { NodeAttachmentManagerOptions } from "@traceai/fi-core/attachments/node"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; + export interface Options { + /** Default OFF. The control endpoint is independent of baseUrl/OTLP. */ + attachments?: NodeAttachmentManagerOptions; + /** Borrow the application's manager; auto never shuts down a borrowed manager. */ + attachmentManager?: AttachmentManager; /** Experimental node:http/node:https propagation; off until explicitly selected. */ propagateOutgoingAdapters?: boolean; /** default/all retain mandatory vendor-host, provider-port and redirect vetoes. */ diff --git a/typescript/packages/traceai_auto/src/index.ts b/typescript/packages/traceai_auto/src/index.ts index 7eea28f79..949ccdf05 100644 --- a/typescript/packages/traceai_auto/src/index.ts +++ b/typescript/packages/traceai_auto/src/index.ts @@ -10,6 +10,10 @@ import { installOutgoingPropagation } from "./outgoing-node"; import { enableLibraries, IntegrationStatus, inventory } from "./libraries"; import { FetchHook, patchFetch, trackCoverage } from "./wire/fetch"; import { captureSecrets, SpanResult } from "./wire/core"; +import type { Span } from "@opentelemetry/api"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import { attachmentOptions, attachmentReport, selectAttachmentManager } from "./attachments"; +export { withAttachmentOwnership, withAttachmentCapturePolicy } from "./wire/attachments"; import { withDeadline } from "./deadline"; import { privacyFilter } from "./export-privacy"; @@ -24,6 +28,7 @@ export type { Options } from "./config"; interface AgentState { config: Config; statuses: IntegrationStatus[]; active: boolean; provider?: any; ownsProvider?: boolean; spanDelivery?: AutoSpanDelivery; + attachments?: AttachmentManager; ownsAttachments?: boolean; ownsSpanExporter?: boolean; processor?: SpanProcessor; processorIsObserver?: boolean; convertedSpanDestination?: ConvertedSpanDestination; detach?: () => void; removeLibraries?: () => void; fetch?: FetchHook; @@ -90,8 +95,10 @@ function removeHooks(agent: AgentState): void { () => agent.removeOutgoing?.(), () => agent.fetch?.remove(), () => agent.removeObservation?.(), () => agent.removeLibraries?.(), ]) { try { remove(); } catch { /* A broken cleanup must not skip the others. */ } } } -export interface Report { active: boolean; config: Record; integrations: IntegrationStatus[]; collector: string } -/** Initialize once. No network calls or thrown startup errors. */ +export interface Report { active: boolean; config: Record; integrations: IntegrationStatus[]; collector: string; attachments: ReturnType } +/** The same manager passed to adapters; await ready() explicitly at startup if needed. */ +export function getAttachmentManager(): AttachmentManager | undefined { return sharedState()?.attachments; } +/** Initialize once. No awaited network calls or thrown startup errors; opted-in attachments bootstrap asynchronously. */ export function init(options: Options = {}): TracerProvider | undefined { const existing = sharedState(); if (existing) { warn("initialized", "Already initialized; keeping the existing provider and configuration."); return existing.active ? existing.provider : undefined; } @@ -142,10 +149,13 @@ export function init(options: Options = {}): TracerProvider | undefined { if (!agent.provider) { agent.provider = register({ projectName: config.projectName, projectType: config.projectType === "observe" ? ProjectType.OBSERVE : ProjectType.EXPERIMENT, setGlobalTracerProvider: false, verbose: false, endpoint: config.baseUrl, batch: true, spanProcessors: [processor], + traceConfig: config.traceConfig, attachments: options.attachmentManager ? { enabled: false } : attachmentOptions(config, options), sampler: new ParentBasedSampler({ root: new TraceIdRatioBasedSampler(config.sampleRate) }) }); agent.ownsProvider = true; if (!existing) agent.ownsGlobalProvider = trace.setGlobalTracerProvider(agent.provider); } + const attachmentRuntime = selectAttachmentManager(agent.provider, config, options, agent.ownsProvider === true); + agent.attachments = attachmentRuntime.manager; agent.ownsAttachments = attachmentRuntime.owned; if (agent.processorIsObserver) { agent.convertedSpanDestination = borrowConvertedSpanDestination(agent.provider, processor); const destination = agent.convertedSpanDestination; @@ -180,7 +190,7 @@ export function init(options: Options = {}): TracerProvider | undefined { const admissionPrivacy = privacyFilter(options.traceConfig, [config.apiKey, config.secretKey].filter((v): v is string => !!v)); const wirePipeline = (tracer as any)._spanProcessor; const wireDestinationReady = () => hasAutoSpanSink(wirePipeline, agent.processorIsObserver ? [processor] : []); - const emit = (result: SpanResult, parent: Context, start: number, end: number) => { + const emit = (result: SpanResult, parent: Context, start: number, end: number, bindMedia?: (span: Span) => void) => { const canDeliver = () => agent.spanDelivery?.canForward(wireDestinationReady) === true; // Credential discovery is independent of which semantic copy wins delivery. let secrets: ReturnType; @@ -210,6 +220,7 @@ export function init(options: Options = {}): TracerProvider | undefined { if (secrets === null) failedCaptures.add(span.getContextSpan()); span.setStatus({ code: result.ok ? SpanStatusCode.OK : SpanStatusCode.ERROR, message: result.description }); const recording = span.isRecording(); + try { bindMedia?.(span); } catch { /* Media never affects semantic delivery. */ } span.end(end); submitted = recording; } catch { /* Internal failures are not application failures. */ } @@ -219,14 +230,14 @@ export function init(options: Options = {}): TracerProvider | undefined { // Adapters stay off unless explicitly selected; destination policy is not activation. agent.removeOutgoing = installOutgoingPropagation(outgoingPolicyForConfig(config), createOutgoingPrivacy(config.traceConfig, credentialsForConfig(config)), agent.statuses, warn, config.propagateOutgoingAdapters); if (config.wireCapture && enabled("wire", config)) agent.removeObservation = observeTransports(); - agent.removeLibraries = enableLibraries(config, options, instrumentationProvider, agent.statuses, warn, processor); + agent.removeLibraries = enableLibraries(config, { ...options, attachmentManager: agent.attachments }, instrumentationProvider, agent.statuses, warn, processor); // Stay outside the installed mapping adapters: Genkit can shorten an SDK- // truncated value when it scrubs a local path or identity. Inspect the native // span first so that its detached export copy cannot erase that loss signal. const mappedEnd = processor.onEnd.bind(processor); processor.onEnd = span => { rememberSpanValueLimit(span); mappedEnd(span); }; if (config.wireCapture && enabled("wire", config) && typeof globalThis.fetch === "function") { - agent.fetch = patchFetch(config.wireMaxBodyBytes, emit, [config.apiKey, config.secretKey].filter((v): v is string => !!v)); agent.statuses.push({ name: "wire", status: "instrumented" }); + agent.fetch = patchFetch(config.wireMaxBodyBytes, emit, [config.apiKey, config.secretKey].filter((v): v is string => !!v), agent.attachments); agent.statuses.push({ name: "wire", status: "instrumented" }); } else agent.statuses.push({ name: "wire", status: "disabled" }); agent.active = true; agent.beforeExit = () => { void forceFlush(); }; @@ -244,7 +255,7 @@ export function init(options: Options = {}): TracerProvider | undefined { try { if (failed.propagator && (propagation as any)._getGlobalPropagator?.() === failed.propagator) propagation.disable(); } catch { /* Leave application globals alone. */ } try { if (failed.ownsGlobalProvider && existingProvider() === failed.provider) trace.disable(); } catch { /* Leave app globals alone. */ } try { if (failed.contextManager && (context as any)._getContextManager?.() === failed.contextManager) context.disable(); } catch { /* Fail open. */ } - void withDeadline(async () => { await failed.processor?.shutdown(); }).catch(() => {}); + void withDeadline(async () => { await Promise.all([failed.processor?.shutdown(), failed.ownsAttachments ? failed.attachments?.shutdown() : undefined]); }).catch(() => {}); // Configuration is terminal for this process, even after failed startup. // Hooks are rolled back, but init never installs a second provider or loader. } @@ -255,16 +266,16 @@ export function report(): Report { sharedState(); if (!state) { const config = resolveConfig(); - return { active: false, config: maskConfig(config), integrations: inventory(config), collector: "not_tested (agent not initialized)" }; + return { active: false, config: maskConfig(config), integrations: inventory(config), collector: "not_tested (agent not initialized)", attachments: attachmentReport() }; } - return { active: state.active, config: maskConfig(state.config), integrations: state.statuses.map(s => ({ ...s })), collector: state.collector }; + return { active: state.active, config: maskConfig(state.config), integrations: state.statuses.map(s => ({ ...s })), collector: state.collector, attachments: attachmentReport(state.attachments) }; } export async function forceFlush(): Promise { sharedState(); try { await withDeadline(async () => { await Promise.all([...(state?.fetch?.pending ?? []), state?.spanDelivery?.drain()]); - await state?.processor?.forceFlush(); + await Promise.all([state?.processor?.forceFlush(), state?.attachments?.flush(2500)]); }); } catch { if (state?.collector !== "failed") warn("flush", "Trace flush failed; the application will continue."); } } @@ -284,7 +295,7 @@ export async function shutdown(): Promise { await withDeadline(async () => { await Promise.all([...(agent.fetch?.pending ?? []), agent.spanDelivery?.drain()]); // Never start a late flush after the outer deadline race has returned. - if (Date.now() < drainDeadline) await agent.processor?.forceFlush(); + if (Date.now() < drainDeadline) await Promise.all([agent.processor?.forceFlush(), agent.attachments?.flush(remaining(drainDeadline))]); }, remaining(drainDeadline)); } catch { warn("shutdown", "Trace shutdown failed; the application will continue."); } finally { @@ -294,7 +305,7 @@ export async function shutdown(): Promise { agent.spanDelivery?.revoke(); agent.convertedSpanDestination?.revoke(); try { agent.detach?.(); } catch { /* Leave the application's processors intact. */ } - try { await withDeadline(async () => { await agent.processor?.shutdown(); }, Math.min(2500, remaining(deadline))); } + try { await withDeadline(async () => { await Promise.all([agent.processor?.shutdown(), agent.ownsAttachments ? agent.attachments?.shutdown(Math.max(1, remaining(deadline))) : undefined]); }, Math.min(2500, remaining(deadline))); } catch { warn("shutdown", "Trace shutdown failed; the application will continue."); } } })(); diff --git a/typescript/packages/traceai_auto/src/libraries.ts b/typescript/packages/traceai_auto/src/libraries.ts index c2b8618b8..11c5fd69f 100644 --- a/typescript/packages/traceai_auto/src/libraries.ts +++ b/typescript/packages/traceai_auto/src/libraries.ts @@ -13,7 +13,7 @@ import { Config, enabled, Options } from "./config"; export type Status = "instrumented" | "pending" | "not_installed" | "missing_instrumentor" | "disabled" | "manual" | "failed"; export interface Integration { name: string; package: string; targets: string[]; modules: string[]; className?: string; manual?: string; enable?: string } -export interface IntegrationStatus { name: string; package?: string; targets?: string[]; status: Status; reason?: string } +export interface IntegrationStatus { name: string; package?: string; targets?: string[]; status: Status; reason?: string; media?: "unqualified" } export const integrations: Integration[] = catalog as Integration[]; export function appRequire() { return createRequire(join(process.cwd(), "__traceai_auto__.cjs")); } function resolvable(name: string): boolean { @@ -22,7 +22,7 @@ function resolvable(name: string): boolean { } } export function inventory(config: Config): IntegrationStatus[] { - return integrations.map(i => ({ name: i.name, package: i.package, targets: i.targets, + return integrations.map(i => ({ name: i.name, package: i.package, targets: i.targets, media: "unqualified", status: !enabled(i.name, config) ? "disabled" : ![...i.targets, ...i.modules].some(resolvable) ? "not_installed" : i.manual && !i.enable ? "manual" : !resolvable(i.package) && i.name !== "openai" ? "missing_instrumentor" : "pending", ...(i.manual ? { reason: i.manual } : {}) })); } @@ -114,7 +114,7 @@ export function enableLibraries(config: Config, options: Options, provider: Trac if (typeof enable !== "function") { status.status = "manual"; status.reason = "Installed instrumentor has no zero-argument enable entry; wire capture remains available"; continue; } // Enable entries receive the selected neutral destination explicitly. // Never synthesize FI credentials to unlock a framework integration. - const result = enable({ tracerProvider: provider, spanProcessor, traceConfig: options.traceConfig }); + const result = enable({ tracerProvider: provider, spanProcessor, traceConfig: options.traceConfig, attachmentManager: options.attachmentManager }); if (result && typeof result.setTracerProvider === "function") result.setTracerProvider(provider); status.status = result === false || result === undefined ? "manual" : result?.status ?? "instrumented"; status.reason = result?.reason; @@ -123,7 +123,7 @@ export function enableLibraries(config: Config, options: Options, provider: Trac } const Instrumentor = exports[entry.className!]; if (typeof Instrumentor !== "function") throw new Error("Missing exported Instrumentation class"); - instance = new Instrumentor({ instrumentationConfig: { enabled: false }, traceConfig: options.traceConfig, tracerProvider: provider, exclusiveProcessor: false }); + instance = new Instrumentor({ instrumentationConfig: { enabled: false }, traceConfig: options.traceConfig, tracerProvider: provider, exclusiveProcessor: false, attachmentManager: options.attachmentManager }); const known = (instance._modules ?? []).map((definition: any) => ({ definition, version: moduleVersion(definition.name) })).filter((item: any) => item.version); if (known.length && known.every(({ definition, version }: any) => !definition.supportedVersions.some((range: string) => satisfies(version, range, { includePrerelease: definition.includePrerelease })))) { instance.disable(); status.status = "failed"; status.reason = "Installed target version is outside the instrumentor's supported range"; diff --git a/typescript/packages/traceai_auto/src/wire/attachments.ts b/typescript/packages/traceai_auto/src/wire/attachments.ts new file mode 100644 index 000000000..7171a9697 --- /dev/null +++ b/typescript/packages/traceai_auto/src/wire/attachments.ts @@ -0,0 +1,127 @@ +import { createContextKey, trace, type Context, type Span } from "@opentelemetry/api"; +import type { AttachmentCorrelation, AttachmentManager, AttachmentOptions, AttachmentCapturePolicy } from "@traceai/fi-core/attachments"; +import type { FetchMediaObserver } from "./fetch-runtime"; + +const MEDIA_POLICY = createContextKey("traceai.auto.attachment_policy.v1"); +/** Supply the integration's capture restrictions; inherited denials always win. */ +export function withAttachmentCapturePolicy(parent: Context, policy: AttachmentCapturePolicy): Context { + const previous = parent.getValue(MEDIA_POLICY) as AttachmentCapturePolicy | undefined; + const merged = { ...policy }; + if (Object.values(policy).some(value => value !== undefined && typeof value !== "boolean")) merged.captureContent = false; + for (const key of ["captureContent", "recordInputs", "recordOutputs", "requested"] as const) { + if (previous?.[key] === false) merged[key] = false; + else if (merged[key] === undefined) merged[key] = previous?.[key]; + } + for (const key of ["muted", "temporaryChat", "modelInput", "containsInput", "containsOutput", "errorDetail", "newRecording"] as const) { + if (previous?.[key]) merged[key] = true; + } + return parent.setValue(MEDIA_POLICY, Object.freeze(merged)); +} + +const MEDIA_OWNER = createContextKey("traceai.auto.attachment_owner.v1"); +/** An adapter that captures media owns it independently of semantic span coverage. */ +export function withAttachmentOwnership(parent: Context): Context { return parent.setValue(MEDIA_OWNER, true); } + +const IMAGE_MIMES = new Set(["image/png", "image/jpeg", "image/webp"]); +const AUDIO_MIMES = new Set(["audio/wav", "audio/ogg", "audio/mpeg", "audio/mp4", "audio/webm", "audio/flac"]); +// Bounded transient decoder scratch, not a second media queue. +const MAX_BASE64 = 4 * Math.ceil(256 * 1024 / 3); + +type Operation = "speech" | "transcription" | "image_edit" | "image_generation"; +function operation(method: string, url: string): Operation | undefined { + try { + const u = new URL(url); + // A compatible path on an unknown host is not authority to capture media. + if (method.toUpperCase() !== "POST" || u.protocol !== "https:" || u.username || u.password || (u.port && u.port !== "443")) return; + if (u.hostname === "api.openai.com") { + if (u.pathname === "/v1/audio/speech") return "speech"; + if (["/v1/audio/transcriptions", "/v1/audio/translations"].includes(u.pathname)) return "transcription"; + if (["/v1/images/edits", "/v1/images/variations"].includes(u.pathname)) return "image_edit"; + if (u.pathname === "/v1/images/generations") return "image_generation"; + } + if (u.hostname === "api.deepgram.com" && u.pathname === "/v1/speak") return "speech"; + } catch { /* Unknown destinations never acquire media ownership. */ } +} +function modelSpan(parent: Context): Span | undefined { + const span = trace.getSpan(parent); + // Do not relink wire media to an unrelated ambient HTTP/tool/root span. + const readable = span as (Span & { attributes?: Record; span?: { attributes?: Record } }) | undefined; + const attributes = readable?.attributes ?? readable?.span?.attributes; + const kind = attributes?.["gen_ai.span.kind"] ?? attributes?.["fi.span.kind"]; + return span?.isRecording() && ["LLM", "EMBEDDING"].includes(String(kind)) ? span : undefined; +} + +/** + * Observe only immutable inputs or results the caller has already materialized. + * No reader, clone, tee, filesystem, URL retrieval, recorder or private uploader. + * Incremental/encoded streaming waits for core's admission/reservation API. + */ +export function wireAttachmentObserver(manager: AttachmentManager, parent: Context, method: string, url: string): FetchMediaObserver | undefined { + if (parent.getValue(MEDIA_OWNER) || manager.status().state !== "queued") return; + const op = operation(method, url); + if (!op) return; + let correlation: AttachmentCorrelation | undefined; + const native = modelSpan(parent); + const capture = parent.getValue(MEDIA_POLICY) as AttachmentCapturePolicy | undefined; + // An external native instrumentor can have stricter per-call privacy than auto. + // It must supply its policy, not silently inherit an unrelated manager opt-in. + if (native && !capture) return; + if (native) correlation = manager.correlation(native); + let outputObserved = false; + const attachBlob = (blob: Blob, options: AttachmentOptions) => { + if (correlation) manager.attach(correlation, { kind: "Blob", data: blob }, { ...options, capture: { ...capture, ...options.capture } }); + }; + return { + bind(span) { + // The emitter supplies its actual identity before ending, never a later ambient span. + if (!correlation) correlation = manager.correlation(span); + }, + request(body) { + // A wire-only span is assigned late. Do not retain a pre-admission FormData. + if (!correlation || !(body instanceof FormData)) return; + let entries = 0; + for (const [name, value] of body.entries()) { + if (++entries > 64) break; + if (!(value instanceof Blob)) continue; + if (op === "transcription" && name === "file" && AUDIO_MIMES.has(value.type)) { + attachBlob(value, { mimeType: value.type, direction: "input", purpose: "model_input" }); + } else if (op === "image_edit" && ["image", "image[]", "mask"].includes(name) && IMAGE_MIMES.has(value.type)) { + attachBlob(value, { mimeType: value.type, direction: "input", purpose: "model_input", capture: { modelInput: true } }); + } + } + }, + materialized(value, methodName, mimeType, successful) { + if (outputObserved || !correlation || !successful) return; + if (op === "speech" && AUDIO_MIMES.has(mimeType)) { + if (methodName === "blob" && value instanceof Blob) { + outputObserved = true; + attachBlob(value, { mimeType, direction: "output", purpose: "model_output" }); + } else if (methodName === "arrayBuffer" && value instanceof ArrayBuffer) { + outputObserved = true; + // No copy/view is constructed until the shared policy gate permits access. + manager.attach(correlation, { kind: "Bytes", get data() { return new Uint8Array(value); } }, + { mimeType, direction: "output", purpose: "model_output", capture }); + } + } else if ((op === "image_generation" || op === "image_edit") && methodName === "json" && mimeType === "application/json") { + outputObserved = true; + // Typed OpenAI response only, never recursively scrape arbitrary JSON. + if (!value || typeof value !== "object") return; + const images: unknown = Object.getOwnPropertyDescriptor(value, "data")?.value; + if (!Array.isArray(images)) return; + for (let index = 0; index < Math.min(images.length, 64); index++) { + const item: unknown = images[index]; + if (!item || typeof item !== "object") continue; + const encoded: unknown = Object.getOwnPropertyDescriptor(item, "b64_json")?.value; + if (typeof encoded !== "string") continue; // Remote URLs are NOT retrieved. + manager.attach(correlation, { kind: "Bytes", get data() { + // Decoding and byte inspection are behind server/local privacy admission. + if (!encoded.length || encoded.length > MAX_BASE64 || encoded.length % 4 || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); + const bytes = Buffer.from(encoded, "base64"); + // This operation only qualifies PNG. Other formats remain unsupported. + return bytes.length <= 256 * 1024 && bytes.length >= 8 && bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) ? bytes : new Uint8Array(); + } }, { mimeType: "image/png", direction: "output", purpose: "model_output", capture }); + } + } + }, + }; +} diff --git a/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts b/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts index 0f80b1d57..abd470389 100644 --- a/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts +++ b/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts @@ -2,7 +2,14 @@ import { build, checkRequest, header, isBinary, match, RequestData, ResponseData import { encodeBytes, byteLength, concatBytes } from "./bytes"; import { BodyDecoder, decodeResponse, responseEncoding } from "./encoding"; import { streamingMilestones } from "./stream-timing"; +import type { Span } from "@opentelemetry/api"; +export interface FetchMediaObserver { + bind(span: Span): void; + request(body: BodyInit | null | undefined): void; + materialized(value: unknown, method: "arrayBuffer" | "blob" | "json", mimeType: string, successful: boolean): void; +} export interface FetchContext { + media?(parent: T, method: string, url: string): FetchMediaObserver | undefined; active(): T; skip(parent: T): boolean; run(parent: T, fn: () => R): R; observe?(promise: Promise, ok: (value: R) => void, fail: (error: unknown) => void): void; observeRequest?(request: RequestData): void; @@ -91,7 +98,7 @@ function observeMethod(object: any, name: PropertyKey, after: (result: any, args } catch { /* Frozen/custom runtime objects stay untouched. */ } } export interface FetchHook { remove(): void; pending: Set> } -export function createFetchHook(max: number, emit: (result: SpanResult, parent: T, start: number, end: number) => void, scope: FetchContext, secrets: string[] = [], decodeBody?: BodyDecoder): FetchHook { +export function createFetchHook(max: number, emit: (result: SpanResult, parent: T, start: number, end: number, bindMedia?: (span: Span) => void) => void, scope: FetchContext, secrets: string[] = [], decodeBody?: BodyDecoder): FetchHook { const original = globalThis.fetch; const observe = scope.observe ?? tap; const pending = new Set>(); @@ -114,11 +121,13 @@ export function createFetchHook(max: number, emit: (result: SpanResult, paren try { skip = scope.skip(parent); } catch { /* Broken context still observes headers/URL. */ } if (skip) return passThrough(); const capture = () => { try { return scope.capture?.(parent) !== false; } catch { return false; } }; + let media: FetchMediaObserver | undefined; let req: RequestData | undefined, found: ReturnType, initial: ReturnType; const inspect = () => { const [input, init] = argArray, info = head(); observeRequest(info); // Every route, not just catalog-matched AI calls. found = match(info.method, info.url); + try { media = scope.media?.(parent, info.method, info.url); media?.request(init?.body); } catch { /* Independent, fail-open media admission. */ } // Covered model operations observe URL/headers only, as before. Selection // is delayed until catalog match; unsupported calls cannot choose a route. if (found && capture()) { @@ -145,6 +154,22 @@ export function createFetchHook(max: number, emit: (result: SpanResult, paren throw hostError; } try { inspect(); } catch { return result; } + if (media) { + const observer = media, observed = new WeakSet(); + const decorate = (response: Response) => { + if (observed.has(response)) return; + observed.add(response); + const mimeType = header(headersOf(response.headers), "content-type"); + for (const method of ["arrayBuffer", "blob", "json"] as const) { + observeMethod(response, method, promise => observe(promise, value => { + if (active) observer.materialized(value, method, mimeType, response.ok); + }, () => {})); + } + // Only follow clones explicitly created by the application. Never call clone ourselves. + observeMethod(response, "clone", clone => decorate(clone)); + }; + observe(result, decorate, () => {}); + } if (!found || !req || !initial) return result; const matched = found, request = req, parsed = initial; let resolve!: () => void; @@ -154,7 +179,7 @@ export function createFetchHook(max: number, emit: (result: SpanResult, paren if (finished) return; finished = true; const complete = () => { if (audio) { response.audioOutputBytes = audio.bytes; response.body = undefined; } - try { const span = build(matched, request, response, parsed); if (span) emit(span, parent, start, end); } catch { /* Capture must not reach application code. */ } + try { const span = build(matched, request, response, parsed); if (span) emit(span, parent, start, end, media ? value => media!.bind(value) : undefined); } catch { /* Capture must not reach application code. */ } finally { resolve(); pending.delete(done); } }; // Only completed responses awaiting bounded decoding are flushable, never unread streams. diff --git a/typescript/packages/traceai_auto/src/wire/fetch.ts b/typescript/packages/traceai_auto/src/wire/fetch.ts index 95f2597e2..bc6a4d873 100644 --- a/typescript/packages/traceai_auto/src/wire/fetch.ts +++ b/typescript/packages/traceai_auto/src/wire/fetch.ts @@ -1,7 +1,9 @@ -import { context, createContextKey, trace, Context } from "@opentelemetry/api"; +import { context, createContextKey, trace, Context, type Span } from "@opentelemetry/api"; import { autoModelResponsibility, autoModelWireAllowed, hasAutoModelCoverage } from "@traceai/fi-core"; import { isTracingSuppressed } from "@opentelemetry/core"; import { createFetchHook } from "./fetch-runtime"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import { wireAttachmentObserver } from "./attachments"; import { decodeNodeBody } from "./encoding-node"; import { observePromise, preparePromiseObservation } from "./promise-node"; import type { SpanResult } from "./core"; @@ -35,9 +37,10 @@ function wireAllowed(ctx: Context, phase: "guard" | "inspect" | "request"): bool export function shouldSkip(ctx: Context, consultOperation = true): boolean { return isTracingSuppressed(ctx) || !!ctx.getValue(IN_FLIGHT) || !wireAllowed(ctx, consultOperation ? "inspect" : "guard"); } -export function patchFetch(max: number, emit: (result: SpanResult, parent: Context, start: number, end: number) => void, secrets: string[] = []) { +export function patchFetch(max: number, emit: (result: SpanResult, parent: Context, start: number, end: number, bindMedia?: (span: Span) => void) => void, secrets: string[] = [], attachments?: AttachmentManager) { preparePromiseObservation(); - return createFetchHook(max, emit, { active: () => context.active(), skip: parent => shouldSkip(parent, false), + return createFetchHook(max, emit, { active: () => context.active(), skip: parent => attachments ? isTracingSuppressed(parent) || !!parent.getValue(IN_FLIGHT) : shouldSkip(parent, false), + media: attachments ? (parent, method, url) => wireAttachmentObserver(attachments, parent, method, url) : undefined, // Same context decision, now after supported URL matching. Only the emit // completion boundary can commit wire; credential observation stays separate. capture: parent => wireAllowed(parent, "request"), observe: observePromise, observeRequest: observeOutgoing, diff --git a/typescript/packages/traceai_auto/test/attachment-lifecycle.test.ts b/typescript/packages/traceai_auto/test/attachment-lifecycle.test.ts new file mode 100644 index 000000000..ddd12d76b --- /dev/null +++ b/typescript/packages/traceai_auto/test/attachment-lifecycle.test.ts @@ -0,0 +1,44 @@ +import { spawnSync } from "node:child_process"; +import { resolve } from "node:path"; + +const root = resolve(__dirname, ".."); +function run(mode: string, preload?: string) { + const code = ` + const assert = require('node:assert/strict'); + const { BasicTracerProvider, InMemorySpanExporter } = require('@opentelemetry/sdk-trace-base'); + const { AttachmentManager } = require('@traceai/fi-core/attachments'); + const auto = require('./dist/src/index.js'); + (async () => { + const manager = new AttachmentManager({ enabled: false, environment: () => ({}) }); + const host = new BasicTracerProvider(); + if (${JSON.stringify(mode)} === 'host') host.attachments = manager; + const options = { exporter: new InMemorySpanExporter(), instrumentations: ['wire'], configFile: '/nonexistent' }; + if (${JSON.stringify(mode)} === 'borrowed') options.attachmentManager = manager; + if (${JSON.stringify(mode)} === 'host') options.provider = host; + let network = 0; + globalThis.fetch = () => { network++; throw new Error('unexpected network'); }; + const provider = auto.init(options); + assert.ok(provider); + const selected = auto.getAttachmentManager(); + assert.ok(selected); + if (['host', 'borrowed'].includes(${JSON.stringify(mode)})) assert.equal(selected, manager); + else assert.equal(selected, provider.attachments); + assert.equal(await selected.ready(100), false); + assert.equal(selected.status().state, 'disabled'); + assert.equal(auto.report().attachments.integration, 'component_only'); + await auto.forceFlush(); + await auto.shutdown(); await auto.shutdown(); + assert.equal(selected.status().state, ${JSON.stringify(mode)} === 'owned' ? 'incomplete' : 'disabled'); + assert.equal(network, 0); + await manager.shutdown(); await host.shutdown(); + process.stdout.write('LIFECYCLE_OK'); + })().catch(() => { process.stderr.write('LIFECYCLE_FAILED'); process.exitCode = 1; }); + `; + return spawnSync(process.execPath, [...(preload ? ['--require', preload] : []), '-e', code], { + cwd: root, encoding: 'utf8', timeout: 15000, + env: { ...process.env, FI_ATTACHMENTS_ENABLED: 'false', FI_API_KEY: '', FI_SECRET_KEY: '', FI_CONFIG_FILE: '/nonexistent', FI_AUTO_DEBUG: 'false' }, + }); +} +test.each(['owned', 'host', 'borrowed'])('%s manager uses one selected runtime and preserves lifecycle ownership', mode => { + const result = run(mode); expect(result.error).toBeUndefined(); expect(result.status).toBe(0); expect(result.stdout).toContain('LIFECYCLE_OK'); +}); diff --git a/typescript/packages/traceai_auto/test/attachments.test.ts b/typescript/packages/traceai_auto/test/attachments.test.ts new file mode 100644 index 000000000..987c09e9d --- /dev/null +++ b/typescript/packages/traceai_auto/test/attachments.test.ts @@ -0,0 +1,223 @@ +import { createServer, type Server } from "node:http"; +import { AddressInfo } from "node:net"; +import { context, trace, ROOT_CONTEXT } from "@opentelemetry/api"; +import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; +import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; +import { AttachmentManager, type AttachmentManagerOptions, type AttachmentTransport } from "@traceai/fi-core/attachments"; +import { patchFetch } from "../src/wire/fetch"; +import { withAttachmentOwnership, withAttachmentCapturePolicy } from "../src/wire/attachments"; +import { attachmentOptions, selectAttachmentManager, attachmentReport } from "../src/attachments"; +import { resolveConfig } from "../src/config"; + +// Synthetic media/credentials only. Both provider and attachment traffic terminate on loopback. +const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64"); +function wav() { + const b = Buffer.alloc(44 + 160); b.write("RIFF"); b.writeUInt32LE(b.length - 8, 4); b.write("WAVEfmt ", 8); + b.writeUInt32LE(16, 16); b.writeUInt16LE(1, 20); b.writeUInt16LE(1, 22); b.writeUInt32LE(8000, 24); + b.writeUInt32LE(16000, 28); b.writeUInt16LE(2, 32); b.writeUInt16LE(16, 34); b.write("data", 36); b.writeUInt32LE(160, 40); return b; +} +const audio = wav(); +const uuid = "11111111-1111-4111-8111-111111111111"; +let original: typeof fetch, server: Server, origin: string, manager: AttachmentManager, hook: ReturnType | undefined; +let uploads: Buffer[], requests: any[], transfers: { path: string; body: Buffer }[], providerBodies: Buffer[]; +let provider: BasicTracerProvider, exporter: InMemorySpanExporter; +let responseStatus: number, responseData: Buffer, responseType: string; +const nativeFetch = globalThis.fetch; +const sessionPolicy = { version: "1", enabled: true, allowed_sources: ["Bytes", "Blob"] as ("Bytes" | "Blob")[], allowed_modalities: ["image", "audio"] as ("image" | "audio")[], require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false }; +async function createManager(options: AttachmentManagerOptions = {}, policy = sessionPolicy) { + const send = async (path: string, body: BodyInit) => { + const response = await nativeFetch(`${origin}${path}`, { method: "POST", body }); + return response.json(); + }; + const transport: AttachmentTransport = { + async bootstrap(request) { await send("/attachment-sessions", JSON.stringify(request)); return { schema_version: 1, scope: { organization_id: uuid, workspace_id: uuid, project_id: uuid }, capture_session_id: request.capture_session_id, policy, limits: { max_upload_bytes: 20971520, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }, expires_at: new Date(Date.now() + 60000).toISOString(), transfer_modes: ["authenticated_stream"] }; }, + async initialize(_session, request) { await send("/uploads", JSON.stringify(request)); requests.push(request); return { upload_id: uuid, attachment_id: request.attachment_id, occurrence_id: request.occurrence_id, state: "pending_upload", reserved_bytes: request.declared_bytes, expires_at: new Date(Date.now() + 60000).toISOString(), transfer_mode: "authenticated_stream", write_capability: "synthetic-capability" }; }, + async upload(_session, _upload, bytes) { await send("/bytes", bytes as BodyInit); }, + async complete() { await send("/complete", "{}"); }, + }; + const result = new AttachmentManager({ enabled: true, projectName: "attachment-test", projectType: "observe", environment: () => ({}), transport, ...options }); + expect(await result.ready()).toBe(options.enabled !== false); + expect(result.status().state).toBe(options.enabled === false ? "disabled" : "queued"); + return result; +} +function install() { + globalThis.fetch = (input: RequestInfo | URL, init?: RequestInit) => { + const u = new URL(input instanceof Request ? input.url : String(input)); + return nativeFetch(`${origin}${u.pathname}${u.search}`, init); + }; + hook = patchFetch(4194304, (result, parent, start, end, bindMedia) => { + const span = provider.getTracer("wire-test").startSpan(result.name, { startTime: start, attributes: result.attributes }, parent); + bindMedia?.(span); span.end(end); + }, [], manager); +} +function inNative(fn: () => T, kind = "LLM"): T { + const span = provider.getTracer("native-test").startSpan("native", { attributes: { "gen_ai.span.kind": kind } }); + const result = context.with(withAttachmentCapturePolicy(trace.setSpan(ROOT_CONTEXT, span), { captureContent: true }), fn); + // Promise work retains this exact request context; do not consult later ambient spans. + if (result instanceof Promise) return result.finally(() => span.end()) as T; + span.end(); return result; +} +async function settle() { await manager.flush(2000); } +beforeAll(() => context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable())); +afterAll(() => context.disable()); +beforeEach(async () => { + original = globalThis.fetch; uploads = []; requests = []; transfers = []; providerBodies = []; + responseStatus = 200; responseData = audio; responseType = "audio/wav"; + server = createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", chunk => chunks.push(chunk)); + req.on("end", () => { + const body = Buffer.concat(chunks); + if (["/attachment-sessions", "/uploads", "/bytes", "/complete"].includes(req.url!)) { + transfers.push({ path: req.url!, body }); if (req.url === "/bytes") uploads.push(body); + res.writeHead(200, { "content-type": "application/json" }); res.end("{}"); + } else { providerBodies.push(body); res.writeHead(responseStatus, { "content-type": responseType }); res.end(responseData); } + }); + }); + await new Promise(done => server.listen(0, "127.0.0.1", done)); origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + exporter = new InMemorySpanExporter(); provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + manager = await createManager(); +}); +afterEach(async () => { + hook?.remove(); hook = undefined; globalThis.fetch = original; + await manager.shutdown(1000); await provider.shutdown(); + server.closeAllConnections(); await new Promise(done => server.close(() => done())); +}); + +test.each(["arrayBuffer", "blob"] as const)("caller-consumed speech %s uploads once with the actual wire identity", async method => { + install(); const response = await fetch("https://api.openai.com/v1/audio/speech", { method: "POST", body: JSON.stringify({ model: "tts-1", input: "synthetic", voice: "alloy" }) }); + const result = await response[method](); + expect(Buffer.from(result instanceof Blob ? await result.arrayBuffer() : result)).toEqual(audio); + await settle(); expect(uploads).toEqual([audio]); + const span = exporter.getFinishedSpans()[0]; expect(requests[0].association.parent.span_id).toBe(span.spanContext().spanId); + expect(span.attributes["fi.attachment.correlation_nonce"]).toBeTruthy(); + // Completion never mutates an ended span; authoritative late links belong to the API. + expect(span.attributes["fi.attachment.refs"]).toBe("[]"); +}); +test("native semantic coverage leaves safe media observation on without a second span", async () => { + install(); await inNative(async () => { const response = await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" }); await response.arrayBuffer(); }); + await settle(); expect(uploads).toEqual([audio]); expect(exporter.getFinishedSpans()).toHaveLength(1); + expect(requests[0].association.parent.span_id).toBe(exporter.getFinishedSpans()[0].spanContext().spanId); +}); +test("adapter media ownership is independent and prevents duplicate uploads", async () => { + install(); await inNative(() => context.with(withAttachmentOwnership(context.active()), async () => { await (await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" })).arrayBuffer(); })); + await settle(); expect(uploads).toHaveLength(0); expect(exporter.getFinishedSpans()).toHaveLength(1); +}); +test.each(["api.openai.com/v1/audio/speech", "api.deepgram.com/v1/speak"])("recognized %s preserves cloned application reads and deduplicates output", async route => { + install(); await inNative(async () => { + const response = await fetch(`https://${route}`, { method: "POST" }); const clone = response.clone(); + const [one, two] = await Promise.all([response.arrayBuffer(), clone.blob()]); expect(Buffer.from(one)).toEqual(audio); expect(Buffer.from(await two.arrayBuffer())).toEqual(audio); + }); await settle(); expect(uploads).toEqual([audio]); +}); +test("unread, cancelled and reader-consumed responses do not create eager media readers", async () => { + install(); await inNative(async () => { + const unread = await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" }); + expect(unread.bodyUsed).toBe(false); await unread.body!.cancel(); + const consumed = await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" }); + const reader = consumed.body!.getReader(); const chunks = []; for (;;) { const x = await reader.read(); if (x.done) break; chunks.push(Buffer.from(x.value)); } + expect(Buffer.concat(chunks)).toEqual(audio); + }); await settle(); expect(uploads).toHaveLength(0); +}); +test("FormData keeps the body, boundary and filename on the provider path only", async () => { + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); + const body = new FormData(); body.set("model", "whisper-1"); body.set("file", new Blob([audio], { type: "audio/wav" }), "PRIVATE_CANARY_FILENAME.wav"); + install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/audio/transcriptions", { method: "POST", body })).json(); }); + await settle(); expect(uploads).toEqual([audio]); expect(providerBodies[0].includes(audio)).toBe(true); expect(providerBodies[0].toString()).toContain("PRIVATE_CANARY_FILENAME.wav"); + expect(JSON.stringify(requests)).not.toContain("PRIVATE_CANARY"); expect(JSON.stringify(exporter.getFinishedSpans().map(s => s.attributes))).not.toContain("PRIVATE_CANARY"); + expect(body.get("file")).toBeInstanceOf(Blob); +}); +test("image edits accept only typed allowlisted FormData fields", async () => { + responseType = "application/json"; responseData = Buffer.from('{"data":[]}'); + const body = new FormData(); body.set("image", new Blob([png], { type: "image/png" })); body.set("untrusted_file", new Blob([png], { type: "image/png" })); + install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body })).json(); }); + await settle(); expect(uploads).toEqual([png]); +}); +test("wire-only FormData remains unsupported without retaining the source until a late span", async () => { + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); + const body = new FormData(); body.set("model", "whisper-1"); body.set("file", new Blob([audio], { type: "audio/wav" })); + install(); await (await fetch("https://api.openai.com/v1/audio/transcriptions", { method: "POST", body })).json(); + await settle(); expect(uploads).toHaveLength(0); +}); +test("application-decoded PNG image JSON uploads bytes, not URLs or base64 span fields", async () => { + responseType = "application/json"; responseData = Buffer.from(JSON.stringify({ data: [{ b64_json: png.toString("base64") }, { url: "https://untrusted.invalid/private?token=CANARY" }] })); + install(); const body = await (await fetch("https://api.openai.com/v1/images/generations", { method: "POST", body: JSON.stringify({ model: "dall-e-3", prompt: "synthetic" }) })).json(); + expect(body.data[0].b64_json).toBe(png.toString("base64")); await settle(); expect(uploads).toEqual([png]); + expect(JSON.stringify(requests)).not.toContain("CANARY"); expect(JSON.stringify(exporter.getFinishedSpans().map(s => s.attributes))).not.toContain(png.toString("base64")); +}); +test.each([ + ["disabled", { enabled: false }], ["hide inputs", { traceConfig: { hideInputs: true } }], ["hide text", { traceConfig: { hideInputText: true } }], + ["hide images", { traceConfig: { hideInputImages: true } }], ["PII", { traceConfig: { piiRedaction: true } }], +] as [string, AttachmentManagerOptions][])("%s prevents image transfer at the network boundary", async (_name, options) => { + await manager.shutdown(); manager = await createManager(options); transfers = []; + responseType = "application/json"; responseData = Buffer.from('{"data":[]}'); + const body = new FormData(); body.set("image", new Blob([png], { type: "image/png" }), "PRIVATE_CANARY.png"); + install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body })).json(); }); + await settle(); expect(transfers).toEqual([]); +}); +test.each(["hideOutputs", "hideOutputMessages", "hideOutputText", "piiRedaction"] as const)("%s prevents output speech transfer", async flag => { + manager.updatePolicy({ traceConfig: { [flag]: true } }); transfers = []; + install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" })).arrayBuffer(); }); + await settle(); expect(transfers).toEqual([]); +}); +test("server-required redaction rejects decoded PNG before the base64 decoder", async () => { + await manager.shutdown(); manager = await createManager({}, { ...sessionPolicy, require_media_redaction: true }); transfers = []; + responseType = "application/json"; responseData = Buffer.from(JSON.stringify({ data: [{ b64_json: png.toString("base64") }] })); + install(); const decode = jest.spyOn(Buffer, "from"); + await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/generations", { method: "POST" })).json(); }); + const calls = decode.mock.calls.filter(args => args[0] === png.toString("base64") && (args as unknown[])[1] === "base64"); decode.mockRestore(); + await settle(); expect(calls).toHaveLength(0); expect(transfers).toEqual([]); +}); +test("unknown provider, non-audio MIME and error bodies do not upload", async () => { + install(); await inNative(async () => { + await (await fetch("https://unknown.invalid/v1/audio/speech", { method: "POST" })).arrayBuffer(); + responseType = "application/octet-stream"; await (await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" })).arrayBuffer(); + responseType = "audio/wav"; responseStatus = 403; await (await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" })).arrayBuffer(); + }); await settle(); expect(uploads).toHaveLength(0); +}); +test("oversized media, raw multipart, and ended/unrelated native contexts fail closed", async () => { + responseType = "application/json"; responseData = Buffer.from('{"data":[]}'); + const body = new FormData(); body.set("image", new Blob([new Uint8Array(20 * 1024 * 1024 + 1)], { type: "image/png" })); + install(); await inNative(async () => { + await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body })).json(); + await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", headers: { "content-type": "multipart/form-data; boundary=x" }, body: "--x\r\nPRIVATE_CANARY" })).json(); + }); + const small = new FormData(); small.set("image", new Blob([png], { type: "image/png" })); + await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body: small })).json(); }, "TOOL"); + await settle(); expect(uploads).toHaveLength(0); +}); +test("policy tightening between request and application consumption prevents capture", async () => { + install(); await inNative(async () => { + const response = await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" }); manager.updatePolicy({ enabled: false }); await response.arrayBuffer(); + }); await settle(); expect(uploads).toHaveLength(0); +}); +test("removal makes retained response wrappers inert", async () => { + install(); await inNative(async () => { const response = await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" }); hook!.remove(); await response.arrayBuffer(); }); + await settle(); expect(uploads).toHaveLength(0); +}); +test("borrowed manager identity and restrictive policy survive auto setup", async () => { + const config = resolveConfig({ traceConfig: { hideOutputs: true }, projectName: "auto-project", configFile: "/nonexistent" }, {}); + const selected = selectAttachmentManager({ attachments: manager }, config, {}, false); + expect(selected.manager).toBe(manager); expect(selected.owned).toBe(false); + expect(attachmentOptions(config, { attachments: { projectName: "wrong-project", traceConfig: { hideOutputs: false } } })).toMatchObject({ projectName: "auto-project", traceConfig: { hideOutputs: true } }); + expect(attachmentReport(manager).integration).toBe("component_only"); + install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" })).arrayBuffer(); }); + await settle(); expect(uploads).toHaveLength(0); +}); +test("real installed OpenAI SDK speech operation preserves WAV bytes and correlation", async () => { + const OpenAI = require(process.env.ATT_OPENAI_MODULE || "openai").default; + install(); const sdk = new OpenAI({ apiKey: "synthetic-provider-key", fetch: globalThis.fetch }); + const response = await sdk.audio.speech.create({ model: "tts-1", voice: "alloy", input: "synthetic", response_format: "wav" }); + expect(Buffer.from(await response.arrayBuffer())).toEqual(audio); + await settle(); expect(uploads).toEqual([audio]); expect(exporter.getFinishedSpans()).toHaveLength(1); +}); + +test("native capture is denied without an explicit integration policy", async () => { + install(); const span = provider.getTracer("external").startSpan("external", { attributes: { "gen_ai.span.kind": "LLM" } }); + await context.with(trace.setSpan(ROOT_CONTEXT, span), async () => { await (await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" })).arrayBuffer(); }); + span.end(); await settle(); expect(uploads).toHaveLength(0); +}); +test.each([{ captureContent: false }, { recordOutputs: false }, { muted: true }, { temporaryChat: true }])("integration restrictions cannot be loosened by child scopes: %j", async restriction => { + install(); await inNative(() => context.with(withAttachmentCapturePolicy(withAttachmentCapturePolicy(context.active(), restriction), { captureContent: true, recordOutputs: true, muted: false, temporaryChat: false }), async () => { + await (await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" })).arrayBuffer(); + })); await settle(); expect(uploads).toHaveLength(0); +}); From f642a52b0ce14fc90388850deaf23022c1582f6d Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 04:53:55 -0800 Subject: [PATCH 09/20] feat(auto): migrate wire attachments to preflight and add consumed-stream capture A02: pass the shared manager to instrumentors as `attachments` (the option every adapter lane uses), keep media status apart from span status, and stop disabling a borrowed manager on an env override (updatePolicy is tighten-only; the core applies the override at admission). With attachments OFF, wire skip/inspection is unchanged. A03: every FormData Blob and speech buffer is preflighted at its exact size before it is read; JSON media (chat/responses inputs, image outputs, computer screenshots) goes through the @traceai/openai adapter so SDK and wire observations of one span share an occurrence. Wire-only JSON input uses the wire path's own bounded request copy. A04: chunks the application reads (reader read() or async iterator next()) feed startAttachmentStream under a native model span with an explicit capture policy. Cancel, early return, pipe, tee and clone abort; nothing partial is uploaded. Wire-only incremental reads and wire-only FormData stay unsupported until the core can preflight a late-bound target. Co-Authored-By: Claude Opus 5.5 --- .../traceai_auto/scripts/postbuild.cjs | 2 +- .../packages/traceai_auto/src/attachments.ts | 67 ++- .../packages/traceai_auto/src/libraries.ts | 14 +- .../traceai_auto/src/wire/attachments.ts | 177 ++++--- .../traceai_auto/src/wire/fetch-runtime.ts | 65 ++- .../packages/traceai_auto/src/wire/fetch.ts | 9 +- .../test/attachment-lifecycle.test.ts | 4 +- .../traceai_auto/test/attachments.test.ts | 489 ++++++++++++------ 8 files changed, 575 insertions(+), 252 deletions(-) diff --git a/typescript/packages/traceai_auto/scripts/postbuild.cjs b/typescript/packages/traceai_auto/scripts/postbuild.cjs index 347c39c71..f743f0e6f 100644 --- a/typescript/packages/traceai_auto/scripts/postbuild.cjs +++ b/typescript/packages/traceai_auto/scripts/postbuild.cjs @@ -2,7 +2,7 @@ const fs = require('node:fs'); for (const dir of ['dist/esm', 'dist/esnext']) { fs.writeFileSync(`${dir}/package.json`, '{"type":"module"}\n'); // A single CJS implementation keeps init idempotent across mixed ESM/CJS apps. - fs.writeFileSync(`${dir}/index.js`, 'export { init, shutdown, forceFlush, report, registerSecret } from "../src/index.js";\n'); + fs.writeFileSync(`${dir}/index.js`, 'export { init, shutdown, forceFlush, report, registerSecret, getAttachmentManager, withAttachmentOwnership, withAttachmentCapturePolicy } from "../src/index.js";\n'); fs.writeFileSync(`${dir}/register.js`, 'import "../src/register.js";\n'); for (const name of ['esbuild', 'vite', 'rollup', 'webpack', 'next']) { const named = name === 'next' ? 'withTraceAI' : name === 'webpack' ? 'traceAI, TraceAIPlugin' : 'traceAI'; diff --git a/typescript/packages/traceai_auto/src/attachments.ts b/typescript/packages/traceai_auto/src/attachments.ts index af19bed06..c52514e60 100644 --- a/typescript/packages/traceai_auto/src/attachments.ts +++ b/typescript/packages/traceai_auto/src/attachments.ts @@ -1,41 +1,74 @@ import { createNodeAttachmentManager, type NodeAttachmentManagerOptions } from "@traceai/fi-core/attachments/node"; -import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import type { AttachmentManager, AttachmentStatus } from "@traceai/fi-core/attachments"; import type { Config, Options } from "./config"; +import { WIRE_MEDIA_ROUTES } from "./wire/attachments"; + +const restrictive = (traceConfig: Record | undefined) => Object.fromEntries(Object.entries(traceConfig ?? {}).filter(([key, value]) => + (key.startsWith("hide") || key === "piiRedaction") && value === true)); export function attachmentOptions(config: Config, options: Options): NodeAttachmentManagerOptions { const explicit = options.attachments ?? {}; - const restrictive = Object.fromEntries(Object.entries(explicit.traceConfig ?? {}).filter(([key, value]) => - (key.startsWith("hide") || key === "piiRedaction") && value === true)); return { ...explicit, environment: () => process.env, projectName: config.projectName, projectType: config.projectType, // Only resolved FI credentials, never model/provider credentials. A neutral // exporter does not imply an attachment destination or credential authority. headers: explicit.headers ?? (config.apiKey && config.secretKey ? { "x-api-key": config.apiKey, "x-secret-key": config.secretKey } : {}), - traceConfig: { ...explicit.traceConfig, ...config.traceConfig, ...restrictive } }; + // A restriction from either side wins; an attachment-level `false` cannot relax a global hide flag. + traceConfig: { ...explicit.traceConfig, ...config.traceConfig, ...restrictive(explicit.traceConfig as Record) } as NodeAttachmentManagerOptions["traceConfig"] }; } interface AttachmentProvider { attachments?: AttachmentManager; getDelegate?(): AttachmentProvider } +/** + * One manager per process: the application's (explicit option, then the provider's), otherwise the one auto's own + * provider created. Adapters and wire observation all receive this same manager. + */ export function selectAttachmentManager(provider: AttachmentProvider, config: Config, options: Options, ownsProvider: boolean): { manager: AttachmentManager; owned: boolean } { - const existing = options.attachmentManager ?? provider?.attachments ?? provider?.getDelegate?.()?.attachments; + let existing: AttachmentManager | undefined; + try { existing = options.attachmentManager ?? provider?.attachments ?? provider?.getDelegate?.()?.attachments; } catch { existing = undefined; } if (existing) { - // Borrowing must not loosen application policy or turn capture on implicitly. - const effective = attachmentOptions(config, options); - const restrictions = Object.fromEntries(Object.entries(effective.traceConfig ?? {}).filter(([key, value]) => - (key.startsWith("hide") || key === "piiRedaction") && value === true)); - const environmentDenies = process.env.FI_ATTACHMENTS_ENABLED !== undefined && !/^(true|1|yes)$/i.test(process.env.FI_ATTACHMENTS_ENABLED.trim()); - existing.updatePolicy({ ...(options.attachments?.enabled === false || environmentDenies ? { enabled: false } : {}), traceConfig: restrictions }); + // Borrowing never turns capture on and never loosens application policy: only auto's hide/PII flags are added. + // The env override (FI_ATTACHMENTS_ENABLED=false) is already applied by the manager at every admission, so a + // borrowed manager is not permanently disabled here (updatePolicy is tighten-only). + try { existing.updatePolicy({ traceConfig: restrictive(attachmentOptions(config, options).traceConfig as Record) as NodeAttachmentManagerOptions["traceConfig"] }); } catch { /* fail closed below */ } return { manager: existing, owned: ownsProvider && !options.attachmentManager }; } return { manager: createNodeAttachmentManager(attachmentOptions(config, options)), owned: true }; } + +/** What auto can capture, independent of whether the manager is on. Nothing here is a playback claim. */ +export const AUTO_ATTACHMENT_SOURCES = Object.freeze([ + "wire FormData Blob (transcription audio, image edit image/mask) under a native model span with an explicit capture policy", + "wire JSON input images/files/computer screenshots (OpenAI chat, responses) via the @traceai/openai adapter", + "wire JSON output images (OpenAI images b64_json, responses image_generation_call) consumed by the application via json()", + "wire speech audio materialized by the application via arrayBuffer()/blob()", + "wire speech audio read incrementally by the application (reader or async iterator) under a native model span with an explicit capture policy", +]); +export const AUTO_ATTACHMENT_UNSUPPORTED = Object.freeze([ + "wire-only FormData (the wire span does not exist before the upload; no source is retained until then)", + "wire-only incremental speech reads (correlation_unavailable until the core supports deferred-target preflight)", + "native model spans without an explicit integration capture policy", + "raw multipart bodies, streamed request bodies, compressed responses consumed before the span is emitted", + "RemoteUrl / ProviderFile retrieval", + "piped or teed response bodies", + "WebSocket / RTC audio", + "new recordings", + "product playback qualification", +]); export interface AttachmentReport { - state: ReturnType["state"] | "not_initialized"; - transport: "shared_manager"; integration: "component_only"; sources: string[]; unsupported: string[]; + state: AttachmentStatus["state"] | "not_initialized"; + status?: Omit; + transport: "shared_manager"; + routes: string[]; + sources: readonly string[]; + unsupported: readonly string[]; } export function attachmentReport(manager?: AttachmentManager): AttachmentReport { + let status: AttachmentStatus | undefined; + try { status = manager?.status(); } catch { status = undefined; } + const { state, ...counts } = status ?? { state: "not_initialized" as const }; return { - state: manager?.status().state ?? "not_initialized", + state, ...(status ? { status: counts as Omit } : {}), transport: "shared_manager", - integration: "component_only", - sources: ["explicit-policy native-span FormData Blob", "consumed response arrayBuffer/blob", "consumed PNG JSON output <=256KiB"], - unsupported: ["unqualified native span policy", "JSON inputs", "wire-only FormData", "incremental/encoded audio", "RemoteUrl/ProviderFile", "WebSocket/RTC", "new recordings", "product playback qualification"], + routes: WIRE_MEDIA_ROUTES.map(route => `POST https://${route.host}${route.path} (${route.operation})`), + sources: AUTO_ATTACHMENT_SOURCES, + unsupported: AUTO_ATTACHMENT_UNSUPPORTED, }; } diff --git a/typescript/packages/traceai_auto/src/libraries.ts b/typescript/packages/traceai_auto/src/libraries.ts index 11c5fd69f..6af51f618 100644 --- a/typescript/packages/traceai_auto/src/libraries.ts +++ b/typescript/packages/traceai_auto/src/libraries.ts @@ -13,7 +13,11 @@ import { Config, enabled, Options } from "./config"; export type Status = "instrumented" | "pending" | "not_installed" | "missing_instrumentor" | "disabled" | "manual" | "failed"; export interface Integration { name: string; package: string; targets: string[]; modules: string[]; className?: string; manual?: string; enable?: string } -export interface IntegrationStatus { name: string; package?: string; targets?: string[]; status: Status; reason?: string; media?: "unqualified" } +/** Media capture is tracked apart from span coverage: an SDK span hook does not imply media capture, and vice versa. */ +export type MediaStatus = "provider_adapter" | "unqualified"; +export interface IntegrationStatus { name: string; package?: string; targets?: string[]; status: Status; reason?: string; media?: MediaStatus } +/** Integrations whose instrumentor captures media through the shared manager it receives as `attachments` (#682). */ +const MEDIA_ADAPTERS = new Map([["openai", "openAIAttachmentCapabilities"]]); export const integrations: Integration[] = catalog as Integration[]; export function appRequire() { return createRequire(join(process.cwd(), "__traceai_auto__.cjs")); } function resolvable(name: string): boolean { @@ -22,7 +26,7 @@ function resolvable(name: string): boolean { } } export function inventory(config: Config): IntegrationStatus[] { - return integrations.map(i => ({ name: i.name, package: i.package, targets: i.targets, media: "unqualified", + return integrations.map(i => ({ name: i.name, package: i.package, targets: i.targets, media: MEDIA_ADAPTERS.has(i.name) ? "provider_adapter" : "unqualified", status: !enabled(i.name, config) ? "disabled" : ![...i.targets, ...i.modules].some(resolvable) ? "not_installed" : i.manual && !i.enable ? "manual" : !resolvable(i.package) && i.name !== "openai" ? "missing_instrumentor" : "pending", ...(i.manual ? { reason: i.manual } : {}) })); } @@ -109,12 +113,14 @@ export function enableLibraries(config: Config, options: Options, provider: Trac // Keep this require inside the enabled branch: disabled integrations register no hooks. const path = entry.enable && resolvable(`${entry.package}/auto`) ? `${entry.package}/auto` : entry.package; const exports = resolvable(path) ? appRequire()(path) : entry.name === "openai" ? require("@traceai/openai") : appRequire()(path); + // Version aware: an installed instrumentor without the capability descriptor captures no media. + if (status.media === "provider_adapter" && !exports?.[MEDIA_ADAPTERS.get(entry.name)!]) status.media = "unqualified"; if (entry.enable) { const enable = exports.enable ?? exports[entry.enable]; if (typeof enable !== "function") { status.status = "manual"; status.reason = "Installed instrumentor has no zero-argument enable entry; wire capture remains available"; continue; } // Enable entries receive the selected neutral destination explicitly. // Never synthesize FI credentials to unlock a framework integration. - const result = enable({ tracerProvider: provider, spanProcessor, traceConfig: options.traceConfig, attachmentManager: options.attachmentManager }); + const result = enable({ tracerProvider: provider, spanProcessor, traceConfig: options.traceConfig, attachments: options.attachmentManager }); if (result && typeof result.setTracerProvider === "function") result.setTracerProvider(provider); status.status = result === false || result === undefined ? "manual" : result?.status ?? "instrumented"; status.reason = result?.reason; @@ -123,7 +129,7 @@ export function enableLibraries(config: Config, options: Options, provider: Trac } const Instrumentor = exports[entry.className!]; if (typeof Instrumentor !== "function") throw new Error("Missing exported Instrumentation class"); - instance = new Instrumentor({ instrumentationConfig: { enabled: false }, traceConfig: options.traceConfig, tracerProvider: provider, exclusiveProcessor: false, attachmentManager: options.attachmentManager }); + instance = new Instrumentor({ instrumentationConfig: { enabled: false }, traceConfig: options.traceConfig, tracerProvider: provider, exclusiveProcessor: false, attachments: options.attachmentManager }); const known = (instance._modules ?? []).map((definition: any) => ({ definition, version: moduleVersion(definition.name) })).filter((item: any) => item.version); if (known.length && known.every(({ definition, version }: any) => !definition.supportedVersions.some((range: string) => satisfies(version, range, { includePrerelease: definition.includePrerelease })))) { instance.disable(); status.status = "failed"; status.reason = "Installed target version is outside the instrumentor's supported range"; diff --git a/typescript/packages/traceai_auto/src/wire/attachments.ts b/typescript/packages/traceai_auto/src/wire/attachments.ts index 7171a9697..1fa6ddd5c 100644 --- a/typescript/packages/traceai_auto/src/wire/attachments.ts +++ b/typescript/packages/traceai_auto/src/wire/attachments.ts @@ -1,6 +1,8 @@ import { createContextKey, trace, type Context, type Span } from "@opentelemetry/api"; -import type { AttachmentCorrelation, AttachmentManager, AttachmentOptions, AttachmentCapturePolicy } from "@traceai/fi-core/attachments"; -import type { FetchMediaObserver } from "./fetch-runtime"; +import type { + AttachmentCapturePolicy, AttachmentCorrelation, AttachmentManager, AttachmentPurpose, AttachmentStream, +} from "@traceai/fi-core/attachments"; +import type { FetchMediaObserver, FetchMediaSink } from "./fetch-runtime"; const MEDIA_POLICY = createContextKey("traceai.auto.attachment_policy.v1"); /** Supply the integration's capture restrictions; inherited denials always win. */ @@ -18,30 +20,52 @@ export function withAttachmentCapturePolicy(parent: Context, policy: AttachmentC return parent.setValue(MEDIA_POLICY, Object.freeze(merged)); } +// `createContextKey` is `Symbol.for`, so adapters in other packages (for example the ElevenLabs +// instrumentation) can set the same marker without importing auto. const MEDIA_OWNER = createContextKey("traceai.auto.attachment_owner.v1"); /** An adapter that captures media owns it independently of semantic span coverage. */ export function withAttachmentOwnership(parent: Context): Context { return parent.setValue(MEDIA_OWNER, true); } const IMAGE_MIMES = new Set(["image/png", "image/jpeg", "image/webp"]); const AUDIO_MIMES = new Set(["audio/wav", "audio/ogg", "audio/mpeg", "audio/mp4", "audio/webm", "audio/flac"]); -// Bounded transient decoder scratch, not a second media queue. -const MAX_BASE64 = 4 * Math.ceil(256 * 1024 / 3); +/** Consumed incremental audio is reserved up front; an object larger than this is discarded whole (too_large). */ +export const WIRE_STREAM_MAX_BYTES = 4 * 1024 * 1024; +const WIRE_STREAM_FALLBACK_BYTES = 1024 * 1024; +/** Request JSON is parsed only from the wire path's own bounded copy, and only up to this size. */ +const MAX_JSON_BYTES = 4 * 1024 * 1024; -type Operation = "speech" | "transcription" | "image_edit" | "image_generation"; +type Operation = "speech" | "transcription" | "image_edit" | "image_generation" | "chat" | "responses"; +/** Allowlisted protocol entries: exact HTTPS origin and path. A compatible path on another host is not authority to capture. */ +export const WIRE_MEDIA_ROUTES: ReadonlyArray> = Object.freeze([ + { host: "api.openai.com", path: "/v1/audio/speech", operation: "speech" }, + { host: "api.openai.com", path: "/v1/audio/transcriptions", operation: "transcription" }, + { host: "api.openai.com", path: "/v1/audio/translations", operation: "transcription" }, + { host: "api.openai.com", path: "/v1/images/edits", operation: "image_edit" }, + { host: "api.openai.com", path: "/v1/images/variations", operation: "image_edit" }, + { host: "api.openai.com", path: "/v1/images/generations", operation: "image_generation" }, + { host: "api.openai.com", path: "/v1/chat/completions", operation: "chat" }, + { host: "api.openai.com", path: "/v1/responses", operation: "responses" }, + { host: "api.deepgram.com", path: "/v1/speak", operation: "speech" }, +].map(route => Object.freeze(route)) as { host: string; path: string; operation: Operation }[]); function operation(method: string, url: string): Operation | undefined { try { const u = new URL(url); - // A compatible path on an unknown host is not authority to capture media. if (method.toUpperCase() !== "POST" || u.protocol !== "https:" || u.username || u.password || (u.port && u.port !== "443")) return; - if (u.hostname === "api.openai.com") { - if (u.pathname === "/v1/audio/speech") return "speech"; - if (["/v1/audio/transcriptions", "/v1/audio/translations"].includes(u.pathname)) return "transcription"; - if (["/v1/images/edits", "/v1/images/variations"].includes(u.pathname)) return "image_edit"; - if (u.pathname === "/v1/images/generations") return "image_generation"; - } - if (u.hostname === "api.deepgram.com" && u.pathname === "/v1/speak") return "speech"; + return WIRE_MEDIA_ROUTES.find(route => route.host === u.hostname && route.path === u.pathname)?.operation; } catch { /* Unknown destinations never acquire media ownership. */ } } +type OpenAIAdapter = typeof import("@traceai/openai").captureOpenAIAttachments; +let openAIAdapter: OpenAIAdapter | null | undefined; +/** Loaded only when attachments are active, so a disabled auto loads no extra instrumentation module. */ +function captureOpenAIAttachments(...args: Parameters): void { + if (openAIAdapter === undefined) { + try { openAIAdapter = (require("@traceai/openai") as typeof import("@traceai/openai")).captureOpenAIAttachments ?? null; } + catch { openAIAdapter = null; } + } + // Without the shared provider adapter (an older @traceai/openai), JSON media is not captured: no private parser. + if (typeof openAIAdapter === "function") openAIAdapter(...args); +} +function mediaType(value: string): string { return value.split(";")[0].trim().toLowerCase(); } function modelSpan(parent: Context): Span | undefined { const span = trace.getSpan(parent); // Do not relink wire media to an unrelated ambient HTTP/tool/root span. @@ -50,77 +74,114 @@ function modelSpan(parent: Context): Span | undefined { const kind = attributes?.["gen_ai.span.kind"] ?? attributes?.["fi.span.kind"]; return span?.isRecording() && ["LLM", "EMBEDDING"].includes(String(kind)) ? span : undefined; } +/** True when the manager could admit anything at all. A disabled (default) manager makes auto behave exactly as before. */ +export function attachmentsActive(manager: AttachmentManager | undefined): manager is AttachmentManager { + try { return !!manager && manager.status().state !== "disabled"; } catch { return false; } +} +function parseJson(bytes: Uint8Array | undefined): unknown { + if (!bytes || !bytes.byteLength || bytes.byteLength > MAX_JSON_BYTES) return undefined; + try { return JSON.parse(new TextDecoder().decode(bytes)); } catch { return undefined; } +} +function imageFormat(value: unknown): "png" | "jpeg" | "webp" | undefined { + return value === "png" || value === "jpeg" || value === "webp" ? value : undefined; +} /** - * Observe only immutable inputs or results the caller has already materialized. - * No reader, clone, tee, filesystem, URL retrieval, recorder or private uploader. - * Incremental/encoded streaming waits for core's admission/reservation API. + * Wire media observation for allowlisted protocol entries (A03/A04). + * + * - Capture happens only against a REAL span: a recording native model span whose integration supplied an explicit + * capture policy, or the wire span itself once it is emitted (`bind`). The ambient parent is never used. + * - Every source passes `manager.preflight` (the full deny-wins lattice plus budget) BEFORE it is read, copied or + * decoded. JSON media goes through the OpenAI provider adapter (#682), so an SDK span and a wire observation of the + * same span and position share one occurrence. + * - No reader, clone, tee, filesystem, URL retrieval or recorder: only values and chunks the application consumed. + * - Before the wire span exists, nothing is retained: wire-only FormData and wire-only incremental reads are not + * captured (`correlation_unavailable`). */ export function wireAttachmentObserver(manager: AttachmentManager, parent: Context, method: string, url: string): FetchMediaObserver | undefined { - if (parent.getValue(MEDIA_OWNER) || manager.status().state !== "queued") return; + if (parent.getValue(MEDIA_OWNER) || manager.status().state !== "ready") return; const op = operation(method, url); if (!op) return; - let correlation: AttachmentCorrelation | undefined; const native = modelSpan(parent); const capture = parent.getValue(MEDIA_POLICY) as AttachmentCapturePolicy | undefined; // An external native instrumentor can have stricter per-call privacy than auto. // It must supply its policy, not silently inherit an unrelated manager opt-in. if (native && !capture) return; - if (native) correlation = manager.correlation(native); - let outputObserved = false; - const attachBlob = (blob: Blob, options: AttachmentOptions) => { - if (correlation) manager.attach(correlation, { kind: "Blob", data: blob }, { ...options, capture: { ...capture, ...options.capture } }); + let correlation: AttachmentCorrelation | undefined = native ? manager.correlation(native) : undefined; + let outputObserved = false, inputObserved = false, format: ReturnType; + + /** Preflight at the exact size, then attach. The source is touched only after admission. */ + const attachExact = (size: number, mimeType: string, direction: "input" | "output", purpose: AttachmentPurpose, + source: () => { kind: "Bytes"; data: Uint8Array } | { kind: "Blob"; data: Blob }, sourceKind: "Bytes" | "Blob") => { + if (!correlation || !(size > 0)) return; + const ticket = manager.preflight(correlation, { mimeType, direction, purpose, maxBytes: size, sourceKind, capture: { ...capture, ...(direction === "input" ? { modelInput: true } : {}) } }); + if (!ticket.admitted) return; + try { ticket.attach(source() as never); } catch { ticket.release(); } + }; + const captureJsonInput = (body: unknown) => { + if (inputObserved || !correlation || !body || typeof body !== "object") return; + inputObserved = true; + format = imageFormat((body as Record).output_format); + if (op === "chat" || op === "responses") captureOpenAIAttachments(manager, correlation, body, { operation: op, direction: "input", capture }); + }; + const formData = (body: FormData) => { + let entries = 0; + for (const [name, value] of body.entries()) { + if (++entries > 64) break; + if (typeof value === "string") { if (name === "output_format") format = imageFormat(value); continue; } + const type = mediaType(value.type); + if (op === "transcription" && name === "file" && AUDIO_MIMES.has(type)) { + attachExact(value.size, type, "input", "model_input", () => ({ kind: "Blob", data: value }), "Blob"); + } else if (op === "image_edit" && ["image", "image[]", "mask"].includes(name) && IMAGE_MIMES.has(type)) { + attachExact(value.size, type, "input", "model_input", () => ({ kind: "Blob", data: value }), "Blob"); + } + } }; return { - bind(span) { + bind(span, request) { // The emitter supplies its actual identity before ending, never a later ambient span. - if (!correlation) correlation = manager.correlation(span); + if (correlation) return; + correlation = manager.correlation(span); + // JSON input uses the wire path's own bounded copy; nothing was retained for media before this point. + if (op === "chat" || op === "responses" || op === "image_generation") captureJsonInput(parseJson(request?.body)); }, request(body) { - // A wire-only span is assigned late. Do not retain a pre-admission FormData. - if (!correlation || !(body instanceof FormData)) return; - let entries = 0; - for (const [name, value] of body.entries()) { - if (++entries > 64) break; - if (!(value instanceof Blob)) continue; - if (op === "transcription" && name === "file" && AUDIO_MIMES.has(value.type)) { - attachBlob(value, { mimeType: value.type, direction: "input", purpose: "model_input" }); - } else if (op === "image_edit" && ["image", "image[]", "mask"].includes(name) && IMAGE_MIMES.has(value.type)) { - attachBlob(value, { mimeType: value.type, direction: "input", purpose: "model_input", capture: { modelInput: true } }); - } + // Wire-only: the span does not exist yet, so FormData is not retained until a late span (fail closed). + if (!correlation) return; + if (body instanceof FormData && (op === "transcription" || op === "image_edit")) formData(body); + else if (typeof body === "string" && (op === "chat" || op === "responses" || op === "image_generation") && body.length <= MAX_JSON_BYTES) { + try { captureJsonInput(JSON.parse(body)); } catch { /* Not JSON: nothing to capture. */ } } }, + consumed(mimeType, successful, contentLength): FetchMediaSink | undefined { + // Incremental reads need the target before the first chunk: only a native span qualifies (A04). + const type = mediaType(mimeType); + if (outputObserved || !correlation || !successful || op !== "speech" || !AUDIO_MIMES.has(type)) return; + const open = (maxBytes: number) => manager.startAttachmentStream(correlation!, { mimeType: type, direction: "output", purpose: "model_output", maxBytes, capture }); + const exact = contentLength !== undefined && contentLength > 0 && contentLength <= WIRE_STREAM_MAX_BYTES ? contentLength : undefined; + let stream: AttachmentStream = open(exact ?? WIRE_STREAM_MAX_BYTES); + // The negotiated upload limit or the owned budget can be below the default reservation. + if (!exact && (stream.reason === "too_large" || stream.reason === "queue_full")) stream = open(WIRE_STREAM_FALLBACK_BYTES); + if (stream.reason !== "queued") return; + outputObserved = true; + return { write: chunk => stream.tryWrite(chunk), end: () => { stream.finish(); }, abort: () => stream.abort("incomplete") }; + }, materialized(value, methodName, mimeType, successful) { if (outputObserved || !correlation || !successful) return; - if (op === "speech" && AUDIO_MIMES.has(mimeType)) { + const type = mediaType(mimeType); + if (op === "speech" && AUDIO_MIMES.has(type)) { if (methodName === "blob" && value instanceof Blob) { outputObserved = true; - attachBlob(value, { mimeType, direction: "output", purpose: "model_output" }); + attachExact(value.size, type, "output", "model_output", () => ({ kind: "Blob", data: value }), "Blob"); } else if (methodName === "arrayBuffer" && value instanceof ArrayBuffer) { outputObserved = true; - // No copy/view is constructed until the shared policy gate permits access. - manager.attach(correlation, { kind: "Bytes", get data() { return new Uint8Array(value); } }, - { mimeType, direction: "output", purpose: "model_output", capture }); + // The view (and the core's synchronous copy) exists only after admission. + attachExact(value.byteLength, type, "output", "model_output", () => ({ kind: "Bytes", data: new Uint8Array(value) }), "Bytes"); } - } else if ((op === "image_generation" || op === "image_edit") && methodName === "json" && mimeType === "application/json") { + } else if (methodName === "json" && type === "application/json" && (op === "image_generation" || op === "image_edit" || op === "responses")) { outputObserved = true; - // Typed OpenAI response only, never recursively scrape arbitrary JSON. - if (!value || typeof value !== "object") return; - const images: unknown = Object.getOwnPropertyDescriptor(value, "data")?.value; - if (!Array.isArray(images)) return; - for (let index = 0; index < Math.min(images.length, 64); index++) { - const item: unknown = images[index]; - if (!item || typeof item !== "object") continue; - const encoded: unknown = Object.getOwnPropertyDescriptor(item, "b64_json")?.value; - if (typeof encoded !== "string") continue; // Remote URLs are NOT retrieved. - manager.attach(correlation, { kind: "Bytes", get data() { - // Decoding and byte inspection are behind server/local privacy admission. - if (!encoded.length || encoded.length > MAX_BASE64 || encoded.length % 4 || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); - const bytes = Buffer.from(encoded, "base64"); - // This operation only qualifies PNG. Other formats remain unsupported. - return bytes.length <= 256 * 1024 && bytes.length >= 8 && bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) ? bytes : new Uint8Array(); - } }, { mimeType: "image/png", direction: "output", purpose: "model_output", capture }); - } + // Typed provider adapter only (#682): no recursive scraping, no URL retrieval, decode after admission. + captureOpenAIAttachments(manager, correlation, value, { operation: op === "responses" ? "responses" : "images", direction: "output", capture, imageFormat: format }); } }, }; diff --git a/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts b/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts index abd470389..0b7f97532 100644 --- a/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts +++ b/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts @@ -3,9 +3,14 @@ import { encodeBytes, byteLength, concatBytes } from "./bytes"; import { BodyDecoder, decodeResponse, responseEncoding } from "./encoding"; import { streamingMilestones } from "./stream-timing"; import type { Span } from "@opentelemetry/api"; +/** Receives only chunks the application already consumed; never pulls. */ +export interface FetchMediaSink { write(chunk: Uint8Array): boolean; end(): void; abort(): void } export interface FetchMediaObserver { - bind(span: Span): void; + /** The wire span, before it ends, with the wire path's own bounded request copy. */ + bind(span: Span, request?: RequestData): void; request(body: BodyInit | null | undefined): void; + /** The application started incremental consumption of a response body (reader or async iterator). */ + consumed?(mimeType: string, successful: boolean, contentLength: number | undefined): FetchMediaSink | undefined; materialized(value: unknown, method: "arrayBuffer" | "blob" | "json", mimeType: string, successful: boolean): void; } export interface FetchContext { @@ -103,6 +108,58 @@ export function createFetchHook(max: number, emit: (result: SpanResult, paren const observe = scope.observe ?? tap; const pending = new Set>(); let active = true; + /** + * A04: forward chunks the application reads from THIS response body (reader `read()` or async-iterator `next()`) + * to the media sink, after the application's own promise fulfils. No read, clone, tee or pipe is ever issued here. + * The sink opens only when the application starts consuming; piping, teeing or a clone of the body is unsupported + * and aborts the object, so nothing partial is uploaded as if complete. + */ + function observeConsumption(response: Response, observer: FetchMediaObserver, mimeType: string, responseHeaders: Record): void { + let stream: ReadableStream | null; + try { stream = response.body; } catch { return; } + if (!stream) return; + const body = stream; + let sink: FetchMediaSink | undefined, state: "idle" | "open" | "closed" = "idle"; + const length = responseHeaders["content-length"]; + const declared = length && /^\d+$/.test(length) && !header(responseHeaders, "content-encoding") ? Number(length) : undefined; + const open = () => { + if (state !== "idle" || !active) return; + try { sink = observer.consumed!(mimeType, response.ok, declared); } catch { sink = undefined; } + state = sink ? "open" : "closed"; + }; + const close = (complete: boolean) => { + if (state !== "open") { state = "closed"; return; } + state = "closed"; + try { if (complete && active) sink!.end(); else sink!.abort(); } catch { /* Media never reaches the application. */ } + }; + const read = (item: ReadableStreamReadResult) => { + if (state !== "open") return; + if (item.done) { close(true); return; } + if (!(item.value instanceof Uint8Array) || !active) { close(false); return; } + // The core copies synchronously, so a reused BYOB buffer cannot change uploaded bytes. + try { if (!sink!.write(item.value)) close(false); } catch { close(false); } + }; + const readers = new WeakSet(); + observeMethod(body, "getReader", reader => { + if (readers.has(reader)) return; readers.add(reader); + open(); + observeMethod(reader, "read", promise => observe(promise, read, () => close(false))); + observeMethod(reader, "cancel", () => close(false)); + observeMethod(reader, "releaseLock", () => close(false)); + }); + const iterator = (it: any) => { + if (readers.has(it)) return; readers.add(it); + open(); + observeMethod(it, "next", promise => observe(promise, read, () => close(false))); + // An early return (break) leaves the object incomplete, cancelled or not. + observeMethod(it, "return", () => close(false)); + }; + observeMethod(body, "values", it => iterator(it)); + observeMethod(body, Symbol.asyncIterator, it => iterator(it)); + observeMethod(body, "cancel", () => close(false)); + for (const name of ["tee", "pipeTo", "pipeThrough"]) observeMethod(body, name, () => close(false)); + observeMethod(response, "clone", () => close(false)); + } const patched = new Proxy(original, { apply(target, receiver, argArray: [RequestInfo | URL, RequestInit?]) { if (!active) return Reflect.apply(target, receiver, argArray); @@ -159,12 +216,14 @@ export function createFetchHook(max: number, emit: (result: SpanResult, paren const decorate = (response: Response) => { if (observed.has(response)) return; observed.add(response); - const mimeType = header(headersOf(response.headers), "content-type"); + const responseHeaders = headersOf(response.headers); + const mimeType = header(responseHeaders, "content-type"); for (const method of ["arrayBuffer", "blob", "json"] as const) { observeMethod(response, method, promise => observe(promise, value => { if (active) observer.materialized(value, method, mimeType, response.ok); }, () => {})); } + if (observer.consumed) observeConsumption(response, observer, mimeType, responseHeaders); // Only follow clones explicitly created by the application. Never call clone ourselves. observeMethod(response, "clone", clone => decorate(clone)); }; @@ -179,7 +238,7 @@ export function createFetchHook(max: number, emit: (result: SpanResult, paren if (finished) return; finished = true; const complete = () => { if (audio) { response.audioOutputBytes = audio.bytes; response.body = undefined; } - try { const span = build(matched, request, response, parsed); if (span) emit(span, parent, start, end, media ? value => media!.bind(value) : undefined); } catch { /* Capture must not reach application code. */ } + try { const span = build(matched, request, response, parsed); if (span) emit(span, parent, start, end, media ? value => media!.bind(value, request) : undefined); } catch { /* Capture must not reach application code. */ } finally { resolve(); pending.delete(done); } }; // Only completed responses awaiting bounded decoding are flushable, never unread streams. diff --git a/typescript/packages/traceai_auto/src/wire/fetch.ts b/typescript/packages/traceai_auto/src/wire/fetch.ts index bc6a4d873..578af28e3 100644 --- a/typescript/packages/traceai_auto/src/wire/fetch.ts +++ b/typescript/packages/traceai_auto/src/wire/fetch.ts @@ -3,7 +3,7 @@ import { autoModelResponsibility, autoModelWireAllowed, hasAutoModelCoverage } f import { isTracingSuppressed } from "@opentelemetry/core"; import { createFetchHook } from "./fetch-runtime"; import type { AttachmentManager } from "@traceai/fi-core/attachments"; -import { wireAttachmentObserver } from "./attachments"; +import { attachmentsActive, wireAttachmentObserver } from "./attachments"; import { decodeNodeBody } from "./encoding-node"; import { observePromise, preparePromiseObservation } from "./promise-node"; import type { SpanResult } from "./core"; @@ -39,8 +39,11 @@ export function shouldSkip(ctx: Context, consultOperation = true): boolean { } export function patchFetch(max: number, emit: (result: SpanResult, parent: Context, start: number, end: number, bindMedia?: (span: Span) => void) => void, secrets: string[] = [], attachments?: AttachmentManager) { preparePromiseObservation(); - return createFetchHook(max, emit, { active: () => context.active(), skip: parent => attachments ? isTracingSuppressed(parent) || !!parent.getValue(IN_FLIGHT) : shouldSkip(parent, false), - media: attachments ? (parent, method, url) => wireAttachmentObserver(attachments, parent, method, url) : undefined, + // With attachments OFF (the default) both decisions are exactly the pre-attachment ones. When ON, a call that + // native coverage keeps out of wire spans still reaches media observation; span capture keeps its own gate below. + return createFetchHook(max, emit, { active: () => context.active(), + skip: parent => attachmentsActive(attachments) ? isTracingSuppressed(parent) || !!parent.getValue(IN_FLIGHT) : shouldSkip(parent, false), + media: attachments ? (parent, method, url) => attachmentsActive(attachments) ? wireAttachmentObserver(attachments, parent, method, url) : undefined : undefined, // Same context decision, now after supported URL matching. Only the emit // completion boundary can commit wire; credential observation stays separate. capture: parent => wireAllowed(parent, "request"), observe: observePromise, observeRequest: observeOutgoing, diff --git a/typescript/packages/traceai_auto/test/attachment-lifecycle.test.ts b/typescript/packages/traceai_auto/test/attachment-lifecycle.test.ts index ddd12d76b..a8fcefab1 100644 --- a/typescript/packages/traceai_auto/test/attachment-lifecycle.test.ts +++ b/typescript/packages/traceai_auto/test/attachment-lifecycle.test.ts @@ -25,10 +25,10 @@ function run(mode: string, preload?: string) { else assert.equal(selected, provider.attachments); assert.equal(await selected.ready(100), false); assert.equal(selected.status().state, 'disabled'); - assert.equal(auto.report().attachments.integration, 'component_only'); + assert.equal(auto.report().attachments.state, 'disabled'); assert.equal(auto.report().attachments.transport, 'shared_manager'); await auto.forceFlush(); await auto.shutdown(); await auto.shutdown(); - assert.equal(selected.status().state, ${JSON.stringify(mode)} === 'owned' ? 'incomplete' : 'disabled'); + assert.equal(selected.status().state, 'disabled'); assert.equal(network, 0); await manager.shutdown(); await host.shutdown(); process.stdout.write('LIFECYCLE_OK'); diff --git a/typescript/packages/traceai_auto/test/attachments.test.ts b/typescript/packages/traceai_auto/test/attachments.test.ts index 987c09e9d..55d067e45 100644 --- a/typescript/packages/traceai_auto/test/attachments.test.ts +++ b/typescript/packages/traceai_auto/test/attachments.test.ts @@ -1,46 +1,67 @@ -import { createServer, type Server } from "node:http"; +import { createServer, request as httpRequest, type Server } from "node:http"; import { AddressInfo } from "node:net"; +import { randomUUID } from "node:crypto"; import { context, trace, ROOT_CONTEXT } from "@opentelemetry/api"; import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; -import { AttachmentManager, type AttachmentManagerOptions, type AttachmentTransport } from "@traceai/fi-core/attachments"; +import { AttachmentManager, createAttachmentHttpTransport, type AttachmentHttpClient, type AttachmentManagerOptions } from "@traceai/fi-core/attachments"; +import { captureOpenAIAttachments } from "@traceai/openai"; import { patchFetch } from "../src/wire/fetch"; import { withAttachmentOwnership, withAttachmentCapturePolicy } from "../src/wire/attachments"; import { attachmentOptions, selectAttachmentManager, attachmentReport } from "../src/attachments"; import { resolveConfig } from "../src/config"; -// Synthetic media/credentials only. Both provider and attachment traffic terminate on loopback. +// Synthetic media/credentials only. Provider and attachment traffic both terminate on loopback. const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64"); -function wav() { - const b = Buffer.alloc(44 + 160); b.write("RIFF"); b.writeUInt32LE(b.length - 8, 4); b.write("WAVEfmt ", 8); +function wav(samples = 80) { + const b = Buffer.alloc(44 + samples * 2); b.write("RIFF"); b.writeUInt32LE(b.length - 8, 4); b.write("WAVEfmt ", 8); b.writeUInt32LE(16, 16); b.writeUInt16LE(1, 20); b.writeUInt16LE(1, 22); b.writeUInt32LE(8000, 24); - b.writeUInt32LE(16000, 28); b.writeUInt16LE(2, 32); b.writeUInt16LE(16, 34); b.write("data", 36); b.writeUInt32LE(160, 40); return b; + b.writeUInt32LE(16000, 28); b.writeUInt16LE(2, 32); b.writeUInt16LE(16, 34); b.write("data", 36); b.writeUInt32LE(samples * 2, 40); + for (let i = 0; i < samples; i++) b.writeInt16LE(Math.round(8000 * Math.sin(i / 3)), 44 + i * 2); + return b; } const audio = wav(); -const uuid = "11111111-1111-4111-8111-111111111111"; let original: typeof fetch, server: Server, origin: string, manager: AttachmentManager, hook: ReturnType | undefined; -let uploads: Buffer[], requests: any[], transfers: { path: string; body: Buffer }[], providerBodies: Buffer[]; +let uploads: Buffer[], requests: any[], transfers: string[], providerBodies: Buffer[], providerRequests: number; let provider: BasicTracerProvider, exporter: InMemorySpanExporter; -let responseStatus: number, responseData: Buffer, responseType: string; +let responseStatus: number, responseData: Buffer, responseType: string, responseChunks: number; const nativeFetch = globalThis.fetch; -const sessionPolicy = { version: "1", enabled: true, allowed_sources: ["Bytes", "Blob"] as ("Bytes" | "Blob")[], allowed_modalities: ["image", "audio"] as ("image" | "audio")[], require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false }; -async function createManager(options: AttachmentManagerOptions = {}, policy = sessionPolicy) { - const send = async (path: string, body: BodyInit) => { - const response = await nativeFetch(`${origin}${path}`, { method: "POST", body }); - return response.json(); - }; - const transport: AttachmentTransport = { - async bootstrap(request) { await send("/attachment-sessions", JSON.stringify(request)); return { schema_version: 1, scope: { organization_id: uuid, workspace_id: uuid, project_id: uuid }, capture_session_id: request.capture_session_id, policy, limits: { max_upload_bytes: 20971520, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }, expires_at: new Date(Date.now() + 60000).toISOString(), transfer_modes: ["authenticated_stream"] }; }, - async initialize(_session, request) { await send("/uploads", JSON.stringify(request)); requests.push(request); return { upload_id: uuid, attachment_id: request.attachment_id, occurrence_id: request.occurrence_id, state: "pending_upload", reserved_bytes: request.declared_bytes, expires_at: new Date(Date.now() + 60000).toISOString(), transfer_mode: "authenticated_stream", write_capability: "synthetic-capability" }; }, - async upload(_session, _upload, bytes) { await send("/bytes", bytes as BodyInit); }, - async complete() { await send("/complete", "{}"); }, - }; - const result = new AttachmentManager({ enabled: true, projectName: "attachment-test", projectType: "observe", environment: () => ({}), transport, ...options }); +const serverPolicy = { version: "fixture-v1", enabled: true, allowed_sources: ["Bytes", "Blob"], allowed_modalities: ["image", "audio", "document", "file"], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false }; +const limits = { max_upload_bytes: 20971520, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, + active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, + upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }; +let policy: Record = serverPolicy; +const scope = { organization_id: randomUUID(), workspace_id: randomUUID(), project_id: randomUUID() }; +// Sessions of one hour or more: the core renews within max(60 s, 20% of lifetime) of expiry. +const expiry = () => new Date(Date.now() + 3600000).toISOString(); +function control(req: import("node:http").IncomingMessage, body: Buffer, res: import("node:http").ServerResponse) { + transfers.push(req.url!); res.setHeader("content-type", "application/json"); + if (req.url!.endsWith("attachment-sessions")) { + const parsed = JSON.parse(body.toString()); + res.end(JSON.stringify({ schema_version: 1, scope, capture_session_id: parsed.capture_session_id, policy, limits, expires_at: expiry(), transfer_modes: ["authenticated_stream"] })); + } else if (req.url!.endsWith("/uploads")) { + const parsed = JSON.parse(body.toString()); requests.push(parsed); res.statusCode = 201; + res.end(JSON.stringify({ upload_id: parsed.attachment_id, attachment_id: parsed.attachment_id, occurrence_id: parsed.occurrence_id, state: "pending_upload", + reserved_bytes: parsed.declared_bytes, expires_at: expiry(), transfer_mode: "authenticated_stream", write_capability: "synthetic-upload-only" })); + } else if (req.url!.endsWith("/content")) { + uploads.push(body); res.end(JSON.stringify({ upload_id: req.url!.split("/").at(-2), state: "pending_upload", reason: "queued" })); + } else { res.statusCode = 202; res.end(JSON.stringify({ upload_id: req.url!.split("/").at(-2), state: "validating", reason: "binding_pending" })); } +} +const client: AttachmentHttpClient = req => new Promise((resolve, reject) => { + const outgoing = httpRequest(`${origin}${new URL(req.url).pathname}`, { method: req.method, headers: { ...req.headers, "x-attachment-control": "1" }, signal: req.signal }, response => { + const chunks: Buffer[] = []; response.on("data", c => chunks.push(c)); response.on("end", () => resolve({ status: response.statusCode!, headers: {}, body: Buffer.concat(chunks) } as any)); + }); outgoing.on("error", reject); outgoing.end(req.body as any); +}); +async function createManager(options: AttachmentManagerOptions = {}, serverOverrides: Record = {}) { + policy = { ...serverPolicy, ...serverOverrides }; + const result = new AttachmentManager({ enabled: true, projectName: "attachment-test", projectType: "observe", environment: () => ({}), + transport: createAttachmentHttpTransport({ endpoint: "https://attachments.example", headers: { authorization: "Bearer synthetic" }, client }), ...options }); expect(await result.ready()).toBe(options.enabled !== false); - expect(result.status().state).toBe(options.enabled === false ? "disabled" : "queued"); + expect(result.status().state).toBe(options.enabled === false ? "disabled" : "ready"); return result; } -function install() { +function install(target = manager) { globalThis.fetch = (input: RequestInfo | URL, init?: RequestInit) => { const u = new URL(input instanceof Request ? input.url : String(input)); return nativeFetch(`${origin}${u.pathname}${u.search}`, init); @@ -48,30 +69,37 @@ function install() { hook = patchFetch(4194304, (result, parent, start, end, bindMedia) => { const span = provider.getTracer("wire-test").startSpan(result.name, { startTime: start, attributes: result.attributes }, parent); bindMedia?.(span); span.end(end); - }, [], manager); + }, [], target); } -function inNative(fn: () => T, kind = "LLM"): T { +let nativeSpanId = ""; +function inNative(fn: () => T, kind = "LLM", policyContext = true): T { const span = provider.getTracer("native-test").startSpan("native", { attributes: { "gen_ai.span.kind": kind } }); - const result = context.with(withAttachmentCapturePolicy(trace.setSpan(ROOT_CONTEXT, span), { captureContent: true }), fn); + nativeSpanId = span.spanContext().spanId; + const parent = trace.setSpan(ROOT_CONTEXT, span); + const result = context.with(policyContext ? withAttachmentCapturePolicy(parent, { captureContent: true }) : parent, fn); // Promise work retains this exact request context; do not consult later ambient spans. if (result instanceof Promise) return result.finally(() => span.end()) as T; span.end(); return result; } async function settle() { await manager.flush(2000); } +const speech = (route = "api.openai.com/v1/audio/speech") => fetch(`https://${route}`, { method: "POST", body: JSON.stringify({ model: "tts-1", input: "synthetic", voice: "alloy" }) }); beforeAll(() => context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable())); afterAll(() => context.disable()); beforeEach(async () => { - original = globalThis.fetch; uploads = []; requests = []; transfers = []; providerBodies = []; - responseStatus = 200; responseData = audio; responseType = "audio/wav"; + original = globalThis.fetch; uploads = []; requests = []; transfers = []; providerBodies = []; providerRequests = 0; + responseStatus = 200; responseData = audio; responseType = "audio/wav"; responseChunks = 1; server = createServer((req, res) => { const chunks: Buffer[] = []; req.on("data", chunk => chunks.push(chunk)); req.on("end", () => { const body = Buffer.concat(chunks); - if (["/attachment-sessions", "/uploads", "/bytes", "/complete"].includes(req.url!)) { - transfers.push({ path: req.url!, body }); if (req.url === "/bytes") uploads.push(body); - res.writeHead(200, { "content-type": "application/json" }); res.end("{}"); - } else { providerBodies.push(body); res.writeHead(responseStatus, { "content-type": responseType }); res.end(responseData); } + if (req.headers["x-attachment-control"]) { control(req, body, res); return; } + providerRequests++; providerBodies.push(body); + res.writeHead(responseStatus, { "content-type": responseType }); + // Several network chunks, so incremental readers see more than one read. + const size = Math.ceil(responseData.length / responseChunks); + for (let i = 0; i < responseData.length; i += size) res.write(responseData.subarray(i, i + size)); + res.end(); }); }); await new Promise(done => server.listen(0, "127.0.0.1", done)); origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; @@ -83,141 +111,274 @@ afterEach(async () => { await manager.shutdown(1000); await provider.shutdown(); server.closeAllConnections(); await new Promise(done => server.close(() => done())); }); +const sessionsOnly = () => transfers.filter(path => !path.endsWith("attachment-sessions")); -test.each(["arrayBuffer", "blob"] as const)("caller-consumed speech %s uploads once with the actual wire identity", async method => { - install(); const response = await fetch("https://api.openai.com/v1/audio/speech", { method: "POST", body: JSON.stringify({ model: "tts-1", input: "synthetic", voice: "alloy" }) }); - const result = await response[method](); - expect(Buffer.from(result instanceof Blob ? await result.arrayBuffer() : result)).toEqual(audio); - await settle(); expect(uploads).toEqual([audio]); - const span = exporter.getFinishedSpans()[0]; expect(requests[0].association.parent.span_id).toBe(span.spanContext().spanId); - expect(span.attributes["fi.attachment.correlation_nonce"]).toBeTruthy(); - // Completion never mutates an ended span; authoritative late links belong to the API. - expect(span.attributes["fi.attachment.refs"]).toBe("[]"); -}); -test("native semantic coverage leaves safe media observation on without a second span", async () => { - install(); await inNative(async () => { const response = await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" }); await response.arrayBuffer(); }); - await settle(); expect(uploads).toEqual([audio]); expect(exporter.getFinishedSpans()).toHaveLength(1); - expect(requests[0].association.parent.span_id).toBe(exporter.getFinishedSpans()[0].spanContext().spanId); -}); -test("adapter media ownership is independent and prevents duplicate uploads", async () => { - install(); await inNative(() => context.with(withAttachmentOwnership(context.active()), async () => { await (await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" })).arrayBuffer(); })); - await settle(); expect(uploads).toHaveLength(0); expect(exporter.getFinishedSpans()).toHaveLength(1); -}); -test.each(["api.openai.com/v1/audio/speech", "api.deepgram.com/v1/speak"])("recognized %s preserves cloned application reads and deduplicates output", async route => { - install(); await inNative(async () => { - const response = await fetch(`https://${route}`, { method: "POST" }); const clone = response.clone(); - const [one, two] = await Promise.all([response.arrayBuffer(), clone.blob()]); expect(Buffer.from(one)).toEqual(audio); expect(Buffer.from(await two.arrayBuffer())).toEqual(audio); - }); await settle(); expect(uploads).toEqual([audio]); -}); -test("unread, cancelled and reader-consumed responses do not create eager media readers", async () => { - install(); await inNative(async () => { - const unread = await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" }); - expect(unread.bodyUsed).toBe(false); await unread.body!.cancel(); - const consumed = await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" }); - const reader = consumed.body!.getReader(); const chunks = []; for (;;) { const x = await reader.read(); if (x.done) break; chunks.push(Buffer.from(x.value)); } - expect(Buffer.concat(chunks)).toEqual(audio); - }); await settle(); expect(uploads).toHaveLength(0); -}); -test("FormData keeps the body, boundary and filename on the provider path only", async () => { - responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); - const body = new FormData(); body.set("model", "whisper-1"); body.set("file", new Blob([audio], { type: "audio/wav" }), "PRIVATE_CANARY_FILENAME.wav"); - install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/audio/transcriptions", { method: "POST", body })).json(); }); - await settle(); expect(uploads).toEqual([audio]); expect(providerBodies[0].includes(audio)).toBe(true); expect(providerBodies[0].toString()).toContain("PRIVATE_CANARY_FILENAME.wav"); - expect(JSON.stringify(requests)).not.toContain("PRIVATE_CANARY"); expect(JSON.stringify(exporter.getFinishedSpans().map(s => s.attributes))).not.toContain("PRIVATE_CANARY"); - expect(body.get("file")).toBeInstanceOf(Blob); -}); -test("image edits accept only typed allowlisted FormData fields", async () => { - responseType = "application/json"; responseData = Buffer.from('{"data":[]}'); - const body = new FormData(); body.set("image", new Blob([png], { type: "image/png" })); body.set("untrusted_file", new Blob([png], { type: "image/png" })); - install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body })).json(); }); - await settle(); expect(uploads).toEqual([png]); -}); -test("wire-only FormData remains unsupported without retaining the source until a late span", async () => { - responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); - const body = new FormData(); body.set("model", "whisper-1"); body.set("file", new Blob([audio], { type: "audio/wav" })); - install(); await (await fetch("https://api.openai.com/v1/audio/transcriptions", { method: "POST", body })).json(); - await settle(); expect(uploads).toHaveLength(0); -}); -test("application-decoded PNG image JSON uploads bytes, not URLs or base64 span fields", async () => { - responseType = "application/json"; responseData = Buffer.from(JSON.stringify({ data: [{ b64_json: png.toString("base64") }, { url: "https://untrusted.invalid/private?token=CANARY" }] })); - install(); const body = await (await fetch("https://api.openai.com/v1/images/generations", { method: "POST", body: JSON.stringify({ model: "dall-e-3", prompt: "synthetic" }) })).json(); - expect(body.data[0].b64_json).toBe(png.toString("base64")); await settle(); expect(uploads).toEqual([png]); - expect(JSON.stringify(requests)).not.toContain("CANARY"); expect(JSON.stringify(exporter.getFinishedSpans().map(s => s.attributes))).not.toContain(png.toString("base64")); -}); -test.each([ - ["disabled", { enabled: false }], ["hide inputs", { traceConfig: { hideInputs: true } }], ["hide text", { traceConfig: { hideInputText: true } }], - ["hide images", { traceConfig: { hideInputImages: true } }], ["PII", { traceConfig: { piiRedaction: true } }], -] as [string, AttachmentManagerOptions][])("%s prevents image transfer at the network boundary", async (_name, options) => { - await manager.shutdown(); manager = await createManager(options); transfers = []; - responseType = "application/json"; responseData = Buffer.from('{"data":[]}'); - const body = new FormData(); body.set("image", new Blob([png], { type: "image/png" }), "PRIVATE_CANARY.png"); - install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body })).json(); }); - await settle(); expect(transfers).toEqual([]); -}); -test.each(["hideOutputs", "hideOutputMessages", "hideOutputText", "piiRedaction"] as const)("%s prevents output speech transfer", async flag => { - manager.updatePolicy({ traceConfig: { [flag]: true } }); transfers = []; - install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" })).arrayBuffer(); }); - await settle(); expect(transfers).toEqual([]); -}); -test("server-required redaction rejects decoded PNG before the base64 decoder", async () => { - await manager.shutdown(); manager = await createManager({}, { ...sessionPolicy, require_media_redaction: true }); transfers = []; - responseType = "application/json"; responseData = Buffer.from(JSON.stringify({ data: [{ b64_json: png.toString("base64") }] })); - install(); const decode = jest.spyOn(Buffer, "from"); - await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/generations", { method: "POST" })).json(); }); - const calls = decode.mock.calls.filter(args => args[0] === png.toString("base64") && (args as unknown[])[1] === "base64"); decode.mockRestore(); - await settle(); expect(calls).toHaveLength(0); expect(transfers).toEqual([]); -}); -test("unknown provider, non-audio MIME and error bodies do not upload", async () => { - install(); await inNative(async () => { - await (await fetch("https://unknown.invalid/v1/audio/speech", { method: "POST" })).arrayBuffer(); - responseType = "application/octet-stream"; await (await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" })).arrayBuffer(); - responseType = "audio/wav"; responseStatus = 403; await (await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" })).arrayBuffer(); - }); await settle(); expect(uploads).toHaveLength(0); -}); -test("oversized media, raw multipart, and ended/unrelated native contexts fail closed", async () => { - responseType = "application/json"; responseData = Buffer.from('{"data":[]}'); - const body = new FormData(); body.set("image", new Blob([new Uint8Array(20 * 1024 * 1024 + 1)], { type: "image/png" })); - install(); await inNative(async () => { - await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body })).json(); - await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", headers: { "content-type": "multipart/form-data; boundary=x" }, body: "--x\r\nPRIVATE_CANARY" })).json(); - }); - const small = new FormData(); small.set("image", new Blob([png], { type: "image/png" })); - await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body: small })).json(); }, "TOOL"); - await settle(); expect(uploads).toHaveLength(0); -}); -test("policy tightening between request and application consumption prevents capture", async () => { - install(); await inNative(async () => { - const response = await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" }); manager.updatePolicy({ enabled: false }); await response.arrayBuffer(); - }); await settle(); expect(uploads).toHaveLength(0); -}); -test("removal makes retained response wrappers inert", async () => { - install(); await inNative(async () => { const response = await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" }); hook!.remove(); await response.arrayBuffer(); }); - await settle(); expect(uploads).toHaveLength(0); +describe("A04 speech output", () => { + test.each(["arrayBuffer", "blob"] as const)("wire-only: application-materialized %s uploads once, bound to the actual wire span", async method => { + install(); const response = await speech(); + const result = await response[method](); + expect(Buffer.from(result instanceof Blob ? await result.arrayBuffer() : result)).toEqual(audio); + await settle(); expect(uploads).toEqual([audio]); + const span = exporter.getFinishedSpans()[0]; + expect(requests[0].association.parent.span_id).toBe(span.spanContext().spanId); + expect(requests[0]).toMatchObject({ direction: "output", purpose: "model_output", mime_type: "audio/wav", declared_bytes: audio.length }); + // The upload commits after the wire span ended: the authoritative link is association.parent, never a late mutation (T-CH-03). + expect(span.attributes["fi.attachment.refs"]).toBeUndefined(); + expect(manager.status()).toMatchObject({ openTickets: 0, openStreams: 0, ownedBytes: 0 }); + }); + test("native span + explicit policy: arrayBuffer is captured once from consumed chunks, with no second span", async () => { + responseChunks = 4; install(); + await inNative(async () => { await (await speech()).arrayBuffer(); }); + await settle(); expect(uploads).toEqual([audio]); expect(exporter.getFinishedSpans()).toHaveLength(1); + expect(requests[0].association.parent.span_id).toBe(nativeSpanId); + const refs = JSON.parse(String(exporter.getFinishedSpans()[0].attributes["fi.attachment.refs"])); + expect(refs).toEqual([expect.objectContaining({ occurrence_id: requests[0].occurrence_id, modality: "audio", direction: "output" })]); + }); + test("T-CS-01 native span: application reader consumption is observed with no extra read, clone or tee", async () => { + responseChunks = 5; install(); + const tee = jest.spyOn(ReadableStream.prototype, "tee"), clone = jest.spyOn(Response.prototype, "clone"); + const reads = jest.spyOn(ReadableStreamDefaultReader.prototype, "read"); + let appReads = 0; + await inNative(async () => { + const response = await speech(); const reader = response.body!.getReader(); const chunks: Buffer[] = []; + for (;;) { appReads++; const x = await reader.read(); if (x.done) break; chunks.push(Buffer.from(x.value)); } + expect(Buffer.concat(chunks)).toEqual(audio); + }); + const underlying = reads.mock.calls.length; + await settle(); + expect(tee).not.toHaveBeenCalled(); expect(clone).not.toHaveBeenCalled(); tee.mockRestore(); clone.mockRestore(); reads.mockRestore(); + expect(appReads).toBeGreaterThan(2); expect(underlying).toBe(appReads); expect(uploads).toEqual([audio]); + }); + test("native span: async-iterator consumption uploads once; a break before the end uploads nothing", async () => { + responseChunks = 4; install(); + await inNative(async () => { + const chunks: Buffer[] = []; for await (const chunk of (await speech()).body as any) chunks.push(Buffer.from(chunk)); + expect(Buffer.concat(chunks)).toEqual(audio); + }); + await settle(); expect(uploads).toEqual([audio]); + uploads = []; + await inNative(async () => { for await (const _chunk of (await speech()).body as any) break; }); + await settle(); expect(uploads).toHaveLength(0); expect(manager.status()).toMatchObject({ openStreams: 0, ownedBytes: 0 }); + }); + test("adapter media ownership is independent and prevents duplicate uploads", async () => { + install(); await inNative(() => context.with(withAttachmentOwnership(context.active()), async () => { await (await speech()).arrayBuffer(); })); + await settle(); expect(uploads).toHaveLength(0); expect(exporter.getFinishedSpans()).toHaveLength(1); + }); + test("the ElevenLabs marker (Symbol.for key, no auto import) is honoured", async () => { + install(); await inNative(() => context.with(context.active().setValue(Symbol.for("traceai.auto.attachment_owner.v1"), true), async () => { await (await speech()).arrayBuffer(); })); + await settle(); expect(uploads).toHaveLength(0); + }); + test.each(["api.openai.com/v1/audio/speech", "api.deepgram.com/v1/speak"])("recognized %s: cloned application reads upload once and stay byte-identical", async route => { + install(); await inNative(async () => { + const response = await speech(route); const copy = response.clone(); + const [one, two] = await Promise.all([response.arrayBuffer(), copy.blob()]); + expect(Buffer.from(one)).toEqual(audio); expect(Buffer.from(await two.arrayBuffer())).toEqual(audio); + }); await settle(); expect(uploads).toEqual([audio]); + }); + test("unread, cancelled and cancelled-midway bodies cause no eager read and upload nothing", async () => { + responseChunks = 4; install(); + await inNative(async () => { + const unread = await speech(); expect(unread.bodyUsed).toBe(false); await unread.body!.cancel(); + const partial = await speech(); const reader = partial.body!.getReader(); await reader.read(); await reader.cancel(); + }); + // Wire-only incremental reads are not captured (the wire span does not exist yet; nothing is retained). + const wireOnly = await speech(); const reader = wireOnly.body!.getReader(); while (!(await reader.read()).done) { /* app reads */ } + await settle(); expect(uploads).toHaveLength(0); expect(manager.status()).toMatchObject({ openStreams: 0, openTickets: 0, ownedBytes: 0 }); + }); + test("piped bodies are not captured", async () => { + install(); await inNative(async () => { + const response = await speech(); const chunks: Uint8Array[] = []; + await response.body!.pipeTo(new WritableStream({ write(chunk) { chunks.push(chunk); } })); + expect(Buffer.concat(chunks)).toEqual(audio); + }); await settle(); expect(uploads).toHaveLength(0); + }); + test("an object above the stream cap is discarded whole (too_large), never truncated", async () => { + responseData = Buffer.concat([audio, Buffer.alloc(4 * 1024 * 1024)]); responseChunks = 8; install(); + await inNative(async () => { const reader = (await speech()).body!.getReader(); while (!(await reader.read()).done) { /* app reads */ } }); + await settle(); expect(uploads).toHaveLength(0); expect(manager.status().ownedBytes).toBe(0); + }); }); -test("borrowed manager identity and restrictive policy survive auto setup", async () => { - const config = resolveConfig({ traceConfig: { hideOutputs: true }, projectName: "auto-project", configFile: "/nonexistent" }, {}); - const selected = selectAttachmentManager({ attachments: manager }, config, {}, false); - expect(selected.manager).toBe(manager); expect(selected.owned).toBe(false); - expect(attachmentOptions(config, { attachments: { projectName: "wrong-project", traceConfig: { hideOutputs: false } } })).toMatchObject({ projectName: "auto-project", traceConfig: { hideOutputs: true } }); - expect(attachmentReport(manager).integration).toBe("component_only"); - install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" })).arrayBuffer(); }); - await settle(); expect(uploads).toHaveLength(0); + +describe("A03 FormData and JSON media", () => { + test("native: transcription FormData keeps body, boundary and filename on the provider path only", async () => { + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); + const body = new FormData(); body.set("model", "whisper-1"); body.set("file", new Blob([audio], { type: "audio/wav" }), "PRIVATE_CANARY_FILENAME.wav"); + install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/audio/transcriptions", { method: "POST", body })).json(); }); + await settle(); expect(uploads).toEqual([audio]); expect(providerBodies[0].includes(audio)).toBe(true); expect(providerBodies[0].toString()).toContain("PRIVATE_CANARY_FILENAME.wav"); + expect(requests[0]).toMatchObject({ direction: "input", purpose: "model_input", source_kind: "Blob" }); + expect(JSON.stringify(requests)).not.toContain("PRIVATE_CANARY"); expect(JSON.stringify(exporter.getFinishedSpans().map(s => s.attributes))).not.toContain("PRIVATE_CANARY"); + expect(body.get("file")).toBeInstanceOf(Blob); + }); + test("native: image edits accept only typed allowlisted FormData fields", async () => { + responseType = "application/json"; responseData = Buffer.from('{"data":[]}'); + const body = new FormData(); body.set("image", new Blob([png], { type: "image/png" })); body.set("untrusted_file", new Blob([png], { type: "image/png" })); + install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body })).json(); }); + await settle(); expect(uploads).toEqual([png]); + }); + test("wire-only FormData is not captured, and its source is not retained until the late span", async () => { + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); + const body = new FormData(); body.set("model", "whisper-1"); body.set("file", new Blob([audio], { type: "audio/wav" })); + install(); await (await fetch("https://api.openai.com/v1/audio/transcriptions", { method: "POST", body })).json(); + await settle(); expect(uploads).toHaveLength(0); + }); + test("wire-only image JSON output uploads decoded bytes through the provider adapter, never URLs", async () => { + responseType = "application/json"; responseData = Buffer.from(JSON.stringify({ data: [{ b64_json: png.toString("base64") }, { url: "https://untrusted.invalid/private?token=CANARY" }] })); + install(); const body = await (await fetch("https://api.openai.com/v1/images/generations", { method: "POST", body: JSON.stringify({ model: "gpt-image-1", prompt: "synthetic" }) })).json(); + expect(body.data[0].b64_json).toBe(png.toString("base64")); await settle(); expect(uploads).toEqual([png]); + expect(providerRequests).toBe(1); + expect(requests[0]).toMatchObject({ direction: "output", purpose: "model_output", mime_type: "image/png", message_index: 0, part_index: 0 }); + expect(JSON.stringify(requests)).not.toContain("CANARY"); expect(JSON.stringify(exporter.getFinishedSpans().map(s => s.attributes))).not.toContain(png.toString("base64")); + }); + test("wire-only chat and responses JSON input: images and computer screenshots via the provider adapter", async () => { + responseType = "application/json"; responseData = Buffer.from('{"id":"synthetic","object":"chat.completion","choices":[]}'); + const uri = `data:image/png;base64,${png.toString("base64")}`; + install(); + await (await fetch("https://api.openai.com/v1/chat/completions", { method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-4o", messages: [{ role: "user", content: [{ type: "text", text: "synthetic" }, { type: "image_url", image_url: { url: uri } }] }] }) })).json(); + responseData = Buffer.from('{"id":"synthetic","object":"response","output":[]}'); + await (await fetch("https://api.openai.com/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "computer-use-preview", input: [{ type: "computer_call_output", call_id: "call_synthetic", output: { type: "computer_screenshot", image_url: uri } }] }) })).json(); + await settle(); expect(uploads).toEqual([png, png]); + expect(requests.map(r => r.purpose)).toEqual(["model_input", "computer_screenshot"]); + expect(requests[1].tool_call_id).toBe("call_synthetic"); + const spans = exporter.getFinishedSpans(); + expect(requests.map(r => r.association.parent.span_id)).toEqual(spans.map(s => s.spanContext().spanId)); + }); + test("a provider adapter capture on the same span and position is not uploaded again by wire (#682 registry)", async () => { + responseType = "application/json"; responseData = Buffer.from('{"id":"synthetic","object":"chat.completion","choices":[]}'); + const payload = { model: "gpt-4o", messages: [{ role: "user", content: [{ type: "image_url", image_url: { url: `data:image/png;base64,${png.toString("base64")}` } }] }] }; + install(); await inNative(async () => { + captureOpenAIAttachments(manager, trace.getSpan(context.active())!, payload, { operation: "chat", direction: "input" }); + await (await fetch("https://api.openai.com/v1/chat/completions", { method: "POST", body: JSON.stringify(payload) })).json(); + }); + await settle(); expect(uploads).toEqual([png]); + }); }); -test("real installed OpenAI SDK speech operation preserves WAV bytes and correlation", async () => { - const OpenAI = require(process.env.ATT_OPENAI_MODULE || "openai").default; - install(); const sdk = new OpenAI({ apiKey: "synthetic-provider-key", fetch: globalThis.fetch }); - const response = await sdk.audio.speech.create({ model: "tts-1", voice: "alloy", input: "synthetic", response_format: "wav" }); - expect(Buffer.from(await response.arrayBuffer())).toEqual(audio); - await settle(); expect(uploads).toEqual([audio]); expect(exporter.getFinishedSpans()).toHaveLength(1); + +describe("privacy negative controls", () => { + test.each([ + ["disabled", { enabled: false }], ["hide inputs", { traceConfig: { hideInputs: true } }], ["hide text", { traceConfig: { hideInputText: true } }], + ["hide images", { traceConfig: { hideInputImages: true } }], ["PII", { traceConfig: { piiRedaction: true } }], + ] as [string, AttachmentManagerOptions][])("%s prevents input image transfer at the network boundary", async (_name, options) => { + await manager.shutdown(); manager = await createManager(options); transfers = []; + responseType = "application/json"; responseData = Buffer.from('{"data":[]}'); + const body = new FormData(); body.set("image", new Blob([png], { type: "image/png" }), "PRIVATE_CANARY.png"); + install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body })).json(); }); + await (await fetch("https://api.openai.com/v1/chat/completions", { method: "POST", + body: JSON.stringify({ messages: [{ role: "user", content: [{ type: "image_url", image_url: { url: `data:image/png;base64,${png.toString("base64")}` } }] }] }) })).json(); + await settle(); expect(sessionsOnly()).toEqual([]); + }); + test.each(["hideOutputs", "hideOutputMessages", "hideOutputText", "piiRedaction"] as const)("%s prevents output speech and image transfer", async flag => { + manager.updatePolicy({ traceConfig: { [flag]: true } }); transfers = []; + install(); await inNative(async () => { await (await speech()).arrayBuffer(); }); + await (await speech()).arrayBuffer(); + responseType = "application/json"; responseData = Buffer.from(JSON.stringify({ data: [{ b64_json: png.toString("base64") }] })); + await (await fetch("https://api.openai.com/v1/images/generations", { method: "POST", body: "{}" })).json(); + await settle(); expect(sessionsOnly()).toEqual([]); + }); + test("T-PF-09 server-required redaction denies before any base64 decode", async () => { + await manager.shutdown(); manager = await createManager({}, { require_media_redaction: true }); transfers = []; + responseType = "application/json"; responseData = Buffer.from(JSON.stringify({ data: [{ b64_json: png.toString("base64") }] })); + install(); const decode = jest.spyOn(Buffer, "from"); + await (await fetch("https://api.openai.com/v1/images/generations", { method: "POST", body: "{}" })).json(); + await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/generations", { method: "POST", body: "{}" })).json(); }); + const calls = decode.mock.calls.filter(args => args[0] === png.toString("base64") && (args as unknown[])[1] === "base64"); decode.mockRestore(); + await settle(); expect(calls).toHaveLength(0); expect(sessionsOnly()).toEqual([]); + }); + test("unknown host, non-audio MIME and error bodies do not upload", async () => { + install(); await inNative(async () => { + await (await speech("unknown.invalid/v1/audio/speech")).arrayBuffer(); + responseType = "application/octet-stream"; await (await speech()).arrayBuffer(); + responseType = "audio/wav"; responseStatus = 403; await (await speech()).arrayBuffer(); + }); + await (await speech("api.openai.com.evil.invalid/v1/audio/speech")).arrayBuffer(); + await settle(); expect(uploads).toHaveLength(0); + }); + test("oversized media, raw multipart, and ended/unrelated native contexts fail closed", async () => { + responseType = "application/json"; responseData = Buffer.from('{"data":[]}'); + const body = new FormData(); body.set("image", new Blob([new Uint8Array(20 * 1024 * 1024 + 1)], { type: "image/png" })); + install(); await inNative(async () => { + await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body })).json(); + await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", headers: { "content-type": "multipart/form-data; boundary=x" }, body: "--x\r\nPRIVATE_CANARY" })).json(); + }); + const small = new FormData(); small.set("image", new Blob([png], { type: "image/png" })); + await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body: small })).json(); }, "TOOL"); + await settle(); expect(uploads).toHaveLength(0); expect(manager.status()).toMatchObject({ openTickets: 0, ownedBytes: 0 }); + }); + test("policy tightening between request and application consumption prevents capture", async () => { + install(); await inNative(async () => { const response = await speech(); manager.updatePolicy({ traceConfig: { hideOutputs: true } }); await response.arrayBuffer(); }); + await settle(); expect(uploads).toHaveLength(0); + }); + test("hook removal makes retained response wrappers inert", async () => { + install(); await inNative(async () => { const response = await speech(); hook!.remove(); await response.arrayBuffer(); }); + await settle(); expect(uploads).toHaveLength(0); + }); + test("native capture is denied without an explicit integration policy", async () => { + install(); await inNative(async () => { await (await speech()).arrayBuffer(); }, "LLM", false); + await settle(); expect(uploads).toHaveLength(0); + }); + test.each([{ captureContent: false }, { recordOutputs: false }, { muted: true }, { temporaryChat: true }])("integration restrictions cannot be loosened by child scopes: %j", async restriction => { + install(); await inNative(() => context.with(withAttachmentCapturePolicy(withAttachmentCapturePolicy(context.active(), restriction), { captureContent: true, recordOutputs: true, muted: false, temporaryChat: false }), async () => { + await (await speech()).arrayBuffer(); + })); await settle(); expect(uploads).toHaveLength(0); + }); + test("a disabled manager installs no media observer: unchanged provider traffic, no control traffic", async () => { + const disabled = new AttachmentManager({ enabled: false, environment: () => ({}) }); + install(disabled); await inNative(async () => { await (await speech()).arrayBuffer(); }); + await (await speech()).arrayBuffer(); + await settle(); expect(transfers).toEqual([]); expect(providerRequests).toBe(2); + }); }); -test("native capture is denied without an explicit integration policy", async () => { - install(); const span = provider.getTracer("external").startSpan("external", { attributes: { "gen_ai.span.kind": "LLM" } }); - await context.with(trace.setSpan(ROOT_CONTEXT, span), async () => { await (await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" })).arrayBuffer(); }); - span.end(); await settle(); expect(uploads).toHaveLength(0); +describe("A02 ownership and configuration", () => { + test("borrowed manager identity survives auto setup; auto adds only restrictions", async () => { + const config = resolveConfig({ traceConfig: { hideOutputs: true }, projectName: "auto-project", configFile: "/nonexistent" }, {}); + const selected = selectAttachmentManager({ attachments: manager }, config, {}, false); + expect(selected.manager).toBe(manager); expect(selected.owned).toBe(false); + expect(attachmentOptions(config, { attachments: { projectName: "wrong-project", traceConfig: { hideOutputs: false } } })).toMatchObject({ projectName: "auto-project", traceConfig: { hideOutputs: true } }); + install(); await inNative(async () => { await (await speech()).arrayBuffer(); }); + await settle(); expect(uploads).toHaveLength(0); + }); + test("an env override does not permanently disable a borrowed manager (updatePolicy is tighten-only)", async () => { + const before = process.env.FI_ATTACHMENTS_ENABLED; process.env.FI_ATTACHMENTS_ENABLED = "false"; + try { selectAttachmentManager({ attachments: manager }, resolveConfig({ configFile: "/nonexistent" }, {}), {}, false); } + finally { if (before === undefined) delete process.env.FI_ATTACHMENTS_ENABLED; else process.env.FI_ATTACHMENTS_ENABLED = before; } + install(); await inNative(async () => { await (await speech()).arrayBuffer(); }); + await settle(); expect(uploads).toEqual([audio]); + }); + test("status report is policy and source aware and claims no playback", () => { + const report = attachmentReport(manager); + expect(report.state).toBe("ready"); expect(report.status).toMatchObject({ openTickets: 0 }); + expect(report.routes).toContain("POST https://api.openai.com/v1/audio/speech (speech)"); + expect(report.unsupported).toEqual(expect.arrayContaining(["WebSocket / RTC audio", "product playback qualification"])); + expect(attachmentReport().state).toBe("not_initialized"); + }); }); -test.each([{ captureContent: false }, { recordOutputs: false }, { muted: true }, { temporaryChat: true }])("integration restrictions cannot be loosened by child scopes: %j", async restriction => { - install(); await inNative(() => context.with(withAttachmentCapturePolicy(withAttachmentCapturePolicy(context.active(), restriction), { captureContent: true, recordOutputs: true, muted: false, temporaryChat: false }), async () => { - await (await fetch("https://api.openai.com/v1/audio/speech", { method: "POST" })).arrayBuffer(); - })); await settle(); expect(uploads).toHaveLength(0); + +describe("real SDKs against loopback", () => { + const sdk = () => require(process.env.ATT_OPENAI_MODULE || "openai"); + test("OpenAI SDK speech (wire-only) preserves WAV bytes and correlation", async () => { + const OpenAI = sdk().default ?? sdk().OpenAI; + install(); const client = new OpenAI({ apiKey: "synthetic-provider-key", fetch: (...args: Parameters) => globalThis.fetch(...args) }); + const response = await client.audio.speech.create({ model: "tts-1", voice: "alloy", input: "synthetic", response_format: "wav" }); + expect(Buffer.from(await response.arrayBuffer())).toEqual(audio); + await settle(); expect(uploads).toEqual([audio]); expect(exporter.getFinishedSpans()).toHaveLength(1); + expect(requests[0].association.parent.span_id).toBe(exporter.getFinishedSpans()[0].spanContext().spanId); + }); + test("OpenAI SDK image generation (wire-only) uploads the decoded PNG", async () => { + const OpenAI = sdk().default ?? sdk().OpenAI; + responseType = "application/json"; responseData = Buffer.from(JSON.stringify({ created: 1, data: [{ b64_json: png.toString("base64") }] })); + install(); const client = new OpenAI({ apiKey: "synthetic-provider-key", fetch: (...args: Parameters) => globalThis.fetch(...args) }); + const result = await client.images.generate({ model: "gpt-image-1", prompt: "synthetic" }); + expect(result.data[0].b64_json).toBe(png.toString("base64")); + await settle(); expect(uploads).toEqual([png]); + }); + test("OpenAI SDK transcription (wire-only) keeps the multipart request intact and captures nothing", async () => { + const mod = sdk(); const OpenAI = mod.default ?? mod.OpenAI; const toFile = mod.toFile; + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); + install(); const client = new OpenAI({ apiKey: "synthetic-provider-key", fetch: (...args: Parameters) => globalThis.fetch(...args) }); + const result = await client.audio.transcriptions.create({ model: "whisper-1", file: await toFile(audio, "synthetic.wav", { type: "audio/wav" }) }); + expect(result.text).toBe("synthetic"); expect(providerBodies[0].includes(audio)).toBe(true); + await settle(); expect(uploads).toHaveLength(0); + }); }); From a3ccf83dc36ae46f9de44ec278891c95c15667a8 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 04:57:18 -0800 Subject: [PATCH 10/20] test(auto): prove the selected manager reaches the OpenAI instrumentor Co-Authored-By: Claude Opus 5.5 --- .../test/attachment-adapters.test.ts | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 typescript/packages/traceai_auto/test/attachment-adapters.test.ts diff --git a/typescript/packages/traceai_auto/test/attachment-adapters.test.ts b/typescript/packages/traceai_auto/test/attachment-adapters.test.ts new file mode 100644 index 000000000..f9935d41c --- /dev/null +++ b/typescript/packages/traceai_auto/test/attachment-adapters.test.ts @@ -0,0 +1,92 @@ +import { spawnSync } from "node:child_process"; +import { resolve } from "node:path"; + +// A02: auto hands the ONE selected manager to the instrumentors it enables, as their `attachments` option. +// A real OpenAI SDK call (loopback only, synthetic image) must upload exactly once, from the native adapter, bound to +// the native LLM span; the wire layer must not upload it again. Runs in a child process because init is process-global. +const root = resolve(__dirname, ".."); +function run(withManager: boolean) { + const code = ` + const assert = require('node:assert/strict'); + const http = require('node:http'); + const { randomUUID } = require('node:crypto'); + const { InMemorySpanExporter } = require('@opentelemetry/sdk-trace-base'); + const { AttachmentManager, createAttachmentHttpTransport } = require('@traceai/fi-core/attachments'); + const auto = require('./dist/src/index.js'); + const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII='; + (async () => { + const initialized = [], uploaded = []; + const scope = { organization_id: randomUUID(), workspace_id: randomUUID(), project_id: randomUUID() }; + const expiry = () => new Date(Date.now() + 3600000).toISOString(); + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const c of req) chunks.push(c); const bytes = Buffer.concat(chunks); + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('attachment-sessions')) { + res.end(JSON.stringify({ schema_version: 1, scope, capture_session_id: JSON.parse(bytes).capture_session_id, + policy: { version: 'fixture-v1', enabled: true, allowed_sources: ['Bytes', 'Blob'], allowed_modalities: ['image', 'audio'], require_media_redaction: false, + require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false }, + limits: { max_upload_bytes: 1048576, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, + active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, + upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }, + expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); + } else if (req.url.endsWith('/uploads')) { + const body = JSON.parse(bytes); initialized.push(body); res.statusCode = 201; + res.end(JSON.stringify({ upload_id: body.attachment_id, attachment_id: body.attachment_id, occurrence_id: body.occurrence_id, state: 'pending_upload', + reserved_bytes: body.declared_bytes, expires_at: expiry(), transfer_mode: 'authenticated_stream', write_capability: 'synthetic-upload-only' })); + } else if (req.url.endsWith('/content')) { + uploaded.push(bytes); res.end(JSON.stringify({ upload_id: req.url.split('/').at(-2), state: 'pending_upload', reason: 'queued' })); + } else if (req.url.includes('/tracer/')) { + res.statusCode = 202; res.end(JSON.stringify({ upload_id: req.url.split('/').at(-2), state: 'validating', reason: 'binding_pending' })); + } else { + // The synthetic model API. + res.end(JSON.stringify({ id: 'chatcmpl-synthetic', object: 'chat.completion', created: 1, model: 'gpt-4o', + choices: [{ index: 0, finish_reason: 'stop', message: { role: 'assistant', content: 'synthetic' } }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })); + } + }); + await new Promise(r => server.listen(0, '127.0.0.1', r)); + const origin = 'http://127.0.0.1:' + server.address().port; + const client = req => new Promise((resolve, reject) => { + const out = http.request(origin + new URL(req.url).pathname, { method: req.method, headers: req.headers, signal: req.signal }, response => { + const c = []; response.on('data', d => c.push(d)); response.on('end', () => resolve({ status: response.statusCode, headers: {}, body: Buffer.concat(c) })); + }); out.on('error', reject); out.end(req.body); + }); + const manager = new AttachmentManager({ enabled: true, projectName: 'synthetic', projectType: 'observe', environment: () => ({}), + transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic' }, client }) }); + assert.equal(await manager.ready(), true); + const exporter = new InMemorySpanExporter(); + const options = { exporter, instrumentations: ['openai', 'wire'], configFile: '/nonexistent', apiKey: '', secretKey: '' }; + if (${withManager}) options.attachmentManager = manager; + auto.init(options); + assert.equal(auto.getAttachmentManager() === manager, ${withManager}); + const openai = auto.report().integrations.find(row => row.name === 'openai'); + assert.equal(openai.status, 'instrumented'); assert.equal(openai.media, 'provider_adapter'); + const OpenAI = require('openai').default ?? require('openai').OpenAI; + const sdk = new OpenAI({ apiKey: 'synthetic-provider-key', baseURL: origin + '/v1' }); + await sdk.chat.completions.create({ model: 'gpt-4o', messages: [{ role: 'user', content: [{ type: 'text', text: 'synthetic' }, { type: 'image_url', image_url: { url: 'data:image/png;base64,' + PNG } }] }] }); + await auto.forceFlush(); await manager.flush(2000); + const spans = exporter.getFinishedSpans().filter(s => s.attributes['gen_ai.span.kind'] === 'LLM' || s.attributes['fi.span.kind'] === 'LLM'); + process.stdout.write(JSON.stringify({ uploads: uploaded.map(b => b.toString('base64')), parents: initialized.map(r => r.association.parent.span_id), + purposes: initialized.map(r => r.purpose), llmSpans: spans.map(s => s.spanContext().spanId) })); + await auto.shutdown(); await manager.shutdown(); server.closeAllConnections(); server.close(); + })().catch(error => { process.stderr.write(String(error && error.stack || error)); process.exitCode = 1; }); + `; + return spawnSync(process.execPath, ["-e", code], { + cwd: root, encoding: "utf8", timeout: 30000, + env: { ...process.env, FI_ATTACHMENTS_ENABLED: "", FI_API_KEY: "", FI_SECRET_KEY: "", FI_CONFIG_FILE: "/nonexistent", FI_AUTO_DEBUG: "false" }, + }); +} +const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + +test("the selected manager reaches the OpenAI instrumentor as `attachments`: one upload, bound to the native span", () => { + const result = run(true); + expect(result.stderr).toBe(""); expect(result.status).toBe(0); + const out = JSON.parse(result.stdout); + expect(out.uploads).toEqual([png]); + expect(out.purposes).toEqual(["model_input"]); + expect(out.llmSpans).toHaveLength(1); expect(out.parents).toEqual(out.llmSpans); +}); +test("without an enabled manager (the default) the same call uploads nothing", () => { + const result = run(false); + expect(result.stderr).toBe(""); expect(result.status).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ uploads: [] }); +}); From f892de86d899bc53688239f8b3a900dc0f546d66 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 05:20:34 -0800 Subject: [PATCH 11/20] fix(auto): capture wire-only JSON output from the wire response copy; fix test doubles The latest OpenAI SDK (7.x) consumes JSON with text(), so wire-only image output is now read from the wire path's own bounded response copy at bind, whatever method the application used. Test doubles read Blob request bodies; T-CS-01 ignores undici's own request-body tee and reads. Co-Authored-By: Claude Opus 5.5 --- .../traceai_auto/src/wire/attachments.ts | 20 +++++++++----- .../traceai_auto/src/wire/fetch-runtime.ts | 6 ++--- .../test/attachment-adapters.test.ts | 17 +++++++----- .../traceai_auto/test/attachments.test.ts | 26 ++++++++++++------- 4 files changed, 43 insertions(+), 26 deletions(-) diff --git a/typescript/packages/traceai_auto/src/wire/attachments.ts b/typescript/packages/traceai_auto/src/wire/attachments.ts index 1fa6ddd5c..19430cc3f 100644 --- a/typescript/packages/traceai_auto/src/wire/attachments.ts +++ b/typescript/packages/traceai_auto/src/wire/attachments.ts @@ -3,6 +3,7 @@ import type { AttachmentCapturePolicy, AttachmentCorrelation, AttachmentManager, AttachmentPurpose, AttachmentStream, } from "@traceai/fi-core/attachments"; import type { FetchMediaObserver, FetchMediaSink } from "./fetch-runtime"; +import { header } from "./core"; const MEDIA_POLICY = createContextKey("traceai.auto.attachment_policy.v1"); /** Supply the integration's capture restrictions; inherited denials always win. */ @@ -124,6 +125,12 @@ export function wireAttachmentObserver(manager: AttachmentManager, parent: Conte format = imageFormat((body as Record).output_format); if (op === "chat" || op === "responses") captureOpenAIAttachments(manager, correlation, body, { operation: op, direction: "input", capture }); }; + const jsonOutput = (value: unknown) => { + if (outputObserved || !correlation || !value || typeof value !== "object" || !(op === "image_generation" || op === "image_edit" || op === "responses")) return; + outputObserved = true; + // Typed provider adapter only (#682): no recursive scraping, no URL retrieval, decode after admission. + captureOpenAIAttachments(manager, correlation, value, { operation: op === "responses" ? "responses" : "images", direction: "output", capture, imageFormat: format }); + }; const formData = (body: FormData) => { let entries = 0; for (const [name, value] of body.entries()) { @@ -138,12 +145,15 @@ export function wireAttachmentObserver(manager: AttachmentManager, parent: Conte } }; return { - bind(span, request) { + bind(span, request, response) { // The emitter supplies its actual identity before ending, never a later ambient span. if (correlation) return; correlation = manager.correlation(span); - // JSON input uses the wire path's own bounded copy; nothing was retained for media before this point. + // JSON uses the wire path's own bounded copies; nothing was retained for media before this point. if (op === "chat" || op === "responses" || op === "image_generation") captureJsonInput(parseJson(request?.body)); + // Output JSON, however the application consumed it (json(), text(), arrayBuffer()); the same span and + // position captured again from json() below resolves to the first occurrence in the adapter's registry. + if (response && (response.status ?? 0) >= 200 && (response.status ?? 0) < 300 && !response.truncated && mediaType(header(response.headers, "content-type")) === "application/json") jsonOutput(parseJson(response.body)); }, request(body) { // Wire-only: the span does not exist yet, so FormData is not retained until a late span (fail closed). @@ -178,11 +188,7 @@ export function wireAttachmentObserver(manager: AttachmentManager, parent: Conte // The view (and the core's synchronous copy) exists only after admission. attachExact(value.byteLength, type, "output", "model_output", () => ({ kind: "Bytes", data: new Uint8Array(value) }), "Bytes"); } - } else if (methodName === "json" && type === "application/json" && (op === "image_generation" || op === "image_edit" || op === "responses")) { - outputObserved = true; - // Typed provider adapter only (#682): no recursive scraping, no URL retrieval, decode after admission. - captureOpenAIAttachments(manager, correlation, value, { operation: op === "responses" ? "responses" : "images", direction: "output", capture, imageFormat: format }); - } + } else if (methodName === "json" && type === "application/json") jsonOutput(value); }, }; } diff --git a/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts b/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts index 0b7f97532..f64628128 100644 --- a/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts +++ b/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts @@ -6,8 +6,8 @@ import type { Span } from "@opentelemetry/api"; /** Receives only chunks the application already consumed; never pulls. */ export interface FetchMediaSink { write(chunk: Uint8Array): boolean; end(): void; abort(): void } export interface FetchMediaObserver { - /** The wire span, before it ends, with the wire path's own bounded request copy. */ - bind(span: Span, request?: RequestData): void; + /** The wire span, before it ends, with the wire path's own bounded request and response copies. */ + bind(span: Span, request?: RequestData, response?: ResponseData): void; request(body: BodyInit | null | undefined): void; /** The application started incremental consumption of a response body (reader or async iterator). */ consumed?(mimeType: string, successful: boolean, contentLength: number | undefined): FetchMediaSink | undefined; @@ -238,7 +238,7 @@ export function createFetchHook(max: number, emit: (result: SpanResult, paren if (finished) return; finished = true; const complete = () => { if (audio) { response.audioOutputBytes = audio.bytes; response.body = undefined; } - try { const span = build(matched, request, response, parsed); if (span) emit(span, parent, start, end, media ? value => media!.bind(value, request) : undefined); } catch { /* Capture must not reach application code. */ } + try { const span = build(matched, request, response, parsed); if (span) emit(span, parent, start, end, media ? value => media!.bind(value, request, response) : undefined); } catch { /* Capture must not reach application code. */ } finally { resolve(); pending.delete(done); } }; // Only completed responses awaiting bounded decoding are flushable, never unread streams. diff --git a/typescript/packages/traceai_auto/test/attachment-adapters.test.ts b/typescript/packages/traceai_auto/test/attachment-adapters.test.ts index f9935d41c..514e8dcb4 100644 --- a/typescript/packages/traceai_auto/test/attachment-adapters.test.ts +++ b/typescript/packages/traceai_auto/test/attachment-adapters.test.ts @@ -45,11 +45,14 @@ function run(withManager: boolean) { }); await new Promise(r => server.listen(0, '127.0.0.1', r)); const origin = 'http://127.0.0.1:' + server.address().port; - const client = req => new Promise((resolve, reject) => { - const out = http.request(origin + new URL(req.url).pathname, { method: req.method, headers: req.headers, signal: req.signal }, response => { - const c = []; response.on('data', d => c.push(d)); response.on('end', () => resolve({ status: response.statusCode, headers: {}, body: Buffer.concat(c) })); - }); out.on('error', reject); out.end(req.body); - }); + const client = async req => { + const body = req.body instanceof Blob ? Buffer.from(await req.body.arrayBuffer()) : req.body; + return new Promise((resolve, reject) => { + const out = http.request(origin + new URL(req.url).pathname, { method: req.method, headers: req.headers, signal: req.signal }, response => { + const c = []; response.on('data', d => c.push(d)); response.on('end', () => resolve({ status: response.statusCode, headers: {}, body: Buffer.concat(c) })); + }); out.on('error', reject); out.end(body); + }); + }; const manager = new AttachmentManager({ enabled: true, projectName: 'synthetic', projectType: 'observe', environment: () => ({}), transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic' }, client }) }); assert.equal(await manager.ready(), true); @@ -58,12 +61,12 @@ function run(withManager: boolean) { if (${withManager}) options.attachmentManager = manager; auto.init(options); assert.equal(auto.getAttachmentManager() === manager, ${withManager}); - const openai = auto.report().integrations.find(row => row.name === 'openai'); - assert.equal(openai.status, 'instrumented'); assert.equal(openai.media, 'provider_adapter'); const OpenAI = require('openai').default ?? require('openai').OpenAI; const sdk = new OpenAI({ apiKey: 'synthetic-provider-key', baseURL: origin + '/v1' }); await sdk.chat.completions.create({ model: 'gpt-4o', messages: [{ role: 'user', content: [{ type: 'text', text: 'synthetic' }, { type: 'image_url', image_url: { url: 'data:image/png;base64,' + PNG } }] }] }); await auto.forceFlush(); await manager.flush(2000); + const openai = auto.report().integrations.find(row => row.name === 'openai'); + assert.equal(openai.status, 'instrumented'); assert.equal(openai.media, 'provider_adapter'); const spans = exporter.getFinishedSpans().filter(s => s.attributes['gen_ai.span.kind'] === 'LLM' || s.attributes['fi.span.kind'] === 'LLM'); process.stdout.write(JSON.stringify({ uploads: uploaded.map(b => b.toString('base64')), parents: initialized.map(r => r.association.parent.span_id), purposes: initialized.map(r => r.purpose), llmSpans: spans.map(s => s.spanContext().spanId) })); diff --git a/typescript/packages/traceai_auto/test/attachments.test.ts b/typescript/packages/traceai_auto/test/attachments.test.ts index 55d067e45..4050b1de0 100644 --- a/typescript/packages/traceai_auto/test/attachments.test.ts +++ b/typescript/packages/traceai_auto/test/attachments.test.ts @@ -48,11 +48,15 @@ function control(req: import("node:http").IncomingMessage, body: Buffer, res: im uploads.push(body); res.end(JSON.stringify({ upload_id: req.url!.split("/").at(-2), state: "pending_upload", reason: "queued" })); } else { res.statusCode = 202; res.end(JSON.stringify({ upload_id: req.url!.split("/").at(-2), state: "validating", reason: "binding_pending" })); } } -const client: AttachmentHttpClient = req => new Promise((resolve, reject) => { +const client: AttachmentHttpClient = async req => { + // Content PUTs may carry the application's own Blob; the double reads it only when the core sends it. + const body = req.body instanceof Blob ? Buffer.from(await req.body.arrayBuffer()) : req.body; + return new Promise((resolve, reject) => { const outgoing = httpRequest(`${origin}${new URL(req.url).pathname}`, { method: req.method, headers: { ...req.headers, "x-attachment-control": "1" }, signal: req.signal }, response => { const chunks: Buffer[] = []; response.on("data", c => chunks.push(c)); response.on("end", () => resolve({ status: response.statusCode!, headers: {}, body: Buffer.concat(chunks) } as any)); - }); outgoing.on("error", reject); outgoing.end(req.body as any); -}); + }); outgoing.on("error", reject); outgoing.end(body as any); + }); +}; async function createManager(options: AttachmentManagerOptions = {}, serverOverrides: Record = {}) { policy = { ...serverPolicy, ...serverOverrides }; const result = new AttachmentManager({ enabled: true, projectName: "attachment-test", projectType: "observe", environment: () => ({}), @@ -138,15 +142,17 @@ describe("A04 speech output", () => { responseChunks = 5; install(); const tee = jest.spyOn(ReadableStream.prototype, "tee"), clone = jest.spyOn(Response.prototype, "clone"); const reads = jest.spyOn(ReadableStreamDefaultReader.prototype, "read"); - let appReads = 0; + let appReads = 0, responseBody: unknown, appReader: unknown; await inNative(async () => { - const response = await speech(); const reader = response.body!.getReader(); const chunks: Buffer[] = []; + const response = await speech(); responseBody = response.body; const reader = response.body!.getReader(); appReader = reader; const chunks: Buffer[] = []; for (;;) { appReads++; const x = await reader.read(); if (x.done) break; chunks.push(Buffer.from(x.value)); } expect(Buffer.concat(chunks)).toEqual(audio); }); - const underlying = reads.mock.calls.length; + // Only this reader's calls: undici reads its own request-body stream internally. + const underlying = reads.mock.contexts.filter(receiver => receiver === appReader).length; await settle(); - expect(tee).not.toHaveBeenCalled(); expect(clone).not.toHaveBeenCalled(); tee.mockRestore(); clone.mockRestore(); reads.mockRestore(); + // undici itself tees the REQUEST body (cloneRequest); the response body must never be teed or cloned. + expect(tee.mock.contexts).not.toContain(responseBody); expect(clone).not.toHaveBeenCalled(); tee.mockRestore(); clone.mockRestore(); reads.mockRestore(); expect(appReads).toBeGreaterThan(2); expect(underlying).toBe(appReads); expect(uploads).toEqual([audio]); }); test("native span: async-iterator consumption uploads once; a break before the end uploads nothing", async () => { @@ -324,7 +330,7 @@ describe("privacy negative controls", () => { }); test("a disabled manager installs no media observer: unchanged provider traffic, no control traffic", async () => { const disabled = new AttachmentManager({ enabled: false, environment: () => ({}) }); - install(disabled); await inNative(async () => { await (await speech()).arrayBuffer(); }); + transfers = []; install(disabled); await inNative(async () => { await (await speech()).arrayBuffer(); }); await (await speech()).arrayBuffer(); await settle(); expect(transfers).toEqual([]); expect(providerRequests).toBe(2); }); @@ -374,7 +380,9 @@ describe("real SDKs against loopback", () => { await settle(); expect(uploads).toEqual([png]); }); test("OpenAI SDK transcription (wire-only) keeps the multipart request intact and captures nothing", async () => { - const mod = sdk(); const OpenAI = mod.default ?? mod.OpenAI; const toFile = mod.toFile; + const mod = sdk(); + // openai 4.x sends multipart as a Node stream, which a WHATWG fetch rejects without `duplex` (SDK limitation, not auto). + if (Number(require(`${process.env.ATT_OPENAI_MODULE || "openai"}/package.json`).version.split(".")[0]) < 5) return; const OpenAI = mod.default ?? mod.OpenAI; const toFile = mod.toFile; responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); install(); const client = new OpenAI({ apiKey: "synthetic-provider-key", fetch: (...args: Parameters) => globalThis.fetch(...args) }); const result = await client.audio.transcriptions.create({ model: "whisper-1", file: await toFile(audio, "synthetic.wav", { type: "audio/wav" }) }); From 0f57469d906b1dcce76625c00fb1ef271759635a Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 05:34:54 -0800 Subject: [PATCH 12/20] docs(auto): document attachment options, sources and lifecycle helpers Co-Authored-By: Claude Opus 5.5 --- .../traceai_auto/docs/CONFIGURATION.md | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/typescript/packages/traceai_auto/docs/CONFIGURATION.md b/typescript/packages/traceai_auto/docs/CONFIGURATION.md index 9c99b8d2f..c174e2d0a 100644 --- a/typescript/packages/traceai_auto/docs/CONFIGURATION.md +++ b/typescript/packages/traceai_auto/docs/CONFIGURATION.md @@ -230,6 +230,31 @@ When error hiding is unset, any input/output value, message or text hide flag en Known-secret redaction is always active on traceAI-owned exports. The registry is bounded to 1024 values of at least eight characters. `registerSecret(value)` adds an exact string without coercing objects. Unknown free-text credentials, transformed secrets and credentials hidden in unparsed bodies may remain unknown; hide sensitive content instead of relying on inference. +## Attachments (images, files, audio) + +Attachments are **off** by default. Upgrading `@traceai/auto` captures no new media. New recordings are a separate switch, also off. + +| Environment | `init` option | Default and behavior | +|---|---|---| +| `FI_ATTACHMENTS_ENABLED` | `attachments.enabled` | `false`. A manager that reads the process environment (the default) also treats a value that is not true as a deny at every admission | +| `FI_ATTACHMENTS_ENDPOINT` | `attachments.endpoint` | Unset. The attachment control origin, independent of `baseUrl` and the OTLP collector; must be HTTPS | +| — | `attachments` | Options for the manager auto creates. Hide and PII flags from either `traceConfig` win; an attachment-level `false` cannot relax a global hide flag | +| — | `attachmentManager` | Borrow the application's manager. Auto never shuts down a borrowed manager and only adds restrictions to it | + +Auto uses ONE manager: `attachmentManager`, else the selected provider's `attachments`, else its own. It passes that manager to every instrumentor it enables as their `attachments` option, and `getAttachmentManager()` returns it. Every source is checked against the deny-wins policy (local hide/PII flags, server policy, sampling, budget) before it is read or decoded; storage stays private. + +Wire capture (`wire`) observes only allowlisted HTTPS protocol entries (`report().attachments.routes`): OpenAI speech, transcription and translation, image generation, edits and variations, chat completions, responses, and Deepgram speak. It never fetches URLs, opens files, clones, tees or reads a response body itself. + +| Source | Wire-only call | Under a native model span | +|---|---|---| +| JSON input images, files, computer screenshots (chat, responses) | Captured, from the wire request copy | By the instrumentor (`@traceai/openai`); the same position is never uploaded twice | +| JSON output images (images `b64_json`, responses `image_generation_call`) | Captured, from the wire response copy | By the instrumentor where it covers the operation; wire capture needs an explicit integration policy (`withAttachmentCapturePolicy`) | +| Speech audio the application read with `arrayBuffer()` / `blob()` | Captured | Needs an explicit integration policy | +| Speech audio the application read incrementally (reader, async iterator) | Not captured: the wire span does not exist yet and nothing is retained | Needs an explicit integration policy; cancel, early return, pipe or tee upload nothing | +| FormData audio / image files | Not captured (no retention before the span exists) | Needs an explicit integration policy | + +An adapter that captures media itself marks its context with `withAttachmentOwnership(context)` (context key `traceai.auto.attachment_owner.v1`), and wire capture then skips that call. `report().attachments` shows the manager state, queue counters, routes, sources and unsupported sources; `report().integrations[].media` shows whether an integration captures media through a provider adapter. No row is a playback claim. + ## Bring your own OpenTelemetry destination With no complete FI key pair, `init({ exporter })`, `init({ spanProcessor })`, `init({ provider })`, or an existing application provider activates instrumentation **without creating an FI export route**. If neither a destination nor a complete pair exists, instrumentation is disabled. With a complete pair and a provider but no explicit exporter/processor, the agent attempts to attach its FI route to that provider. @@ -275,6 +300,9 @@ Standard `OTEL_SERVICE_NAME` and `OTEL_RESOURCE_ATTRIBUTES` preserve application | `forceFlush()` | Flush completed spans with a five-second deadline; does not consume unread responses | | `shutdown()` | Idempotent hook removal and bounded drain/shutdown; no supported reinitialization afterwards | | `registerSecret(value)` | Register an exact sensitive string of at least eight characters; ignore other values | +| `getAttachmentManager()` | The one attachment manager auto selected and passed to instrumentors, or `undefined` before `init` | +| `withAttachmentCapturePolicy(context, policy)` | Supply an integration's capture restrictions for wire media under its native span; inherited denials always win | +| `withAttachmentOwnership(context)` | Mark a call whose media an adapter captures itself, so wire capture does not upload it again | ## CLI From 21f18e6638d039a443ea81edf73b63d31ee76f2b Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 05:40:48 -0800 Subject: [PATCH 13/20] test(auto): space loopback chunks and read the SDK version without its exports map Co-Authored-By: Claude Opus 5.5 --- .../traceai_auto/test/attachments.test.ts | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/typescript/packages/traceai_auto/test/attachments.test.ts b/typescript/packages/traceai_auto/test/attachments.test.ts index 4050b1de0..f6c1d9d6e 100644 --- a/typescript/packages/traceai_auto/test/attachments.test.ts +++ b/typescript/packages/traceai_auto/test/attachments.test.ts @@ -1,6 +1,8 @@ import { createServer, request as httpRequest, type Server } from "node:http"; import { AddressInfo } from "node:net"; import { randomUUID } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; import { context, trace, ROOT_CONTEXT } from "@opentelemetry/api"; import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; @@ -100,10 +102,12 @@ beforeEach(async () => { if (req.headers["x-attachment-control"]) { control(req, body, res); return; } providerRequests++; providerBodies.push(body); res.writeHead(responseStatus, { "content-type": responseType }); - // Several network chunks, so incremental readers see more than one read. - const size = Math.ceil(responseData.length / responseChunks); - for (let i = 0; i < responseData.length; i += size) res.write(responseData.subarray(i, i + size)); - res.end(); + // Several network chunks, spaced so loopback does not coalesce them: incremental readers see several reads. + const size = Math.ceil(responseData.length / responseChunks), data = responseData; + void (async () => { + for (let i = 0; i < data.length; i += size) { res.write(data.subarray(i, i + size)); if (responseChunks > 1) await new Promise(r => setTimeout(r, 15)); } + res.end(); + })(); }); }); await new Promise(done => server.listen(0, "127.0.0.1", done)); origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; @@ -363,6 +367,14 @@ describe("A02 ownership and configuration", () => { describe("real SDKs against loopback", () => { const sdk = () => require(process.env.ATT_OPENAI_MODULE || "openai"); + // The package's exports map hides package.json; walk up from the resolved entry instead. + const sdkVersion = (): string => { + let dir = dirname(require.resolve(process.env.ATT_OPENAI_MODULE || "openai")); + for (let i = 0; i < 8; i++, dir = dirname(dir)) { + try { const pkg = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")); if (pkg.name === "openai") return pkg.version; } catch { /* keep walking */ } + } + return "0.0.0"; + }; test("OpenAI SDK speech (wire-only) preserves WAV bytes and correlation", async () => { const OpenAI = sdk().default ?? sdk().OpenAI; install(); const client = new OpenAI({ apiKey: "synthetic-provider-key", fetch: (...args: Parameters) => globalThis.fetch(...args) }); @@ -382,7 +394,7 @@ describe("real SDKs against loopback", () => { test("OpenAI SDK transcription (wire-only) keeps the multipart request intact and captures nothing", async () => { const mod = sdk(); // openai 4.x sends multipart as a Node stream, which a WHATWG fetch rejects without `duplex` (SDK limitation, not auto). - if (Number(require(`${process.env.ATT_OPENAI_MODULE || "openai"}/package.json`).version.split(".")[0]) < 5) return; const OpenAI = mod.default ?? mod.OpenAI; const toFile = mod.toFile; + if (Number(sdkVersion().split(".")[0]) < 5) return; const OpenAI = mod.default ?? mod.OpenAI; const toFile = mod.toFile; responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); install(); const client = new OpenAI({ apiKey: "synthetic-provider-key", fetch: (...args: Parameters) => globalThis.fetch(...args) }); const result = await client.audio.transcriptions.create({ model: "whisper-1", file: await toFile(audio, "synthetic.wav", { type: "audio/wav" }) }); From 9da7e9f0359cb500c9ca4a6d4df1fafa35dc4154 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 05:54:05 -0800 Subject: [PATCH 14/20] test(auto): lookalike host and port controls under a native span Co-Authored-By: Claude Opus 5.5 --- typescript/packages/traceai_auto/test/attachments.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/typescript/packages/traceai_auto/test/attachments.test.ts b/typescript/packages/traceai_auto/test/attachments.test.ts index f6c1d9d6e..9ff27ef2f 100644 --- a/typescript/packages/traceai_auto/test/attachments.test.ts +++ b/typescript/packages/traceai_auto/test/attachments.test.ts @@ -298,10 +298,13 @@ describe("privacy negative controls", () => { test("unknown host, non-audio MIME and error bodies do not upload", async () => { install(); await inNative(async () => { await (await speech("unknown.invalid/v1/audio/speech")).arrayBuffer(); + // A lookalike host or port under a native span: only the media allowlist stands between it and capture. + await (await speech("api.openai.com.evil.invalid/v1/audio/speech")).arrayBuffer(); + await (await speech("api.openai.com:8443/v1/audio/speech")).arrayBuffer(); responseType = "application/octet-stream"; await (await speech()).arrayBuffer(); responseType = "audio/wav"; responseStatus = 403; await (await speech()).arrayBuffer(); }); - await (await speech("api.openai.com.evil.invalid/v1/audio/speech")).arrayBuffer(); + responseStatus = 200; await (await speech("api.openai.com.evil.invalid/v1/audio/speech")).arrayBuffer(); await settle(); expect(uploads).toHaveLength(0); }); test("oversized media, raw multipart, and ended/unrelated native contexts fail closed", async () => { From 0697ed96d9334859e8aad80a53fc7201dd04feb1 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:26:19 -0800 Subject: [PATCH 15/20] feat(auto): capture wire-only FormData and incremental speech through deferred targets Merges feat/attachments-ts-core 930876ea and adopts its deferred-target preflight in the wire media observer: - Wire-only FormData audio (OpenAI transcriptions/translations, the openai 5.x+ SDK route) and wire-only incremental speech reads (reader or async iterator; OpenAI speech, Deepgram speak) are admitted against a deferred target before the wire span exists. The core holds bounded bytes, reads and sends nothing, and re-checks policy, environment and sampling when the wire span binds. - Every path on which no wire span will be bound (unmatched route, sync or async failure, emitter delivery gate, build failure, invalid span identity) cancels the target at once; the 300 s bind deadline is only the backstop. A late bind returning false is treated as terminal. - JSON media is unchanged: captured at bind from the wire path's own bounded copies, never through a deferred target. - report().integrations[].media reports provider_adapter for anthropic (version aware, like openai). Tests: wire-only FormData and reader/iterator rows flipped from "not captured" to one upload bound to the exported wire span; no-span, unsampled, image-edit (no wire span) and policy-tightened-while-held rows; deferred path inside the privacy matrix; the env-override test now builds a process-env manager and proves the kill switch at admission; new child-process rows drive auto.init() with an enabled manager against allowlisted HTTPS hosts (and with the kill switch flipped after bootstrap), and the installed @traceai/anthropic instrumentor end to end. Co-Authored-By: Claude Opus 5.5 --- .../traceai_auto/docs/CONFIGURATION.md | 8 +- .../packages/traceai_auto/src/attachments.ts | 13 +- .../packages/traceai_auto/src/libraries.ts | 2 +- .../traceai_auto/src/wire/attachments.ts | 56 +++-- .../traceai_auto/src/wire/fetch-runtime.ts | 12 +- .../test/attachment-adapters.test.ts | 210 ++++++++++++++++++ .../traceai_auto/test/attachments.test.ts | 124 +++++++++-- 7 files changed, 385 insertions(+), 40 deletions(-) diff --git a/typescript/packages/traceai_auto/docs/CONFIGURATION.md b/typescript/packages/traceai_auto/docs/CONFIGURATION.md index c174e2d0a..33722abd7 100644 --- a/typescript/packages/traceai_auto/docs/CONFIGURATION.md +++ b/typescript/packages/traceai_auto/docs/CONFIGURATION.md @@ -250,8 +250,12 @@ Wire capture (`wire`) observes only allowlisted HTTPS protocol entries (`report( | JSON input images, files, computer screenshots (chat, responses) | Captured, from the wire request copy | By the instrumentor (`@traceai/openai`); the same position is never uploaded twice | | JSON output images (images `b64_json`, responses `image_generation_call`) | Captured, from the wire response copy | By the instrumentor where it covers the operation; wire capture needs an explicit integration policy (`withAttachmentCapturePolicy`) | | Speech audio the application read with `arrayBuffer()` / `blob()` | Captured | Needs an explicit integration policy | -| Speech audio the application read incrementally (reader, async iterator) | Not captured: the wire span does not exist yet and nothing is retained | Needs an explicit integration policy; cancel, early return, pipe or tee upload nothing | -| FormData audio / image files | Not captured (no retention before the span exists) | Needs an explicit integration policy | +| Speech audio the application read incrementally (reader, async iterator) | Captured: held in a bounded buffer until the wire span exists, then checked again | Needs an explicit integration policy; cancel, early return, pipe or tee upload nothing | +| FormData audio / image files | Transcription and translation audio captured: held (not read) until the wire span exists, then checked again. Image edits and variations have no wire span, so their files are not captured | Needs an explicit integration policy | + +A wire-only capture waits for the wire span, which exists only once the response completed. When the span is emitted, policy, the environment switch and sampling are checked again before anything is sent. If no span will come (the call failed, was not traced, or its span was not delivered), the held capture is dropped at once; a span that arrives after five minutes binds nothing. Bytes held this way are capped at half the owned budget, so a larger file is not captured (`queue_full`). + +Media in Anthropic `messages.create` input (inline base64 images and documents) is captured by `@traceai/anthropic` when auto enables it; `report().integrations[].media` reports `provider_adapter` for `openai` and `anthropic` when the installed instrumentor exposes its capability descriptor. An adapter that captures media itself marks its context with `withAttachmentOwnership(context)` (context key `traceai.auto.attachment_owner.v1`), and wire capture then skips that call. `report().attachments` shows the manager state, queue counters, routes, sources and unsupported sources; `report().integrations[].media` shows whether an integration captures media through a provider adapter. No row is a playback claim. diff --git a/typescript/packages/traceai_auto/src/attachments.ts b/typescript/packages/traceai_auto/src/attachments.ts index c52514e60..04fb7298f 100644 --- a/typescript/packages/traceai_auto/src/attachments.ts +++ b/typescript/packages/traceai_auto/src/attachments.ts @@ -25,8 +25,10 @@ export function selectAttachmentManager(provider: AttachmentProvider, config: Co try { existing = options.attachmentManager ?? provider?.attachments ?? provider?.getDelegate?.()?.attachments; } catch { existing = undefined; } if (existing) { // Borrowing never turns capture on and never loosens application policy: only auto's hide/PII flags are added. - // The env override (FI_ATTACHMENTS_ENABLED=false) is already applied by the manager at every admission, so a - // borrowed manager is not permanently disabled here (updatePolicy is tighten-only). + // A borrowed manager is not permanently disabled here (updatePolicy is tighten-only). The env override + // (FI_ATTACHMENTS_ENABLED=false) is applied at every admission by a manager whose `environment` reads + // `process.env` (auto's own and the `createNodeAttachmentManager` default); a borrowed manager built with a custom + // `environment` sees only what that function returns. try { existing.updatePolicy({ traceConfig: restrictive(attachmentOptions(config, options).traceConfig as Record) as NodeAttachmentManagerOptions["traceConfig"] }); } catch { /* fail closed below */ } return { manager: existing, owned: ownsProvider && !options.attachmentManager }; } @@ -36,14 +38,17 @@ export function selectAttachmentManager(provider: AttachmentProvider, config: Co /** What auto can capture, independent of whether the manager is on. Nothing here is a playback claim. */ export const AUTO_ATTACHMENT_SOURCES = Object.freeze([ "wire FormData Blob (transcription audio, image edit image/mask) under a native model span with an explicit capture policy", + "wire-only FormData Blob (transcription and translation audio), held bounded until the wire span exists", "wire JSON input images/files/computer screenshots (OpenAI chat, responses) via the @traceai/openai adapter", "wire JSON output images (OpenAI images b64_json, responses image_generation_call) consumed by the application via json()", "wire speech audio materialized by the application via arrayBuffer()/blob()", "wire speech audio read incrementally by the application (reader or async iterator) under a native model span with an explicit capture policy", + "wire-only speech audio read incrementally by the application (reader or async iterator), held bounded until the wire span exists", ]); export const AUTO_ATTACHMENT_UNSUPPORTED = Object.freeze([ - "wire-only FormData (the wire span does not exist before the upload; no source is retained until then)", - "wire-only incremental speech reads (correlation_unavailable until the core supports deferred-target preflight)", + "wire-only image edits and variations (no wire span is emitted for these routes; held files are dropped at once)", + "wire-only FormData or incremental speech above the pre-bind budget (half the owned budget): queue_full", + "wire-only captures whose wire span is never emitted, or binds after the deadline: correlation_unavailable", "native model spans without an explicit integration capture policy", "raw multipart bodies, streamed request bodies, compressed responses consumed before the span is emitted", "RemoteUrl / ProviderFile retrieval", diff --git a/typescript/packages/traceai_auto/src/libraries.ts b/typescript/packages/traceai_auto/src/libraries.ts index 6af51f618..4b3ebf466 100644 --- a/typescript/packages/traceai_auto/src/libraries.ts +++ b/typescript/packages/traceai_auto/src/libraries.ts @@ -17,7 +17,7 @@ export interface Integration { name: string; package: string; targets: string[]; export type MediaStatus = "provider_adapter" | "unqualified"; export interface IntegrationStatus { name: string; package?: string; targets?: string[]; status: Status; reason?: string; media?: MediaStatus } /** Integrations whose instrumentor captures media through the shared manager it receives as `attachments` (#682). */ -const MEDIA_ADAPTERS = new Map([["openai", "openAIAttachmentCapabilities"]]); +const MEDIA_ADAPTERS = new Map([["openai", "openAIAttachmentCapabilities"], ["anthropic", "anthropicAttachmentCapabilities"]]); export const integrations: Integration[] = catalog as Integration[]; export function appRequire() { return createRequire(join(process.cwd(), "__traceai_auto__.cjs")); } function resolvable(name: string): boolean { diff --git a/typescript/packages/traceai_auto/src/wire/attachments.ts b/typescript/packages/traceai_auto/src/wire/attachments.ts index 19430cc3f..7b7384a2b 100644 --- a/typescript/packages/traceai_auto/src/wire/attachments.ts +++ b/typescript/packages/traceai_auto/src/wire/attachments.ts @@ -1,6 +1,6 @@ import { createContextKey, trace, type Context, type Span } from "@opentelemetry/api"; import type { - AttachmentCapturePolicy, AttachmentCorrelation, AttachmentManager, AttachmentPurpose, AttachmentStream, + AttachmentCapturePolicy, AttachmentCorrelation, AttachmentDeferredCorrelation, AttachmentManager, AttachmentPurpose, AttachmentStream, } from "@traceai/fi-core/attachments"; import type { FetchMediaObserver, FetchMediaSink } from "./fetch-runtime"; import { header } from "./core"; @@ -34,6 +34,12 @@ export const WIRE_STREAM_MAX_BYTES = 4 * 1024 * 1024; const WIRE_STREAM_FALLBACK_BYTES = 1024 * 1024; /** Request JSON is parsed only from the wire path's own bounded copy, and only up to this size. */ const MAX_JSON_BYTES = 4 * 1024 * 1024; +/** + * How long a wire-only capture may wait for its wire span, which exists only once the response completed. The core + * maximum, because a transcription of a long recording can take minutes; every path on which no span will be emitted + * cancels the target at once (`settle`), so this bound is only the backstop (for example a hook removed mid-flight). + */ +export const WIRE_DEFERRED_BIND_TIMEOUT_MS = 300000; type Operation = "speech" | "transcription" | "image_edit" | "image_generation" | "chat" | "responses"; /** Allowlisted protocol entries: exact HTTPS origin and path. A compatible path on another host is not authority to capture. */ @@ -96,8 +102,10 @@ function imageFormat(value: unknown): "png" | "jpeg" | "webp" | undefined { * decoded. JSON media goes through the OpenAI provider adapter (#682), so an SDK span and a wire observation of the * same span and position share one occurrence. * - No reader, clone, tee, filesystem, URL retrieval or recorder: only values and chunks the application consumed. - * - Before the wire span exists, nothing is retained: wire-only FormData and wire-only incremental reads are not - * captured (`correlation_unavailable`). + * - Wire-only FormData files and wire-only incremental speech reads are admitted against a deferred target before the + * wire span exists (the core holds bounded bytes and reads or sends nothing). `bind` binds the wire span, where policy + * and sampling are evaluated again; a span that never comes (`settle`), a late bind or a timeout uploads nothing. + * JSON media never uses the deferred target: it is captured at `bind`, from the wire path's own bounded copies. */ export function wireAttachmentObserver(manager: AttachmentManager, parent: Context, method: string, url: string): FetchMediaObserver | undefined { if (parent.getValue(MEDIA_OWNER) || manager.status().state !== "ready") return; @@ -110,12 +118,22 @@ export function wireAttachmentObserver(manager: AttachmentManager, parent: Conte if (native && !capture) return; let correlation: AttachmentCorrelation | undefined = native ? manager.correlation(native) : undefined; let outputObserved = false, inputObserved = false, format: ReturnType; + // Wire-only: one deferred target per call, created only when a FormData file or an incremental read needs it. + let deferred: AttachmentDeferredCorrelation | undefined, settled = false; + /** The capture target now: the native or bound wire span, else (until the wire span exists or never will) the deferred one. */ + const target = (): AttachmentCorrelation | undefined => { + if (correlation || settled) return correlation; + try { deferred ??= manager.deferredCorrelation({ bindTimeoutMs: WIRE_DEFERRED_BIND_TIMEOUT_MS }); } catch { return undefined; } + return deferred; + }; /** Preflight at the exact size, then attach. The source is touched only after admission. */ const attachExact = (size: number, mimeType: string, direction: "input" | "output", purpose: AttachmentPurpose, source: () => { kind: "Bytes"; data: Uint8Array } | { kind: "Blob"; data: Blob }, sourceKind: "Bytes" | "Blob") => { - if (!correlation || !(size > 0)) return; - const ticket = manager.preflight(correlation, { mimeType, direction, purpose, maxBytes: size, sourceKind, capture: { ...capture, ...(direction === "input" ? { modelInput: true } : {}) } }); + if (!(size > 0)) return; + const into = target(); + if (!into) return; + const ticket = manager.preflight(into, { mimeType, direction, purpose, maxBytes: size, sourceKind, capture: { ...capture, ...(direction === "input" ? { modelInput: true } : {}) } }); if (!ticket.admitted) return; try { ticket.attach(source() as never); } catch { ticket.release(); } }; @@ -147,7 +165,11 @@ export function wireAttachmentObserver(manager: AttachmentManager, parent: Conte return { bind(span, request, response) { // The emitter supplies its actual identity before ending, never a later ambient span. - if (correlation) return; + if (correlation || settled) return; + settled = true; + // Pending wire-only captures are re-evaluated against this span now (policy, environment, sampling). `false` means + // late, cancelled or an invalid span identity: cancel (a no-op once cancelled) so nothing waits for the deadline. + try { if (deferred && !deferred.bind(span)) deferred.cancel(); } catch { /* never throws; fail closed */ } correlation = manager.correlation(span); // JSON uses the wire path's own bounded copies; nothing was retained for media before this point. if (op === "chat" || op === "responses" || op === "image_generation") captureJsonInput(parseJson(request?.body)); @@ -155,19 +177,27 @@ export function wireAttachmentObserver(manager: AttachmentManager, parent: Conte // position captured again from json() below resolves to the first occurrence in the adapter's registry. if (response && (response.status ?? 0) >= 200 && (response.status ?? 0) < 300 && !response.truncated && mediaType(header(response.headers, "content-type")) === "application/json") jsonOutput(parseJson(response.body)); }, + settle() { + // No wire span will be bound (unmatched or failed call, delivery gate, build failure): end pending captures now. + if (correlation || settled) return; + settled = true; + try { deferred?.cancel(); } catch { /* never throws */ } + }, request(body) { - // Wire-only: the span does not exist yet, so FormData is not retained until a late span (fail closed). - if (!correlation) return; + // FormData files are admitted before the span exists (deferred target when wire-only); JSON waits for `bind`. if (body instanceof FormData && (op === "transcription" || op === "image_edit")) formData(body); + else if (!correlation) return; else if (typeof body === "string" && (op === "chat" || op === "responses" || op === "image_generation") && body.length <= MAX_JSON_BYTES) { try { captureJsonInput(JSON.parse(body)); } catch { /* Not JSON: nothing to capture. */ } } }, consumed(mimeType, successful, contentLength): FetchMediaSink | undefined { - // Incremental reads need the target before the first chunk: only a native span qualifies (A04). + // Incremental reads need the target before the first chunk: the native span, else the deferred wire target (A04). const type = mediaType(mimeType); - if (outputObserved || !correlation || !successful || op !== "speech" || !AUDIO_MIMES.has(type)) return; - const open = (maxBytes: number) => manager.startAttachmentStream(correlation!, { mimeType: type, direction: "output", purpose: "model_output", maxBytes, capture }); + if (outputObserved || !successful || op !== "speech" || !AUDIO_MIMES.has(type)) return; + const into = target(); + if (!into) return; + const open = (maxBytes: number) => manager.startAttachmentStream(into, { mimeType: type, direction: "output", purpose: "model_output", maxBytes, capture }); const exact = contentLength !== undefined && contentLength > 0 && contentLength <= WIRE_STREAM_MAX_BYTES ? contentLength : undefined; let stream: AttachmentStream = open(exact ?? WIRE_STREAM_MAX_BYTES); // The negotiated upload limit or the owned budget can be below the default reservation. @@ -177,7 +207,7 @@ export function wireAttachmentObserver(manager: AttachmentManager, parent: Conte return { write: chunk => stream.tryWrite(chunk), end: () => { stream.finish(); }, abort: () => stream.abort("incomplete") }; }, materialized(value, methodName, mimeType, successful) { - if (outputObserved || !correlation || !successful) return; + if (outputObserved || !successful) return; const type = mediaType(mimeType); if (op === "speech" && AUDIO_MIMES.has(type)) { if (methodName === "blob" && value instanceof Blob) { @@ -188,7 +218,7 @@ export function wireAttachmentObserver(manager: AttachmentManager, parent: Conte // The view (and the core's synchronous copy) exists only after admission. attachExact(value.byteLength, type, "output", "model_output", () => ({ kind: "Bytes", data: new Uint8Array(value) }), "Bytes"); } - } else if (methodName === "json" && type === "application/json") jsonOutput(value); + } else if (correlation && methodName === "json" && type === "application/json") jsonOutput(value); }, }; } diff --git a/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts b/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts index f64628128..3662f96cb 100644 --- a/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts +++ b/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts @@ -8,6 +8,8 @@ export interface FetchMediaSink { write(chunk: Uint8Array): boolean; end(): void export interface FetchMediaObserver { /** The wire span, before it ends, with the wire path's own bounded request and response copies. */ bind(span: Span, request?: RequestData, response?: ResponseData): void; + /** No wire span will be bound for this call: end anything held for it now (never waits for a timeout). */ + settle?(): void; request(body: BodyInit | null | undefined): void; /** The application started incremental consumption of a response body (reader or async iterator). */ consumed?(mimeType: string, successful: boolean, contentLength: number | undefined): FetchMediaSink | undefined; @@ -208,9 +210,11 @@ export function createFetchHook(max: number, emit: (result: SpanResult, paren const span = found && req && initial && build(found, req, { headers: {}, size: 0, error: hostError instanceof Error ? hostError : new Error("Transport failure") }, initial); if (span) emit(span, parent, start, errorAt); } catch { /* Preserve the host error identity. */ } + finally { try { media?.settle?.(); } catch { /* Media never reaches the application. */ } } throw hostError; } - try { inspect(); } catch { return result; } + const settle = () => { try { media?.settle?.(); } catch { /* Media never reaches the application. */ } }; + try { inspect(); } catch { settle(); return result; } if (media) { const observer = media, observed = new WeakSet(); const decorate = (response: Response) => { @@ -229,7 +233,8 @@ export function createFetchHook(max: number, emit: (result: SpanResult, paren }; observe(result, decorate, () => {}); } - if (!found || !req || !initial) return result; + // No wire span for this call: a deferred wire-only capture can never bind (a native target is unaffected). + if (!found || !req || !initial) { settle(); return result; } const matched = found, request = req, parsed = initial; let resolve!: () => void; const done = new Promise(r => { resolve = r; }); @@ -239,7 +244,8 @@ export function createFetchHook(max: number, emit: (result: SpanResult, paren const complete = () => { if (audio) { response.audioOutputBytes = audio.bytes; response.body = undefined; } try { const span = build(matched, request, response, parsed); if (span) emit(span, parent, start, end, media ? value => media!.bind(value, request, response) : undefined); } catch { /* Capture must not reach application code. */ } - finally { resolve(); pending.delete(done); } + // `emit` binds only when it starts the wire span; any other outcome cancels what was held for it. + finally { settle(); resolve(); pending.delete(done); } }; // Only completed responses awaiting bounded decoding are flushable, never unread streams. // Compressed JSON/SSE audio must be decoded before its base64 length scanner. diff --git a/typescript/packages/traceai_auto/test/attachment-adapters.test.ts b/typescript/packages/traceai_auto/test/attachment-adapters.test.ts index 514e8dcb4..1a56694e0 100644 --- a/typescript/packages/traceai_auto/test/attachment-adapters.test.ts +++ b/typescript/packages/traceai_auto/test/attachment-adapters.test.ts @@ -93,3 +93,213 @@ test("without an enabled manager (the default) the same call uploads nothing", ( expect(result.stderr).toBe(""); expect(result.status).toBe(0); expect(JSON.parse(result.stdout)).toMatchObject({ uploads: [] }); }); + +// Shared loopback double for the attachment control plane (synthetic credentials and scope only). +const controlServer = ` + const http = require('node:http'); + const { randomUUID } = require('node:crypto'); + const initialized = [], uploaded = [], transfers = []; + const scope = { organization_id: randomUUID(), workspace_id: randomUUID(), project_id: randomUUID() }; + const expiry = () => new Date(Date.now() + 3600000).toISOString(); + function control(req, bytes, res) { + transfers.push(req.url); res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('attachment-sessions')) { + res.end(JSON.stringify({ schema_version: 1, scope, capture_session_id: JSON.parse(bytes).capture_session_id, + policy: { version: 'fixture-v1', enabled: true, allowed_sources: ['Bytes', 'Blob'], allowed_modalities: ['image', 'audio', 'document'], require_media_redaction: false, + require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false }, + limits: { max_upload_bytes: 1048576, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, + active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, + upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }, + expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); + } else if (req.url.endsWith('/uploads')) { + const body = JSON.parse(bytes); initialized.push(body); res.statusCode = 201; + res.end(JSON.stringify({ upload_id: body.attachment_id, attachment_id: body.attachment_id, occurrence_id: body.occurrence_id, state: 'pending_upload', + reserved_bytes: body.declared_bytes, expires_at: expiry(), transfer_mode: 'authenticated_stream', write_capability: 'synthetic-upload-only' })); + } else if (req.url.endsWith('/content')) { + uploaded.push(bytes); res.end(JSON.stringify({ upload_id: req.url.split('/').at(-2), state: 'pending_upload', reason: 'queued' })); + } else { res.statusCode = 202; res.end(JSON.stringify({ upload_id: req.url.split('/').at(-2), state: 'validating', reason: 'binding_pending' })); } + } + function controlClient(origin) { + return async req => { + const body = req.body instanceof Blob ? Buffer.from(await req.body.arrayBuffer()) : req.body; + return new Promise((resolve, reject) => { + const out = http.request(origin + new URL(req.url).pathname, { method: req.method, headers: { ...req.headers, 'x-attachment-control': '1' }, signal: req.signal }, response => { + const c = []; response.on('data', d => c.push(d)); response.on('end', () => resolve({ status: response.statusCode, headers: {}, body: Buffer.concat(c) })); + }); out.on('error', reject); out.end(body); + }); + }; + } +`; +function wavBase64(samples = 80) { + const b = Buffer.alloc(44 + samples * 2); b.write("RIFF"); b.writeUInt32LE(b.length - 8, 4); b.write("WAVEfmt ", 8); + b.writeUInt32LE(16, 16); b.writeUInt16LE(1, 20); b.writeUInt16LE(1, 22); b.writeUInt32LE(8000, 24); + b.writeUInt32LE(16000, 28); b.writeUInt16LE(2, 32); b.writeUInt16LE(16, 34); b.write("data", 36); b.writeUInt32LE(samples * 2, 40); + for (let i = 0; i < samples; i++) b.writeInt16LE(Math.round(8000 * Math.sin(i / 3)), 44 + i * 2); + return b.toString("base64"); +} + +// auto.init() with an ENABLED manager against allowlisted HTTPS hosts, wire only (no native instrumentor). The +// application's fetch is redirected to loopback BEFORE init, so auto's wire hook sees the real provider URLs. +function runWire(env: Record = {}) { + const code = ` + const assert = require('node:assert/strict'); + const { InMemorySpanExporter } = require('@opentelemetry/sdk-trace-base'); + const { AttachmentManager, createAttachmentHttpTransport } = require('@traceai/fi-core/attachments'); + const auto = require('./dist/src/index.js'); + ${controlServer} + const WAV = Buffer.from('${wavBase64()}', 'base64'); + (async () => { + let providerCalls = 0; + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const c of req) chunks.push(c); const bytes = Buffer.concat(chunks); + if (req.headers['x-attachment-control']) { control(req, bytes, res); return; } + providerCalls++; + const path = req.url.split('?')[0]; + if (path === '/v1/audio/transcriptions') { res.setHeader('content-type', 'application/json'); res.end('{"text":"synthetic"}'); return; } + // Speech: several spaced chunks, so incremental readers see several reads. + res.writeHead(200, { 'content-type': 'audio/wav' }); + const size = Math.ceil(WAV.length / 4); + for (let i = 0; i < WAV.length; i += size) { res.write(WAV.subarray(i, i + size)); await new Promise(r => setTimeout(r, 15)); } + res.end(); + }); + await new Promise(r => server.listen(0, '127.0.0.1', r)); + const origin = 'http://127.0.0.1:' + server.address().port; + const nativeFetch = globalThis.fetch; + globalThis.fetch = (input, init) => { const u = new URL(input instanceof Request ? input.url : String(input)); return nativeFetch(origin + u.pathname + u.search, init); }; + const manager = new AttachmentManager({ enabled: true, projectName: 'synthetic', projectType: 'observe', environment: () => process.env, + transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic' }, client: controlClient(origin) }) }); + const ready = await manager.ready(); + const exporter = new InMemorySpanExporter(); + auto.init({ exporter, instrumentations: ['wire'], configFile: '/nonexistent', apiKey: '', secretKey: '', attachmentManager: manager }); + assert.equal(auto.getAttachmentManager(), manager); + // The kill switch flipped after bootstrap: a process-env manager must deny at admission. + if (process.env.TRACEAI_TEST_KILL_SWITCH === '1') process.env.FI_ATTACHMENTS_ENABLED = 'false'; + const json = { 'content-type': 'application/json' }; + // 1. Speech the application materialized. + const speech = await fetch('https://api.openai.com/v1/audio/speech', { method: 'POST', headers: json, body: JSON.stringify({ model: 'tts-1', input: 'synthetic', voice: 'alloy' }) }); + assert.deepEqual(Buffer.from(await speech.arrayBuffer()), WAV); + // 2. Wire-only FormData (the OpenAI SDK transcription route on openai 5.x+). + const form = new FormData(); form.set('model', 'whisper-1'); form.set('file', new Blob([WAV], { type: 'audio/wav' }), 'synthetic.wav'); + assert.deepEqual(await (await fetch('https://api.openai.com/v1/audio/transcriptions', { method: 'POST', body: form })).json(), { text: 'synthetic' }); + // 3. Wire-only incremental read (the Deepgram SDK speak route). + const speak = await fetch('https://api.deepgram.com/v1/speak?model=aura-asteria-en', { method: 'POST', headers: json, body: JSON.stringify({ text: 'synthetic' }) }); + const reader = speak.body.getReader(), parts = []; + for (;;) { const x = await reader.read(); if (x.done) break; parts.push(Buffer.from(x.value)); } + assert.deepEqual(Buffer.concat(parts), WAV); + await auto.forceFlush(); await manager.flush(2000); + const report = auto.report().attachments; + const spans = exporter.getFinishedSpans(); + process.stdout.write(JSON.stringify({ ready, state: report.state, providerCalls, uploads: uploaded.map(b => b.toString('base64')), + initialized: initialized.map(r => ({ parent: r.association.parent.span_id, direction: r.direction, purpose: r.purpose, mime: r.mime_type, source: r.source_kind })), + spans: spans.map(s => s.spanContext().spanId), controlCalls: transfers.filter(p => !p.endsWith('attachment-sessions')).length, + status: manager.status() })); + await auto.shutdown(); await manager.shutdown(); server.closeAllConnections(); server.close(); + })().catch(error => { process.stderr.write(String(error && error.stack || error)); process.exitCode = 1; }); + `; + return spawnSync(process.execPath, ["-e", code], { + cwd: root, encoding: "utf8", timeout: 30000, + env: { ...process.env, FI_ATTACHMENTS_ENABLED: "", FI_API_KEY: "", FI_SECRET_KEY: "", FI_CONFIG_FILE: "/nonexistent", FI_AUTO_DEBUG: "false", ...env }, + }); +} +test("auto.init + enabled manager + allowlisted hosts: speech, wire-only FormData and wire-only reads each upload once, bound to their wire span", () => { + const result = runWire(); + expect(result.stderr).toBe(""); expect(result.status).toBe(0); + const out = JSON.parse(result.stdout); + expect(out).toMatchObject({ ready: true, state: "ready", providerCalls: 3 }); + expect(out.uploads).toEqual([wavBase64(), wavBase64(), wavBase64()]); + expect(out.spans).toHaveLength(3); + // Each upload is bound to a distinct wire span that auto actually exported. + expect(new Set(out.initialized.map((r: any) => r.parent))).toEqual(new Set(out.spans)); + expect(out.initialized.map((r: any) => `${r.direction}/${r.purpose}/${r.mime}`).sort()).toEqual(["input/model_input/audio/wav", "output/model_output/audio/wav", "output/model_output/audio/wav"]); + expect(out.status).toMatchObject({ openTickets: 0, openStreams: 0, ownedBytes: 0 }); +}); +test("the same calls with FI_ATTACHMENTS_ENABLED=false set after bootstrap send nothing (kill switch at admission)", () => { + const result = runWire({ TRACEAI_TEST_KILL_SWITCH: "1" }); + expect(result.stderr).toBe(""); expect(result.status).toBe(0); + const out = JSON.parse(result.stdout); + // Provider traffic and wire spans are unchanged; no attachment control traffic beyond the bootstrap session. + expect(out).toMatchObject({ ready: true, providerCalls: 3, uploads: [], initialized: [], controlCalls: 0 }); + expect(out.spans).toHaveLength(3); + expect(out.status).toMatchObject({ openTickets: 0, openStreams: 0, ownedBytes: 0 }); +}); + +// Anthropic: auto enables the INSTALLED @traceai/anthropic (resolved from the application directory, like a real app) +// and hands it the same manager. The application directory links only the SDK and the instrumentor. +function anthropicApp(): string { + const { mkdtempSync, mkdirSync, symlinkSync, readFileSync } = require("node:fs") as typeof import("node:fs"); + const { tmpdir } = require("node:os") as typeof import("node:os"); + const { dirname, join } = require("node:path") as typeof import("node:path"); + const app = mkdtempSync(join(tmpdir(), "traceai-auto-anthropic-")); + const packageRoot = (name: string, from: string) => { + let dir = dirname(require.resolve(name, { paths: [from] })); + for (;;) { + try { if (JSON.parse(readFileSync(join(dir, "package.json"), "utf8")).name === name) return dir; } catch { /* ascend */ } + const parent = dirname(dir); if (parent === dir) throw new Error(`Cannot locate ${name}`); dir = parent; + } + }; + for (const [name, target] of [["@anthropic-ai/sdk", packageRoot("@anthropic-ai/sdk", root)], ["@traceai/anthropic", resolve(root, "../traceai_anthropic")]]) { + mkdirSync(join(app, "node_modules", dirname(name)), { recursive: true }); + symlinkSync(target, join(app, "node_modules", name), "dir"); + } + return app; +} +function runAnthropic(app: string) { + const fiCore = require.resolve("@traceai/fi-core/attachments", { paths: [root] }); + const sdkTrace = require.resolve("@opentelemetry/sdk-trace-base", { paths: [root] }); + const code = ` + const assert = require('node:assert/strict'); + const { InMemorySpanExporter } = require(${JSON.stringify(sdkTrace)}); + const { AttachmentManager, createAttachmentHttpTransport } = require(${JSON.stringify(fiCore)}); + const auto = require(${JSON.stringify(resolve(root, "dist/src/index.js"))}); + ${controlServer} + const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII='; + (async () => { + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const c of req) chunks.push(c); const bytes = Buffer.concat(chunks); + if (req.headers['x-attachment-control']) { control(req, bytes, res); return; } + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({ id: 'msg_synthetic', type: 'message', role: 'assistant', model: 'claude-synthetic', stop_reason: 'end_turn', stop_sequence: null, + content: [{ type: 'text', text: 'synthetic' }], usage: { input_tokens: 1, output_tokens: 1 } })); + }); + await new Promise(r => server.listen(0, '127.0.0.1', r)); + const origin = 'http://127.0.0.1:' + server.address().port; + const manager = new AttachmentManager({ enabled: true, projectName: 'synthetic', projectType: 'observe', environment: () => ({}), + transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic' }, client: controlClient(origin) }) }); + assert.equal(await manager.ready(), true); + const exporter = new InMemorySpanExporter(); + auto.init({ exporter, instrumentations: ['anthropic', 'wire'], configFile: '/nonexistent', apiKey: '', secretKey: '', attachmentManager: manager }); + const Anthropic = require('@anthropic-ai/sdk').default ?? require('@anthropic-ai/sdk').Anthropic; + const sdk = new Anthropic({ apiKey: 'synthetic-provider-key', baseURL: origin }); + await sdk.messages.create({ model: 'claude-synthetic', max_tokens: 8, messages: [{ role: 'user', content: [ + { type: 'text', text: 'synthetic' }, { type: 'image', source: { type: 'base64', media_type: 'image/png', data: PNG } }] }] }); + await auto.forceFlush(); await manager.flush(2000); + const row = auto.report().integrations.find(r => r.name === 'anthropic'); + const spans = exporter.getFinishedSpans().filter(s => s.attributes['gen_ai.span.kind'] === 'LLM' || s.attributes['fi.span.kind'] === 'LLM'); + process.stdout.write(JSON.stringify({ status: row && row.status, media: row && row.media, uploads: uploaded.map(b => b.toString('base64')), + parents: initialized.map(r => r.association.parent.span_id), purposes: initialized.map(r => r.purpose), llmSpans: spans.map(s => s.spanContext().spanId) })); + await auto.shutdown(); await manager.shutdown(); server.closeAllConnections(); server.close(); + })().catch(error => { process.stderr.write(String(error && error.stack || error)); process.exitCode = 1; }); + `; + return spawnSync(process.execPath, ["-e", code], { + cwd: app, encoding: "utf8", timeout: 30000, + env: { ...process.env, FI_ATTACHMENTS_ENABLED: "", FI_API_KEY: "", FI_SECRET_KEY: "", FI_CONFIG_FILE: "/nonexistent", FI_AUTO_DEBUG: "false" }, + }); +} +test("anthropic: report().integrations[].media is provider_adapter, and the installed instrumentor uploads the image once, bound to its span", () => { + const app = anthropicApp(); + try { + const result = runAnthropic(app); + expect(result.stderr).toBe(""); expect(result.status).toBe(0); + const out = JSON.parse(result.stdout); + expect(out).toMatchObject({ status: "instrumented", media: "provider_adapter", uploads: [png], purposes: ["model_input"] }); + expect(out.llmSpans).toHaveLength(1); expect(out.parents).toEqual(out.llmSpans); + } finally { require("node:fs").rmSync(app, { recursive: true, force: true }); } +}); +test("inventory reports media capture per integration, apart from span status", () => { + const { inventory } = require("../src/libraries") as typeof import("../src/libraries"); + const { resolveConfig } = require("../src/config") as typeof import("../src/config"); + const rows = inventory(resolveConfig({ configFile: "/nonexistent" }, {})); + const media = (name: string) => rows.find(row => row.name === name)?.media; + expect(media("openai")).toBe("provider_adapter"); expect(media("anthropic")).toBe("provider_adapter"); + expect(rows.filter(row => row.media === "provider_adapter").map(row => row.name).sort()).toEqual(["anthropic", "openai"]); +}); diff --git a/typescript/packages/traceai_auto/test/attachments.test.ts b/typescript/packages/traceai_auto/test/attachments.test.ts index 9ff27ef2f..4aa75de1e 100644 --- a/typescript/packages/traceai_auto/test/attachments.test.ts +++ b/typescript/packages/traceai_auto/test/attachments.test.ts @@ -5,7 +5,7 @@ import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { context, trace, ROOT_CONTEXT } from "@opentelemetry/api"; import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; -import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; +import { AlwaysOffSampler, BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; import { AttachmentManager, createAttachmentHttpTransport, type AttachmentHttpClient, type AttachmentManagerOptions } from "@traceai/fi-core/attachments"; import { captureOpenAIAttachments } from "@traceai/openai"; import { patchFetch } from "../src/wire/fetch"; @@ -67,16 +67,28 @@ async function createManager(options: AttachmentManagerOptions = {}, serverOverr expect(result.status().state).toBe(options.enabled === false ? "disabled" : "ready"); return result; } -function install(target = manager) { +/** `emitSpans: false` models an emitter that never starts the wire span (for example its delivery gate refused it). */ +function install(target = manager, emitSpans = true) { globalThis.fetch = (input: RequestInfo | URL, init?: RequestInit) => { const u = new URL(input instanceof Request ? input.url : String(input)); return nativeFetch(`${origin}${u.pathname}${u.search}`, init); }; hook = patchFetch(4194304, (result, parent, start, end, bindMedia) => { + if (!emitSpans) return; const span = provider.getTracer("wire-test").startSpan(result.name, { startTime: start, attributes: result.attributes }, parent); bindMedia?.(span); span.end(end); }, [], target); } +const transcription = () => { + const body = new FormData(); body.set("model", "whisper-1"); body.set("file", new Blob([audio], { type: "audio/wav" }), "PRIVATE_CANARY_FILENAME.wav"); + return fetch("https://api.openai.com/v1/audio/transcriptions", { method: "POST", body }); +}; +async function readAll(response: Response, mode: "reader" | "iterator" = "reader"): Promise { + const chunks: Buffer[] = []; + if (mode === "iterator") { for await (const chunk of response.body as any) chunks.push(Buffer.from(chunk)); } + else { const reader = response.body!.getReader(); for (;;) { const x = await reader.read(); if (x.done) break; chunks.push(Buffer.from(x.value)); } } + return Buffer.concat(chunks); +} let nativeSpanId = ""; function inNative(fn: () => T, kind = "LLM", policyContext = true): T { const span = provider.getTracer("native-test").startSpan("native", { attributes: { "gen_ai.span.kind": kind } }); @@ -191,10 +203,34 @@ describe("A04 speech output", () => { const unread = await speech(); expect(unread.bodyUsed).toBe(false); await unread.body!.cancel(); const partial = await speech(); const reader = partial.body!.getReader(); await reader.read(); await reader.cancel(); }); - // Wire-only incremental reads are not captured (the wire span does not exist yet; nothing is retained). - const wireOnly = await speech(); const reader = wireOnly.body!.getReader(); while (!(await reader.read()).done) { /* app reads */ } + // Wire-only (deferred target): a reader cancelled midway uploads nothing, and nothing stays held. + const wireOnly = await speech(); const reader = wireOnly.body!.getReader(); await reader.read(); await reader.cancel(); await settle(); expect(uploads).toHaveLength(0); expect(manager.status()).toMatchObject({ openStreams: 0, openTickets: 0, ownedBytes: 0 }); }); + test.each([ + ["api.openai.com/v1/audio/speech", { model: "tts-1", input: "synthetic", voice: "alloy" }, "reader"], + ["api.openai.com/v1/audio/speech", { model: "tts-1", input: "synthetic", voice: "alloy" }, "iterator"], + ["api.deepgram.com/v1/speak", { text: "synthetic" }, "reader"], + ] as const)("wire-only %s (%j, %s): incremental reads upload once, bound to the wire span through a deferred target", async (route, body, mode) => { + responseChunks = 4; install(); + const response = await fetch(`https://${route}`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); + expect(await readAll(response, mode)).toEqual(audio); + await settle(); expect(uploads).toEqual([audio]); + const spans = exporter.getFinishedSpans(); expect(spans).toHaveLength(1); + expect(requests).toHaveLength(1); + expect(requests[0].association.parent.span_id).toBe(spans[0].spanContext().spanId); + expect(requests[0]).toMatchObject({ direction: "output", purpose: "model_output", mime_type: "audio/wav", declared_bytes: audio.length }); + expect(manager.status()).toMatchObject({ openStreams: 0, openTickets: 0, ownedBytes: 0 }); + }); + test("wire-only captures whose wire span is never emitted are dropped when the call completes, not after the deadline", async () => { + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); install(manager, false); + await (await transcription()).json(); + responseType = "audio/wav"; responseData = audio; responseChunks = 4; + expect(await readAll(await speech())).toEqual(audio); + await new Promise(done => setImmediate(done)); + expect(manager.status()).toMatchObject({ openStreams: 0, openTickets: 0, ownedBytes: 0 }); + await settle(); expect(uploads).toHaveLength(0); expect(sessionsOnly()).toEqual([]); expect(exporter.getFinishedSpans()).toHaveLength(0); + }); test("piped bodies are not captured", async () => { install(); await inNative(async () => { const response = await speech(); const chunks: Uint8Array[] = []; @@ -225,11 +261,34 @@ describe("A03 FormData and JSON media", () => { install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body })).json(); }); await settle(); expect(uploads).toEqual([png]); }); - test("wire-only FormData is not captured, and its source is not retained until the late span", async () => { + test("wire-only transcription FormData is held (unread) until the wire span exists, then uploads once bound to it", async () => { responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); - const body = new FormData(); body.set("model", "whisper-1"); body.set("file", new Blob([audio], { type: "audio/wav" })); - install(); await (await fetch("https://api.openai.com/v1/audio/transcriptions", { method: "POST", body })).json(); - await settle(); expect(uploads).toHaveLength(0); + install(); const pending = transcription(); + // Admitted before the span exists: bytes are reserved, nothing is sent until bind. + expect(manager.status().ownedBytes).toBe(audio.length); expect(sessionsOnly()).toEqual([]); + expect(await (await pending).json()).toEqual({ text: "synthetic" }); + await settle(); expect(uploads).toEqual([audio]); expect(providerBodies[0].includes(audio)).toBe(true); + const spans = exporter.getFinishedSpans(); expect(spans).toHaveLength(1); + expect(requests[0]).toMatchObject({ direction: "input", purpose: "model_input", source_kind: "Blob", mime_type: "audio/wav" }); + expect(requests[0].association.parent.span_id).toBe(spans[0].spanContext().spanId); + expect(JSON.stringify(requests)).not.toContain("PRIVATE_CANARY"); + expect(manager.status()).toMatchObject({ openTickets: 0, ownedBytes: 0 }); + }); + test("wire-only image edits have no wire span: the held file is dropped when the call completes and nothing uploads", async () => { + responseType = "application/json"; responseData = Buffer.from('{"data":[]}'); + const body = new FormData(); body.set("image", new Blob([png], { type: "image/png" })); + install(); await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body })).json(); + expect(manager.status()).toMatchObject({ openTickets: 0, ownedBytes: 0 }); + await settle(); expect(uploads).toHaveLength(0); expect(sessionsOnly()).toEqual([]); + }); + test("a wire span that is not sampled uploads nothing held for it (re-checked at bind)", async () => { + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); + await provider.shutdown(); exporter = new InMemorySpanExporter(); + provider = new BasicTracerProvider({ sampler: new AlwaysOffSampler(), spanProcessors: [new SimpleSpanProcessor(exporter)] }); + install(); await (await transcription()).json(); + responseType = "audio/wav"; responseData = audio; responseChunks = 4; expect(await readAll(await speech())).toEqual(audio); + await settle(); expect(uploads).toHaveLength(0); expect(sessionsOnly()).toEqual([]); + expect(manager.status()).toMatchObject({ openStreams: 0, openTickets: 0, ownedBytes: 0 }); }); test("wire-only image JSON output uploads decoded bytes through the provider adapter, never URLs", async () => { responseType = "application/json"; responseData = Buffer.from(JSON.stringify({ data: [{ b64_json: png.toString("base64") }, { url: "https://untrusted.invalid/private?token=CANARY" }] })); @@ -276,12 +335,23 @@ describe("privacy negative controls", () => { install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body })).json(); }); await (await fetch("https://api.openai.com/v1/chat/completions", { method: "POST", body: JSON.stringify({ messages: [{ role: "user", content: [{ type: "image_url", image_url: { url: `data:image/png;base64,${png.toString("base64")}` } }] }] }) })).json(); - await settle(); expect(sessionsOnly()).toEqual([]); + // Wire-only FormData audio through the deferred target: denied at admission, nothing held (image hiding is image-only). + if (!options.traceConfig?.hideInputImages) { responseData = Buffer.from('{"text":"synthetic"}'); await (await transcription()).json(); } + await settle(); expect(sessionsOnly()).toEqual([]); expect(manager.status()).toMatchObject({ openTickets: 0, ownedBytes: 0 }); + }); + test.each(["hideInputs", "hideInputMessages", "piiRedaction"] as const)("%s set after a wire-only file was held denies it at bind", async flag => { + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); + install(); const pending = transcription(); + expect(manager.status().ownedBytes).toBe(audio.length); + manager.updatePolicy({ traceConfig: { [flag]: true } }); transfers = []; + await (await pending).json(); + await settle(); expect(sessionsOnly()).toEqual([]); expect(uploads).toHaveLength(0); expect(manager.status()).toMatchObject({ openTickets: 0, ownedBytes: 0 }); }); test.each(["hideOutputs", "hideOutputMessages", "hideOutputText", "piiRedaction"] as const)("%s prevents output speech and image transfer", async flag => { manager.updatePolicy({ traceConfig: { [flag]: true } }); transfers = []; install(); await inNative(async () => { await (await speech()).arrayBuffer(); }); await (await speech()).arrayBuffer(); + responseChunks = 4; await readAll(await speech()); responseChunks = 1; responseType = "application/json"; responseData = Buffer.from(JSON.stringify({ data: [{ b64_json: png.toString("base64") }] })); await (await fetch("https://api.openai.com/v1/images/generations", { method: "POST", body: "{}" })).json(); await settle(); expect(sessionsOnly()).toEqual([]); @@ -352,18 +422,35 @@ describe("A02 ownership and configuration", () => { install(); await inNative(async () => { await (await speech()).arrayBuffer(); }); await settle(); expect(uploads).toHaveLength(0); }); - test("an env override does not permanently disable a borrowed manager (updatePolicy is tighten-only)", async () => { - const before = process.env.FI_ATTACHMENTS_ENABLED; process.env.FI_ATTACHMENTS_ENABLED = "false"; - try { selectAttachmentManager({ attachments: manager }, resolveConfig({ configFile: "/nonexistent" }, {}), {}, false); } - finally { if (before === undefined) delete process.env.FI_ATTACHMENTS_ENABLED; else process.env.FI_ATTACHMENTS_ENABLED = before; } - install(); await inNative(async () => { await (await speech()).arrayBuffer(); }); - await settle(); expect(uploads).toEqual([audio]); + test("FI_ATTACHMENTS_ENABLED=false denies at admission on a process-env manager, without permanently disabling a borrowed one", async () => { + const before = process.env.FI_ATTACHMENTS_ENABLED; + delete process.env.FI_ATTACHMENTS_ENABLED; + try { + // A manager whose `environment` reads process.env, as auto's own and the Node factory's default do. + await manager.shutdown(); manager = await createManager({ environment: () => process.env }); + process.env.FI_ATTACHMENTS_ENABLED = "false"; + selectAttachmentManager({ attachments: manager }, resolveConfig({ configFile: "/nonexistent" }, {}), {}, false); + transfers = []; install(); + // Kill switch at admission: materialized, incremental and FormData wire captures, native and wire-only, all denied. + await inNative(async () => { await (await speech()).arrayBuffer(); }); + await (await speech()).arrayBuffer(); + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); await (await transcription()).json(); + await settle(); expect(sessionsOnly()).toEqual([]); expect(uploads).toHaveLength(0); + expect(manager.status()).toMatchObject({ openTickets: 0, openStreams: 0, ownedBytes: 0 }); + // The switch is read at each admission: lifting it resumes capture on the same borrowed manager. + delete process.env.FI_ATTACHMENTS_ENABLED; + responseType = "audio/wav"; responseData = audio; + await inNative(async () => { await (await speech()).arrayBuffer(); }); + await settle(); expect(uploads).toEqual([audio]); + } finally { if (before === undefined) delete process.env.FI_ATTACHMENTS_ENABLED; else process.env.FI_ATTACHMENTS_ENABLED = before; } }); test("status report is policy and source aware and claims no playback", () => { const report = attachmentReport(manager); expect(report.state).toBe("ready"); expect(report.status).toMatchObject({ openTickets: 0 }); expect(report.routes).toContain("POST https://api.openai.com/v1/audio/speech (speech)"); expect(report.unsupported).toEqual(expect.arrayContaining(["WebSocket / RTC audio", "product playback qualification"])); + expect(report.unsupported.join("\n")).not.toMatch(/deferred-target preflight/); + expect(report.sources).toEqual(expect.arrayContaining([expect.stringContaining("wire-only FormData Blob"), expect.stringContaining("wire-only speech audio read incrementally")])); expect(attachmentReport().state).toBe("not_initialized"); }); }); @@ -394,7 +481,7 @@ describe("real SDKs against loopback", () => { expect(result.data[0].b64_json).toBe(png.toString("base64")); await settle(); expect(uploads).toEqual([png]); }); - test("OpenAI SDK transcription (wire-only) keeps the multipart request intact and captures nothing", async () => { + test("OpenAI SDK transcription (wire-only) keeps the multipart request intact and uploads the file once, bound to the wire span", async () => { const mod = sdk(); // openai 4.x sends multipart as a Node stream, which a WHATWG fetch rejects without `duplex` (SDK limitation, not auto). if (Number(sdkVersion().split(".")[0]) < 5) return; const OpenAI = mod.default ?? mod.OpenAI; const toFile = mod.toFile; @@ -402,6 +489,9 @@ describe("real SDKs against loopback", () => { install(); const client = new OpenAI({ apiKey: "synthetic-provider-key", fetch: (...args: Parameters) => globalThis.fetch(...args) }); const result = await client.audio.transcriptions.create({ model: "whisper-1", file: await toFile(audio, "synthetic.wav", { type: "audio/wav" }) }); expect(result.text).toBe("synthetic"); expect(providerBodies[0].includes(audio)).toBe(true); - await settle(); expect(uploads).toHaveLength(0); + await settle(); expect(uploads).toEqual([audio]); + const spans = exporter.getFinishedSpans(); expect(spans).toHaveLength(1); + expect(requests[0]).toMatchObject({ direction: "input", purpose: "model_input", source_kind: "Blob" }); + expect(requests[0].association.parent.span_id).toBe(spans[0].spanContext().spanId); }); }); From 64d2a02d88675e25eabc7dee0e8a80ec6a589ece Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:29:45 -0800 Subject: [PATCH 16/20] test(auto): an invalid wire span identity cancels the deferred target at once Co-Authored-By: Claude Opus 5.5 --- .../traceai_auto/test/attachments.test.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/typescript/packages/traceai_auto/test/attachments.test.ts b/typescript/packages/traceai_auto/test/attachments.test.ts index 4aa75de1e..2a603b4bf 100644 --- a/typescript/packages/traceai_auto/test/attachments.test.ts +++ b/typescript/packages/traceai_auto/test/attachments.test.ts @@ -3,7 +3,7 @@ import { AddressInfo } from "node:net"; import { randomUUID } from "node:crypto"; import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; -import { context, trace, ROOT_CONTEXT } from "@opentelemetry/api"; +import { context, trace, ROOT_CONTEXT, INVALID_SPAN_CONTEXT } from "@opentelemetry/api"; import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; import { AlwaysOffSampler, BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; import { AttachmentManager, createAttachmentHttpTransport, type AttachmentHttpClient, type AttachmentManagerOptions } from "@traceai/fi-core/attachments"; @@ -67,15 +67,18 @@ async function createManager(options: AttachmentManagerOptions = {}, serverOverr expect(result.status().state).toBe(options.enabled === false ? "disabled" : "ready"); return result; } -/** `emitSpans: false` models an emitter that never starts the wire span (for example its delivery gate refused it). */ -function install(target = manager, emitSpans = true) { +/** + * `emit: "none"` models an emitter that never starts the wire span (for example its delivery gate refused it); + * `"invalid"` binds a span with an invalid identity, which a deferred target refuses to bind. + */ +function install(target = manager, emit: "span" | "none" | "invalid" = "span") { globalThis.fetch = (input: RequestInfo | URL, init?: RequestInit) => { const u = new URL(input instanceof Request ? input.url : String(input)); return nativeFetch(`${origin}${u.pathname}${u.search}`, init); }; hook = patchFetch(4194304, (result, parent, start, end, bindMedia) => { - if (!emitSpans) return; - const span = provider.getTracer("wire-test").startSpan(result.name, { startTime: start, attributes: result.attributes }, parent); + if (emit === "none") return; + const span = emit === "invalid" ? trace.wrapSpanContext(INVALID_SPAN_CONTEXT) : provider.getTracer("wire-test").startSpan(result.name, { startTime: start, attributes: result.attributes }, parent); bindMedia?.(span); span.end(end); }, [], target); } @@ -222,8 +225,8 @@ describe("A04 speech output", () => { expect(requests[0]).toMatchObject({ direction: "output", purpose: "model_output", mime_type: "audio/wav", declared_bytes: audio.length }); expect(manager.status()).toMatchObject({ openStreams: 0, openTickets: 0, ownedBytes: 0 }); }); - test("wire-only captures whose wire span is never emitted are dropped when the call completes, not after the deadline", async () => { - responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); install(manager, false); + test.each(["none", "invalid"] as const)("wire-only captures whose wire span is never bound (emit: %s) are dropped when the call completes, not after the deadline", async emit => { + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); install(manager, emit); await (await transcription()).json(); responseType = "audio/wav"; responseData = audio; responseChunks = 4; expect(await readAll(await speech())).toEqual(audio); From 60ef0c907ad0d143400e235f16263ebfcbe071bb Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:43:52 -0800 Subject: [PATCH 17/20] test(auto): unset the kill switch for the process-env child manager; wire-only materialized refs now project onto the wire span Co-Authored-By: Claude Opus 5.5 --- .../traceai_auto/test/attachment-adapters.test.ts | 8 ++++---- typescript/packages/traceai_auto/test/attachments.test.ts | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/typescript/packages/traceai_auto/test/attachment-adapters.test.ts b/typescript/packages/traceai_auto/test/attachment-adapters.test.ts index 1a56694e0..0be17ab23 100644 --- a/typescript/packages/traceai_auto/test/attachment-adapters.test.ts +++ b/typescript/packages/traceai_auto/test/attachment-adapters.test.ts @@ -196,10 +196,10 @@ function runWire(env: Record = {}) { await auto.shutdown(); await manager.shutdown(); server.closeAllConnections(); server.close(); })().catch(error => { process.stderr.write(String(error && error.stack || error)); process.exitCode = 1; }); `; - return spawnSync(process.execPath, ["-e", code], { - cwd: root, encoding: "utf8", timeout: 30000, - env: { ...process.env, FI_ATTACHMENTS_ENABLED: "", FI_API_KEY: "", FI_SECRET_KEY: "", FI_CONFIG_FILE: "/nonexistent", FI_AUTO_DEBUG: "false", ...env }, - }); + // This manager reads process.env at each admission, so the switch must be UNSET (an empty value is a deny). + const childEnv: Record = { ...process.env, FI_API_KEY: "", FI_SECRET_KEY: "", FI_CONFIG_FILE: "/nonexistent", FI_AUTO_DEBUG: "false", ...env }; + delete childEnv.FI_ATTACHMENTS_ENABLED; + return spawnSync(process.execPath, ["-e", code], { cwd: root, encoding: "utf8", timeout: 30000, env: childEnv as NodeJS.ProcessEnv }); } test("auto.init + enabled manager + allowlisted hosts: speech, wire-only FormData and wire-only reads each upload once, bound to their wire span", () => { const result = runWire(); diff --git a/typescript/packages/traceai_auto/test/attachments.test.ts b/typescript/packages/traceai_auto/test/attachments.test.ts index 2a603b4bf..6408f6d25 100644 --- a/typescript/packages/traceai_auto/test/attachments.test.ts +++ b/typescript/packages/traceai_auto/test/attachments.test.ts @@ -145,8 +145,10 @@ describe("A04 speech output", () => { const span = exporter.getFinishedSpans()[0]; expect(requests[0].association.parent.span_id).toBe(span.spanContext().spanId); expect(requests[0]).toMatchObject({ direction: "output", purpose: "model_output", mime_type: "audio/wav", declared_bytes: audio.length }); - // The upload commits after the wire span ended: the authoritative link is association.parent, never a late mutation (T-CH-03). - expect(span.attributes["fi.attachment.refs"]).toBeUndefined(); + // The body is consumed through a reader the runtime opens, so the bytes arrive through a deferred target that binds + // inside the emitter, before the wire span ends: the reference is projected onto that span, once. + const refs = JSON.parse(String(span.attributes["fi.attachment.refs"])); + expect(refs).toEqual([expect.objectContaining({ occurrence_id: requests[0].occurrence_id, modality: "audio", direction: "output" })]); expect(manager.status()).toMatchObject({ openTickets: 0, openStreams: 0, ownedBytes: 0 }); }); test("native span + explicit policy: arrayBuffer is captured once from consumed chunks, with no second span", async () => { From 9e0a44a94b0588686ba7200173eeac828ed1e117 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:37:33 -0800 Subject: [PATCH 18/20] fix(auto): a failed or aborted wire call drops the held wire-only capture A fetch that rejected (connection drop, AbortSignal abort) or whose response body failed while read still emits an error wire span, but the deferred target held for wire-only FormData audio or incremental speech is no longer bound to it: emit gets no media binder, and settle() cancels the target at once, as on the synchronous-throw path. A call that completed with an HTTP error status is a finished call and still binds. Regression rows: socket reset, abort after send and a response body that fails mid-read each upload nothing and free the held bytes; a completed 400 binds the held audio to its wire span. CONFIGURATION.md and the settle() comment now state the exact boundary. Co-Authored-By: Claude Opus 5.5 --- .../traceai_auto/docs/CONFIGURATION.md | 2 +- .../traceai_auto/src/wire/attachments.ts | 3 +- .../traceai_auto/src/wire/fetch-runtime.ts | 6 ++- .../traceai_auto/test/attachments.test.ts | 42 +++++++++++++++++-- 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/typescript/packages/traceai_auto/docs/CONFIGURATION.md b/typescript/packages/traceai_auto/docs/CONFIGURATION.md index 33722abd7..b6a71e20a 100644 --- a/typescript/packages/traceai_auto/docs/CONFIGURATION.md +++ b/typescript/packages/traceai_auto/docs/CONFIGURATION.md @@ -253,7 +253,7 @@ Wire capture (`wire`) observes only allowlisted HTTPS protocol entries (`report( | Speech audio the application read incrementally (reader, async iterator) | Captured: held in a bounded buffer until the wire span exists, then checked again | Needs an explicit integration policy; cancel, early return, pipe or tee upload nothing | | FormData audio / image files | Transcription and translation audio captured: held (not read) until the wire span exists, then checked again. Image edits and variations have no wire span, so their files are not captured | Needs an explicit integration policy | -A wire-only capture waits for the wire span, which exists only once the response completed. When the span is emitted, policy, the environment switch and sampling are checked again before anything is sent. If no span will come (the call failed, was not traced, or its span was not delivered), the held capture is dropped at once; a span that arrives after five minutes binds nothing. Bytes held this way are capped at half the owned budget, so a larger file is not captured (`queue_full`). +A wire-only capture waits for the wire span, which exists only once the response completed. When the span is emitted, policy, the environment switch and sampling are checked again before anything is sent. If the call fails or is aborted (fetch throws or rejects, the connection drops, the request is aborted through its `AbortSignal`, or the response body fails while it is read), the held capture is dropped at once: the wire span is still recorded with an error status, but nothing is attached to it. The same happens when no span will come (the call was not traced, or its span was not delivered). A call that completed with an HTTP error status (4xx/5xx) did finish, so its held input is attached to that span, as with a native instrumentor span. A span that arrives after five minutes binds nothing. Bytes held this way are capped at half the owned budget, so a larger file is not captured (`queue_full`). Media in Anthropic `messages.create` input (inline base64 images and documents) is captured by `@traceai/anthropic` when auto enables it; `report().integrations[].media` reports `provider_adapter` for `openai` and `anthropic` when the installed instrumentor exposes its capability descriptor. diff --git a/typescript/packages/traceai_auto/src/wire/attachments.ts b/typescript/packages/traceai_auto/src/wire/attachments.ts index 7b7384a2b..19c794305 100644 --- a/typescript/packages/traceai_auto/src/wire/attachments.ts +++ b/typescript/packages/traceai_auto/src/wire/attachments.ts @@ -178,7 +178,8 @@ export function wireAttachmentObserver(manager: AttachmentManager, parent: Conte if (response && (response.status ?? 0) >= 200 && (response.status ?? 0) < 300 && !response.truncated && mediaType(header(response.headers, "content-type")) === "application/json") jsonOutput(parseJson(response.body)); }, settle() { - // No wire span will be bound (unmatched or failed call, delivery gate, build failure): end pending captures now. + // No wire span will be bound (unmatched call; a thrown, rejected or aborted call, including a response body that + // failed or was aborted while read, whose error wire span binds nothing; delivery gate; build failure): end now. if (correlation || settled) return; settled = true; try { deferred?.cancel(); } catch { /* never throws */ } diff --git a/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts b/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts index 3662f96cb..ada04d071 100644 --- a/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts +++ b/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts @@ -243,7 +243,11 @@ export function createFetchHook(max: number, emit: (result: SpanResult, paren if (finished) return; finished = true; const complete = () => { if (audio) { response.audioOutputBytes = audio.bytes; response.body = undefined; } - try { const span = build(matched, request, response, parsed); if (span) emit(span, parent, start, end, media ? value => media!.bind(value, request, response) : undefined); } catch { /* Capture must not reach application code. */ } + // A transport failure or abort (fetch rejected, or the response body failed or was aborted while read) still + // emits an error wire span, but nothing held is bound to it, as on the synchronous-throw path: the held + // capture is cancelled below. A completed HTTP error response (4xx/5xx) is a finished call and binds. + const bindMedia = media && !response.error ? (span: Span) => media!.bind(span, request, response) : undefined; + try { const span = build(matched, request, response, parsed); if (span) emit(span, parent, start, end, bindMedia); } catch { /* Capture must not reach application code. */ } // `emit` binds only when it starts the wire span; any other outcome cancels what was held for it. finally { settle(); resolve(); pending.delete(done); } }; diff --git a/typescript/packages/traceai_auto/test/attachments.test.ts b/typescript/packages/traceai_auto/test/attachments.test.ts index 6408f6d25..0097b61a0 100644 --- a/typescript/packages/traceai_auto/test/attachments.test.ts +++ b/typescript/packages/traceai_auto/test/attachments.test.ts @@ -27,6 +27,8 @@ let original: typeof fetch, server: Server, origin: string, manager: AttachmentM let uploads: Buffer[], requests: any[], transfers: string[], providerBodies: Buffer[], providerRequests: number; let provider: BasicTracerProvider, exporter: InMemorySpanExporter; let responseStatus: number, responseData: Buffer, responseType: string, responseChunks: number; +/** `reset` destroys the socket once the request body arrived; `hang` never answers; `partial` sends half the body, then destroys. */ +let responseMode: "normal" | "reset" | "hang" | "partial"; const nativeFetch = globalThis.fetch; const serverPolicy = { version: "fixture-v1", enabled: true, allowed_sources: ["Bytes", "Blob"], allowed_modalities: ["image", "audio", "document", "file"], require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false }; @@ -82,10 +84,15 @@ function install(target = manager, emit: "span" | "none" | "invalid" = "span") { bindMedia?.(span); span.end(end); }, [], target); } -const transcription = () => { +const transcription = (signal?: AbortSignal) => { const body = new FormData(); body.set("model", "whisper-1"); body.set("file", new Blob([audio], { type: "audio/wav" }), "PRIVATE_CANARY_FILENAME.wav"); - return fetch("https://api.openai.com/v1/audio/transcriptions", { method: "POST", body }); + return fetch("https://api.openai.com/v1/audio/transcriptions", { method: "POST", body, signal }); }; +/** Polls a condition well inside the 300 s deferred deadline, so a pass proves the call's own exit released it. */ +async function eventually(condition: () => boolean, ms = 2000) { + for (const until = Date.now() + ms; !condition() && Date.now() < until;) await new Promise(r => setTimeout(r, 5)); + expect(condition()).toBe(true); +} async function readAll(response: Response, mode: "reader" | "iterator" = "reader"): Promise { const chunks: Buffer[] = []; if (mode === "iterator") { for await (const chunk of response.body as any) chunks.push(Buffer.from(chunk)); } @@ -108,7 +115,7 @@ beforeAll(() => context.setGlobalContextManager(new AsyncLocalStorageContextMana afterAll(() => context.disable()); beforeEach(async () => { original = globalThis.fetch; uploads = []; requests = []; transfers = []; providerBodies = []; providerRequests = 0; - responseStatus = 200; responseData = audio; responseType = "audio/wav"; responseChunks = 1; + responseStatus = 200; responseData = audio; responseType = "audio/wav"; responseChunks = 1; responseMode = "normal"; server = createServer((req, res) => { const chunks: Buffer[] = []; req.on("data", chunk => chunks.push(chunk)); @@ -116,7 +123,10 @@ beforeEach(async () => { const body = Buffer.concat(chunks); if (req.headers["x-attachment-control"]) { control(req, body, res); return; } providerRequests++; providerBodies.push(body); + if (responseMode === "reset") { req.socket.destroy(); return; } + if (responseMode === "hang") return; res.writeHead(responseStatus, { "content-type": responseType }); + if (responseMode === "partial") { res.write(responseData.subarray(0, Math.ceil(responseData.length / 2)), () => setTimeout(() => req.socket.destroy(), 15)); return; } // Several network chunks, spaced so loopback does not coalesce them: incremental readers see several reads. const size = Math.ceil(responseData.length / responseChunks), data = responseData; void (async () => { @@ -279,6 +289,32 @@ describe("A03 FormData and JSON media", () => { expect(JSON.stringify(requests)).not.toContain("PRIVATE_CANARY"); expect(manager.status()).toMatchObject({ openTickets: 0, ownedBytes: 0 }); }); + test.each(["reset", "abort", "partial"] as const)("wire-only transcription whose call fails after the send (%s) drops the held audio at once: the error wire span binds nothing", async mode => { + // reset: the connection drops, fetch rejects. abort: the application aborts after the send, fetch rejects (AbortError). + // partial: the response body fails while the application reads it. In each case the provider received the audio. + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic transcription"}'); + responseMode = mode === "abort" ? "hang" : mode; + install(); const controller = new AbortController(); const pending = transcription(controller.signal); + expect(manager.status().ownedBytes).toBe(audio.length); + if (mode === "abort") { await eventually(() => providerRequests === 1); controller.abort(); } + const failure = await pending.then(response => response.json()).then(() => undefined, (error: unknown) => error); + expect(failure).toBeInstanceOf(Error); + if (mode === "abort") expect((failure as Error).name).toBe("AbortError"); + await eventually(() => manager.status().ownedBytes === 0); + expect(manager.status()).toMatchObject({ openTickets: 0, openStreams: 0, ownedBytes: 0 }); + expect(providerRequests).toBe(1); expect(providerBodies[0].includes(audio)).toBe(true); + // The error wire span is still emitted; nothing held was bound to it. + await eventually(() => exporter.getFinishedSpans().length === 1); + await settle(); expect(uploads).toHaveLength(0); expect(requests).toHaveLength(0); expect(sessionsOnly()).toEqual([]); + }); + test("wire-only transcription answered with an HTTP error (completed 400) is a finished call: the held audio binds to its wire span", async () => { + responseStatus = 400; responseType = "application/json"; responseData = Buffer.from('{"error":{"message":"synthetic","type":"invalid_request_error"}}'); + install(); const response = await transcription(); expect(response.status).toBe(400); await response.json(); + await settle(); expect(uploads).toEqual([audio]); + const spans = exporter.getFinishedSpans(); expect(spans).toHaveLength(1); + expect(requests[0].association.parent.span_id).toBe(spans[0].spanContext().spanId); + expect(manager.status()).toMatchObject({ openTickets: 0, ownedBytes: 0 }); + }); test("wire-only image edits have no wire span: the held file is dropped when the call completes and nothing uploads", async () => { responseType = "application/json"; responseData = Buffer.from('{"data":[]}'); const body = new FormData(); body.set("image", new Blob([png], { type: "image/png" })); From 1afa12d6cd2267930703cb8e8a4b31e2345dfd56 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:02:31 -0800 Subject: [PATCH 19/20] test(auto): check the failure's name, not instanceof Error (undici errors come from Node's realm) Co-Authored-By: Claude Opus 5.5 --- typescript/packages/traceai_auto/test/attachments.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/typescript/packages/traceai_auto/test/attachments.test.ts b/typescript/packages/traceai_auto/test/attachments.test.ts index 0097b61a0..205cd70a7 100644 --- a/typescript/packages/traceai_auto/test/attachments.test.ts +++ b/typescript/packages/traceai_auto/test/attachments.test.ts @@ -298,8 +298,8 @@ describe("A03 FormData and JSON media", () => { expect(manager.status().ownedBytes).toBe(audio.length); if (mode === "abort") { await eventually(() => providerRequests === 1); controller.abort(); } const failure = await pending.then(response => response.json()).then(() => undefined, (error: unknown) => error); - expect(failure).toBeInstanceOf(Error); - if (mode === "abort") expect((failure as Error).name).toBe("AbortError"); + // Undici's errors come from Node's realm (not jest's), so check the shape, not `instanceof Error`. + expect((failure as Error | undefined)?.name).toBe(mode === "abort" ? "AbortError" : "TypeError"); await eventually(() => manager.status().ownedBytes === 0); expect(manager.status()).toMatchObject({ openTickets: 0, openStreams: 0, ownedBytes: 0 }); expect(providerRequests).toBe(1); expect(providerBodies[0].includes(audio)).toBe(true); From 93a2aa46862e7310a6f3e2eae81ed4d6c8073a0a Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 05:20:11 -0800 Subject: [PATCH 20/20] test(auto): owner mark keeps wire capture out of adapter-owned calls One call seen by two layers: an adapter that captures claims the call with Symbol.for("traceai.auto.attachment_owner.v1") around the provider SDK call, and the wire layer then uploads nothing for that call. This adds the rows the suite did not have: wire-only incremental reads (reader and async iterator, now captured through a deferred target) and FormData under the claim, for the OpenAI speech and Deepgram speak routes, with and without an outer capture policy, an outer owner with no adapter upload, a concurrent owned and unowned call (the claim does not leak), and the real OpenAI SDK. The provider-media claim is stubbed through the public Symbol key, so the file needs no adapter package. Co-Authored-By: Claude Opus 5.5 --- .../test/attachment-owner.test.ts | 203 ++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 typescript/packages/traceai_auto/test/attachment-owner.test.ts diff --git a/typescript/packages/traceai_auto/test/attachment-owner.test.ts b/typescript/packages/traceai_auto/test/attachment-owner.test.ts new file mode 100644 index 000000000..9d8faf872 --- /dev/null +++ b/typescript/packages/traceai_auto/test/attachment-owner.test.ts @@ -0,0 +1,203 @@ +/* + * Owner mark vs the wire layer, for one call seen by two layers (an adapter span around a provider SDK call, and auto's + * wire capture of the same HTTP call). The adapter that captures claims the call with the shared context key + * Symbol.for("traceai.auto.attachment_owner.v1"); the wire layer must then upload nothing for that call, including + * wire-only incremental speech reads that would otherwise capture through a deferred target. + * + * The provider-media claim (@traceai/provider-media) is stubbed here through the public Symbol key, so this file + * needs no adapter package: the stub claims exactly as an adapter does (around the SDK call only; the application + * reads the body later, outside that context) and uploads its own copy, bound to its own span. + * Provider and attachment traffic both terminate on loopback. Synthetic data only. + */ +import { createServer, request as httpRequest, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import { AddressInfo } from "node:net"; +import { randomUUID } from "node:crypto"; +import { context, trace, ROOT_CONTEXT, type Span } from "@opentelemetry/api"; +import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; +import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; +import { AttachmentManager, createAttachmentHttpTransport, type AttachmentHttpClient } from "@traceai/fi-core/attachments"; +import { patchFetch } from "../src/wire/fetch"; +import { withAttachmentCapturePolicy } from "../src/wire/attachments"; + +const OWNER = Symbol.for("traceai.auto.attachment_owner.v1"); +function wav(samples: number, step: number) { + const b = Buffer.alloc(44 + samples * 2); b.write("RIFF"); b.writeUInt32LE(b.length - 8, 4); b.write("WAVEfmt ", 8); + b.writeUInt32LE(16, 16); b.writeUInt16LE(1, 20); b.writeUInt16LE(1, 22); b.writeUInt32LE(8000, 24); + b.writeUInt32LE(16000, 28); b.writeUInt16LE(2, 32); b.writeUInt16LE(16, 34); b.write("data", 36); b.writeUInt32LE(samples * 2, 40); + for (let i = 0; i < samples; i++) b.writeInt16LE(Math.round(8000 * Math.sin(i / step)), 44 + i * 2); + return b; +} +// Distinguishable payloads per provider route. +const openaiAudio = wav(80, 3), deepgramAudio = wav(96, 5); +const OPENAI_SPEECH = "https://api.openai.com/v1/audio/speech", DEEPGRAM_SPEAK = "https://api.deepgram.com/v1/speak?model=aura-asteria-en"; + +let server: Server, origin: string, manager: AttachmentManager, hook: ReturnType | undefined; +let provider: BasicTracerProvider, exporter: InMemorySpanExporter; +let uploads: Array<{ data: Buffer; request: any }>, pending: Map, wireSpans: string[]; +const nativeFetch = globalThis.fetch; +const serverPolicy = { version: "fixture-v1", enabled: true, allowed_sources: ["Bytes", "Blob"], allowed_modalities: ["image", "audio", "document", "file"], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false }; +const limits = { max_upload_bytes: 20971520, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, + active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, + upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }; +const scope = { organization_id: randomUUID(), workspace_id: randomUUID(), project_id: randomUUID() }; +const expiry = () => new Date(Date.now() + 3600000).toISOString(); +function control(req: IncomingMessage, body: Buffer, res: ServerResponse) { + res.setHeader("content-type", "application/json"); + if (req.url!.endsWith("attachment-sessions")) { + const parsed = JSON.parse(body.toString()); + res.end(JSON.stringify({ schema_version: 1, scope, capture_session_id: parsed.capture_session_id, policy: serverPolicy, limits, expires_at: expiry(), transfer_modes: ["authenticated_stream"] })); + } else if (req.url!.endsWith("/uploads")) { + const parsed = JSON.parse(body.toString()); pending.set(parsed.attachment_id, parsed); res.statusCode = 201; + res.end(JSON.stringify({ upload_id: parsed.attachment_id, attachment_id: parsed.attachment_id, occurrence_id: parsed.occurrence_id, state: "pending_upload", + reserved_bytes: parsed.declared_bytes, expires_at: expiry(), transfer_mode: "authenticated_stream", write_capability: "synthetic-upload-only" })); + } else if (req.url!.endsWith("/content")) { + const id = req.url!.split("/").at(-2)!; uploads.push({ data: body, request: pending.get(id) }); + res.end(JSON.stringify({ upload_id: id, state: "pending_upload", reason: "queued" })); + } else { res.statusCode = 202; res.end(JSON.stringify({ upload_id: req.url!.split("/").at(-2), state: "validating", reason: "binding_pending" })); } +} +const client: AttachmentHttpClient = async req => { + const body = req.body instanceof Blob ? Buffer.from(await req.body.arrayBuffer()) : req.body; + return new Promise((resolve, reject) => { + const outgoing = httpRequest(`${origin}${new URL(req.url).pathname}`, { method: req.method, headers: { ...req.headers, "x-attachment-control": "1" }, signal: req.signal }, response => { + const chunks: Buffer[] = []; response.on("data", c => chunks.push(c)); response.on("end", () => resolve({ status: response.statusCode!, headers: {}, body: Buffer.concat(chunks) } as any)); + }); outgoing.on("error", reject); outgoing.end(body as any); + }); +}; + +beforeAll(() => context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable())); +afterAll(() => context.disable()); +beforeEach(async () => { + uploads = []; pending = new Map(); wireSpans = []; + server = createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", c => chunks.push(c)); + req.on("end", () => { + const body = Buffer.concat(chunks); + if (req.headers["x-attachment-control"]) { control(req, body, res); return; } + if (req.url!.startsWith("/v1/audio/transcriptions")) { res.writeHead(200, { "content-type": "application/json" }); res.end('{"text":"synthetic"}'); return; } + const data = req.url!.startsWith("/v1/speak") ? deepgramAudio : openaiAudio; + res.writeHead(200, { "content-type": "audio/wav" }); + // Several spaced network chunks, so incremental readers see several reads. + void (async () => { for (let i = 0; i < data.length; i += 24) { res.write(data.subarray(i, i + 24)); await new Promise(r => setTimeout(r, 10)); } res.end(); })(); + }); + }); + await new Promise(done => server.listen(0, "127.0.0.1", done)); origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + exporter = new InMemorySpanExporter(); provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + manager = new AttachmentManager({ enabled: true, projectName: "attachment-owner-test", projectType: "observe", environment: () => ({}), + transport: createAttachmentHttpTransport({ endpoint: "https://attachments.example", headers: { authorization: "Bearer synthetic" }, client }) }); + expect(await manager.ready()).toBe(true); + // Auto's real fetch hook over a loopback redirect; the emitter binds media to the actual wire span. + globalThis.fetch = (input: RequestInfo | URL, init?: RequestInit) => { + const u = new URL(input instanceof Request ? input.url : String(input)); + return nativeFetch(`${origin}${u.pathname}${u.search}`, init); + }; + hook = patchFetch(4194304, (result, parent, start, end, bindMedia) => { + const span = provider.getTracer("wire").startSpan(result.name, { startTime: start, attributes: result.attributes }, parent); + wireSpans.push(span.spanContext().spanId); bindMedia?.(span); span.end(end); + }, [], manager); +}); +afterEach(async () => { + hook?.remove(); hook = undefined; globalThis.fetch = nativeFetch; + await manager.shutdown(1000); await provider.shutdown(); + server.closeAllConnections(); await new Promise(done => server.close(() => done())); +}); + +/** + * A provider adapter (stand-in for @traceai/provider-media): its own span, the owner claim around the provider call + * only, and its own upload of the bytes the application read. Returns what the call returned. + */ +function adapter() { + const span: Span = provider.getTracer("adapter").startSpan("Adapter Audio speech", { attributes: { "fi.span.kind": "LLM" } }); + return { + span, + claim: (fn: () => T): T => context.with(trace.setSpan(context.active(), span).setValue(OWNER, true), fn), + capture(data: Buffer) { + const ticket = manager.preflight(manager.correlation(span), { mimeType: "audio/wav", direction: "output", purpose: "model_output", maxBytes: data.length, sourceKind: "Bytes" }); + if (ticket.admitted) ticket.attach({ kind: "Bytes", data: new Uint8Array(data) }); + span.end(); + }, + }; +} +const post = (url: string) => fetch(url, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ text: "synthetic", input: "synthetic" }) }); +async function read(response: Response, mode: "reader" | "iterator" | "arrayBuffer"): Promise { + if (mode === "arrayBuffer") return Buffer.from(await response.arrayBuffer()); + const chunks: Buffer[] = []; + if (mode === "iterator") { for await (const chunk of response.body as any) chunks.push(Buffer.from(chunk)); return Buffer.concat(chunks); } + const reader = response.body!.getReader(); + for (;;) { const x = await reader.read(); if (x.done) return Buffer.concat(chunks); chunks.push(Buffer.from(x.value)); } +} +const parentOf = (data: Buffer) => uploads.find(u => Buffer.compare(u.data, data) === 0)?.request.association.parent.span_id; +/** "policy": an outer integration supplied an explicit capture policy, so without the claim the wire layer would also capture against the adapter's model span. */ +const outerContext = (p: "policy" | "no-policy") => p === "policy" ? withAttachmentCapturePolicy(ROOT_CONTEXT, { captureContent: true }) : ROOT_CONTEXT; +const settle = () => manager.flush(2000); +const idle = () => expect(manager.status()).toMatchObject({ openTickets: 0, openStreams: 0, ownedBytes: 0 }); + +describe("one call, two layers: the owner mark keeps wire capture out", () => { + test.each([[OPENAI_SPEECH, openaiAudio], [DEEPGRAM_SPEAK, deepgramAudio]] as const)("positive control: unowned wire-only %s read incrementally uploads once, bound to the wire span", async (url, data) => { + expect(await read(await post(url), "reader")).toEqual(data); + await settle(); expect(uploads.map(u => u.data)).toEqual([data]); + expect(wireSpans).toContain(parentOf(data)); idle(); + }); + + for (const [route, url, data] of [["OpenAI speech", OPENAI_SPEECH, openaiAudio], ["Deepgram speak", DEEPGRAM_SPEAK, deepgramAudio]] as const) { + for (const mode of ["reader", "iterator", "arrayBuffer"] as const) for (const p of ["policy", "no-policy"] as const) { + test(`adapter-owned ${route} (${mode}, ${p}, body read after the claim scope ended): exactly one upload, the adapter's`, async () => { + await context.with(outerContext(p), async () => { + const a = adapter(); + const response = await a.claim(() => post(url)); + const bytes = await read(response, mode); expect(bytes).toEqual(data); + a.capture(bytes); + await settle(); + expect(uploads.map(u => u.data)).toEqual([data]); + expect(parentOf(data)).toBe(a.span.spanContext().spanId); + expect(wireSpans).not.toContain(parentOf(data)); idle(); + }); + }); + } + test(`an outer owner alone (the ElevenLabs marker, no adapter upload) on ${route}: nothing uploads, nothing is held`, async () => { + const response = await context.with(ROOT_CONTEXT.setValue(OWNER, true), () => post(url)); + expect(await read(response, "reader")).toEqual(data); + await settle(); expect(uploads).toHaveLength(0); idle(); + }); + } + + test.each(["policy", "no-policy"] as const)("adapter-owned transcription FormData (%s): the file path is skipped too", async p => { + await context.with(outerContext(p), async () => { + const a = adapter(); + const body = new FormData(); body.set("model", "whisper-1"); body.set("file", new Blob([openaiAudio], { type: "audio/wav" }), "synthetic.wav"); + await (await a.claim(() => fetch("https://api.openai.com/v1/audio/transcriptions", { method: "POST", body }))).json(); + a.capture(openaiAudio); + await settle(); expect(uploads.map(u => u.data)).toEqual([openaiAudio]); + expect(parentOf(openaiAudio)).toBe(a.span.spanContext().spanId); idle(); + }); + }); + + test.each(["policy", "no-policy"] as const)("concurrent (%s): an owned OpenAI read and an unowned Deepgram read upload once each; the claim does not leak", async p => { + await context.with(outerContext(p), async () => { + const a = adapter(); + const [owned, unowned] = await Promise.all([ + (async () => { const bytes = await read(await a.claim(() => post(OPENAI_SPEECH)), "reader"); a.capture(bytes); return bytes; })(), + (async () => read(await post(DEEPGRAM_SPEAK), "reader"))(), + ]); + expect(owned).toEqual(openaiAudio); expect(unowned).toEqual(deepgramAudio); + await settle(); + expect(uploads).toHaveLength(2); + expect(parentOf(openaiAudio)).toBe(a.span.spanContext().spanId); + expect(wireSpans).toContain(parentOf(deepgramAudio)); idle(); + }); + }); + + test.each(["policy", "no-policy"] as const)("real OpenAI SDK (%s): the claim around speech.create covers the later incremental read", async p => { + const mod = require(process.env.ATT_OPENAI_MODULE || "openai"); const OpenAI = mod.default ?? mod.OpenAI; + const sdk = new OpenAI({ apiKey: "synthetic-provider-key", fetch: (...args: Parameters) => globalThis.fetch(...args) }); + await context.with(outerContext(p), async () => { + const a = adapter(); + const response = await a.claim(() => sdk.audio.speech.create({ model: "tts-1", voice: "alloy", input: "synthetic", response_format: "wav" })); + const bytes = await read(response, "reader"); expect(bytes).toEqual(openaiAudio); + a.capture(bytes); + await settle(); expect(uploads.map(u => u.data)).toEqual([openaiAudio]); + expect(parentOf(openaiAudio)).toBe(a.span.spanContext().spanId); idle(); + }); + }); +});