From e02c24ab005c826763687a8b003f9e470bbeed67 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 19:59:31 -0800 Subject: [PATCH 1/7] feat(openai,anthropic): capture typed inline media through the shared attachment manager OpenAI chat/responses input images, files and computer_call_output screenshots, Responses image_generation_call and Images generate b64 outputs; Anthropic Messages image/document blocks and nested tool_result media with computer-tool screenshot classification. Decode is lazy behind manager admission, inline only and capped at 256 KiB per part. Telemetry payloads are projected so media bytes and filenames never reach OTLP. Co-Authored-By: Claude Opus 5.5 --- .../traceai_anthropic/src/attachments.ts | 119 ++++++++ .../packages/traceai_anthropic/src/index.ts | 3 +- .../traceai_anthropic/src/instrumentation.ts | 18 +- .../traceai_openai/src/attachments.ts | 159 +++++++++++ .../packages/traceai_openai/src/index.ts | 3 +- .../traceai_openai/src/instrumentation.ts | 82 ++++-- .../test/attachments-providers.test.cjs | 256 ++++++++++++++++++ 7 files changed, 610 insertions(+), 30 deletions(-) create mode 100644 typescript/packages/traceai_anthropic/src/attachments.ts create mode 100644 typescript/packages/traceai_openai/src/attachments.ts create mode 100644 typescript/packages/traceai_openai/test/attachments-providers.test.cjs diff --git a/typescript/packages/traceai_anthropic/src/attachments.ts b/typescript/packages/traceai_anthropic/src/attachments.ts new file mode 100644 index 000000000..4c3d1e00e --- /dev/null +++ b/typescript/packages/traceai_anthropic/src/attachments.ts @@ -0,0 +1,119 @@ +import type { TraceConfigOptions } from "@traceai/fi-core"; +import type { AttachmentManager, AttachmentSpan, AttachmentCorrelation, AttachmentCapturePolicy, AttachmentOptions } from "@traceai/fi-core/attachments"; + +/** Native SDK span ownership is deliberately not overridden by this adapter. */ +export const anthropicAttachmentCapabilities = Object.freeze({ + schemaVersion: 1, + operations: ["messages.create.input", "tool_result.input"] as const, + sources: ["inline_base64"] as const, + unsupported: ["remote_url", "provider_file", "native_span_hookup", "generated_image", "beta_messages_hookup"] as const, + maxInlineBytes: 256 * 1024, +}); +type RecordValue = Record; +const record = (value: unknown): RecordValue => value !== null && typeof value === "object" ? value as RecordValue : {}; +const registryKey = Symbol.for("@traceai/anthropic/attachment-observations/v1"); +type Registry = WeakMap>>; +const host = globalThis as typeof globalThis & { [registryKey]?: Registry }; +const observations = (host[registryKey] ??= new WeakMap()); + +/** Observe only Messages' typed image/document and one-level tool_result blocks. + * The manager owns policy, upload, lifecycle and references. A native integration + * must supply its actual emitted span/correlation, never an ambient guessed span. + */ +export function captureAnthropicAttachments(manager: AttachmentManager | undefined, + span: AttachmentSpan | AttachmentCorrelation, payload: unknown, + options: { traceConfig?: TraceConfigOptions; capture?: AttachmentCapturePolicy } = {}): void { + if (!manager) return; + try { + const correlation = "snapshot" in span ? span : manager.correlation(span); + const identity = correlation.snapshot(); + if (!identity?.sampled) return; + const messages = record(payload).messages; + if (!Array.isArray(messages)) return; + let spans = observations.get(manager); + if (!spans) { spans = new Map(); observations.set(manager, spans); } + const spanKey = `${identity.traceId}/${identity.spanId}`; + const config = options.traceConfig ?? {}; + const hidden = config.hideInputs || config.hideInputMessages || config.hideInputText || config.piiRedaction; + const computerNames = new Set(); + const tools = record(payload).tools; + if (Array.isArray(tools)) for (const value of tools.slice(0, 256)) { + const tool = record(value); + if (typeof tool.type === "string" && /^computer_\d{8}$/.test(tool.type) && typeof tool.name === "string") computerNames.add(tool.name); + } + const computerCalls = new Set(); + for (const value of messages.slice(0, 256)) { + const message = record(value); + if (message.role !== "assistant" || !Array.isArray(message.content)) continue; + for (const value of message.content.slice(0, 256)) { + const block = record(value); + if (block.type === "tool_use" && typeof block.id === "string" && typeof block.name === "string" && computerNames.has(block.name)) computerCalls.add(block.id); + } + } + let visited = 0; + messages.slice(0, 256).forEach((value, messageIndex) => { + const message = record(value); + if (!Array.isArray(message.content)) return; + // Nested positions are relative to their tool result, identified by toolCallId. + const visit = (value: unknown, index: number, position: string, toolCallId?: string, errorDetail = false) => { + if (++visited > 256) return; + const part = record(value); + if (part.type !== "image" && part.type !== "document") return; + const source = record(part.source); + if (source.type !== "base64") return; + const mime = source.media_type; + if (typeof mime !== "string" || !(part.type === "image" ? ["image/png", "image/jpeg", "image/webp"] : ["application/pdf"]).includes(mime)) return; + const key = `input/${messageIndex}/${position}`; + if (spans!.get(spanKey)?.has(key)) return; + const role: AttachmentOptions["role"] = toolCallId ? "tool" : message.role === "assistant" ? "assistant" : "user"; + const handle = manager.attach(correlation, { kind: "Bytes", get data() { + const encoded = source.data; + if (typeof encoded !== "string" || !encoded.length || encoded.length % 4 !== 0 || + encoded.length > 4 * Math.ceil(anthropicAttachmentCapabilities.maxInlineBytes / 3) || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); + const bytes = Buffer.from(encoded, "base64"); + return bytes.length <= anthropicAttachmentCapabilities.maxInlineBytes && bytes.toString("base64") === encoded ? bytes : new Uint8Array(); + } }, { mimeType: mime, direction: "input", messageIndex, partIndex: index, role, + purpose: part.type === "document" ? "document" : toolCallId ? computerCalls.has(toolCallId) ? "computer_screenshot" : "tool_result" : "model_input", + ...(toolCallId ? { toolCallId } : {}), + capture: { ...options.capture, modelInput: true, errorDetail: errorDetail || options.capture?.errorDetail, + ...(hidden || (part.type === "image" && config.hideInputImages) || (errorDetail && config.hideErrorDetails) ? { captureContent: false } : {}) }, + }); + if (handle.admission === "queued") { + let positions = spans!.get(spanKey); + if (!positions) { + if (spans!.size >= 1024) spans!.delete(spans!.keys().next().value!); + positions = new Set(); spans!.set(spanKey, positions); + } + if (positions.size < 256) positions.add(key); + } + }; + message.content.slice(0, 256).forEach((value, index) => { + const block = record(value); + if (block.type === "tool_result" && Array.isArray(block.content)) { + // Invalid tool IDs fail closed instead of losing causal identity. + if (typeof block.tool_use_id !== "string" || !/^[a-zA-Z0-9_-]{1,128}$/.test(block.tool_use_id)) return; + block.content.slice(0, 256).forEach((child, childIndex) => visit(child, childIndex, `${index}/${childIndex}`, block.tool_use_id as string, block.is_error === true)); + } else visit(value, index, String(index)); + }); + }); + } catch { /* No source-bearing exceptions or changes to the application's result. */ } +} + +/** A telemetry-only projection; never mutates the request passed to the SDK. */ +export function projectAnthropicMedia(payload: unknown): any { + const body = record(payload); + if (!Array.isArray(body.messages)) return payload; + const project = (value: unknown, nested = false): unknown => { + const block = record(value); + if (block.type === "image" || block.type === "document") return { type: block.type }; + if (block.type === "tool_result" && Array.isArray(block.content)) return { + type: block.type, tool_use_id: block.tool_use_id, is_error: block.is_error, + content: nested ? [] : block.content.map(child => project(child, true)), + }; + return block; + }; + return { ...body, messages: body.messages.map(value => { + const message = record(value); + return Array.isArray(message.content) ? { ...message, content: message.content.map(value => project(value)) } : message; + }) }; +} diff --git a/typescript/packages/traceai_anthropic/src/index.ts b/typescript/packages/traceai_anthropic/src/index.ts index 378f38976..bc14a48fb 100644 --- a/typescript/packages/traceai_anthropic/src/index.ts +++ b/typescript/packages/traceai_anthropic/src/index.ts @@ -1,2 +1,3 @@ export * from './version'; -export * from './instrumentation'; \ No newline at end of file +export * from './instrumentation'; +export * from './attachments'; diff --git a/typescript/packages/traceai_anthropic/src/instrumentation.ts b/typescript/packages/traceai_anthropic/src/instrumentation.ts index 77d132430..7df2e6d6f 100644 --- a/typescript/packages/traceai_anthropic/src/instrumentation.ts +++ b/typescript/packages/traceai_anthropic/src/instrumentation.ts @@ -1,4 +1,6 @@ import Anthropic from "@anthropic-ai/sdk"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import { captureAnthropicAttachments, projectAnthropicMedia } from "./attachments"; import { InstrumentationBase, InstrumentationConfig, @@ -334,16 +336,22 @@ function finishInstrumentationStream( export class AnthropicInstrumentation extends InstrumentationBase{ private fiTracer!: FITracer; private _traceConfig?: TraceConfigOptions; + private attachments?: AttachmentManager; constructor({ instrumentationConfig, traceConfig, + attachments, }: { instrumentationConfig?: InstrumentationConfig; traceConfig?: TraceConfigOptions; + /** The provider-owned shared attachment manager. */ + attachments?: AttachmentManager; } = {}) { super("@traceai/instrumentation-anthropic", VERSION,Object.assign({}, instrumentationConfig)); this._traceConfig = traceConfig; + this.attachments = attachments; + this.fiTracer = new FITracer({ tracer: this.tracer, traceConfig: this._traceConfig }); } public override enable(): void { @@ -404,12 +412,13 @@ export class AnthropicInstrumentation extends InstrumentationBase{ // Not async: the SDK returns an APIPromise whose withResponse(), asResponse() // and lazy body parsing callers (including messages.stream()) depend on. return function patchedMessagesCreate(this: Anthropic.Messages, ...args: Parameters) { - const params = args[0] as Anthropic.MessageCreateParams; + const originalParams = args[0] as Anthropic.MessageCreateParams; + const params = projectAnthropicMedia(originalParams) as Anthropic.MessageCreateParams; const { messages: _messages, system: _system, tools: _tools, ...invocationParameters } = params; - const span = instrumentation.fiTracer.startSpan("Anthropic Messages Create", { - kind: SpanKind.INTERNAL, - attributes: { + const span = instrumentation.fiTracer.startSpan("Anthropic Messages Create", { kind: SpanKind.INTERNAL }); + captureAnthropicAttachments(instrumentation.attachments, span, originalParams, { traceConfig: instrumentation._traceConfig }); + span.setAttributes({ [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, [SemanticConventions.LLM_MODEL_NAME]: params.model, [SemanticConventions.LLM_PROVIDER]: LLMProvider.ANTHROPIC, @@ -418,7 +427,6 @@ export class AnthropicInstrumentation extends InstrumentationBase{ ...getAnthropicInputMessagesAttributes(params), ...getAnthropicToolsAttributes(params), [SemanticConventions.RAW_INPUT]: safelyJSONStringify(params) ?? "", - }, }); const execContext = getExecContext(span); diff --git a/typescript/packages/traceai_openai/src/attachments.ts b/typescript/packages/traceai_openai/src/attachments.ts new file mode 100644 index 000000000..d47e00ce0 --- /dev/null +++ b/typescript/packages/traceai_openai/src/attachments.ts @@ -0,0 +1,159 @@ +import type { TraceConfigOptions } from "@traceai/fi-core"; +import type { AttachmentManager, AttachmentOptions, AttachmentSpan, AttachmentCorrelation, AttachmentCapturePolicy } from "@traceai/fi-core/attachments"; + +/** Declarative handoff to auto/wire owners; this is not a native/wire hookup. */ +export const openAIAttachmentCapabilities = Object.freeze({ + schemaVersion: 1, + operations: ["chat.completions.create.input", "responses.create.input", "responses.create.output", "images.generate.output"] as const, + sources: ["inline_base64"] as const, + unsupported: ["remote_url", "provider_file", "image_edit_stream", "speech", "partial_image"] as const, + maxInlineBytes: 256 * 1024, +}); + +type RecordValue = Record; +const record = (value: unknown): RecordValue => value !== null && typeof value === "object" ? value as RecordValue : {}; +const list = (value: unknown): unknown[] => Array.isArray(value) ? value : []; +const MAX_PARTS = 256; +// Shared across CJS/ESM copies. Retain only admitted span IDs and positional keys, never sources. +const registryKey = Symbol.for("@traceai/openai/attachment-observations/v1"); +type Registry = WeakMap>>; +const host = globalThis as typeof globalThis & { [registryKey]?: Registry }; +const observations = (host[registryKey] ??= new WeakMap()); + +export interface OpenAIAttachmentCaptureOptions { + operation: "chat" | "responses" | "images"; + direction: "input" | "output"; + traceConfig?: TraceConfigOptions; + capture?: AttachmentCapturePolicy; + /** The Images request's typed output_format; never inferred from filenames. */ + imageFormat?: "png" | "jpeg" | "webp"; +} + +function restrictions(options: OpenAIAttachmentCaptureOptions): AttachmentCapturePolicy { + const config = options.traceConfig ?? {}; + const input = options.direction === "input"; + const hidden = input + ? config.hideInputs || config.hideInputMessages || config.hideInputText + : config.hideOutputs || config.hideOutputMessages || config.hideOutputText; + return { ...options.capture, + ...(hidden || config.piiRedaction ? { captureContent: false } : {}), + ...(input ? { modelInput: true } : {}), + }; +} + +/** Typed, synchronous observation only. No provider downloads or background stream consumption. + * Pass the SAME emitted identity to SDK and wire observations to deduplicate positions. + * Denied sources are never read: the shared manager calls the getter only after admission. + */ +export function captureOpenAIAttachments(manager: AttachmentManager | undefined, + span: AttachmentSpan | AttachmentCorrelation, payload: unknown, options: OpenAIAttachmentCaptureOptions): void { + if (!manager) return; + try { + const correlation = "snapshot" in span ? span : manager.correlation(span); + const identity = correlation.snapshot(); + if (!identity?.sampled) return; + let spans = observations.get(manager); + if (!spans) { spans = new Map(); observations.set(manager, spans); } + const spanKey = `${identity.traceId}/${identity.spanId}`; + const capture = restrictions(options); + let visited = 0; + const attach = (read: () => unknown, mimeTypes: string[], messageIndex: number, partIndex: number, + purpose: AttachmentOptions["purpose"], toolCallId?: string, raw = false, role: AttachmentOptions["role"] = "user") => { + if (++visited > MAX_PARTS) return; + const key = `${options.operation}/${options.direction}/${messageIndex}/${partIndex}`; + if (spans!.get(spanKey)?.has(key)) return; + for (const mimeType of mimeTypes) { + const handle = manager.attach(correlation, { kind: "Bytes", get data() { + const value = read(); + if (typeof value !== "string") return new Uint8Array(); + const prefix = `data:${mimeType};base64,`; + if (!raw && !value.startsWith(prefix)) return new Uint8Array(); + // Bound allocation BEFORE slicing/decoding. Padding and alphabet are strict. + const start = raw ? 0 : prefix.length; + const length = value.length - start; + if (length <= 0 || length > 4 * Math.ceil(openAIAttachmentCapabilities.maxInlineBytes / 3) || length % 4 !== 0) return new Uint8Array(); + const encoded = value.slice(start); + if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); + const bytes = Buffer.from(encoded, "base64"); + if (bytes.length > openAIAttachmentCapabilities.maxInlineBytes || bytes.toString("base64") !== encoded) return new Uint8Array(); + return bytes; + } }, { mimeType, direction: options.direction, purpose, messageIndex, partIndex, role, + ...(toolCallId ? { toolCallId } : {}), capture: { + ...capture, + ...(options.direction === "input" && mimeType.startsWith("image/") && options.traceConfig?.hideInputImages ? { captureContent: false } : {}), + } }); + if (handle.admission === "queued") { + let positions = spans!.get(spanKey); + if (!positions) { + if (spans!.size >= 1024) spans!.delete(spans!.keys().next().value!); + positions = new Set(); spans!.set(spanKey, positions); + } + if (positions.size < MAX_PARTS) positions.add(key); + return; + } + if (handle.admission !== "too_large" && handle.admission !== "source_unavailable") return; + } + }; + const images = ["image/png", "image/jpeg", "image/webp"]; + const parts = (content: unknown, messageIndex: number, role: AttachmentOptions["role"]) => { + list(content).slice(0, MAX_PARTS).forEach((value, partIndex) => { + const part = record(value); + if (part.type === "image_url") attach(() => record(part.image_url).url, images, messageIndex, partIndex, "model_input", undefined, false, role); + else if (part.type === "input_image") attach(() => part.image_url, images, messageIndex, partIndex, "model_input", undefined, false, role); + else if (part.type === "input_file" || part.type === "file") attach(() => part.type === "file" ? record(part.file).file_data : part.file_data, + ["application/pdf", "application/octet-stream"], messageIndex, partIndex, "document", undefined, false, role); + }); + }; + const body = record(payload); + if (options.direction === "output" && body.error) return; + if (options.direction === "input") { + const messages = options.operation === "chat" ? body.messages : body.input; + list(messages).slice(0, MAX_PARTS).forEach((value, messageIndex) => { + const message = record(value); + if (message.type === "computer_call_output") { + const output = record(message.output); + if (output.type === "computer_screenshot") attach(() => output.image_url, images, messageIndex, 0, "computer_screenshot", + typeof message.call_id === "string" ? message.call_id : undefined, false, "tool"); + } else if (!message.type || message.type === "message") { + const role = ["user", "assistant", "tool", "system"].includes(String(message.role)) ? message.role as AttachmentOptions["role"] : "unknown"; + parts(message.content, messageIndex, role); + } + }); + } else if (options.operation === "images") { + list(body.data).slice(0, MAX_PARTS).forEach((value, index) => { + const item = record(value); + // The Images API defaults to PNG. Explicit response format wins where provided. + const format = body.output_format ?? options.imageFormat; + const mime = format === "jpeg" ? "image/jpeg" : format === "webp" ? "image/webp" : "image/png"; + attach(() => item.b64_json, [mime], index, 0, "model_output", undefined, true, "assistant"); + }); + } else if (options.operation === "responses") { + list(body.output).slice(0, MAX_PARTS).forEach((value, index) => { + const item = record(value); + if (item.type === "image_generation_call" && item.status === "completed") { + const mime = item.output_format === "jpeg" ? "image/jpeg" : item.output_format === "webp" ? "image/webp" : "image/png"; + attach(() => item.result, [mime], index, 0, "model_output", undefined, true, "assistant"); + } + }); + } + } catch { /* Instrumentation must not throw or log source-bearing errors. */ } +} + +/** Telemetry-only typed projection. Never send media bytes, filenames or source URLs to OTLP. */ +export function projectOpenAIMedia(payload: unknown, operation: "chat" | "responses" | "images", direction: "input" | "output", redactImages = true): any { + const body = record(payload); + const projectPart = (value: unknown): unknown => { + const part = record(value); + return (["input_file", "file"].includes(String(part.type)) || redactImages && ["image_url", "input_image"].includes(String(part.type))) ? { type: part.type } : value; + }; + if (operation === "images") return { created: body.created }; + const field = direction === "input" ? operation === "chat" ? "messages" : "input" : "output"; + if (!Array.isArray(body[field])) return payload; + return { ...body, [field]: list(body[field]).map(value => { + const message = record(value); + if (message.type === "computer_call_output") return { type: message.type, call_id: message.call_id, output: { type: "computer_screenshot" } }; + if (message.type === "image_generation_call") return { type: message.type, id: message.id, status: message.status }; + if (Array.isArray(message.content)) return { ...message, content: message.content.map(projectPart) }; + return value; + }) }; +} diff --git a/typescript/packages/traceai_openai/src/index.ts b/typescript/packages/traceai_openai/src/index.ts index a38448910..e886a4c31 100644 --- a/typescript/packages/traceai_openai/src/index.ts +++ b/typescript/packages/traceai_openai/src/index.ts @@ -1,2 +1,3 @@ export * from './instrumentation'; -export * from './version'; \ No newline at end of file +export * from './version'; +export * from './attachments'; diff --git a/typescript/packages/traceai_openai/src/instrumentation.ts b/typescript/packages/traceai_openai/src/instrumentation.ts index a04bc72d0..1951022ea 100644 --- a/typescript/packages/traceai_openai/src/instrumentation.ts +++ b/typescript/packages/traceai_openai/src/instrumentation.ts @@ -1,4 +1,6 @@ import openai from "openai"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import { captureOpenAIAttachments, projectOpenAIMedia } from "./attachments"; import { InstrumentationBase, InstrumentationConfig, @@ -107,6 +109,7 @@ function getExecContext(span: Span) { export class OpenAIInstrumentation extends InstrumentationBase{ private fiTracer!: FITracer; private _traceConfig?: TraceConfigOptions; + private attachments?: AttachmentManager; // Lazily initialized: InstrumentationBase can patch during super(). // A Set is needed so disable() also undoes every manually patched ESM SDK. private patchedModules?: Set; @@ -114,6 +117,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ constructor({ instrumentationConfig, traceConfig, + attachments, }: { /** * The config for the instrumentation @@ -125,6 +129,8 @@ export class OpenAIInstrumentation extends InstrumentationBase{ * @see {@link TraceConfigOptions} */ traceConfig?: TraceConfigOptions; + /** The shared manager from register(...).attachments; never a per-adapter uploader. */ + attachments?: AttachmentManager; } = {}) { super( "@traceai/fi-instrumentation-openai", @@ -132,6 +138,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ Object.assign({}, instrumentationConfig), ); this._traceConfig = traceConfig; + this.attachments = attachments; // The base constructor can call enable() before our trace options exist. this.fiTracer = new FITracer({ tracer: this.tracer, traceConfig: this._traceConfig }); } @@ -223,7 +230,8 @@ export class OpenAIInstrumentation extends InstrumentationBase{ this: unknown, ...args: Parameters ) { - const body = args[0]; + const originalBody = args[0]; + const body = projectOpenAIMedia(originalBody, "chat", "input", !!instrumentation.attachments); // Prediction text is input content, captured only in maskable payloads. const { messages: _messages, prediction: _prediction, ...invocationParameters } = body; @@ -237,11 +245,11 @@ export class OpenAIInstrumentation extends InstrumentationBase{ // diag.debug(`@traceai/openai: this.fiTracer type: ${instrumentation.fiTracer?.constructor?.name}`); // --- END ADDED LOGS --- - const span = instrumentation.fiTracer.startSpan( - `OpenAI Chat Completions`, - { - kind: SpanKind.INTERNAL, - attributes: { + const span = instrumentation.fiTracer.startSpan("OpenAI Chat Completions", { kind: SpanKind.INTERNAL }); + captureOpenAIAttachments(instrumentation.attachments, span, originalBody, { + operation: "chat", direction: "input", traceConfig: instrumentation._traceConfig, + }); + span.setAttributes({ [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, [SemanticConventions.LLM_MODEL_NAME]: body.model, @@ -254,9 +262,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ ...getLLMInputMessagesAttributes(body), ...getLLMToolsJSONSchema(body), [SemanticConventions.RAW_INPUT]: safelyJSONStringify(body) ?? "", - }, - }, - ); + }); // --- ADD LOG --- // diag.debug(`@traceai/openai: ChatCompletion span STARTED: ${span.spanContext().spanId}`); @@ -519,21 +525,22 @@ export class OpenAIInstrumentation extends InstrumentationBase{ }; // Telemetry-only copy: MCP credentials must never reach any exported // duplicate, but the original SDK request must remain untouched. + const projectedBody = projectOpenAIMedia(body, "responses", "input", !!instrumentation.attachments); const telemetryBody = { - ...body, + ...projectedBody, ...(body.tools && { tools: sanitizeToolDefinitions(body.tools) }), - ...(body.input && { input: sanitizeResponsesInput(body.input) }), + ...(projectedBody.input && { input: sanitizeResponsesInput(projectedBody.input) }), }; const { input: _messages, instructions: _instructions, prompt, ...parameters } = telemetryBody; const invocationParameters = { ...parameters, ...(prompt && { prompt: { id: prompt.id, version: prompt.version } }), }; - const span = instrumentation.fiTracer.startSpan( - `OpenAI Responses`, - { - kind: SpanKind.INTERNAL, - attributes: { + const span = instrumentation.fiTracer.startSpan("OpenAI Responses", { kind: SpanKind.INTERNAL }); + captureOpenAIAttachments(instrumentation.attachments, span, body, { + operation: "responses", direction: "input", traceConfig: instrumentation._traceConfig, + }); + span.setAttributes({ [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, [SemanticConventions.LLM_MODEL_NAME]: body.model, @@ -544,12 +551,10 @@ export class OpenAIInstrumentation extends InstrumentationBase{ [SemanticConventions.PROMPT_TEMPLATE_VARIABLES]: safelyJSONStringify(prompt?.variables) ?? undefined, [SemanticConventions.LLM_PROVIDER]: providerForResource(this), [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", - ...getResponsesInputMessagesAttributes(body), + ...getResponsesInputMessagesAttributes(telemetryBody), ...getLLMToolsJSONSchema(body), [SemanticConventions.RAW_INPUT]: safelyJSONStringify(telemetryBody) ?? "", - }, - }, - ); + }); const contextSpan = span.getContextSpan(); const execContext = getExecContext(contextSpan); const requestStartedAt = performance.now(); @@ -581,16 +586,20 @@ export class OpenAIInstrumentation extends InstrumentationBase{ if (endSpan) span.end(); return; } + captureOpenAIAttachments(instrumentation.attachments, span, result, { + operation: "responses", direction: "output", traceConfig: instrumentation._traceConfig, + }); + const projected = projectOpenAIMedia(result, "responses", "output"); const output = { id: result.id, object: result.object, status: result.status, - model: result.model, output: result.output, usage: result.usage, + model: result.model, output: projected.output, usage: result.usage, }; span.setAttributes({ [SemanticConventions.OUTPUT_VALUE]: JSON.stringify(output), [SemanticConventions.OUTPUT_MIME_TYPE]: MimeType.JSON, [SemanticConventions.GEN_AI_RESPONSE_MODEL]: result.model, [SemanticConventions.GEN_AI_RESPONSE_ID]: result.id, - ...getResponsesOutputMessagesAttributes(result), + ...getResponsesOutputMessagesAttributes(projected), ...getResponsesUsageAttributes(result), [SemanticConventions.RAW_OUTPUT]: safelyJSONStringify(output) ?? "", }); @@ -628,6 +637,32 @@ export class OpenAIInstrumentation extends InstrumentationBase{ ); } + if (typeof module.OpenAI.Images?.prototype?.generate === "function") { + this._wrap(module.OpenAI.Images.prototype, "generate", (original: typeof module.OpenAI.Images.prototype.generate): any => { + return function(this: unknown, ...args: Parameters) { + const span = instrumentation.fiTracer.startSpan("OpenAI Images", { kind: SpanKind.INTERNAL, attributes: { + [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, + [SemanticConventions.LLM_PROVIDER]: providerForResource(this), + [SemanticConventions.GEN_AI_OPERATION_NAME]: "image_generation", + } }); + try { + const execContext = trace.setSpan(context.active(), span); + const promise = context.with(execContext, () => original.apply(this, args)); + return observeAPIPromise(promise, result => { + if (span.isRecording() && !isStream(result)) { + captureOpenAIAttachments(instrumentation.attachments, span, result, { + operation: "images", direction: "output", traceConfig: instrumentation._traceConfig, + imageFormat: (args[0] as { output_format?: "png" | "jpeg" | "webp" }).output_format, + }); + } + span.end(); + return result; + }, error => endSpanWithError(span, error), response => endRawResponseSpan(span, response)); + } catch (error) { endSpanWithError(span, error); throw error; } + }; + }); + } + // Keep the legacy diagnostic property, but never use it as the guard. // Immutable class exports still work via the identity map. try { (sdk as typeof openai & { fiPatched?: boolean }).fiPatched = true; } catch { /* immutable SDK */ } @@ -647,6 +682,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ this._unwrap(sdk.Completions.prototype, "create"); this._unwrap(sdk.Embeddings.prototype, "create"); if (sdk.Responses) this._unwrap(sdk.Responses.prototype, "create"); + if (sdk.Images) this._unwrap(sdk.Images.prototype, "generate"); try { (sdk as typeof openai & { fiPatched?: boolean }).fiPatched = false; } catch { /* immutable SDK */ } patchRegistry.owners.delete(sdk); // An automatic require hook may use a different namespace for this class. @@ -983,7 +1019,7 @@ function serializeChatCompletionInputMessage( if (part.type === "text") { return { type: "text", text: part.text }; } else if (part.type === "image_url") { - return { type: "image_url", image_url: { url: part.image_url.url } }; + return part.image_url ? { type: "image_url", image_url: { url: part.image_url.url } } : { type: "image_url" }; } return { type: part.type }; }); diff --git a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs new file mode 100644 index 000000000..dcba8d149 --- /dev/null +++ b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs @@ -0,0 +1,256 @@ +/* Cloud component qualification: real SDKs, synthetic media, loopback-only APIs. */ +const { test, after } = require('node:test'); +const assert = require('node:assert/strict'); +const http = require('node:http'); +const { once } = require('node:events'); +const { randomUUID } = require('node:crypto'); +const { pathToFileURL } = require('node:url'); +const path = require('node:path'); +const { AttachmentManager, createAttachmentHttpTransport } = require('@traceai/fi-core/attachments'); +const { BasicTracerProvider, SimpleSpanProcessor, InMemorySpanExporter } = require('@opentelemetry/sdk-trace-base'); +const { OpenAIInstrumentation } = require('../dist/src/instrumentation.js'); +const { AnthropicInstrumentation } = require('../../traceai_anthropic/dist/src/instrumentation.js'); +const oa = require('../dist/src/attachments.js'); +const an = require('../../traceai_anthropic/dist/src/attachments.js'); +const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aD1sAAAAASUVORK5CYII='; +const PDF = Buffer.from('%PDF-1.7\n% synthetic document\n%%EOF').toString('base64'); +const image = () => ({ type: 'image', source: { type: 'base64', media_type: 'image/png', data: PNG } }); +const document = () => ({ type: 'document', title: 'synthetic-private-filename', source: { type: 'base64', media_type: 'application/pdf', data: PDF } }); +const identity = () => ({ traceId: randomUUID().replace(/-/g, ''), spanId: randomUUID().replace(/-/g, '').slice(0, 16), traceFlags: 1 }); +const keepalive = setInterval(() => {}, 1000); +after(() => clearInterval(keepalive)); + +async function fixture(config = {}, serverPolicy = {}) { + const calls = [], initialized = [], uploaded = []; + const scope = { organization_id: randomUUID(), workspace_id: randomUUID(), project_id: randomUUID() }; + const expiry = () => new Date(Date.now() + 60000).toISOString(); + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const c of req) chunks.push(c); + const bytes = Buffer.concat(chunks); calls.push(req.url); + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('attachment-sessions')) { + const body = JSON.parse(bytes); + res.end(JSON.stringify({ schema_version: 1, scope, capture_session_id: body.capture_session_id, + policy: { version: 'fixture-v1', enabled: true, allowed_sources: ['Bytes'], allowed_modalities: ['image', 'document', 'file'], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, + record_audio: false, record_video: false, ...serverPolicy }, + limits: { max_upload_bytes: 1024 * 1024, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, + recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, + initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, + transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }, expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); + } else if (req.url.endsWith('/uploads')) { + const body = JSON.parse(bytes); initialized.push(body); + res.statusCode = 201; + res.end(JSON.stringify({ upload_id: body.attachment_id, attachment_id: body.attachment_id, occurrence_id: body.occurrence_id, + state: 'pending_upload', reserved_bytes: body.declared_bytes, expires_at: expiry(), transfer_mode: 'authenticated_stream', write_capability: 'synthetic-upload-only' })); + } else if (req.url.endsWith('/content')) { + assert.equal(req.headers.authorization, 'Bearer synthetic-upload-only'); uploaded.push(bytes); + res.end(JSON.stringify({ upload_id: req.url.split('/').at(-2), state: 'pending_upload', reason: 'queued' })); + } else { + res.statusCode = 202; res.end(JSON.stringify({ upload_id: req.url.split('/').at(-2), state: 'validating', reason: 'binding_pending' })); + } + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const client = req => new Promise((resolve, reject) => { + const request = http.request(`http://127.0.0.1:${server.address().port}${new URL(req.url).pathname}`, { method: req.method, headers: req.headers, signal: req.signal }, response => { + const chunks = []; response.on('data', c => chunks.push(c)); response.on('end', () => resolve({ status: response.statusCode, body: Buffer.concat(chunks) })); + }); request.on('error', reject); request.end(req.body); + }); + const manager = new AttachmentManager({ enabled: true, projectName: 'synthetic', projectType: 'observe', + transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic' }, client }), ...config }); + assert.equal(await manager.ready(), config.enabled !== false); + return { manager, calls, initialized, uploaded, async close() { await manager.shutdown(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } }; +} +const oaInput = () => ({ input: [ + { role: 'user', content: [{ type: 'input_image', image_url: `data:image/png;base64,${PNG}` }, { type: 'input_file', filename: 'synthetic-private-filename', file_data: `data:application/pdf;base64,${PDF}` }] }, + { type: 'computer_call_output', call_id: 'call_computer', output: { type: 'computer_screenshot', image_url: `data:image/png;base64,${PNG}` } }, +] }); +const anInput = () => ({ tools: [{ type: 'computer_20250124', name: 'computer', display_width_px: 800, display_height_px: 600 }], messages: [ + { role: 'assistant', content: [{ type: 'tool_use', id: 'tool_computer', name: 'computer', input: { action: 'screenshot' } }] }, + { role: 'user', content: [image(), document(), { type: 'tool_result', tool_use_id: 'tool_computer', content: [image(), image()] }] }, +] }); + +for (const provider of ['openai', 'anthropic']) { + const capture = (m, s, p, extra = {}) => provider === 'openai' + ? oa.captureOpenAIAttachments(m, s, p, { operation: 'responses', direction: 'input', ...extra }) + : an.captureAnthropicAttachments(m, s, p, extra); + const input = provider === 'openai' ? oaInput : anInput; + test(`${provider}: typed image/document/screenshots preserve bytes, causal positions and distinct repeated frames`, async () => { + const f = await fixture(); + try { + const id = identity(), body = input(), saved = JSON.stringify(body); + capture(f.manager, id, body); capture(f.manager, { ...id }, body); + await f.manager.flush(); + assert.equal(f.initialized.length, provider === 'openai' ? 3 : 4); + assert.equal(f.uploaded.length, f.initialized.length); + assert.deepEqual(f.uploaded.map(b => b.toString('base64')).sort(), (provider === 'openai' ? [PNG, PDF, PNG] : [PNG, PDF, PNG, PNG]).sort()); + assert.equal(new Set(f.initialized.map(r => r.occurrence_id)).size, f.initialized.length); + assert.ok(f.initialized.every(r => r.direction === 'input' && r.association.parent.span_id === id.spanId)); + assert.equal(f.initialized.filter(r => r.purpose === 'computer_screenshot').length, provider === 'openai' ? 1 : 2); + assert.ok(!JSON.stringify(f.initialized).includes('synthetic-private-filename')); + assert.equal(JSON.stringify(body), saved); + } finally { await f.close(); } + }); + for (const key of ['disabled', 'hideInputs', 'hideInputMessages', 'hideInputText', 'piiRedaction', 'hideInputImages', 'serverRedaction', 'serverMetadata']) { + test(`${provider}: ${key} denies protected sources without reading or uploading`, async () => { + const f = await fixture(key === 'disabled' ? { enabled: false } : { traceConfig: { [key]: true } }, + key === 'serverRedaction' ? { require_media_redaction: true } : key === 'serverMetadata' ? { require_metadata_removal: true } : {}); + try { + let read = false; + const source = provider === 'openai' ? { input: [{ type: 'computer_call_output', call_id: 'call_computer', output: { type: 'computer_screenshot', get image_url() { read = true; throw Error('synthetic-secret'); } } }] } + : { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_computer', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', get data() { read = true; throw Error('synthetic-secret'); } } }] }] }] }; + const before = f.calls.length; capture(f.manager, identity(), source); + await f.manager.flush(); assert.equal(read, false); assert.equal(f.calls.length, before); + if (key === 'disabled') assert.equal(f.calls.length, 0); + } finally { await f.close(); } + }); + } + for (const capturePolicy of [{ captureContent: false }, { recordInputs: false }, { muted: true }, { temporaryChat: true }]) { + test(`${provider}: integration restriction ${Object.keys(capturePolicy)[0]} wins`, async () => { + const f = await fixture(); try { capture(f.manager, identity(), input(), { capture: capturePolicy }); await f.manager.flush(); assert.equal(f.initialized.length, 0); } finally { await f.close(); } + }); + } + test(`${provider}: malformed and oversized inline data is bounded; foreign URLs/files are not retrieved`, async () => { + const f = await fixture(); + try { + for (const data of ['!', 'a===', 'AB==', 'AAAA'.repeat(100000), 'https://127.0.0.1:1/?synthetic-secret']) { + const body = provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_image', image_url: data }] }] } + : { messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data } }] }] }; + capture(f.manager, identity(), body); + } + capture(f.manager, identity(), provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_file', file_id: 'file_secret' }] }] } + : { messages: [{ role: 'user', content: [{ type: 'document', source: { type: 'file', file_id: 'file_secret' } }, null, { type: 'tool_result', tool_use_id: {}, content: [image()] }] }] }); + await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } + }); + test(`${provider}: unsampled identity and queued policy tightening cannot upload`, async () => { + const f = await fixture(); try { + capture(f.manager, { ...identity(), traceFlags: 0 }, input()); assert.equal(f.manager.status().ownedBytes, 0); + capture(f.manager, identity(), input()); f.manager.updatePolicy({ traceConfig: { hideInputs: true } }); await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } + }); +} + +test('OpenAI generated images obey output and PII restrictions', async () => { + for (const traceConfig of [{}, { hideOutputs: true }, { hideOutputMessages: true }, { hideOutputText: true }, { piiRedaction: true }]) { + const f = await fixture({ traceConfig }); try { + oa.captureOpenAIAttachments(f.manager, identity(), { data: [{ b64_json: PNG }] }, { operation: 'images', direction: 'output' }); + await f.manager.flush(); assert.equal(f.uploaded.length, Object.keys(traceConfig).length ? 0 : 1); + } finally { await f.close(); } + } +}); + +test('Telemetry projections strip typed bytes, filenames and signed source locations without mutating requests', () => { + for (const [body, project] of [[oaInput(), x => oa.projectOpenAIMedia(x, 'responses', 'input')], [anInput(), an.projectAnthropicMedia]]) { + const before = JSON.stringify(body), projected = JSON.stringify(project(body)); + assert.equal(JSON.stringify(body), before); + for (const secret of [PNG, PDF, 'synthetic-private-filename']) assert.ok(!projected.includes(secret)); + } +}); + +test('CJS/ESM observations share deduplication without creating a second capture', async () => { + const f = await fixture(); try { + const esm = await import(pathToFileURL(path.resolve(__dirname, '../dist/esm/attachments.js'))); + const s = identity(); oa.captureOpenAIAttachments(f.manager, s, oaInput(), { operation: 'responses', direction: 'input' }); + esm.captureOpenAIAttachments(f.manager, { ...s }, oaInput(), { operation: 'responses', direction: 'input' }); + const anthropicESM = await import(pathToFileURL(path.resolve(__dirname, '../../traceai_anthropic/dist/esm/attachments.js'))); + const a = identity(); an.captureAnthropicAttachments(f.manager, a, anInput()); anthropicESM.captureAnthropicAttachments(f.manager, { ...a }, anInput()); + await f.manager.flush(); assert.equal(f.uploaded.length, 7); + } finally { await f.close(); } +}); + +test('Anthropic tool-result images are not labelled computer screenshots without a typed computer tool', async () => { + const f = await fixture(); try { + an.captureAnthropicAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_photo', content: [image()] }] }] }); + await f.manager.flush(); assert.equal(f.initialized[0].purpose, 'tool_result'); assert.equal(f.initialized[0].part_index, 0); + } finally { await f.close(); } +}); + +test('Anthropic error-tool media respects hideErrorDetails before source access', async () => { + const f = await fixture({ traceConfig: { hideErrorDetails: true } }); try { + let read = false; + an.captureAnthropicAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_error', is_error: true, + content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', get data() { read = true; return PNG; } } }] }] }] }); + await f.manager.flush(); assert.equal(read, false); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } +}); + +test('OpenAI image error bodies are not treated as generated media', async () => { + const f = await fixture(); try { + oa.captureOpenAIAttachments(f.manager, identity(), { error: { message: 'synthetic' }, data: [{ b64_json: PNG }] }, { operation: 'images', direction: 'output' }); + await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } +}); + +test('Local instrumentor privacy restrictions cannot be loosened by a permissive shared manager', async () => { + const f = await fixture(); try { + oa.captureOpenAIAttachments(f.manager, identity(), oaInput(), { operation: 'responses', direction: 'input', traceConfig: { hideInputs: true } }); + an.captureAnthropicAttachments(f.manager, identity(), anInput(), { traceConfig: { piiRedaction: true } }); + await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } +}); + +async function providerServer() { + const requests = []; + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const c of req) chunks.push(c); + const body = JSON.parse(Buffer.concat(chunks)); requests.push({ path: req.url, body }); + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('/chat/completions')) res.end(JSON.stringify({ id: 'chat_fixture', object: 'chat.completion', model: body.model, choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })); + else if (req.url.endsWith('/responses')) res.end(JSON.stringify({ id: 'resp_fixture', object: 'response', status: 'completed', model: body.model, output: [{ type: 'computer_call', id: 'computer_fixture', call_id: 'call_computer', status: 'completed', action: { type: 'screenshot' } }, { type: 'image_generation_call', id: 'img_fixture', status: 'completed', result: PNG }], usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } })); + else if (req.url.endsWith('/images/generations')) res.end(JSON.stringify({ created: 1, data: [{ b64_json: PNG }] })); + else res.end(JSON.stringify({ id: 'msg_fixture', type: 'message', role: 'assistant', model: body.model, content: [{ type: 'text', text: 'ok' }], stop_reason: 'end_turn', stop_sequence: null, usage: { input_tokens: 1, output_tokens: 1 } })); + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + return { requests, url: `http://127.0.0.1:${server.address().port}/v1`, async close() { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } }; +} + +test('Real SDKs preserve APIPromise/request/response behavior and upload only through the shared manager', async () => { + const sdkModule = require(process.env.SDK_OPENAI || 'openai'); + const anthropicModule = require(process.env.SDK_ANTHROPIC || '../../traceai_anthropic/node_modules/@anthropic-ai/sdk'); + const OpenAI = sdkModule.OpenAI || sdkModule.default || sdkModule; + const Anthropic = anthropicModule.Anthropic || anthropicModule.default || anthropicModule; + const native = 'openTelemetry' in Anthropic.prototype; + const f = await fixture(), p = await providerServer(); + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + const oi = new OpenAIInstrumentation({ instrumentationConfig: { enabled: false }, attachments: f.manager }); + const ai = new AnthropicInstrumentation({ instrumentationConfig: { enabled: false }, attachments: f.manager }); + oi.setTracerProvider(provider); ai.setTracerProvider(provider); + oi.manuallyInstrument(sdkModule); ai.manuallyInstrument(anthropicModule); + try { + const client = new OpenAI({ apiKey: 'synthetic-provider-key', baseURL: p.url, maxRetries: 0 }); + const body = { model: 'synthetic-model', messages: [{ role: 'user', content: [{ type: 'image_url', image_url: { url: `data:image/png;base64,${PNG}` } }] }] }; + const pending = client.chat.completions.create(body); assert.equal(typeof pending.withResponse, 'function'); + assert.equal((await pending.withResponse()).data.choices[0].message.content, 'ok'); + let expected = 1; + if (client.responses) { + const result = await client.responses.create({ model: 'synthetic-model', ...oaInput() }); + assert.equal(result.output[0].call_id, 'call_computer'); expected += 4; + } + const generated = await client.images.generate({ prompt: 'synthetic', model: 'synthetic-model', response_format: 'b64_json' }); + assert.equal(generated.data[0].b64_json, PNG); expected++; + const ac = new Anthropic({ apiKey: 'synthetic-provider-key', baseURL: p.url, maxRetries: 0 }); + const request = ac.messages.create({ model: 'synthetic-model', max_tokens: 10, ...anInput() }); + assert.equal(typeof request.withResponse, 'function'); assert.equal((await request.withResponse()).data.content[0].text, 'ok'); + if (!native) expected += 4; + await f.manager.flush(); await provider.forceFlush(); + assert.equal(f.uploaded.length, expected); + assert.ok(p.requests.some(r => JSON.stringify(r.body).includes(PNG))); + assert.ok(!JSON.stringify(exporter.getFinishedSpans().map(s => s.attributes)).includes(PNG)); + assert.ok(exporter.getFinishedSpans().some(s => s.attributes['fi.attachment.refs'])); + // A raw-response caller owns the body; instrumentation must not parse for media. + const raw = await client.images.generate({ prompt: 'synthetic', response_format: 'b64_json' }).asResponse(); + assert.equal(raw.bodyUsed, false); await raw.arrayBuffer(); + await f.manager.flush(); assert.equal(f.uploaded.length, expected); + // Requests still reach the provider unchanged after privacy tightening; media egress stops. + f.manager.updatePolicy({ traceConfig: { hideInputs: true, piiRedaction: true } }); + await client.chat.completions.create(body); + await ac.messages.create({ model: 'synthetic-model', max_tokens: 10, ...anInput() }); + await client.images.generate({ prompt: 'synthetic', response_format: 'b64_json' }); + await f.manager.flush(); assert.equal(f.uploaded.length, expected); + console.log(JSON.stringify({ nativeAnthropic: native, responsesAvailable: !!client.responses, uploads: expected, spans: exporter.getFinishedSpans().length })); + } finally { + oi.disable(); ai.disable(); await provider.shutdown(); await f.close(); await p.close(); + } +}); From 52b95b0ff500fc4a50d69195a21ec5c5e6dbabd5 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 20:11:55 -0800 Subject: [PATCH 2/7] fix(openai,anthropic): project only admitted media and keep existing telemetry otherwise Capture now runs before the telemetry projection and returns the admitted positions. Only those parts are replaced by their fi-attachment occurrence URI; denied, disabled or absent capture leaves the span payload exactly as before. A second observer of the same span position reuses the first occurrence. Streamed Responses output media and the Images edit and variation endpoints are explicitly unsupported. Adds a network-isolated CI workflow against real SDK floor and latest releases. Co-Authored-By: Claude Opus 5.5 --- .../openai-anthropic-attachments-tests.yml | 70 ++++++++ .../traceai_anthropic/src/attachments.ts | 124 ++++++++----- .../traceai_anthropic/src/instrumentation.ts | 17 +- .../traceai_openai/src/attachments.ts | 167 +++++++++++------- .../traceai_openai/src/instrumentation.ts | 121 +++++++------ .../test/attachments-providers.test.cjs | 119 ++++++++++++- 6 files changed, 440 insertions(+), 178 deletions(-) create mode 100644 .github/workflows/openai-anthropic-attachments-tests.yml diff --git a/.github/workflows/openai-anthropic-attachments-tests.yml b/.github/workflows/openai-anthropic-attachments-tests.yml new file mode 100644 index 000000000..a8335241c --- /dev/null +++ b/.github/workflows/openai-anthropic-attachments-tests.yml @@ -0,0 +1,70 @@ +name: OpenAI and Anthropic TypeScript attachments +on: + pull_request: + paths: + - 'typescript/packages/traceai_openai/**' + - 'typescript/packages/traceai_anthropic/**' + - 'typescript/packages/fi-core/**' + - 'typescript/packages/fi-semantic-conventions/**' + - 'typescript/pnpm-lock.yaml' + - '.github/workflows/openai-anthropic-attachments-tests.yml' + workflow_dispatch: +permissions: + contents: read +jobs: + attachments: + name: OpenAI ${{ matrix.openai }} / Anthropic ${{ matrix.anthropic }} + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - { openai: '4.0.0', anthropic: '0.20.0' } + - { openai: '5.23.2', anthropic: '0.27.3' } + - { openai: '6.0.0', anthropic: '0.133.0' } + - { openai: '7.47.0', anthropic: '0.134.0' } + - { openai: 'latest', anthropic: 'latest' } + defaults: + run: + shell: bash + working-directory: typescript + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22.23.4' + - name: Install pinned workspace dependencies and real SDK fixtures + run: | + mkdir -p "$RUNNER_TEMP/corepack-bin" + corepack enable --install-directory "$RUNNER_TEMP/corepack-bin" pnpm + echo "$RUNNER_TEMP/corepack-bin" >> "$GITHUB_PATH" + corepack pnpm --filter @traceai/openai... --filter @traceai/anthropic... install --frozen-lockfile --ignore-scripts + npm install --prefix "$RUNNER_TEMP/sdks" --ignore-scripts --no-audit --no-fund \ + "openai-under-test@npm:openai@${{ matrix.openai }}" \ + "anthropic-under-test@npm:@anthropic-ai/sdk@${{ matrix.anthropic }}" + - name: Build CJS and ESM + run: | + pnpm run build:deps + pnpm --filter @traceai/openai run build + pnpm --filter @traceai/anthropic run build + - name: Network-isolated attachment adapters against real SDKs and loopback mocks + run: | + sudo unshare --net -- bash -c 'ip link set lo up; exec "$@"' bash \ + env PATH="$PATH" \ + SDK_OPENAI="$RUNNER_TEMP/sdks/node_modules/openai-under-test" \ + SDK_ANTHROPIC="$RUNNER_TEMP/sdks/node_modules/anthropic-under-test" \ + node --test packages/traceai_openai/test/attachments-providers.test.cjs + - name: Network-isolated package unit suites + if: matrix.openai == 'latest' + run: | + sudo unshare --net -- bash -c 'ip link set lo up; exec "$@"' bash \ + env PATH="$PATH" node packages/traceai_openai/node_modules/jest/bin/jest.js \ + --config packages/traceai_openai/jest.config.js --runInBand + sudo unshare --net -- bash -c 'ip link set lo up; exec "$@"' bash \ + env PATH="$PATH" node node_modules/jest/bin/jest.js \ + --config packages/traceai_anthropic/jest.config.js --runInBand + git diff --exit-code -- pnpm-lock.yaml diff --git a/typescript/packages/traceai_anthropic/src/attachments.ts b/typescript/packages/traceai_anthropic/src/attachments.ts index 4c3d1e00e..876ac52ae 100644 --- a/typescript/packages/traceai_anthropic/src/attachments.ts +++ b/typescript/packages/traceai_anthropic/src/attachments.ts @@ -6,114 +6,142 @@ export const anthropicAttachmentCapabilities = Object.freeze({ schemaVersion: 1, operations: ["messages.create.input", "tool_result.input"] as const, sources: ["inline_base64"] as const, - unsupported: ["remote_url", "provider_file", "native_span_hookup", "generated_image", "beta_messages_hookup"] as const, + /** Fail closed: no capture is attempted for these. */ + unsupported: ["remote_url", "provider_file", "native_span_hookup", "generated_image", "beta_messages_hookup", "streamed_output"] as const, maxInlineBytes: 256 * 1024, }); + +/** Admitted media positions for one observation: position key -> occurrence. */ +export type AnthropicCapturedMedia = ReadonlyMap>; +type Captured = Map>; type RecordValue = Record; const record = (value: unknown): RecordValue => value !== null && typeof value === "object" ? value as RecordValue : {}; +const MAX_PARTS = 256; +const MAX_SPANS = 1024; +const EMPTY: AnthropicCapturedMedia = new Map(); const registryKey = Symbol.for("@traceai/anthropic/attachment-observations/v1"); -type Registry = WeakMap>>; +type Registry = WeakMap>; const host = globalThis as typeof globalThis & { [registryKey]?: Registry }; const observations = (host[registryKey] ??= new WeakMap()); +const TOOL_ID = /^[a-zA-Z0-9_-]{1,128}$/; + +function decode(encoded: unknown): Uint8Array { + const max = anthropicAttachmentCapabilities.maxInlineBytes; + // Bound allocation BEFORE decoding. Padding and alphabet are strict. + if (typeof encoded !== "string" || !encoded.length || encoded.length % 4 !== 0 || + encoded.length > 4 * Math.ceil(max / 3) || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); + const bytes = Buffer.from(encoded, "base64"); + return bytes.length <= max && bytes.toString("base64") === encoded ? bytes : new Uint8Array(); +} -/** Observe only Messages' typed image/document and one-level tool_result blocks. +/** Observe only Messages' typed image/document blocks and one level of tool_result content. * The manager owns policy, upload, lifecycle and references. A native integration * must supply its actual emitted span/correlation, never an ambient guessed span. + * Returns the admitted positions so the caller can project them out of span telemetry. */ export function captureAnthropicAttachments(manager: AttachmentManager | undefined, span: AttachmentSpan | AttachmentCorrelation, payload: unknown, - options: { traceConfig?: TraceConfigOptions; capture?: AttachmentCapturePolicy } = {}): void { - if (!manager) return; + options: { traceConfig?: TraceConfigOptions; capture?: AttachmentCapturePolicy } = {}): AnthropicCapturedMedia { + if (!manager) return EMPTY; + const captured: Captured = new Map(); try { const correlation = "snapshot" in span ? span : manager.correlation(span); const identity = correlation.snapshot(); - if (!identity?.sampled) return; + if (!identity?.sampled) return EMPTY; const messages = record(payload).messages; - if (!Array.isArray(messages)) return; + if (!Array.isArray(messages)) return EMPTY; let spans = observations.get(manager); if (!spans) { spans = new Map(); observations.set(manager, spans); } const spanKey = `${identity.traceId}/${identity.spanId}`; const config = options.traceConfig ?? {}; const hidden = config.hideInputs || config.hideInputMessages || config.hideInputText || config.piiRedaction; + // Screenshot classification requires a typed computer tool AND an assistant tool_use naming it. const computerNames = new Set(); const tools = record(payload).tools; - if (Array.isArray(tools)) for (const value of tools.slice(0, 256)) { + if (Array.isArray(tools)) for (const value of tools.slice(0, MAX_PARTS)) { const tool = record(value); if (typeof tool.type === "string" && /^computer_\d{8}$/.test(tool.type) && typeof tool.name === "string") computerNames.add(tool.name); } const computerCalls = new Set(); - for (const value of messages.slice(0, 256)) { + for (const value of messages.slice(0, MAX_PARTS)) { const message = record(value); if (message.role !== "assistant" || !Array.isArray(message.content)) continue; - for (const value of message.content.slice(0, 256)) { + for (const value of message.content.slice(0, MAX_PARTS)) { const block = record(value); if (block.type === "tool_use" && typeof block.id === "string" && typeof block.name === "string" && computerNames.has(block.name)) computerCalls.add(block.id); } } let visited = 0; - messages.slice(0, 256).forEach((value, messageIndex) => { + messages.slice(0, MAX_PARTS).forEach((value, messageIndex) => { const message = record(value); if (!Array.isArray(message.content)) return; // Nested positions are relative to their tool result, identified by toolCallId. const visit = (value: unknown, index: number, position: string, toolCallId?: string, errorDetail = false) => { - if (++visited > 256) return; + if (++visited > MAX_PARTS) return; const part = record(value); if (part.type !== "image" && part.type !== "document") return; const source = record(part.source); if (source.type !== "base64") return; const mime = source.media_type; if (typeof mime !== "string" || !(part.type === "image" ? ["image/png", "image/jpeg", "image/webp"] : ["application/pdf"]).includes(mime)) return; - const key = `input/${messageIndex}/${position}`; - if (spans!.get(spanKey)?.has(key)) return; + const positionKey = `input/${messageIndex}/${position}`; + const previous = spans!.get(spanKey)?.get(positionKey); + if (previous) { captured.set(positionKey, previous); return; } const role: AttachmentOptions["role"] = toolCallId ? "tool" : message.role === "assistant" ? "assistant" : "user"; - const handle = manager.attach(correlation, { kind: "Bytes", get data() { - const encoded = source.data; - if (typeof encoded !== "string" || !encoded.length || encoded.length % 4 !== 0 || - encoded.length > 4 * Math.ceil(anthropicAttachmentCapabilities.maxInlineBytes / 3) || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); - const bytes = Buffer.from(encoded, "base64"); - return bytes.length <= anthropicAttachmentCapabilities.maxInlineBytes && bytes.toString("base64") === encoded ? bytes : new Uint8Array(); - } }, { mimeType: mime, direction: "input", messageIndex, partIndex: index, role, - purpose: part.type === "document" ? "document" : toolCallId ? computerCalls.has(toolCallId) ? "computer_screenshot" : "tool_result" : "model_input", - ...(toolCallId ? { toolCallId } : {}), - capture: { ...options.capture, modelInput: true, errorDetail: errorDetail || options.capture?.errorDetail, - ...(hidden || (part.type === "image" && config.hideInputImages) || (errorDetail && config.hideErrorDetails) ? { captureContent: false } : {}) }, - }); + const handle = manager.attach(correlation, { kind: "Bytes", get data() { return decode(source.data); } }, + { mimeType: mime, direction: "input", messageIndex, partIndex: index, role, + purpose: part.type === "document" ? "document" : toolCallId ? computerCalls.has(toolCallId) ? "computer_screenshot" : "tool_result" : "model_input", + ...(toolCallId ? { toolCallId } : {}), + capture: { ...options.capture, modelInput: true, errorDetail: errorDetail || options.capture?.errorDetail, + ...(hidden || (part.type === "image" && config.hideInputImages) || (errorDetail && config.hideErrorDetails) ? { captureContent: false } : {}) }, + }); if (handle.admission === "queued") { + const occurrence = Object.freeze({ occurrenceId: handle.occurrenceId, mimeType: mime }); let positions = spans!.get(spanKey); if (!positions) { - if (spans!.size >= 1024) spans!.delete(spans!.keys().next().value!); - positions = new Set(); spans!.set(spanKey, positions); + if (spans!.size >= MAX_SPANS) spans!.delete(spans!.keys().next().value!); + positions = new Map(); spans!.set(spanKey, positions); } - if (positions.size < 256) positions.add(key); + if (positions.size < MAX_PARTS) positions.set(positionKey, occurrence); + captured.set(positionKey, occurrence); } }; - message.content.slice(0, 256).forEach((value, index) => { + message.content.slice(0, MAX_PARTS).forEach((value, index) => { const block = record(value); if (block.type === "tool_result" && Array.isArray(block.content)) { // Invalid tool IDs fail closed instead of losing causal identity. - if (typeof block.tool_use_id !== "string" || !/^[a-zA-Z0-9_-]{1,128}$/.test(block.tool_use_id)) return; - block.content.slice(0, 256).forEach((child, childIndex) => visit(child, childIndex, `${index}/${childIndex}`, block.tool_use_id as string, block.is_error === true)); + if (typeof block.tool_use_id !== "string" || !TOOL_ID.test(block.tool_use_id)) return; + block.content.slice(0, MAX_PARTS).forEach((child, childIndex) => visit(child, childIndex, `${index}/${childIndex}`, block.tool_use_id as string, block.is_error === true)); } else visit(value, index, String(index)); }); }); } catch { /* No source-bearing exceptions or changes to the application's result. */ } + return captured; } -/** A telemetry-only projection; never mutates the request passed to the SDK. */ -export function projectAnthropicMedia(payload: unknown): any { - const body = record(payload); - if (!Array.isArray(body.messages)) return payload; - const project = (value: unknown, nested = false): unknown => { - const block = record(value); - if (block.type === "image" || block.type === "document") return { type: block.type }; - if (block.type === "tool_result" && Array.isArray(block.content)) return { - type: block.type, tool_use_id: block.tool_use_id, is_error: block.is_error, - content: nested ? [] : block.content.map(child => project(child, true)), +/** Telemetry-only projection. ONLY admitted positions are replaced with a `fi-attachment:` occurrence URI + * source, so uploaded bytes are not exported twice. Everything else is returned exactly as before, and the + * request passed to the SDK is never mutated. + */ +export function projectAnthropicMedia(payload: unknown, captured: AnthropicCapturedMedia): any { + if (!captured.size) return payload; + try { + const body = record(payload); + if (!Array.isArray(body.messages)) return payload; + const replace = (value: unknown, position: string): unknown => { + const occurrence = captured.get(position); + return occurrence ? { ...record(value), source: { type: "url", url: `fi-attachment:${occurrence.occurrenceId}` } } : value; }; - return block; - }; - return { ...body, messages: body.messages.map(value => { - const message = record(value); - return Array.isArray(message.content) ? { ...message, content: message.content.map(value => project(value)) } : message; - }) }; + return { ...body, messages: body.messages.map((value, messageIndex) => { + const message = record(value); + if (!Array.isArray(message.content)) return value; + return { ...message, content: message.content.map((value, index) => { + const block = record(value); + if (block.type === "tool_result" && Array.isArray(block.content)) { + return { ...block, content: block.content.map((child, childIndex) => replace(child, `input/${messageIndex}/${index}/${childIndex}`)) }; + } + return replace(value, `input/${messageIndex}/${index}`); + }) }; + }) }; + } catch { return payload; } } diff --git a/typescript/packages/traceai_anthropic/src/instrumentation.ts b/typescript/packages/traceai_anthropic/src/instrumentation.ts index 7df2e6d6f..91561e08c 100644 --- a/typescript/packages/traceai_anthropic/src/instrumentation.ts +++ b/typescript/packages/traceai_anthropic/src/instrumentation.ts @@ -413,16 +413,21 @@ export class AnthropicInstrumentation extends InstrumentationBase{ // and lazy body parsing callers (including messages.stream()) depend on. return function patchedMessagesCreate(this: Anthropic.Messages, ...args: Parameters) { const originalParams = args[0] as Anthropic.MessageCreateParams; - const params = projectAnthropicMedia(originalParams) as Anthropic.MessageCreateParams; - const { messages: _messages, system: _system, tools: _tools, ...invocationParameters } = params; + const { messages: _messages, system: _system, tools: _tools, ...invocationParameters } = originalParams; - const span = instrumentation.fiTracer.startSpan("Anthropic Messages Create", { kind: SpanKind.INTERNAL }); - captureAnthropicAttachments(instrumentation.attachments, span, originalParams, { traceConfig: instrumentation._traceConfig }); - span.setAttributes({ + // Content attributes are set after capture so admitted media can be projected out first. + const span = instrumentation.fiTracer.startSpan("Anthropic Messages Create", { + kind: SpanKind.INTERNAL, + attributes: { [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, - [SemanticConventions.LLM_MODEL_NAME]: params.model, + [SemanticConventions.LLM_MODEL_NAME]: originalParams.model, [SemanticConventions.LLM_PROVIDER]: LLMProvider.ANTHROPIC, [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", + }, + }); + const captured = captureAnthropicAttachments(instrumentation.attachments, span, originalParams, { traceConfig: instrumentation._traceConfig }); + const params = projectAnthropicMedia(originalParams, captured) as Anthropic.MessageCreateParams; + span.setAttributes({ [SemanticConventions.LLM_INVOCATION_PARAMETERS]: safelyJSONStringify(invocationParameters) ?? "", ...getAnthropicInputMessagesAttributes(params), ...getAnthropicToolsAttributes(params), diff --git a/typescript/packages/traceai_openai/src/attachments.ts b/typescript/packages/traceai_openai/src/attachments.ts index d47e00ce0..319628c03 100644 --- a/typescript/packages/traceai_openai/src/attachments.ts +++ b/typescript/packages/traceai_openai/src/attachments.ts @@ -6,17 +6,25 @@ export const openAIAttachmentCapabilities = Object.freeze({ schemaVersion: 1, operations: ["chat.completions.create.input", "responses.create.input", "responses.create.output", "images.generate.output"] as const, sources: ["inline_base64"] as const, - unsupported: ["remote_url", "provider_file", "image_edit_stream", "speech", "partial_image"] as const, + /** Fail closed: no capture is attempted for these. */ + unsupported: ["remote_url", "provider_file", "streamed_output", "partial_image", "images.edit", "images.createVariation", "speech"] as const, maxInlineBytes: 256 * 1024, }); +/** Admitted media positions for one observation: position key -> occurrence. */ +export type CapturedMedia = ReadonlyMap>; +type Captured = Map>; type RecordValue = Record; const record = (value: unknown): RecordValue => value !== null && typeof value === "object" ? value as RecordValue : {}; const list = (value: unknown): unknown[] => Array.isArray(value) ? value : []; const MAX_PARTS = 256; -// Shared across CJS/ESM copies. Retain only admitted span IDs and positional keys, never sources. +const MAX_SPANS = 1024; +const EMPTY: CapturedMedia = new Map(); +const IMAGES = ["image/png", "image/jpeg", "image/webp"]; +const FILES = ["application/pdf", "application/octet-stream"]; +// Shared across CJS/ESM copies. Retains only admitted span IDs, positions and occurrence IDs, never sources. const registryKey = Symbol.for("@traceai/openai/attachment-observations/v1"); -type Registry = WeakMap>>; +type Registry = WeakMap>; const host = globalThis as typeof globalThis & { [registryKey]?: Registry }; const observations = (host[registryKey] ??= new WeakMap()); @@ -29,6 +37,7 @@ export interface OpenAIAttachmentCaptureOptions { imageFormat?: "png" | "jpeg" | "webp"; } +/** The local instrumentor's restrictions; the shared manager applies its own policy as well (deny wins). */ function restrictions(options: OpenAIAttachmentCaptureOptions): AttachmentCapturePolicy { const config = options.traceConfig ?? {}; const input = options.direction === "input"; @@ -41,17 +50,38 @@ function restrictions(options: OpenAIAttachmentCaptureOptions): AttachmentCaptur }; } -/** Typed, synchronous observation only. No provider downloads or background stream consumption. - * Pass the SAME emitted identity to SDK and wire observations to deduplicate positions. +function decode(value: unknown, mimeType: string, raw: boolean): Uint8Array { + if (typeof value !== "string") return new Uint8Array(); + const prefix = `data:${mimeType};base64,`; + if (!raw && !value.startsWith(prefix)) return new Uint8Array(); + // Bound allocation BEFORE slicing/decoding. Padding and alphabet are strict. + const start = raw ? 0 : prefix.length; + const length = value.length - start; + if (length <= 0 || length > 4 * Math.ceil(openAIAttachmentCapabilities.maxInlineBytes / 3) || length % 4 !== 0) return new Uint8Array(); + const encoded = value.slice(start); + if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); + const bytes = Buffer.from(encoded, "base64"); + if (bytes.length > openAIAttachmentCapabilities.maxInlineBytes || bytes.toString("base64") !== encoded) return new Uint8Array(); + return bytes; +} + +const key = (options: Pick, messageIndex: number, partIndex: number) => + `${options.operation}/${options.direction}/${messageIndex}/${partIndex}`; + +/** Typed, synchronous observation only. No provider downloads or stream consumption. + * Pass the SAME emitted identity to SDK and wire observations: a second observation of a position + * returns the first occurrence instead of uploading again. * Denied sources are never read: the shared manager calls the getter only after admission. + * Returns the admitted positions so the caller can project them out of span telemetry. */ export function captureOpenAIAttachments(manager: AttachmentManager | undefined, - span: AttachmentSpan | AttachmentCorrelation, payload: unknown, options: OpenAIAttachmentCaptureOptions): void { - if (!manager) return; + span: AttachmentSpan | AttachmentCorrelation, payload: unknown, options: OpenAIAttachmentCaptureOptions): CapturedMedia { + if (!manager) return EMPTY; + const captured: Captured = new Map(); try { const correlation = "snapshot" in span ? span : manager.correlation(span); const identity = correlation.snapshot(); - if (!identity?.sampled) return; + if (!identity?.sampled) return EMPTY; let spans = observations.get(manager); if (!spans) { spans = new Map(); observations.set(manager, spans); } const spanKey = `${identity.traceId}/${identity.spanId}`; @@ -60,71 +90,63 @@ export function captureOpenAIAttachments(manager: AttachmentManager | undefined, const attach = (read: () => unknown, mimeTypes: string[], messageIndex: number, partIndex: number, purpose: AttachmentOptions["purpose"], toolCallId?: string, raw = false, role: AttachmentOptions["role"] = "user") => { if (++visited > MAX_PARTS) return; - const key = `${options.operation}/${options.direction}/${messageIndex}/${partIndex}`; - if (spans!.get(spanKey)?.has(key)) return; + const position = key(options, messageIndex, partIndex); + const previous = spans!.get(spanKey)?.get(position); + if (previous) { captured.set(position, previous); return; } for (const mimeType of mimeTypes) { - const handle = manager.attach(correlation, { kind: "Bytes", get data() { - const value = read(); - if (typeof value !== "string") return new Uint8Array(); - const prefix = `data:${mimeType};base64,`; - if (!raw && !value.startsWith(prefix)) return new Uint8Array(); - // Bound allocation BEFORE slicing/decoding. Padding and alphabet are strict. - const start = raw ? 0 : prefix.length; - const length = value.length - start; - if (length <= 0 || length > 4 * Math.ceil(openAIAttachmentCapabilities.maxInlineBytes / 3) || length % 4 !== 0) return new Uint8Array(); - const encoded = value.slice(start); - if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); - const bytes = Buffer.from(encoded, "base64"); - if (bytes.length > openAIAttachmentCapabilities.maxInlineBytes || bytes.toString("base64") !== encoded) return new Uint8Array(); - return bytes; - } }, { mimeType, direction: options.direction, purpose, messageIndex, partIndex, role, - ...(toolCallId ? { toolCallId } : {}), capture: { - ...capture, - ...(options.direction === "input" && mimeType.startsWith("image/") && options.traceConfig?.hideInputImages ? { captureContent: false } : {}), - } }); + const handle = manager.attach(correlation, { kind: "Bytes", get data() { return decode(read(), mimeType, raw); } }, + { mimeType, direction: options.direction, purpose, messageIndex, partIndex, role, + ...(toolCallId ? { toolCallId } : {}), capture: { + ...capture, + ...(options.direction === "input" && mimeType.startsWith("image/") && options.traceConfig?.hideInputImages ? { captureContent: false } : {}), + } }); if (handle.admission === "queued") { + const occurrence = Object.freeze({ occurrenceId: handle.occurrenceId, mimeType }); let positions = spans!.get(spanKey); if (!positions) { - if (spans!.size >= 1024) spans!.delete(spans!.keys().next().value!); - positions = new Set(); spans!.set(spanKey, positions); + if (spans!.size >= MAX_SPANS) spans!.delete(spans!.keys().next().value!); + positions = new Map(); spans!.set(spanKey, positions); } - if (positions.size < MAX_PARTS) positions.add(key); + if (positions.size < MAX_PARTS) positions.set(position, occurrence); + captured.set(position, occurrence); return; } - if (handle.admission !== "too_large" && handle.admission !== "source_unavailable") return; + // Only a non-matching data-URI type (decoded as empty) tries the next candidate. Denials stop. + if (handle.admission !== "too_large") return; } }; - const images = ["image/png", "image/jpeg", "image/webp"]; + const toolCallId = (value: unknown) => typeof value === "string" && /^[a-zA-Z0-9_-]{1,128}$/.test(value) ? value : undefined; const parts = (content: unknown, messageIndex: number, role: AttachmentOptions["role"]) => { list(content).slice(0, MAX_PARTS).forEach((value, partIndex) => { const part = record(value); - if (part.type === "image_url") attach(() => record(part.image_url).url, images, messageIndex, partIndex, "model_input", undefined, false, role); - else if (part.type === "input_image") attach(() => part.image_url, images, messageIndex, partIndex, "model_input", undefined, false, role); - else if (part.type === "input_file" || part.type === "file") attach(() => part.type === "file" ? record(part.file).file_data : part.file_data, - ["application/pdf", "application/octet-stream"], messageIndex, partIndex, "document", undefined, false, role); + if (part.type === "image_url") attach(() => record(part.image_url).url, IMAGES, messageIndex, partIndex, "model_input", undefined, false, role); + else if (part.type === "input_image") attach(() => part.image_url, IMAGES, messageIndex, partIndex, "model_input", undefined, false, role); + else if (part.type === "input_file") attach(() => part.file_data, FILES, messageIndex, partIndex, "document", undefined, false, role); + else if (part.type === "file") attach(() => record(part.file).file_data, FILES, messageIndex, partIndex, "document", undefined, false, role); }); }; const body = record(payload); - if (options.direction === "output" && body.error) return; + if (options.direction === "output" && body.error) return EMPTY; if (options.direction === "input") { - const messages = options.operation === "chat" ? body.messages : body.input; + const messages = options.operation === "chat" ? body.messages : options.operation === "responses" ? body.input : undefined; list(messages).slice(0, MAX_PARTS).forEach((value, messageIndex) => { const message = record(value); if (message.type === "computer_call_output") { const output = record(message.output); - if (output.type === "computer_screenshot") attach(() => output.image_url, images, messageIndex, 0, "computer_screenshot", - typeof message.call_id === "string" ? message.call_id : undefined, false, "tool"); + // A screenshot is model-bound input. Its call id keeps the causal link to the computer_call. + if (output.type === "computer_screenshot") attach(() => output.image_url, IMAGES, messageIndex, 0, "computer_screenshot", + toolCallId(message.call_id), false, "tool"); } else if (!message.type || message.type === "message") { const role = ["user", "assistant", "tool", "system"].includes(String(message.role)) ? message.role as AttachmentOptions["role"] : "unknown"; parts(message.content, messageIndex, role); } }); } else if (options.operation === "images") { + // The Images API defaults to PNG. The typed request/response format wins where provided. + const format = body.output_format ?? options.imageFormat; + const mime = format === "jpeg" ? "image/jpeg" : format === "webp" ? "image/webp" : "image/png"; list(body.data).slice(0, MAX_PARTS).forEach((value, index) => { const item = record(value); - // The Images API defaults to PNG. Explicit response format wins where provided. - const format = body.output_format ?? options.imageFormat; - const mime = format === "jpeg" ? "image/jpeg" : format === "webp" ? "image/webp" : "image/png"; attach(() => item.b64_json, [mime], index, 0, "model_output", undefined, true, "assistant"); }); } else if (options.operation === "responses") { @@ -137,23 +159,44 @@ export function captureOpenAIAttachments(manager: AttachmentManager | undefined, }); } } catch { /* Instrumentation must not throw or log source-bearing errors. */ } + return captured; } -/** Telemetry-only typed projection. Never send media bytes, filenames or source URLs to OTLP. */ -export function projectOpenAIMedia(payload: unknown, operation: "chat" | "responses" | "images", direction: "input" | "output", redactImages = true): any { - const body = record(payload); - const projectPart = (value: unknown): unknown => { - const part = record(value); - return (["input_file", "file"].includes(String(part.type)) || redactImages && ["image_url", "input_image"].includes(String(part.type))) ? { type: part.type } : value; - }; - if (operation === "images") return { created: body.created }; - const field = direction === "input" ? operation === "chat" ? "messages" : "input" : "output"; - if (!Array.isArray(body[field])) return payload; - return { ...body, [field]: list(body[field]).map(value => { - const message = record(value); - if (message.type === "computer_call_output") return { type: message.type, call_id: message.call_id, output: { type: "computer_screenshot" } }; - if (message.type === "image_generation_call") return { type: message.type, id: message.id, status: message.status }; - if (Array.isArray(message.content)) return { ...message, content: message.content.map(projectPart) }; - return value; - }) }; +const uri = (occurrence: { occurrenceId: string }) => `fi-attachment:${occurrence.occurrenceId}`; + +/** Telemetry-only projection. ONLY admitted positions are replaced with their `fi-attachment:` occurrence + * URI, so uploaded bytes are not exported twice. Everything else is returned exactly as before, and the + * request passed to the SDK is never mutated. + */ +export function projectOpenAIMedia(payload: unknown, captured: CapturedMedia, + operation: "chat" | "responses", direction: "input" | "output"): any { + if (!captured.size) return payload; + try { + const body = record(payload); + const field = direction === "input" ? operation === "chat" ? "messages" : "input" : "output"; + if (!Array.isArray(body[field])) return payload; + const at = (messageIndex: number, partIndex: number) => captured.get(key({ operation, direction }, messageIndex, partIndex)); + return { ...body, [field]: list(body[field]).map((value, messageIndex) => { + const message = record(value); + if (message.type === "computer_call_output") { + const occurrence = at(messageIndex, 0); + return occurrence ? { ...message, output: { ...record(message.output), image_url: uri(occurrence) } } : value; + } + if (message.type === "image_generation_call") { + const occurrence = at(messageIndex, 0); + return occurrence ? { ...message, result: uri(occurrence) } : value; + } + if (!Array.isArray(message.content)) return value; + return { ...message, content: message.content.map((value, partIndex) => { + const occurrence = at(messageIndex, partIndex); + if (!occurrence) return value; + const part = record(value); + if (part.type === "image_url") return { ...part, image_url: { ...record(part.image_url), url: uri(occurrence) } }; + if (part.type === "input_image") return { ...part, image_url: uri(occurrence) }; + if (part.type === "input_file") return { ...part, file_data: uri(occurrence) }; + if (part.type === "file") return { ...part, file: { ...record(part.file), file_data: uri(occurrence) } }; + return value; + }) }; + }) }; + } catch { return payload; } } diff --git a/typescript/packages/traceai_openai/src/instrumentation.ts b/typescript/packages/traceai_openai/src/instrumentation.ts index 1951022ea..855577d88 100644 --- a/typescript/packages/traceai_openai/src/instrumentation.ts +++ b/typescript/packages/traceai_openai/src/instrumentation.ts @@ -231,34 +231,29 @@ export class OpenAIInstrumentation extends InstrumentationBase{ ...args: Parameters ) { const originalBody = args[0]; - const body = projectOpenAIMedia(originalBody, "chat", "input", !!instrumentation.attachments); - // Prediction text is input content, captured only in maskable payloads. - const { messages: _messages, prediction: _prediction, ...invocationParameters } = body; - // diag.debug("@traceai/openai: ChatCompletion patch CALLED. Starting span..."); - - // --- ADD LOGS FOR TRACER AND CONTEXT --- - // const activeContextForSuppressionCheck = context.active(); - // const isSuppressed = isTracingSuppressed(activeContextForSuppressionCheck); - // diag.debug(`@traceai/openai: Is tracing suppressed? ${isSuppressed}`); - // diag.debug(`@traceai/openai: this.tracer type: ${instrumentation.tracer?.constructor?.name}`); // Accessing the base tracer - // diag.debug(`@traceai/openai: this.fiTracer type: ${instrumentation.fiTracer?.constructor?.name}`); - // --- END ADDED LOGS --- - const span = instrumentation.fiTracer.startSpan("OpenAI Chat Completions", { kind: SpanKind.INTERNAL }); - captureOpenAIAttachments(instrumentation.attachments, span, originalBody, { + // Content attributes are set after capture so admitted media can be projected out first. + const span = instrumentation.fiTracer.startSpan("OpenAI Chat Completions", { + kind: SpanKind.INTERNAL, + attributes: { + [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, + [SemanticConventions.LLM_MODEL_NAME]: originalBody.model, + [SemanticConventions.LLM_PROVIDER]: providerForResource(this), + [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", + }, + }); + const captured = captureOpenAIAttachments(instrumentation.attachments, span, originalBody, { operation: "chat", direction: "input", traceConfig: instrumentation._traceConfig, }); + const body = projectOpenAIMedia(originalBody, captured, "chat", "input") as typeof originalBody; + // Prediction text is input content, captured only in maskable payloads. + const { messages: _messages, prediction: _prediction, ...invocationParameters } = body; span.setAttributes({ - [SemanticConventions.FI_SPAN_KIND]: - FISpanKind.LLM, - [SemanticConventions.LLM_MODEL_NAME]: body.model, [SemanticConventions.INPUT_VALUE]: JSON.stringify(body), [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON, [SemanticConventions.LLM_INVOCATION_PARAMETERS]: JSON.stringify(invocationParameters), - [SemanticConventions.LLM_PROVIDER]: providerForResource(this), - [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", ...getLLMInputMessagesAttributes(body), ...getLLMToolsJSONSchema(body), [SemanticConventions.RAW_INPUT]: safelyJSONStringify(body) ?? "", @@ -523,9 +518,22 @@ export class OpenAIInstrumentation extends InstrumentationBase{ const body = args[0] as ResponseCreateParamsBase & { prompt?: { id: string; version?: string; variables?: Record }; }; + // Content attributes are set after capture so admitted media can be projected out first. + const span = instrumentation.fiTracer.startSpan("OpenAI Responses", { + kind: SpanKind.INTERNAL, + attributes: { + [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, + [SemanticConventions.LLM_MODEL_NAME]: body.model, + [SemanticConventions.LLM_PROVIDER]: providerForResource(this), + [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", + }, + }); + const captured = captureOpenAIAttachments(instrumentation.attachments, span, body, { + operation: "responses", direction: "input", traceConfig: instrumentation._traceConfig, + }); // Telemetry-only copy: MCP credentials must never reach any exported // duplicate, but the original SDK request must remain untouched. - const projectedBody = projectOpenAIMedia(body, "responses", "input", !!instrumentation.attachments); + const projectedBody = projectOpenAIMedia(body, captured, "responses", "input") as typeof body; const telemetryBody = { ...projectedBody, ...(body.tools && { tools: sanitizeToolDefinitions(body.tools) }), @@ -536,21 +544,12 @@ export class OpenAIInstrumentation extends InstrumentationBase{ ...parameters, ...(prompt && { prompt: { id: prompt.id, version: prompt.version } }), }; - const span = instrumentation.fiTracer.startSpan("OpenAI Responses", { kind: SpanKind.INTERNAL }); - captureOpenAIAttachments(instrumentation.attachments, span, body, { - operation: "responses", direction: "input", traceConfig: instrumentation._traceConfig, - }); span.setAttributes({ - [SemanticConventions.FI_SPAN_KIND]: - FISpanKind.LLM, - [SemanticConventions.LLM_MODEL_NAME]: body.model, [SemanticConventions.INPUT_VALUE]: JSON.stringify(telemetryBody), [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON, [SemanticConventions.LLM_INVOCATION_PARAMETERS]: JSON.stringify(invocationParameters), [SemanticConventions.PROMPT_TEMPLATE_VARIABLES]: safelyJSONStringify(prompt?.variables) ?? undefined, - [SemanticConventions.LLM_PROVIDER]: providerForResource(this), - [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", ...getResponsesInputMessagesAttributes(telemetryBody), ...getLLMToolsJSONSchema(body), [SemanticConventions.RAW_INPUT]: safelyJSONStringify(telemetryBody) ?? "", @@ -586,10 +585,12 @@ export class OpenAIInstrumentation extends InstrumentationBase{ if (endSpan) span.end(); return; } - captureOpenAIAttachments(instrumentation.attachments, span, result, { + // endSpan is false only on the streamed path. Streamed output media is unsupported + // until the core has a consumed-stream observation API. + const outputMedia = endSpan ? captureOpenAIAttachments(instrumentation.attachments, span, result, { operation: "responses", direction: "output", traceConfig: instrumentation._traceConfig, - }); - const projected = projectOpenAIMedia(result, "responses", "output"); + }) : new Map(); + const projected = projectOpenAIMedia(result, outputMedia, "responses", "output") as ResponseType; const output = { id: result.id, object: result.object, status: result.status, model: result.model, output: projected.output, usage: result.usage, @@ -637,28 +638,42 @@ export class OpenAIInstrumentation extends InstrumentationBase{ ); } + // images.generate: generated b64 outputs. Streamed partial images, edits and variations stay unsupported. if (typeof module.OpenAI.Images?.prototype?.generate === "function") { - this._wrap(module.OpenAI.Images.prototype, "generate", (original: typeof module.OpenAI.Images.prototype.generate): any => { - return function(this: unknown, ...args: Parameters) { - const span = instrumentation.fiTracer.startSpan("OpenAI Images", { kind: SpanKind.INTERNAL, attributes: { - [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, - [SemanticConventions.LLM_PROVIDER]: providerForResource(this), - [SemanticConventions.GEN_AI_OPERATION_NAME]: "image_generation", - } }); - try { - const execContext = trace.setSpan(context.active(), span); - const promise = context.with(execContext, () => original.apply(this, args)); - return observeAPIPromise(promise, result => { - if (span.isRecording() && !isStream(result)) { - captureOpenAIAttachments(instrumentation.attachments, span, result, { - operation: "images", direction: "output", traceConfig: instrumentation._traceConfig, - imageFormat: (args[0] as { output_format?: "png" | "jpeg" | "webp" }).output_format, - }); - } - span.end(); - return result; - }, error => endSpanWithError(span, error), response => endRawResponseSpan(span, response)); - } catch (error) { endSpanWithError(span, error); throw error; } + type ImagesGenerateType = typeof module.OpenAI.Images.prototype.generate; + this._wrap(module.OpenAI.Images.prototype, "generate", (original: ImagesGenerateType): any => { + return function patchedImagesGenerate(this: unknown, ...args: Parameters) { + const request = (args[0] ?? {}) as { model?: string; output_format?: "png" | "jpeg" | "webp" }; + const span = instrumentation.fiTracer.startSpan("OpenAI Images", { + kind: SpanKind.INTERNAL, + attributes: { + [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, + [SemanticConventions.LLM_PROVIDER]: providerForResource(this), + [SemanticConventions.GEN_AI_OPERATION_NAME]: "image_generation", + ...(typeof request.model === "string" ? { [SemanticConventions.LLM_MODEL_NAME]: request.model } : {}), + [SemanticConventions.INPUT_VALUE]: safelyJSONStringify(args[0]) ?? "", + [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON, + }, + }); + const contextSpan = span.getContextSpan(); + const execContext = getExecContext(contextSpan); + const execPromise = safeExecuteInTheMiddle>( + () => context.with(trace.setSpan(execContext, contextSpan), () => original.apply(this, args)), + (error) => { if (error) endSpanWithError(span, error); }, + ); + const wrappedPromise = observeAPIPromise(execPromise, (result) => { + if (!span.isRecording()) return result; + if (!isStream(result)) { + captureOpenAIAttachments(instrumentation.attachments, span, result, { + operation: "images", direction: "output", traceConfig: instrumentation._traceConfig, + imageFormat: request.output_format, + }); + } + span.setStatus({ code: SpanStatusCode.OK }); + span.end(); + return result; + }, (error) => endSpanWithError(span, error), (response) => endRawResponseSpan(span, response)); + return context.bind(execContext, wrappedPromise); }; }); } diff --git a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs index dcba8d149..5a0ceab82 100644 --- a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs +++ b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs @@ -140,21 +140,70 @@ test('OpenAI generated images obey output and PII restrictions', async () => { } }); -test('Telemetry projections strip typed bytes, filenames and signed source locations without mutating requests', () => { - for (const [body, project] of [[oaInput(), x => oa.projectOpenAIMedia(x, 'responses', 'input')], [anInput(), an.projectAnthropicMedia]]) { - const before = JSON.stringify(body), projected = JSON.stringify(project(body)); - assert.equal(JSON.stringify(body), before); - for (const secret of [PNG, PDF, 'synthetic-private-filename']) assert.ok(!projected.includes(secret)); - } +test('Telemetry projections replace only admitted media with occurrence URIs and never mutate requests', async () => { + const f = await fixture(); try { + const cases = [ + [oaInput(), (m, s, x) => oa.captureOpenAIAttachments(m, s, x, { operation: 'responses', direction: 'input' }), (x, c) => oa.projectOpenAIMedia(x, c, 'responses', 'input'), 3], + [anInput(), (m, s, x) => an.captureAnthropicAttachments(m, s, x), (x, c) => an.projectAnthropicMedia(x, c), 4], + ]; + for (const [body, capture, project, count] of cases) { + const before = JSON.stringify(body); + // Nothing admitted: the exact same payload object is returned, so existing telemetry is unchanged. + assert.equal(project(body, new Map()), body); + const captured = capture(f.manager, identity(), body); + assert.equal(captured.size, count); + const projected = JSON.stringify(project(body, captured)); + assert.equal(JSON.stringify(body), before); + for (const secret of [PNG, PDF]) assert.ok(!projected.includes(secret)); + for (const { occurrenceId } of captured.values()) assert.ok(projected.includes(`fi-attachment:${occurrenceId}`)); + } + await f.manager.flush(); + assert.equal(f.uploaded.length, 7); + } finally { await f.close(); } +}); + +test('Denied media keeps the existing telemetry payload untouched', async () => { + const f = await fixture({ traceConfig: { hideInputImages: true } }); try { + const body = oaInput(), captured = oa.captureOpenAIAttachments(f.manager, identity(), body, { operation: 'responses', direction: 'input' }); + // Only the PDF is admitted; both images stay exactly as the application sent them. + assert.deepEqual([...captured.values()].map(v => v.mimeType), ['application/pdf']); + const projected = oa.projectOpenAIMedia(body, captured, 'responses', 'input'); + assert.equal(projected.input[0].content[0], body.input[0].content[0]); + assert.equal(projected.input[1], body.input[1]); + assert.ok(projected.input[0].content[1].file_data.startsWith('fi-attachment:')); + } finally { await f.close(); } +}); + +test('Hostile payloads fail capture without throwing into the application', async () => { + const f = await fixture(); try { + const hostile = new Proxy({}, { get() { throw Error('synthetic-secret'); }, has() { throw Error('synthetic-secret'); } }); + for (const payload of [hostile, { input: [hostile] }, { messages: [hostile] }, { messages: [{ role: 'user', content: [hostile] }] }, null, 7, 'x']) { + assert.equal(oa.captureOpenAIAttachments(f.manager, identity(), payload, { operation: 'responses', direction: 'input' }).size, 0); + assert.equal(oa.captureOpenAIAttachments(f.manager, identity(), payload, { operation: 'chat', direction: 'input' }).size, 0); + assert.equal(an.captureAnthropicAttachments(f.manager, identity(), payload).size, 0); + assert.doesNotThrow(() => an.projectAnthropicMedia(payload, new Map([['input/0/0', { occurrenceId: 'x', mimeType: 'image/png' }]]))); + assert.doesNotThrow(() => oa.projectOpenAIMedia(payload, new Map([['responses/input/0/0', { occurrenceId: 'x', mimeType: 'image/png' }]]), 'responses', 'input')); + } + await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } +}); + +test('Capability descriptors name every unsupported source explicitly', () => { + for (const name of ['remote_url', 'provider_file', 'streamed_output', 'partial_image', 'images.edit', 'speech']) assert.ok(oa.openAIAttachmentCapabilities.unsupported.includes(name)); + for (const name of ['remote_url', 'provider_file', 'native_span_hookup', 'generated_image', 'streamed_output']) assert.ok(an.anthropicAttachmentCapabilities.unsupported.includes(name)); + assert.ok(Object.isFrozen(oa.openAIAttachmentCapabilities) && Object.isFrozen(an.anthropicAttachmentCapabilities)); }); test('CJS/ESM observations share deduplication without creating a second capture', async () => { const f = await fixture(); try { const esm = await import(pathToFileURL(path.resolve(__dirname, '../dist/esm/attachments.js'))); - const s = identity(); oa.captureOpenAIAttachments(f.manager, s, oaInput(), { operation: 'responses', direction: 'input' }); - esm.captureOpenAIAttachments(f.manager, { ...s }, oaInput(), { operation: 'responses', direction: 'input' }); + const s = identity(), first = oa.captureOpenAIAttachments(f.manager, s, oaInput(), { operation: 'responses', direction: 'input' }); + const second = esm.captureOpenAIAttachments(f.manager, { ...s }, oaInput(), { operation: 'responses', direction: 'input' }); + // The second observer reuses the first occurrence, so both projections point at one upload. + assert.deepEqual([...second.entries()], [...first.entries()]); const anthropicESM = await import(pathToFileURL(path.resolve(__dirname, '../../traceai_anthropic/dist/esm/attachments.js'))); - const a = identity(); an.captureAnthropicAttachments(f.manager, a, anInput()); anthropicESM.captureAnthropicAttachments(f.manager, { ...a }, anInput()); + const a = identity(), cjs = an.captureAnthropicAttachments(f.manager, a, anInput()), mjs = anthropicESM.captureAnthropicAttachments(f.manager, { ...a }, anInput()); + assert.deepEqual([...mjs.entries()], [...cjs.entries()]); await f.manager.flush(); assert.equal(f.uploaded.length, 7); } finally { await f.close(); } }); @@ -195,6 +244,14 @@ async function providerServer() { const server = http.createServer(async (req, res) => { const chunks = []; for await (const c of req) chunks.push(c); const body = JSON.parse(Buffer.concat(chunks)); requests.push({ path: req.url, body }); + if (req.url.endsWith('/responses') && body.stream) { + // A streamed Responses call whose completed response carries a generated image. + const response = { id: 'resp_stream', object: 'response', status: 'completed', model: body.model, output: [{ type: 'image_generation_call', id: 'img_stream', status: 'completed', result: PNG }], usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } }; + res.setHeader('content-type', 'text/event-stream'); + res.write(`event: response.created\ndata: ${JSON.stringify({ type: 'response.created', sequence_number: 0, response: { ...response, status: 'in_progress', output: [] } })}\n\n`); + res.end(`event: response.completed\ndata: ${JSON.stringify({ type: 'response.completed', sequence_number: 1, response })}\n\n`); + return; + } res.setHeader('content-type', 'application/json'); if (req.url.endsWith('/chat/completions')) res.end(JSON.stringify({ id: 'chat_fixture', object: 'chat.completion', model: body.model, choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })); else if (req.url.endsWith('/responses')) res.end(JSON.stringify({ id: 'resp_fixture', object: 'response', status: 'completed', model: body.model, output: [{ type: 'computer_call', id: 'computer_fixture', call_id: 'call_computer', status: 'completed', action: { type: 'screenshot' } }, { type: 'image_generation_call', id: 'img_fixture', status: 'completed', result: PNG }], usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } })); @@ -239,6 +296,18 @@ test('Real SDKs preserve APIPromise/request/response behavior and upload only th assert.ok(p.requests.some(r => JSON.stringify(r.body).includes(PNG))); assert.ok(!JSON.stringify(exporter.getFinishedSpans().map(s => s.attributes)).includes(PNG)); assert.ok(exporter.getFinishedSpans().some(s => s.attributes['fi.attachment.refs'])); + // Each admitted reference appears as an occurrence URI in its own span's telemetry. + for (const span of exporter.getFinishedSpans()) { + const refs = JSON.parse(span.attributes['fi.attachment.refs'] ?? '[]'), text = JSON.stringify(span.attributes); + for (const ref of refs.filter(r => r.direction === 'input')) assert.ok(text.includes(`fi-attachment:${ref.occurrence_id}`), span.name); + } + if (client.responses) { + // Streamed output media is unsupported: the stream is delivered unchanged and nothing uploads. + const stream = await client.responses.create({ model: 'synthetic-model', input: 'synthetic', stream: true }); + const events = []; for await (const event of stream) events.push(event.type); + assert.deepEqual(events, ['response.created', 'response.completed']); + await f.manager.flush(); assert.equal(f.uploaded.length, expected); + } // A raw-response caller owns the body; instrumentation must not parse for media. const raw = await client.images.generate({ prompt: 'synthetic', response_format: 'b64_json' }).asResponse(); assert.equal(raw.bodyUsed, false); await raw.arrayBuffer(); @@ -254,3 +323,35 @@ test('Real SDKs preserve APIPromise/request/response behavior and upload only th oi.disable(); ai.disable(); await provider.shutdown(); await f.close(); await p.close(); } }); + +test('A configured but disabled manager leaves existing span telemetry byte-identical', async () => { + const sdkModule = require(process.env.SDK_OPENAI || 'openai'); + const anthropicModule = require(process.env.SDK_ANTHROPIC || '../../traceai_anthropic/node_modules/@anthropic-ai/sdk'); + const OpenAI = sdkModule.OpenAI || sdkModule.default || sdkModule; + const Anthropic = anthropicModule.Anthropic || anthropicModule.default || anthropicModule; + const native = 'openTelemetry' in Anthropic.prototype; + const p = await providerServer(), disabled = await fixture({ enabled: false }); + const run = async attachments => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + const oi = new OpenAIInstrumentation({ instrumentationConfig: { enabled: false }, attachments }); + const ai = new AnthropicInstrumentation({ instrumentationConfig: { enabled: false }, attachments }); + oi.setTracerProvider(provider); ai.setTracerProvider(provider); + oi.manuallyInstrument(sdkModule); ai.manuallyInstrument(anthropicModule); + try { + const client = new OpenAI({ apiKey: 'synthetic-provider-key', baseURL: p.url, maxRetries: 0 }); + await client.chat.completions.create({ model: 'synthetic-model', messages: [{ role: 'user', content: [{ type: 'image_url', image_url: { url: `data:image/png;base64,${PNG}` } }] }] }); + if (client.responses) await client.responses.create({ model: 'synthetic-model', ...oaInput() }); + if (!native) await new Anthropic({ apiKey: 'synthetic-provider-key', baseURL: p.url, maxRetries: 0 }).messages.create({ model: 'synthetic-model', max_tokens: 10, ...anInput() }); + await provider.forceFlush(); + const strip = ({ name, attributes }) => ({ name, attributes: Object.fromEntries(Object.entries(attributes).filter(([k]) => !/latency|duration|time/i.test(k))) }); + return exporter.getFinishedSpans().map(strip); + } finally { oi.disable(); ai.disable(); await provider.shutdown(); } + }; + try { + const baseline = await run(undefined), off = await run(disabled.manager); + assert.ok(baseline.length >= 1); + assert.deepEqual(off, baseline); + assert.equal(disabled.calls.length, 0); + } finally { await disabled.close(); await p.close(); } +}); From 92662aecc430a07ffaffcc3ef6d0d023b229b063 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 20:13:48 -0800 Subject: [PATCH 3/7] test(openai,anthropic): cover local-only privacy restrictions independently of the manager Co-Authored-By: Claude Opus 5.5 --- .../test/attachments-providers.test.cjs | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs index 5a0ceab82..5008ace23 100644 --- a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs +++ b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs @@ -232,9 +232,31 @@ test('OpenAI image error bodies are not treated as generated media', async () => }); test('Local instrumentor privacy restrictions cannot be loosened by a permissive shared manager', async () => { + for (const traceConfig of [{ hideInputs: true }, { hideInputMessages: true }, { hideInputText: true }, { piiRedaction: true }]) { + const f = await fixture(); try { + assert.equal(oa.captureOpenAIAttachments(f.manager, identity(), oaInput(), { operation: 'responses', direction: 'input', traceConfig }).size, 0); + assert.equal(an.captureAnthropicAttachments(f.manager, identity(), anInput(), { traceConfig }).size, 0); + await f.manager.flush(); assert.equal(f.initialized.length, 0, JSON.stringify(traceConfig)); + } finally { await f.close(); } + } + for (const traceConfig of [{ hideOutputs: true }, { piiRedaction: true }]) { + const f = await fixture(); try { + oa.captureOpenAIAttachments(f.manager, identity(), { data: [{ b64_json: PNG }] }, { operation: 'images', direction: 'output', traceConfig }); + await f.manager.flush(); assert.equal(f.initialized.length, 0, JSON.stringify(traceConfig)); + } finally { await f.close(); } + } + // Image-only restriction: the PDF still uploads, no image (including screenshots) does. + const f = await fixture(); try { + oa.captureOpenAIAttachments(f.manager, identity(), oaInput(), { operation: 'responses', direction: 'input', traceConfig: { hideInputImages: true } }); + an.captureAnthropicAttachments(f.manager, identity(), anInput(), { traceConfig: { hideInputImages: true } }); + await f.manager.flush(); assert.deepEqual(f.initialized.map(r => r.mime_type), ['application/pdf', 'application/pdf']); + } finally { await f.close(); } +}); + +test('Local hideErrorDetails denies Anthropic error tool media even when the manager allows it', async () => { const f = await fixture(); try { - oa.captureOpenAIAttachments(f.manager, identity(), oaInput(), { operation: 'responses', direction: 'input', traceConfig: { hideInputs: true } }); - an.captureAnthropicAttachments(f.manager, identity(), anInput(), { traceConfig: { piiRedaction: true } }); + an.captureAnthropicAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_error', is_error: true, content: [image()] }] }] }, + { traceConfig: { hideErrorDetails: true } }); await f.manager.flush(); assert.equal(f.initialized.length, 0); } finally { await f.close(); } }); From 059e0b91516d2b53901d39fd01fdc5b66660aad4 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 20:19:33 -0800 Subject: [PATCH 4/7] fix(openai): keep the Responses telemetry copy loosely typed for older SDK typings Co-Authored-By: Claude Opus 5.5 --- typescript/packages/traceai_openai/src/instrumentation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typescript/packages/traceai_openai/src/instrumentation.ts b/typescript/packages/traceai_openai/src/instrumentation.ts index 855577d88..c457aaf20 100644 --- a/typescript/packages/traceai_openai/src/instrumentation.ts +++ b/typescript/packages/traceai_openai/src/instrumentation.ts @@ -533,7 +533,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ }); // Telemetry-only copy: MCP credentials must never reach any exported // duplicate, but the original SDK request must remain untouched. - const projectedBody = projectOpenAIMedia(body, captured, "responses", "input") as typeof body; + const projectedBody = projectOpenAIMedia(body, captured, "responses", "input"); const telemetryBody = { ...projectedBody, ...(body.tools && { tools: sanitizeToolDefinitions(body.tools) }), From 4b52c19ca5e6f7c47827ca93ebde41cf4bc73b77 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 20:39:13 -0800 Subject: [PATCH 5/7] test(openai,anthropic): pin the 256 KiB inline decode bound with valid data URIs The cloud mutation run showed the OpenAI cap was not exercised: the oversized case was not a data URI, so it failed the prefix check first. Co-Authored-By: Claude Opus 5.5 --- .../test/attachments-providers.test.cjs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs index 5008ace23..c9a9be53a 100644 --- a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs +++ b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs @@ -131,6 +131,19 @@ for (const provider of ['openai', 'anthropic']) { }); } +test('Inline parts are capped at exactly 256 KiB of decoded bytes for both providers', async () => { + const f = await fixture(); try { + const at = Buffer.alloc(256 * 1024, 7).toString('base64'), over = Buffer.alloc(256 * 1024 + 3, 7).toString('base64'); + for (const data of [at, over]) { + oa.captureOpenAIAttachments(f.manager, identity(), { input: [{ role: 'user', content: [{ type: 'input_image', image_url: `data:image/png;base64,${data}` }] }] }, { operation: 'responses', direction: 'input' }); + oa.captureOpenAIAttachments(f.manager, identity(), { data: [{ b64_json: data }] }, { operation: 'images', direction: 'output' }); + an.captureAnthropicAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data } }] }] }); + } + await f.manager.flush(); + assert.deepEqual(f.uploaded.map(b => b.length), [262144, 262144, 262144]); + } finally { await f.close(); } +}); + test('OpenAI generated images obey output and PII restrictions', async () => { for (const traceConfig of [{}, { hideOutputs: true }, { hideOutputMessages: true }, { hideOutputText: true }, { piiRedaction: true }]) { const f = await fixture({ traceConfig }); try { From 912c4fb831ba1608796530be482967ef7d4514ab Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:44:36 -0800 Subject: [PATCH 6/7] feat(openai,anthropic): preflight every media part; capture streamed Responses output Adopt the core preflight API (sdk-streaming-design section 1) in place of lazy `get data()` getters: - Each inline part is preflighted (deny-wins, budget reserved) before its value is read or decoded. Denied parts are never read. - An admitted ticket that is not used (malformed base64, MIME outside the allowlist, decode failure) is released at once, not left to the TTL sweep. - OpenAI data URIs naming a sibling MIME of the same modality are released and re-admitted under the exact MIME before decoding. - When the server negotiates an upload limit below the 256 KiB inline cap, preflight returns too_large only after every privacy step passed, so the part is re-preflighted with its exact decoded length. Small media keeps capturing as before. Streamed Responses: the generated image in the terminal response.completed event that the application consumed from its own iterator is now captured and projected out of the span output, before the span ends. Partial (preview) images and streamed images.generate stay unsupported and are named in the capability descriptor. Test fixtures mint one-hour sessions (core renewal window), assert no FI_ATTACHMENTS_* degradation warnings and no leaked tickets or bytes. Co-Authored-By: Claude Opus 5.5 --- .../traceai_anthropic/src/attachments.ts | 50 ++++++--- .../traceai_openai/src/attachments.ts | 100 ++++++++++++------ .../traceai_openai/src/instrumentation.ts | 8 +- .../test/attachments-providers.test.cjs | 71 +++++++++++-- 4 files changed, 172 insertions(+), 57 deletions(-) diff --git a/typescript/packages/traceai_anthropic/src/attachments.ts b/typescript/packages/traceai_anthropic/src/attachments.ts index 876ac52ae..ce45cd139 100644 --- a/typescript/packages/traceai_anthropic/src/attachments.ts +++ b/typescript/packages/traceai_anthropic/src/attachments.ts @@ -25,17 +25,26 @@ const host = globalThis as typeof globalThis & { [registryKey]?: Registry }; const observations = (host[registryKey] ??= new WeakMap()); const TOOL_ID = /^[a-zA-Z0-9_-]{1,128}$/; -function decode(encoded: unknown): Uint8Array { - const max = anthropicAttachmentCapabilities.maxInlineBytes; - // Bound allocation BEFORE decoding. Padding and alphabet are strict. - if (typeof encoded !== "string" || !encoded.length || encoded.length % 4 !== 0 || - encoded.length > 4 * Math.ceil(max / 3) || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); - const bytes = Buffer.from(encoded, "base64"); - return bytes.length <= max && bytes.toString("base64") === encoded ? bytes : new Uint8Array(); +const MAX_ENCODED = 4 * Math.ceil(anthropicAttachmentCapabilities.maxInlineBytes / 3); + +/** Exact decoded length from the encoded string's shape only (no decode); 0 when out of bounds. */ +function bound(encoded: unknown): number { + if (typeof encoded !== "string" || !encoded.length || encoded.length % 4 !== 0 || encoded.length > MAX_ENCODED) return 0; + return (encoded.length / 4) * 3 - (encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0); +} + +/** Strict decode after admission. The ticket's budget is a hard bound; padding and alphabet are strict. */ +function decode(encoded: unknown, maxBytes: number): Uint8Array | undefined { + // Bound allocation BEFORE decoding. + if (!bound(encoded) || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded as string)) return undefined; + const bytes = Buffer.from(encoded as string, "base64"); + return bytes.length && bytes.length <= maxBytes && bytes.length <= anthropicAttachmentCapabilities.maxInlineBytes && + bytes.toString("base64") === encoded ? bytes : undefined; } /** Observe only Messages' typed image/document blocks and one level of tool_result content. - * The manager owns policy, upload, lifecycle and references. A native integration + * Every part is preflighted (deny-wins, budget reserved) before its base64 is read or decoded; an admitted + * ticket that is not used is released at once. The manager owns policy, upload, lifecycle and references. A native integration * must supply its actual emitted span/correlation, never an ambient guessed span. * Returns the admitted positions so the caller can project them out of span telemetry. */ @@ -88,13 +97,24 @@ export function captureAnthropicAttachments(manager: AttachmentManager | undefin const previous = spans!.get(spanKey)?.get(positionKey); if (previous) { captured.set(positionKey, previous); return; } const role: AttachmentOptions["role"] = toolCallId ? "tool" : message.role === "assistant" ? "assistant" : "user"; - const handle = manager.attach(correlation, { kind: "Bytes", get data() { return decode(source.data); } }, - { mimeType: mime, direction: "input", messageIndex, partIndex: index, role, - purpose: part.type === "document" ? "document" : toolCallId ? computerCalls.has(toolCallId) ? "computer_screenshot" : "tool_result" : "model_input", - ...(toolCallId ? { toolCallId } : {}), - capture: { ...options.capture, modelInput: true, errorDetail: errorDetail || options.capture?.errorDetail, - ...(hidden || (part.type === "image" && config.hideInputImages) || (errorDetail && config.hideErrorDetails) ? { captureContent: false } : {}) }, - }); + const request = { mimeType: mime, direction: "input" as const, + purpose: (part.type === "document" ? "document" : toolCallId ? computerCalls.has(toolCallId) ? "computer_screenshot" : "tool_result" : "model_input") as AttachmentOptions["purpose"], + capture: { ...options.capture, modelInput: true, errorDetail: errorDetail || options.capture?.errorDetail, + ...(hidden || (part.type === "image" && config.hideInputImages) || (errorDetail && config.hideErrorDetails) ? { captureContent: false } : {}) } }; + // Preflight (deny-wins, budget reserved) before the base64 source is read or decoded (design section 1). + // The source is read at most once. + let data: unknown, dataRead = false; + const read = () => { if (!dataRead) { dataRead = true; data = source.data; } return data; }; + let ticket = manager.preflight(correlation, { ...request, maxBytes: anthropicAttachmentCapabilities.maxInlineBytes }); + // too_large is evaluated after every privacy, correlation and sampling step, so sizing the source exactly is safe here. + if (ticket.reason === "too_large") { + const exact = bound(read()); + if (exact > 0) ticket = manager.preflight(correlation, { ...request, maxBytes: exact }); + } + if (!ticket.admitted) return; + const bytes = decode(read(), ticket.maxBytes); + if (!bytes) { ticket.release(); return; } + const handle = ticket.attach({ kind: "Bytes", data: bytes }, { messageIndex, partIndex: index, role, ...(toolCallId ? { toolCallId } : {}) }); if (handle.admission === "queued") { const occurrence = Object.freeze({ occurrenceId: handle.occurrenceId, mimeType: mime }); let positions = spans!.get(spanKey); diff --git a/typescript/packages/traceai_openai/src/attachments.ts b/typescript/packages/traceai_openai/src/attachments.ts index 319628c03..cf0cac04c 100644 --- a/typescript/packages/traceai_openai/src/attachments.ts +++ b/typescript/packages/traceai_openai/src/attachments.ts @@ -4,10 +4,11 @@ import type { AttachmentManager, AttachmentOptions, AttachmentSpan, AttachmentCo /** Declarative handoff to auto/wire owners; this is not a native/wire hookup. */ export const openAIAttachmentCapabilities = Object.freeze({ schemaVersion: 1, - operations: ["chat.completions.create.input", "responses.create.input", "responses.create.output", "images.generate.output"] as const, + operations: ["chat.completions.create.input", "responses.create.input", "responses.create.output", "responses.stream.output", "images.generate.output"] as const, sources: ["inline_base64"] as const, - /** Fail closed: no capture is attempted for these. */ - unsupported: ["remote_url", "provider_file", "streamed_output", "partial_image", "images.edit", "images.createVariation", "speech"] as const, + /** Fail closed: no capture is attempted for these. Streamed Responses output is captured only from the + * terminal `response.completed` event the application consumed; partial (preview) images never are. */ + unsupported: ["remote_url", "provider_file", "partial_image", "images.generate.stream", "images.edit", "images.createVariation", "speech"] as const, maxInlineBytes: 256 * 1024, }); @@ -50,18 +51,35 @@ function restrictions(options: OpenAIAttachmentCaptureOptions): AttachmentCaptur }; } -function decode(value: unknown, mimeType: string, raw: boolean): Uint8Array { - if (typeof value !== "string") return new Uint8Array(); - const prefix = `data:${mimeType};base64,`; - if (!raw && !value.startsWith(prefix)) return new Uint8Array(); - // Bound allocation BEFORE slicing/decoding. Padding and alphabet are strict. - const start = raw ? 0 : prefix.length; +type Parsed = { mimeType: string; encoded: string; bound: number }; +const MAX_ENCODED = 4 * Math.ceil(openAIAttachmentCapabilities.maxInlineBytes / 3); + +/** Structural parse only: no decode or allocation beyond the string the application already holds. + * Returns the candidate MIME named by a data URI (or the typed MIME for raw base64) and the exact decoded length. + */ +function parse(value: unknown, mimeTypes: string[], raw: boolean): Parsed | undefined { + if (typeof value !== "string") return undefined; + let mimeType = mimeTypes[0], start = 0; + if (!raw) { + const comma = value.indexOf(","); + const header = comma > 5 && comma < 128 ? value.slice(0, comma) : ""; + const match = /^data:([a-z]+\/[a-z0-9.+-]+);base64$/.exec(header); + if (!match || !mimeTypes.includes(match[1])) return undefined; + mimeType = match[1]; start = comma + 1; + } const length = value.length - start; - if (length <= 0 || length > 4 * Math.ceil(openAIAttachmentCapabilities.maxInlineBytes / 3) || length % 4 !== 0) return new Uint8Array(); + // Bound allocation BEFORE slicing/decoding. + if (length <= 0 || length > MAX_ENCODED || length % 4 !== 0) return undefined; const encoded = value.slice(start); - if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return new Uint8Array(); + const padding = encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0; + return { mimeType, encoded, bound: (length / 4) * 3 - padding }; +} + +/** Strict decode after admission. Padding and alphabet are strict; the ticket's budget is a hard bound. */ +function decode(encoded: string, maxBytes: number): Uint8Array | undefined { + if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return undefined; const bytes = Buffer.from(encoded, "base64"); - if (bytes.length > openAIAttachmentCapabilities.maxInlineBytes || bytes.toString("base64") !== encoded) return new Uint8Array(); + if (!bytes.length || bytes.length > maxBytes || bytes.length > openAIAttachmentCapabilities.maxInlineBytes || bytes.toString("base64") !== encoded) return undefined; return bytes; } @@ -71,7 +89,8 @@ const key = (options: Pick= MAX_SPANS) spans!.delete(spans!.keys().next().value!); - positions = new Map(); spans!.set(spanKey, positions); - } - if (positions.size < MAX_PARTS) positions.set(position, occurrence); - captured.set(position, occurrence); - return; + // The source is read at most once, and only after a preflight admitted this modality (design section 1). + let source: Parsed | undefined, sourceRead = false; + const parsed = () => { if (!sourceRead) { sourceRead = true; source = parse(read(), mimeTypes, raw); } return source; }; + const preflight = (mimeType: string) => { + const request = { mimeType, direction: options.direction, purpose, capture: { + ...capture, + ...(options.direction === "input" && mimeType.startsWith("image/") && options.traceConfig?.hideInputImages ? { captureContent: false } : {}), + } }; + let ticket = manager.preflight(correlation, { ...request, maxBytes: openAIAttachmentCapabilities.maxInlineBytes }); + // too_large is evaluated after every privacy, correlation and sampling step, so sizing the source exactly is safe here. + if (ticket.reason === "too_large") { + const bound = parsed()?.bound ?? 0; + if (bound > 0) ticket = manager.preflight(correlation, { ...request, maxBytes: bound }); } - // Only a non-matching data-URI type (decoded as empty) tries the next candidate. Denials stop. - if (handle.admission !== "too_large") return; + return ticket; + }; + let ticket = preflight(mimeTypes[0]); + if (!ticket.admitted) return; + const found = parsed(); + if (!found) { ticket.release(); return; } + // A data URI names a sibling MIME of the same modality: re-admit under the exact type before decoding. + if (found.mimeType !== mimeTypes[0]) { + ticket.release(); + ticket = preflight(found.mimeType); + if (!ticket.admitted) return; + } + const bytes = decode(found.encoded, ticket.maxBytes); + if (!bytes) { ticket.release(); return; } + const handle = ticket.attach({ kind: "Bytes", data: bytes }, { messageIndex, partIndex, role, ...(toolCallId ? { toolCallId } : {}) }); + if (handle.admission !== "queued") return; + const occurrence = Object.freeze({ occurrenceId: handle.occurrenceId, mimeType: found.mimeType }); + let positions = spans!.get(spanKey); + if (!positions) { + if (spans!.size >= MAX_SPANS) spans!.delete(spans!.keys().next().value!); + positions = new Map(); spans!.set(spanKey, positions); } + if (positions.size < MAX_PARTS) positions.set(position, occurrence); + captured.set(position, occurrence); }; const toolCallId = (value: unknown) => typeof value === "string" && /^[a-zA-Z0-9_-]{1,128}$/.test(value) ? value : undefined; const parts = (content: unknown, messageIndex: number, role: AttachmentOptions["role"]) => { diff --git a/typescript/packages/traceai_openai/src/instrumentation.ts b/typescript/packages/traceai_openai/src/instrumentation.ts index c457aaf20..d2d05e7a4 100644 --- a/typescript/packages/traceai_openai/src/instrumentation.ts +++ b/typescript/packages/traceai_openai/src/instrumentation.ts @@ -585,11 +585,11 @@ export class OpenAIInstrumentation extends InstrumentationBase{ if (endSpan) span.end(); return; } - // endSpan is false only on the streamed path. Streamed output media is unsupported - // until the core has a consumed-stream observation API. - const outputMedia = endSpan ? captureOpenAIAttachments(instrumentation.attachments, span, result, { + // On the streamed path (endSpan false) `result` is the terminal response the application consumed + // from its own iterator, observed before the span ends. Partial (preview) images are never captured. + const outputMedia = captureOpenAIAttachments(instrumentation.attachments, span, result, { operation: "responses", direction: "output", traceConfig: instrumentation._traceConfig, - }) : new Map(); + }); const projected = projectOpenAIMedia(result, outputMedia, "responses", "output") as ResponseType; const output = { id: result.id, object: result.object, status: result.status, diff --git a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs index c9a9be53a..368cf49fd 100644 --- a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs +++ b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs @@ -19,11 +19,19 @@ const document = () => ({ type: 'document', title: 'synthetic-private-filename', const identity = () => ({ traceId: randomUUID().replace(/-/g, ''), spanId: randomUUID().replace(/-/g, '').slice(0, 16), traceFlags: 1 }); const keepalive = setInterval(() => {}, 1000); after(() => clearInterval(keepalive)); +// The core reports a silently degraded fixture (legacy transport, rejected F01 shape) as a process warning. +const coreWarnings = []; +process.on('warning', warning => { if (/^FI_ATTACHMENTS_/.test(warning.code ?? '')) coreWarnings.push(warning.code); }); +// Preflight tickets that are not used must be released at once, never left to the TTL sweep. +const assertNoOpenTickets = async manager => { + assert.equal(manager.status().openTickets, 0, 'open tickets'); await manager.flush(); assert.equal(manager.status().ownedBytes, 0, 'owned bytes'); +}; -async function fixture(config = {}, serverPolicy = {}) { +async function fixture(config = {}, serverPolicy = {}, serverLimits = {}) { const calls = [], initialized = [], uploaded = []; const scope = { organization_id: randomUUID(), workspace_id: randomUUID(), project_id: randomUUID() }; - const expiry = () => new Date(Date.now() + 60000).toISOString(); + // Sessions of one hour or more: the core renews within max(60 s, 20% of lifetime) of expiry. + const expiry = () => new Date(Date.now() + 3600000).toISOString(); const server = http.createServer(async (req, res) => { const chunks = []; for await (const c of req) chunks.push(c); const bytes = Buffer.concat(chunks); calls.push(req.url); @@ -37,7 +45,7 @@ async function fixture(config = {}, serverPolicy = {}) { limits: { max_upload_bytes: 1024 * 1024, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, - transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }, expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); + transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5, ...serverLimits }, expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); } else if (req.url.endsWith('/uploads')) { const body = JSON.parse(bytes); initialized.push(body); res.statusCode = 201; @@ -121,6 +129,23 @@ for (const provider of ['openai', 'anthropic']) { capture(f.manager, identity(), provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_file', file_id: 'file_secret' }] }] } : { messages: [{ role: 'user', content: [{ type: 'document', source: { type: 'file', file_id: 'file_secret' } }, null, { type: 'tool_result', tool_use_id: {}, content: [image()] }] }] }); await f.manager.flush(); assert.equal(f.initialized.length, 0); + await assertNoOpenTickets(f.manager); + } finally { await f.close(); } + }); + test(`${provider}: a negotiated upload limit below the inline cap still captures small media exactly`, async () => { + const f = await fixture({}, {}, { max_upload_bytes: 100 * 1024 }); + try { + const id = identity(), captured = capture(f.manager, id, input()); + await f.manager.flush(); + assert.equal(captured.size, provider === 'openai' ? 3 : 4); + assert.equal(f.uploaded.length, captured.size); + assert.ok(f.initialized.every(r => r.declared_bytes <= 100 * 1024)); + // Above the negotiated limit nothing is decoded or uploaded, and the reservation is returned. + const big = Buffer.alloc(150 * 1024, 7).toString('base64'); + capture(f.manager, identity(), provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_image', image_url: `data:image/png;base64,${big}` }] }] } + : { messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: big } }] }] }); + await f.manager.flush(); assert.equal(f.uploaded.length, captured.size); + await assertNoOpenTickets(f.manager); } finally { await f.close(); } }); test(`${provider}: unsampled identity and queued policy tightening cannot upload`, async () => { @@ -202,7 +227,10 @@ test('Hostile payloads fail capture without throwing into the application', asyn }); test('Capability descriptors name every unsupported source explicitly', () => { - for (const name of ['remote_url', 'provider_file', 'streamed_output', 'partial_image', 'images.edit', 'speech']) assert.ok(oa.openAIAttachmentCapabilities.unsupported.includes(name)); + for (const name of ['remote_url', 'provider_file', 'partial_image', 'images.generate.stream', 'images.edit', 'speech']) assert.ok(oa.openAIAttachmentCapabilities.unsupported.includes(name)); + // Streamed Responses output is captured from the consumed terminal event; partial images never are. + assert.ok(oa.openAIAttachmentCapabilities.operations.includes('responses.stream.output')); + assert.ok(!oa.openAIAttachmentCapabilities.unsupported.includes('streamed_output')); for (const name of ['remote_url', 'provider_file', 'native_span_hookup', 'generated_image', 'streamed_output']) assert.ok(an.anthropicAttachmentCapabilities.unsupported.includes(name)); assert.ok(Object.isFrozen(oa.openAIAttachmentCapabilities) && Object.isFrozen(an.anthropicAttachmentCapabilities)); }); @@ -337,10 +365,22 @@ test('Real SDKs preserve APIPromise/request/response behavior and upload only th for (const ref of refs.filter(r => r.direction === 'input')) assert.ok(text.includes(`fi-attachment:${ref.occurrence_id}`), span.name); } if (client.responses) { - // Streamed output media is unsupported: the stream is delivered unchanged and nothing uploads. + // Streamed Responses: the stream is delivered unchanged; the generated image in the terminal event the + // application consumed uploads once and is projected out of that span's output telemetry. const stream = await client.responses.create({ model: 'synthetic-model', input: 'synthetic', stream: true }); - const events = []; for await (const event of stream) events.push(event.type); + const events = []; for await (const event of stream) { events.push(event.type); if (event.type === 'response.completed') assert.equal(event.response.output[0].result, PNG); } assert.deepEqual(events, ['response.created', 'response.completed']); + expected++; + await f.manager.flush(); await provider.forceFlush(); assert.equal(f.uploaded.length, expected); + assert.equal(f.uploaded.at(-1).toString('base64'), PNG); + const streamed = exporter.getFinishedSpans().filter(s => s.name === 'OpenAI Responses').at(-1); + const refs = JSON.parse(streamed.attributes['fi.attachment.refs'] ?? '[]'); + assert.equal(refs.length, 1); assert.equal(refs[0].direction, 'output'); assert.equal(refs[0].purpose, 'model_output'); + assert.ok(String(streamed.attributes['output.value']).includes(`fi-attachment:${refs[0].occurrence_id}`)); + assert.ok(!JSON.stringify(streamed.attributes).includes(PNG)); + // A consumer that stops early never reaches the terminal event: nothing is captured. + const early = await client.responses.create({ model: 'synthetic-model', input: 'synthetic', stream: true }); + for await (const event of early) { if (event.type === 'response.created') break; } await f.manager.flush(); assert.equal(f.uploaded.length, expected); } // A raw-response caller owns the body; instrumentation must not parse for media. @@ -390,3 +430,22 @@ test('A configured but disabled manager leaves existing span telemetry byte-iden assert.equal(disabled.calls.length, 0); } finally { await disabled.close(); await p.close(); } }); + +test('OpenAI data URIs of a sibling image type are admitted under their exact MIME', async () => { + const f = await fixture(); try { + const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 1, 2, 3, 0xff, 0xd9]).toString('base64'); + const captured = oa.captureOpenAIAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [ + { type: 'image_url', image_url: { url: `data:image/jpeg;base64,${jpeg}` } }, + { type: 'image_url', image_url: { url: `data:image/gif;base64,${jpeg}` } }, + ] }] }, { operation: 'chat', direction: 'input' }); + await f.manager.flush(); + assert.deepEqual([...captured.values()].map(v => v.mimeType), ['image/jpeg']); + assert.deepEqual(f.initialized.map(r => r.mime_type), ['image/jpeg']); + assert.equal(f.uploaded[0].toString('base64'), jpeg); + await assertNoOpenTickets(f.manager); + } finally { await f.close(); } +}); + +test('No core degradation warning was raised by any fixture', () => { + assert.deepEqual(coreWarnings, []); +}); From 88a4c233baa27a56e87fb359398bd0baec63f9e5 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 03:07:34 -0800 Subject: [PATCH 7/7] fix(openai,anthropic): retry a queue_full preflight at the part's exact size A cap-sized (256 KiB) preflight can return queue_full when the owned-byte budget is nearly used, where the previous attach() reserved only the actual length and admitted small media. queue_full, like too_large, is evaluated after every privacy, correlation and sampling step, so the part is sized exactly and preflighted once more. New test row for both providers. Co-Authored-By: Claude Opus 5.5 --- .../packages/traceai_anthropic/src/attachments.ts | 5 +++-- .../packages/traceai_openai/src/attachments.ts | 5 +++-- .../test/attachments-providers.test.cjs | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/typescript/packages/traceai_anthropic/src/attachments.ts b/typescript/packages/traceai_anthropic/src/attachments.ts index ce45cd139..cb72aee72 100644 --- a/typescript/packages/traceai_anthropic/src/attachments.ts +++ b/typescript/packages/traceai_anthropic/src/attachments.ts @@ -106,8 +106,9 @@ export function captureAnthropicAttachments(manager: AttachmentManager | undefin let data: unknown, dataRead = false; const read = () => { if (!dataRead) { dataRead = true; data = source.data; } return data; }; let ticket = manager.preflight(correlation, { ...request, maxBytes: anthropicAttachmentCapabilities.maxInlineBytes }); - // too_large is evaluated after every privacy, correlation and sampling step, so sizing the source exactly is safe here. - if (ticket.reason === "too_large") { + // too_large (negotiated limit below the cap) and queue_full (owned budget nearly used) are evaluated after every + // privacy, correlation and sampling step, so sizing the source exactly and retrying once is safe here. + if (ticket.reason === "too_large" || ticket.reason === "queue_full") { const exact = bound(read()); if (exact > 0) ticket = manager.preflight(correlation, { ...request, maxBytes: exact }); } diff --git a/typescript/packages/traceai_openai/src/attachments.ts b/typescript/packages/traceai_openai/src/attachments.ts index cf0cac04c..b185f6294 100644 --- a/typescript/packages/traceai_openai/src/attachments.ts +++ b/typescript/packages/traceai_openai/src/attachments.ts @@ -121,8 +121,9 @@ export function captureOpenAIAttachments(manager: AttachmentManager | undefined, ...(options.direction === "input" && mimeType.startsWith("image/") && options.traceConfig?.hideInputImages ? { captureContent: false } : {}), } }; let ticket = manager.preflight(correlation, { ...request, maxBytes: openAIAttachmentCapabilities.maxInlineBytes }); - // too_large is evaluated after every privacy, correlation and sampling step, so sizing the source exactly is safe here. - if (ticket.reason === "too_large") { + // too_large (negotiated limit below the cap) and queue_full (owned budget nearly used) are evaluated after every + // privacy, correlation and sampling step, so sizing the source exactly and retrying once is safe here. + if (ticket.reason === "too_large" || ticket.reason === "queue_full") { const bound = parsed()?.bound ?? 0; if (bound > 0) ticket = manager.preflight(correlation, { ...request, maxBytes: bound }); } diff --git a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs index 368cf49fd..fbb05607c 100644 --- a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs +++ b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs @@ -148,6 +148,20 @@ for (const provider of ['openai', 'anthropic']) { await assertNoOpenTickets(f.manager); } finally { await f.close(); } }); + test(`${provider}: a nearly used owned-byte budget still admits small media at its exact size`, async () => { + const f = await fixture({ maxOwnedBytes: 300 * 1024 }); + try { + const big = Buffer.alloc(200 * 1024, 7).toString('base64'); + const one = data => provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_image', image_url: `data:image/png;base64,${data}` }] }] } + : { messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data } }] }] }; + // Same tick: the 200 KiB part still owns its bytes, so a cap-sized reservation would not fit. + const first = capture(f.manager, identity(), one(big)), second = capture(f.manager, identity(), one(PNG)); + assert.equal(first.size, 1); assert.equal(second.size, 1); + await f.manager.flush(); + assert.deepEqual(f.uploaded.map(b => b.length).sort((a, b) => a - b), [Buffer.from(PNG, 'base64').length, 200 * 1024]); + await assertNoOpenTickets(f.manager); + } finally { await f.close(); } + }); test(`${provider}: unsampled identity and queued policy tightening cannot upload`, async () => { const f = await fixture(); try { capture(f.manager, { ...identity(), traceFlags: 0 }, input()); assert.equal(f.manager.status().ownedBytes, 0);