diff --git a/.github/workflows/openai-anthropic-attachments-tests.yml b/.github/workflows/openai-anthropic-attachments-tests.yml new file mode 100644 index 000000000..a8335241c --- /dev/null +++ b/.github/workflows/openai-anthropic-attachments-tests.yml @@ -0,0 +1,70 @@ +name: OpenAI and Anthropic TypeScript attachments +on: + pull_request: + paths: + - 'typescript/packages/traceai_openai/**' + - 'typescript/packages/traceai_anthropic/**' + - 'typescript/packages/fi-core/**' + - 'typescript/packages/fi-semantic-conventions/**' + - 'typescript/pnpm-lock.yaml' + - '.github/workflows/openai-anthropic-attachments-tests.yml' + workflow_dispatch: +permissions: + contents: read +jobs: + attachments: + name: OpenAI ${{ matrix.openai }} / Anthropic ${{ matrix.anthropic }} + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - { openai: '4.0.0', anthropic: '0.20.0' } + - { openai: '5.23.2', anthropic: '0.27.3' } + - { openai: '6.0.0', anthropic: '0.133.0' } + - { openai: '7.47.0', anthropic: '0.134.0' } + - { openai: 'latest', anthropic: 'latest' } + defaults: + run: + shell: bash + working-directory: typescript + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22.23.4' + - name: Install pinned workspace dependencies and real SDK fixtures + run: | + mkdir -p "$RUNNER_TEMP/corepack-bin" + corepack enable --install-directory "$RUNNER_TEMP/corepack-bin" pnpm + echo "$RUNNER_TEMP/corepack-bin" >> "$GITHUB_PATH" + corepack pnpm --filter @traceai/openai... --filter @traceai/anthropic... install --frozen-lockfile --ignore-scripts + npm install --prefix "$RUNNER_TEMP/sdks" --ignore-scripts --no-audit --no-fund \ + "openai-under-test@npm:openai@${{ matrix.openai }}" \ + "anthropic-under-test@npm:@anthropic-ai/sdk@${{ matrix.anthropic }}" + - name: Build CJS and ESM + run: | + pnpm run build:deps + pnpm --filter @traceai/openai run build + pnpm --filter @traceai/anthropic run build + - name: Network-isolated attachment adapters against real SDKs and loopback mocks + run: | + sudo unshare --net -- bash -c 'ip link set lo up; exec "$@"' bash \ + env PATH="$PATH" \ + SDK_OPENAI="$RUNNER_TEMP/sdks/node_modules/openai-under-test" \ + SDK_ANTHROPIC="$RUNNER_TEMP/sdks/node_modules/anthropic-under-test" \ + node --test packages/traceai_openai/test/attachments-providers.test.cjs + - name: Network-isolated package unit suites + if: matrix.openai == 'latest' + run: | + sudo unshare --net -- bash -c 'ip link set lo up; exec "$@"' bash \ + env PATH="$PATH" node packages/traceai_openai/node_modules/jest/bin/jest.js \ + --config packages/traceai_openai/jest.config.js --runInBand + sudo unshare --net -- bash -c 'ip link set lo up; exec "$@"' bash \ + env PATH="$PATH" node node_modules/jest/bin/jest.js \ + --config packages/traceai_anthropic/jest.config.js --runInBand + git diff --exit-code -- pnpm-lock.yaml diff --git a/typescript/packages/traceai_anthropic/src/attachments.ts b/typescript/packages/traceai_anthropic/src/attachments.ts new file mode 100644 index 000000000..cb72aee72 --- /dev/null +++ b/typescript/packages/traceai_anthropic/src/attachments.ts @@ -0,0 +1,168 @@ +import type { TraceConfigOptions } from "@traceai/fi-core"; +import type { AttachmentManager, AttachmentSpan, AttachmentCorrelation, AttachmentCapturePolicy, AttachmentOptions } from "@traceai/fi-core/attachments"; + +/** Native SDK span ownership is deliberately not overridden by this adapter. */ +export const anthropicAttachmentCapabilities = Object.freeze({ + schemaVersion: 1, + operations: ["messages.create.input", "tool_result.input"] as const, + sources: ["inline_base64"] as const, + /** Fail closed: no capture is attempted for these. */ + unsupported: ["remote_url", "provider_file", "native_span_hookup", "generated_image", "beta_messages_hookup", "streamed_output"] as const, + maxInlineBytes: 256 * 1024, +}); + +/** Admitted media positions for one observation: position key -> occurrence. */ +export type AnthropicCapturedMedia = ReadonlyMap>; +type Captured = Map>; +type RecordValue = Record; +const record = (value: unknown): RecordValue => value !== null && typeof value === "object" ? value as RecordValue : {}; +const MAX_PARTS = 256; +const MAX_SPANS = 1024; +const EMPTY: AnthropicCapturedMedia = new Map(); +const registryKey = Symbol.for("@traceai/anthropic/attachment-observations/v1"); +type Registry = WeakMap>; +const host = globalThis as typeof globalThis & { [registryKey]?: Registry }; +const observations = (host[registryKey] ??= new WeakMap()); +const TOOL_ID = /^[a-zA-Z0-9_-]{1,128}$/; + +const MAX_ENCODED = 4 * Math.ceil(anthropicAttachmentCapabilities.maxInlineBytes / 3); + +/** Exact decoded length from the encoded string's shape only (no decode); 0 when out of bounds. */ +function bound(encoded: unknown): number { + if (typeof encoded !== "string" || !encoded.length || encoded.length % 4 !== 0 || encoded.length > MAX_ENCODED) return 0; + return (encoded.length / 4) * 3 - (encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0); +} + +/** Strict decode after admission. The ticket's budget is a hard bound; padding and alphabet are strict. */ +function decode(encoded: unknown, maxBytes: number): Uint8Array | undefined { + // Bound allocation BEFORE decoding. + if (!bound(encoded) || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded as string)) return undefined; + const bytes = Buffer.from(encoded as string, "base64"); + return bytes.length && bytes.length <= maxBytes && bytes.length <= anthropicAttachmentCapabilities.maxInlineBytes && + bytes.toString("base64") === encoded ? bytes : undefined; +} + +/** Observe only Messages' typed image/document blocks and one level of tool_result content. + * Every part is preflighted (deny-wins, budget reserved) before its base64 is read or decoded; an admitted + * ticket that is not used is released at once. The manager owns policy, upload, lifecycle and references. A native integration + * must supply its actual emitted span/correlation, never an ambient guessed span. + * Returns the admitted positions so the caller can project them out of span telemetry. + */ +export function captureAnthropicAttachments(manager: AttachmentManager | undefined, + span: AttachmentSpan | AttachmentCorrelation, payload: unknown, + options: { traceConfig?: TraceConfigOptions; capture?: AttachmentCapturePolicy } = {}): AnthropicCapturedMedia { + if (!manager) return EMPTY; + const captured: Captured = new Map(); + try { + const correlation = "snapshot" in span ? span : manager.correlation(span); + const identity = correlation.snapshot(); + if (!identity?.sampled) return EMPTY; + const messages = record(payload).messages; + if (!Array.isArray(messages)) return EMPTY; + let spans = observations.get(manager); + if (!spans) { spans = new Map(); observations.set(manager, spans); } + const spanKey = `${identity.traceId}/${identity.spanId}`; + const config = options.traceConfig ?? {}; + const hidden = config.hideInputs || config.hideInputMessages || config.hideInputText || config.piiRedaction; + // Screenshot classification requires a typed computer tool AND an assistant tool_use naming it. + const computerNames = new Set(); + const tools = record(payload).tools; + if (Array.isArray(tools)) for (const value of tools.slice(0, MAX_PARTS)) { + const tool = record(value); + if (typeof tool.type === "string" && /^computer_\d{8}$/.test(tool.type) && typeof tool.name === "string") computerNames.add(tool.name); + } + const computerCalls = new Set(); + for (const value of messages.slice(0, MAX_PARTS)) { + const message = record(value); + if (message.role !== "assistant" || !Array.isArray(message.content)) continue; + for (const value of message.content.slice(0, MAX_PARTS)) { + const block = record(value); + if (block.type === "tool_use" && typeof block.id === "string" && typeof block.name === "string" && computerNames.has(block.name)) computerCalls.add(block.id); + } + } + let visited = 0; + messages.slice(0, MAX_PARTS).forEach((value, messageIndex) => { + const message = record(value); + if (!Array.isArray(message.content)) return; + // Nested positions are relative to their tool result, identified by toolCallId. + const visit = (value: unknown, index: number, position: string, toolCallId?: string, errorDetail = false) => { + if (++visited > MAX_PARTS) return; + const part = record(value); + if (part.type !== "image" && part.type !== "document") return; + const source = record(part.source); + if (source.type !== "base64") return; + const mime = source.media_type; + if (typeof mime !== "string" || !(part.type === "image" ? ["image/png", "image/jpeg", "image/webp"] : ["application/pdf"]).includes(mime)) return; + const positionKey = `input/${messageIndex}/${position}`; + const previous = spans!.get(spanKey)?.get(positionKey); + if (previous) { captured.set(positionKey, previous); return; } + const role: AttachmentOptions["role"] = toolCallId ? "tool" : message.role === "assistant" ? "assistant" : "user"; + const request = { mimeType: mime, direction: "input" as const, + purpose: (part.type === "document" ? "document" : toolCallId ? computerCalls.has(toolCallId) ? "computer_screenshot" : "tool_result" : "model_input") as AttachmentOptions["purpose"], + capture: { ...options.capture, modelInput: true, errorDetail: errorDetail || options.capture?.errorDetail, + ...(hidden || (part.type === "image" && config.hideInputImages) || (errorDetail && config.hideErrorDetails) ? { captureContent: false } : {}) } }; + // Preflight (deny-wins, budget reserved) before the base64 source is read or decoded (design section 1). + // The source is read at most once. + let data: unknown, dataRead = false; + const read = () => { if (!dataRead) { dataRead = true; data = source.data; } return data; }; + let ticket = manager.preflight(correlation, { ...request, maxBytes: anthropicAttachmentCapabilities.maxInlineBytes }); + // too_large (negotiated limit below the cap) and queue_full (owned budget nearly used) are evaluated after every + // privacy, correlation and sampling step, so sizing the source exactly and retrying once is safe here. + if (ticket.reason === "too_large" || ticket.reason === "queue_full") { + const exact = bound(read()); + if (exact > 0) ticket = manager.preflight(correlation, { ...request, maxBytes: exact }); + } + if (!ticket.admitted) return; + const bytes = decode(read(), ticket.maxBytes); + if (!bytes) { ticket.release(); return; } + const handle = ticket.attach({ kind: "Bytes", data: bytes }, { messageIndex, partIndex: index, role, ...(toolCallId ? { toolCallId } : {}) }); + if (handle.admission === "queued") { + const occurrence = Object.freeze({ occurrenceId: handle.occurrenceId, mimeType: mime }); + let positions = spans!.get(spanKey); + if (!positions) { + if (spans!.size >= MAX_SPANS) spans!.delete(spans!.keys().next().value!); + positions = new Map(); spans!.set(spanKey, positions); + } + if (positions.size < MAX_PARTS) positions.set(positionKey, occurrence); + captured.set(positionKey, occurrence); + } + }; + message.content.slice(0, MAX_PARTS).forEach((value, index) => { + const block = record(value); + if (block.type === "tool_result" && Array.isArray(block.content)) { + // Invalid tool IDs fail closed instead of losing causal identity. + if (typeof block.tool_use_id !== "string" || !TOOL_ID.test(block.tool_use_id)) return; + block.content.slice(0, MAX_PARTS).forEach((child, childIndex) => visit(child, childIndex, `${index}/${childIndex}`, block.tool_use_id as string, block.is_error === true)); + } else visit(value, index, String(index)); + }); + }); + } catch { /* No source-bearing exceptions or changes to the application's result. */ } + return captured; +} + +/** Telemetry-only projection. ONLY admitted positions are replaced with a `fi-attachment:` occurrence URI + * source, so uploaded bytes are not exported twice. Everything else is returned exactly as before, and the + * request passed to the SDK is never mutated. + */ +export function projectAnthropicMedia(payload: unknown, captured: AnthropicCapturedMedia): any { + if (!captured.size) return payload; + try { + const body = record(payload); + if (!Array.isArray(body.messages)) return payload; + const replace = (value: unknown, position: string): unknown => { + const occurrence = captured.get(position); + return occurrence ? { ...record(value), source: { type: "url", url: `fi-attachment:${occurrence.occurrenceId}` } } : value; + }; + return { ...body, messages: body.messages.map((value, messageIndex) => { + const message = record(value); + if (!Array.isArray(message.content)) return value; + return { ...message, content: message.content.map((value, index) => { + const block = record(value); + if (block.type === "tool_result" && Array.isArray(block.content)) { + return { ...block, content: block.content.map((child, childIndex) => replace(child, `input/${messageIndex}/${index}/${childIndex}`)) }; + } + return replace(value, `input/${messageIndex}/${index}`); + }) }; + }) }; + } catch { return payload; } +} diff --git a/typescript/packages/traceai_anthropic/src/index.ts b/typescript/packages/traceai_anthropic/src/index.ts index 378f38976..bc14a48fb 100644 --- a/typescript/packages/traceai_anthropic/src/index.ts +++ b/typescript/packages/traceai_anthropic/src/index.ts @@ -1,2 +1,3 @@ export * from './version'; -export * from './instrumentation'; \ No newline at end of file +export * from './instrumentation'; +export * from './attachments'; diff --git a/typescript/packages/traceai_anthropic/src/instrumentation.ts b/typescript/packages/traceai_anthropic/src/instrumentation.ts index 77d132430..91561e08c 100644 --- a/typescript/packages/traceai_anthropic/src/instrumentation.ts +++ b/typescript/packages/traceai_anthropic/src/instrumentation.ts @@ -1,4 +1,6 @@ import Anthropic from "@anthropic-ai/sdk"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import { captureAnthropicAttachments, projectAnthropicMedia } from "./attachments"; import { InstrumentationBase, InstrumentationConfig, @@ -334,16 +336,22 @@ function finishInstrumentationStream( export class AnthropicInstrumentation extends InstrumentationBase{ private fiTracer!: FITracer; private _traceConfig?: TraceConfigOptions; + private attachments?: AttachmentManager; constructor({ instrumentationConfig, traceConfig, + attachments, }: { instrumentationConfig?: InstrumentationConfig; traceConfig?: TraceConfigOptions; + /** The provider-owned shared attachment manager. */ + attachments?: AttachmentManager; } = {}) { super("@traceai/instrumentation-anthropic", VERSION,Object.assign({}, instrumentationConfig)); this._traceConfig = traceConfig; + this.attachments = attachments; + this.fiTracer = new FITracer({ tracer: this.tracer, traceConfig: this._traceConfig }); } public override enable(): void { @@ -404,21 +412,26 @@ export class AnthropicInstrumentation extends InstrumentationBase{ // Not async: the SDK returns an APIPromise whose withResponse(), asResponse() // and lazy body parsing callers (including messages.stream()) depend on. return function patchedMessagesCreate(this: Anthropic.Messages, ...args: Parameters) { - const params = args[0] as Anthropic.MessageCreateParams; - const { messages: _messages, system: _system, tools: _tools, ...invocationParameters } = params; + const originalParams = args[0] as Anthropic.MessageCreateParams; + const { messages: _messages, system: _system, tools: _tools, ...invocationParameters } = originalParams; + // Content attributes are set after capture so admitted media can be projected out first. const span = instrumentation.fiTracer.startSpan("Anthropic Messages Create", { kind: SpanKind.INTERNAL, attributes: { [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, - [SemanticConventions.LLM_MODEL_NAME]: params.model, + [SemanticConventions.LLM_MODEL_NAME]: originalParams.model, [SemanticConventions.LLM_PROVIDER]: LLMProvider.ANTHROPIC, [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", + }, + }); + const captured = captureAnthropicAttachments(instrumentation.attachments, span, originalParams, { traceConfig: instrumentation._traceConfig }); + const params = projectAnthropicMedia(originalParams, captured) as Anthropic.MessageCreateParams; + span.setAttributes({ [SemanticConventions.LLM_INVOCATION_PARAMETERS]: safelyJSONStringify(invocationParameters) ?? "", ...getAnthropicInputMessagesAttributes(params), ...getAnthropicToolsAttributes(params), [SemanticConventions.RAW_INPUT]: safelyJSONStringify(params) ?? "", - }, }); const execContext = getExecContext(span); diff --git a/typescript/packages/traceai_openai/src/attachments.ts b/typescript/packages/traceai_openai/src/attachments.ts new file mode 100644 index 000000000..b185f6294 --- /dev/null +++ b/typescript/packages/traceai_openai/src/attachments.ts @@ -0,0 +1,239 @@ +import type { TraceConfigOptions } from "@traceai/fi-core"; +import type { AttachmentManager, AttachmentOptions, AttachmentSpan, AttachmentCorrelation, AttachmentCapturePolicy } from "@traceai/fi-core/attachments"; + +/** Declarative handoff to auto/wire owners; this is not a native/wire hookup. */ +export const openAIAttachmentCapabilities = Object.freeze({ + schemaVersion: 1, + operations: ["chat.completions.create.input", "responses.create.input", "responses.create.output", "responses.stream.output", "images.generate.output"] as const, + sources: ["inline_base64"] as const, + /** Fail closed: no capture is attempted for these. Streamed Responses output is captured only from the + * terminal `response.completed` event the application consumed; partial (preview) images never are. */ + unsupported: ["remote_url", "provider_file", "partial_image", "images.generate.stream", "images.edit", "images.createVariation", "speech"] as const, + maxInlineBytes: 256 * 1024, +}); + +/** Admitted media positions for one observation: position key -> occurrence. */ +export type CapturedMedia = ReadonlyMap>; +type Captured = Map>; +type RecordValue = Record; +const record = (value: unknown): RecordValue => value !== null && typeof value === "object" ? value as RecordValue : {}; +const list = (value: unknown): unknown[] => Array.isArray(value) ? value : []; +const MAX_PARTS = 256; +const MAX_SPANS = 1024; +const EMPTY: CapturedMedia = new Map(); +const IMAGES = ["image/png", "image/jpeg", "image/webp"]; +const FILES = ["application/pdf", "application/octet-stream"]; +// Shared across CJS/ESM copies. Retains only admitted span IDs, positions and occurrence IDs, never sources. +const registryKey = Symbol.for("@traceai/openai/attachment-observations/v1"); +type Registry = WeakMap>; +const host = globalThis as typeof globalThis & { [registryKey]?: Registry }; +const observations = (host[registryKey] ??= new WeakMap()); + +export interface OpenAIAttachmentCaptureOptions { + operation: "chat" | "responses" | "images"; + direction: "input" | "output"; + traceConfig?: TraceConfigOptions; + capture?: AttachmentCapturePolicy; + /** The Images request's typed output_format; never inferred from filenames. */ + imageFormat?: "png" | "jpeg" | "webp"; +} + +/** The local instrumentor's restrictions; the shared manager applies its own policy as well (deny wins). */ +function restrictions(options: OpenAIAttachmentCaptureOptions): AttachmentCapturePolicy { + const config = options.traceConfig ?? {}; + const input = options.direction === "input"; + const hidden = input + ? config.hideInputs || config.hideInputMessages || config.hideInputText + : config.hideOutputs || config.hideOutputMessages || config.hideOutputText; + return { ...options.capture, + ...(hidden || config.piiRedaction ? { captureContent: false } : {}), + ...(input ? { modelInput: true } : {}), + }; +} + +type Parsed = { mimeType: string; encoded: string; bound: number }; +const MAX_ENCODED = 4 * Math.ceil(openAIAttachmentCapabilities.maxInlineBytes / 3); + +/** Structural parse only: no decode or allocation beyond the string the application already holds. + * Returns the candidate MIME named by a data URI (or the typed MIME for raw base64) and the exact decoded length. + */ +function parse(value: unknown, mimeTypes: string[], raw: boolean): Parsed | undefined { + if (typeof value !== "string") return undefined; + let mimeType = mimeTypes[0], start = 0; + if (!raw) { + const comma = value.indexOf(","); + const header = comma > 5 && comma < 128 ? value.slice(0, comma) : ""; + const match = /^data:([a-z]+\/[a-z0-9.+-]+);base64$/.exec(header); + if (!match || !mimeTypes.includes(match[1])) return undefined; + mimeType = match[1]; start = comma + 1; + } + const length = value.length - start; + // Bound allocation BEFORE slicing/decoding. + if (length <= 0 || length > MAX_ENCODED || length % 4 !== 0) return undefined; + const encoded = value.slice(start); + const padding = encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0; + return { mimeType, encoded, bound: (length / 4) * 3 - padding }; +} + +/** Strict decode after admission. Padding and alphabet are strict; the ticket's budget is a hard bound. */ +function decode(encoded: string, maxBytes: number): Uint8Array | undefined { + if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return undefined; + const bytes = Buffer.from(encoded, "base64"); + if (!bytes.length || bytes.length > maxBytes || bytes.length > openAIAttachmentCapabilities.maxInlineBytes || bytes.toString("base64") !== encoded) return undefined; + return bytes; +} + +const key = (options: Pick, messageIndex: number, partIndex: number) => + `${options.operation}/${options.direction}/${messageIndex}/${partIndex}`; + +/** Typed, synchronous observation only. No provider downloads or stream consumption. + * Pass the SAME emitted identity to SDK and wire observations: a second observation of a position + * returns the first occurrence instead of uploading again. + * Denied sources are never read: every part is preflighted (deny-wins, budget reserved) before its value is + * read or decoded, and an admitted ticket that is not used is released at once. + * Returns the admitted positions so the caller can project them out of span telemetry. + */ +export function captureOpenAIAttachments(manager: AttachmentManager | undefined, + span: AttachmentSpan | AttachmentCorrelation, payload: unknown, options: OpenAIAttachmentCaptureOptions): CapturedMedia { + if (!manager) return EMPTY; + const captured: Captured = new Map(); + try { + const correlation = "snapshot" in span ? span : manager.correlation(span); + const identity = correlation.snapshot(); + if (!identity?.sampled) return EMPTY; + let spans = observations.get(manager); + if (!spans) { spans = new Map(); observations.set(manager, spans); } + const spanKey = `${identity.traceId}/${identity.spanId}`; + const capture = restrictions(options); + let visited = 0; + const attach = (read: () => unknown, mimeTypes: string[], messageIndex: number, partIndex: number, + purpose: AttachmentOptions["purpose"], toolCallId?: string, raw = false, role: AttachmentOptions["role"] = "user") => { + if (++visited > MAX_PARTS) return; + const position = key(options, messageIndex, partIndex); + const previous = spans!.get(spanKey)?.get(position); + if (previous) { captured.set(position, previous); return; } + // The source is read at most once, and only after a preflight admitted this modality (design section 1). + let source: Parsed | undefined, sourceRead = false; + const parsed = () => { if (!sourceRead) { sourceRead = true; source = parse(read(), mimeTypes, raw); } return source; }; + const preflight = (mimeType: string) => { + const request = { mimeType, direction: options.direction, purpose, capture: { + ...capture, + ...(options.direction === "input" && mimeType.startsWith("image/") && options.traceConfig?.hideInputImages ? { captureContent: false } : {}), + } }; + let ticket = manager.preflight(correlation, { ...request, maxBytes: openAIAttachmentCapabilities.maxInlineBytes }); + // too_large (negotiated limit below the cap) and queue_full (owned budget nearly used) are evaluated after every + // privacy, correlation and sampling step, so sizing the source exactly and retrying once is safe here. + if (ticket.reason === "too_large" || ticket.reason === "queue_full") { + const bound = parsed()?.bound ?? 0; + if (bound > 0) ticket = manager.preflight(correlation, { ...request, maxBytes: bound }); + } + return ticket; + }; + let ticket = preflight(mimeTypes[0]); + if (!ticket.admitted) return; + const found = parsed(); + if (!found) { ticket.release(); return; } + // A data URI names a sibling MIME of the same modality: re-admit under the exact type before decoding. + if (found.mimeType !== mimeTypes[0]) { + ticket.release(); + ticket = preflight(found.mimeType); + if (!ticket.admitted) return; + } + const bytes = decode(found.encoded, ticket.maxBytes); + if (!bytes) { ticket.release(); return; } + const handle = ticket.attach({ kind: "Bytes", data: bytes }, { messageIndex, partIndex, role, ...(toolCallId ? { toolCallId } : {}) }); + if (handle.admission !== "queued") return; + const occurrence = Object.freeze({ occurrenceId: handle.occurrenceId, mimeType: found.mimeType }); + let positions = spans!.get(spanKey); + if (!positions) { + if (spans!.size >= MAX_SPANS) spans!.delete(spans!.keys().next().value!); + positions = new Map(); spans!.set(spanKey, positions); + } + if (positions.size < MAX_PARTS) positions.set(position, occurrence); + captured.set(position, occurrence); + }; + const toolCallId = (value: unknown) => typeof value === "string" && /^[a-zA-Z0-9_-]{1,128}$/.test(value) ? value : undefined; + const parts = (content: unknown, messageIndex: number, role: AttachmentOptions["role"]) => { + list(content).slice(0, MAX_PARTS).forEach((value, partIndex) => { + const part = record(value); + if (part.type === "image_url") attach(() => record(part.image_url).url, IMAGES, messageIndex, partIndex, "model_input", undefined, false, role); + else if (part.type === "input_image") attach(() => part.image_url, IMAGES, messageIndex, partIndex, "model_input", undefined, false, role); + else if (part.type === "input_file") attach(() => part.file_data, FILES, messageIndex, partIndex, "document", undefined, false, role); + else if (part.type === "file") attach(() => record(part.file).file_data, FILES, messageIndex, partIndex, "document", undefined, false, role); + }); + }; + const body = record(payload); + if (options.direction === "output" && body.error) return EMPTY; + if (options.direction === "input") { + const messages = options.operation === "chat" ? body.messages : options.operation === "responses" ? body.input : undefined; + list(messages).slice(0, MAX_PARTS).forEach((value, messageIndex) => { + const message = record(value); + if (message.type === "computer_call_output") { + const output = record(message.output); + // A screenshot is model-bound input. Its call id keeps the causal link to the computer_call. + if (output.type === "computer_screenshot") attach(() => output.image_url, IMAGES, messageIndex, 0, "computer_screenshot", + toolCallId(message.call_id), false, "tool"); + } else if (!message.type || message.type === "message") { + const role = ["user", "assistant", "tool", "system"].includes(String(message.role)) ? message.role as AttachmentOptions["role"] : "unknown"; + parts(message.content, messageIndex, role); + } + }); + } else if (options.operation === "images") { + // The Images API defaults to PNG. The typed request/response format wins where provided. + const format = body.output_format ?? options.imageFormat; + const mime = format === "jpeg" ? "image/jpeg" : format === "webp" ? "image/webp" : "image/png"; + list(body.data).slice(0, MAX_PARTS).forEach((value, index) => { + const item = record(value); + attach(() => item.b64_json, [mime], index, 0, "model_output", undefined, true, "assistant"); + }); + } else if (options.operation === "responses") { + list(body.output).slice(0, MAX_PARTS).forEach((value, index) => { + const item = record(value); + if (item.type === "image_generation_call" && item.status === "completed") { + const mime = item.output_format === "jpeg" ? "image/jpeg" : item.output_format === "webp" ? "image/webp" : "image/png"; + attach(() => item.result, [mime], index, 0, "model_output", undefined, true, "assistant"); + } + }); + } + } catch { /* Instrumentation must not throw or log source-bearing errors. */ } + return captured; +} + +const uri = (occurrence: { occurrenceId: string }) => `fi-attachment:${occurrence.occurrenceId}`; + +/** Telemetry-only projection. ONLY admitted positions are replaced with their `fi-attachment:` occurrence + * URI, so uploaded bytes are not exported twice. Everything else is returned exactly as before, and the + * request passed to the SDK is never mutated. + */ +export function projectOpenAIMedia(payload: unknown, captured: CapturedMedia, + operation: "chat" | "responses", direction: "input" | "output"): any { + if (!captured.size) return payload; + try { + const body = record(payload); + const field = direction === "input" ? operation === "chat" ? "messages" : "input" : "output"; + if (!Array.isArray(body[field])) return payload; + const at = (messageIndex: number, partIndex: number) => captured.get(key({ operation, direction }, messageIndex, partIndex)); + return { ...body, [field]: list(body[field]).map((value, messageIndex) => { + const message = record(value); + if (message.type === "computer_call_output") { + const occurrence = at(messageIndex, 0); + return occurrence ? { ...message, output: { ...record(message.output), image_url: uri(occurrence) } } : value; + } + if (message.type === "image_generation_call") { + const occurrence = at(messageIndex, 0); + return occurrence ? { ...message, result: uri(occurrence) } : value; + } + if (!Array.isArray(message.content)) return value; + return { ...message, content: message.content.map((value, partIndex) => { + const occurrence = at(messageIndex, partIndex); + if (!occurrence) return value; + const part = record(value); + if (part.type === "image_url") return { ...part, image_url: { ...record(part.image_url), url: uri(occurrence) } }; + if (part.type === "input_image") return { ...part, image_url: uri(occurrence) }; + if (part.type === "input_file") return { ...part, file_data: uri(occurrence) }; + if (part.type === "file") return { ...part, file: { ...record(part.file), file_data: uri(occurrence) } }; + return value; + }) }; + }) }; + } catch { return payload; } +} diff --git a/typescript/packages/traceai_openai/src/index.ts b/typescript/packages/traceai_openai/src/index.ts index a38448910..e886a4c31 100644 --- a/typescript/packages/traceai_openai/src/index.ts +++ b/typescript/packages/traceai_openai/src/index.ts @@ -1,2 +1,3 @@ export * from './instrumentation'; -export * from './version'; \ No newline at end of file +export * from './version'; +export * from './attachments'; diff --git a/typescript/packages/traceai_openai/src/instrumentation.ts b/typescript/packages/traceai_openai/src/instrumentation.ts index a04bc72d0..d2d05e7a4 100644 --- a/typescript/packages/traceai_openai/src/instrumentation.ts +++ b/typescript/packages/traceai_openai/src/instrumentation.ts @@ -1,4 +1,6 @@ import openai from "openai"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import { captureOpenAIAttachments, projectOpenAIMedia } from "./attachments"; import { InstrumentationBase, InstrumentationConfig, @@ -107,6 +109,7 @@ function getExecContext(span: Span) { export class OpenAIInstrumentation extends InstrumentationBase{ private fiTracer!: FITracer; private _traceConfig?: TraceConfigOptions; + private attachments?: AttachmentManager; // Lazily initialized: InstrumentationBase can patch during super(). // A Set is needed so disable() also undoes every manually patched ESM SDK. private patchedModules?: Set; @@ -114,6 +117,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ constructor({ instrumentationConfig, traceConfig, + attachments, }: { /** * The config for the instrumentation @@ -125,6 +129,8 @@ export class OpenAIInstrumentation extends InstrumentationBase{ * @see {@link TraceConfigOptions} */ traceConfig?: TraceConfigOptions; + /** The shared manager from register(...).attachments; never a per-adapter uploader. */ + attachments?: AttachmentManager; } = {}) { super( "@traceai/fi-instrumentation-openai", @@ -132,6 +138,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ Object.assign({}, instrumentationConfig), ); this._traceConfig = traceConfig; + this.attachments = attachments; // The base constructor can call enable() before our trace options exist. this.fiTracer = new FITracer({ tracer: this.tracer, traceConfig: this._traceConfig }); } @@ -223,40 +230,34 @@ export class OpenAIInstrumentation extends InstrumentationBase{ this: unknown, ...args: Parameters ) { - const body = args[0]; - // Prediction text is input content, captured only in maskable payloads. - const { messages: _messages, prediction: _prediction, ...invocationParameters } = body; - + const originalBody = args[0]; // diag.debug("@traceai/openai: ChatCompletion patch CALLED. Starting span..."); - - // --- ADD LOGS FOR TRACER AND CONTEXT --- - // const activeContextForSuppressionCheck = context.active(); - // const isSuppressed = isTracingSuppressed(activeContextForSuppressionCheck); - // diag.debug(`@traceai/openai: Is tracing suppressed? ${isSuppressed}`); - // diag.debug(`@traceai/openai: this.tracer type: ${instrumentation.tracer?.constructor?.name}`); // Accessing the base tracer - // diag.debug(`@traceai/openai: this.fiTracer type: ${instrumentation.fiTracer?.constructor?.name}`); - // --- END ADDED LOGS --- - const span = instrumentation.fiTracer.startSpan( - `OpenAI Chat Completions`, - { - kind: SpanKind.INTERNAL, - attributes: { - [SemanticConventions.FI_SPAN_KIND]: - FISpanKind.LLM, - [SemanticConventions.LLM_MODEL_NAME]: body.model, + // Content attributes are set after capture so admitted media can be projected out first. + const span = instrumentation.fiTracer.startSpan("OpenAI Chat Completions", { + kind: SpanKind.INTERNAL, + attributes: { + [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, + [SemanticConventions.LLM_MODEL_NAME]: originalBody.model, + [SemanticConventions.LLM_PROVIDER]: providerForResource(this), + [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", + }, + }); + const captured = captureOpenAIAttachments(instrumentation.attachments, span, originalBody, { + operation: "chat", direction: "input", traceConfig: instrumentation._traceConfig, + }); + const body = projectOpenAIMedia(originalBody, captured, "chat", "input") as typeof originalBody; + // Prediction text is input content, captured only in maskable payloads. + const { messages: _messages, prediction: _prediction, ...invocationParameters } = body; + span.setAttributes({ [SemanticConventions.INPUT_VALUE]: JSON.stringify(body), [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON, [SemanticConventions.LLM_INVOCATION_PARAMETERS]: JSON.stringify(invocationParameters), - [SemanticConventions.LLM_PROVIDER]: providerForResource(this), - [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", ...getLLMInputMessagesAttributes(body), ...getLLMToolsJSONSchema(body), [SemanticConventions.RAW_INPUT]: safelyJSONStringify(body) ?? "", - }, - }, - ); + }); // --- ADD LOG --- // diag.debug(`@traceai/openai: ChatCompletion span STARTED: ${span.spanContext().spanId}`); @@ -517,39 +518,42 @@ export class OpenAIInstrumentation extends InstrumentationBase{ const body = args[0] as ResponseCreateParamsBase & { prompt?: { id: string; version?: string; variables?: Record }; }; + // Content attributes are set after capture so admitted media can be projected out first. + const span = instrumentation.fiTracer.startSpan("OpenAI Responses", { + kind: SpanKind.INTERNAL, + attributes: { + [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, + [SemanticConventions.LLM_MODEL_NAME]: body.model, + [SemanticConventions.LLM_PROVIDER]: providerForResource(this), + [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", + }, + }); + const captured = captureOpenAIAttachments(instrumentation.attachments, span, body, { + operation: "responses", direction: "input", traceConfig: instrumentation._traceConfig, + }); // Telemetry-only copy: MCP credentials must never reach any exported // duplicate, but the original SDK request must remain untouched. + const projectedBody = projectOpenAIMedia(body, captured, "responses", "input"); const telemetryBody = { - ...body, + ...projectedBody, ...(body.tools && { tools: sanitizeToolDefinitions(body.tools) }), - ...(body.input && { input: sanitizeResponsesInput(body.input) }), + ...(projectedBody.input && { input: sanitizeResponsesInput(projectedBody.input) }), }; const { input: _messages, instructions: _instructions, prompt, ...parameters } = telemetryBody; const invocationParameters = { ...parameters, ...(prompt && { prompt: { id: prompt.id, version: prompt.version } }), }; - const span = instrumentation.fiTracer.startSpan( - `OpenAI Responses`, - { - kind: SpanKind.INTERNAL, - attributes: { - [SemanticConventions.FI_SPAN_KIND]: - FISpanKind.LLM, - [SemanticConventions.LLM_MODEL_NAME]: body.model, + span.setAttributes({ [SemanticConventions.INPUT_VALUE]: JSON.stringify(telemetryBody), [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON, [SemanticConventions.LLM_INVOCATION_PARAMETERS]: JSON.stringify(invocationParameters), [SemanticConventions.PROMPT_TEMPLATE_VARIABLES]: safelyJSONStringify(prompt?.variables) ?? undefined, - [SemanticConventions.LLM_PROVIDER]: providerForResource(this), - [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", - ...getResponsesInputMessagesAttributes(body), + ...getResponsesInputMessagesAttributes(telemetryBody), ...getLLMToolsJSONSchema(body), [SemanticConventions.RAW_INPUT]: safelyJSONStringify(telemetryBody) ?? "", - }, - }, - ); + }); const contextSpan = span.getContextSpan(); const execContext = getExecContext(contextSpan); const requestStartedAt = performance.now(); @@ -581,16 +585,22 @@ export class OpenAIInstrumentation extends InstrumentationBase{ if (endSpan) span.end(); return; } + // On the streamed path (endSpan false) `result` is the terminal response the application consumed + // from its own iterator, observed before the span ends. Partial (preview) images are never captured. + const outputMedia = captureOpenAIAttachments(instrumentation.attachments, span, result, { + operation: "responses", direction: "output", traceConfig: instrumentation._traceConfig, + }); + const projected = projectOpenAIMedia(result, outputMedia, "responses", "output") as ResponseType; const output = { id: result.id, object: result.object, status: result.status, - model: result.model, output: result.output, usage: result.usage, + model: result.model, output: projected.output, usage: result.usage, }; span.setAttributes({ [SemanticConventions.OUTPUT_VALUE]: JSON.stringify(output), [SemanticConventions.OUTPUT_MIME_TYPE]: MimeType.JSON, [SemanticConventions.GEN_AI_RESPONSE_MODEL]: result.model, [SemanticConventions.GEN_AI_RESPONSE_ID]: result.id, - ...getResponsesOutputMessagesAttributes(result), + ...getResponsesOutputMessagesAttributes(projected), ...getResponsesUsageAttributes(result), [SemanticConventions.RAW_OUTPUT]: safelyJSONStringify(output) ?? "", }); @@ -628,6 +638,46 @@ export class OpenAIInstrumentation extends InstrumentationBase{ ); } + // images.generate: generated b64 outputs. Streamed partial images, edits and variations stay unsupported. + if (typeof module.OpenAI.Images?.prototype?.generate === "function") { + type ImagesGenerateType = typeof module.OpenAI.Images.prototype.generate; + this._wrap(module.OpenAI.Images.prototype, "generate", (original: ImagesGenerateType): any => { + return function patchedImagesGenerate(this: unknown, ...args: Parameters) { + const request = (args[0] ?? {}) as { model?: string; output_format?: "png" | "jpeg" | "webp" }; + const span = instrumentation.fiTracer.startSpan("OpenAI Images", { + kind: SpanKind.INTERNAL, + attributes: { + [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, + [SemanticConventions.LLM_PROVIDER]: providerForResource(this), + [SemanticConventions.GEN_AI_OPERATION_NAME]: "image_generation", + ...(typeof request.model === "string" ? { [SemanticConventions.LLM_MODEL_NAME]: request.model } : {}), + [SemanticConventions.INPUT_VALUE]: safelyJSONStringify(args[0]) ?? "", + [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON, + }, + }); + const contextSpan = span.getContextSpan(); + const execContext = getExecContext(contextSpan); + const execPromise = safeExecuteInTheMiddle>( + () => context.with(trace.setSpan(execContext, contextSpan), () => original.apply(this, args)), + (error) => { if (error) endSpanWithError(span, error); }, + ); + const wrappedPromise = observeAPIPromise(execPromise, (result) => { + if (!span.isRecording()) return result; + if (!isStream(result)) { + captureOpenAIAttachments(instrumentation.attachments, span, result, { + operation: "images", direction: "output", traceConfig: instrumentation._traceConfig, + imageFormat: request.output_format, + }); + } + span.setStatus({ code: SpanStatusCode.OK }); + span.end(); + return result; + }, (error) => endSpanWithError(span, error), (response) => endRawResponseSpan(span, response)); + return context.bind(execContext, wrappedPromise); + }; + }); + } + // Keep the legacy diagnostic property, but never use it as the guard. // Immutable class exports still work via the identity map. try { (sdk as typeof openai & { fiPatched?: boolean }).fiPatched = true; } catch { /* immutable SDK */ } @@ -647,6 +697,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ this._unwrap(sdk.Completions.prototype, "create"); this._unwrap(sdk.Embeddings.prototype, "create"); if (sdk.Responses) this._unwrap(sdk.Responses.prototype, "create"); + if (sdk.Images) this._unwrap(sdk.Images.prototype, "generate"); try { (sdk as typeof openai & { fiPatched?: boolean }).fiPatched = false; } catch { /* immutable SDK */ } patchRegistry.owners.delete(sdk); // An automatic require hook may use a different namespace for this class. @@ -983,7 +1034,7 @@ function serializeChatCompletionInputMessage( if (part.type === "text") { return { type: "text", text: part.text }; } else if (part.type === "image_url") { - return { type: "image_url", image_url: { url: part.image_url.url } }; + return part.image_url ? { type: "image_url", image_url: { url: part.image_url.url } } : { type: "image_url" }; } return { type: part.type }; }); diff --git a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs new file mode 100644 index 000000000..fbb05607c --- /dev/null +++ b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs @@ -0,0 +1,465 @@ +/* Cloud component qualification: real SDKs, synthetic media, loopback-only APIs. */ +const { test, after } = require('node:test'); +const assert = require('node:assert/strict'); +const http = require('node:http'); +const { once } = require('node:events'); +const { randomUUID } = require('node:crypto'); +const { pathToFileURL } = require('node:url'); +const path = require('node:path'); +const { AttachmentManager, createAttachmentHttpTransport } = require('@traceai/fi-core/attachments'); +const { BasicTracerProvider, SimpleSpanProcessor, InMemorySpanExporter } = require('@opentelemetry/sdk-trace-base'); +const { OpenAIInstrumentation } = require('../dist/src/instrumentation.js'); +const { AnthropicInstrumentation } = require('../../traceai_anthropic/dist/src/instrumentation.js'); +const oa = require('../dist/src/attachments.js'); +const an = require('../../traceai_anthropic/dist/src/attachments.js'); +const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aD1sAAAAASUVORK5CYII='; +const PDF = Buffer.from('%PDF-1.7\n% synthetic document\n%%EOF').toString('base64'); +const image = () => ({ type: 'image', source: { type: 'base64', media_type: 'image/png', data: PNG } }); +const document = () => ({ type: 'document', title: 'synthetic-private-filename', source: { type: 'base64', media_type: 'application/pdf', data: PDF } }); +const identity = () => ({ traceId: randomUUID().replace(/-/g, ''), spanId: randomUUID().replace(/-/g, '').slice(0, 16), traceFlags: 1 }); +const keepalive = setInterval(() => {}, 1000); +after(() => clearInterval(keepalive)); +// The core reports a silently degraded fixture (legacy transport, rejected F01 shape) as a process warning. +const coreWarnings = []; +process.on('warning', warning => { if (/^FI_ATTACHMENTS_/.test(warning.code ?? '')) coreWarnings.push(warning.code); }); +// Preflight tickets that are not used must be released at once, never left to the TTL sweep. +const assertNoOpenTickets = async manager => { + assert.equal(manager.status().openTickets, 0, 'open tickets'); await manager.flush(); assert.equal(manager.status().ownedBytes, 0, 'owned bytes'); +}; + +async function fixture(config = {}, serverPolicy = {}, serverLimits = {}) { + const calls = [], initialized = [], uploaded = []; + const scope = { organization_id: randomUUID(), workspace_id: randomUUID(), project_id: randomUUID() }; + // Sessions of one hour or more: the core renews within max(60 s, 20% of lifetime) of expiry. + const expiry = () => new Date(Date.now() + 3600000).toISOString(); + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const c of req) chunks.push(c); + const bytes = Buffer.concat(chunks); calls.push(req.url); + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('attachment-sessions')) { + const body = JSON.parse(bytes); + res.end(JSON.stringify({ schema_version: 1, scope, capture_session_id: body.capture_session_id, + policy: { version: 'fixture-v1', enabled: true, allowed_sources: ['Bytes'], allowed_modalities: ['image', 'document', 'file'], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, + record_audio: false, record_video: false, ...serverPolicy }, + limits: { max_upload_bytes: 1024 * 1024, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, + recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, + initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, + transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5, ...serverLimits }, expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); + } else if (req.url.endsWith('/uploads')) { + const body = JSON.parse(bytes); initialized.push(body); + res.statusCode = 201; + res.end(JSON.stringify({ upload_id: body.attachment_id, attachment_id: body.attachment_id, occurrence_id: body.occurrence_id, + state: 'pending_upload', reserved_bytes: body.declared_bytes, expires_at: expiry(), transfer_mode: 'authenticated_stream', write_capability: 'synthetic-upload-only' })); + } else if (req.url.endsWith('/content')) { + assert.equal(req.headers.authorization, 'Bearer synthetic-upload-only'); uploaded.push(bytes); + res.end(JSON.stringify({ upload_id: req.url.split('/').at(-2), state: 'pending_upload', reason: 'queued' })); + } else { + res.statusCode = 202; res.end(JSON.stringify({ upload_id: req.url.split('/').at(-2), state: 'validating', reason: 'binding_pending' })); + } + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const client = req => new Promise((resolve, reject) => { + const request = http.request(`http://127.0.0.1:${server.address().port}${new URL(req.url).pathname}`, { method: req.method, headers: req.headers, signal: req.signal }, response => { + const chunks = []; response.on('data', c => chunks.push(c)); response.on('end', () => resolve({ status: response.statusCode, body: Buffer.concat(chunks) })); + }); request.on('error', reject); request.end(req.body); + }); + const manager = new AttachmentManager({ enabled: true, projectName: 'synthetic', projectType: 'observe', + transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic' }, client }), ...config }); + assert.equal(await manager.ready(), config.enabled !== false); + return { manager, calls, initialized, uploaded, async close() { await manager.shutdown(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } }; +} +const oaInput = () => ({ input: [ + { role: 'user', content: [{ type: 'input_image', image_url: `data:image/png;base64,${PNG}` }, { type: 'input_file', filename: 'synthetic-private-filename', file_data: `data:application/pdf;base64,${PDF}` }] }, + { type: 'computer_call_output', call_id: 'call_computer', output: { type: 'computer_screenshot', image_url: `data:image/png;base64,${PNG}` } }, +] }); +const anInput = () => ({ tools: [{ type: 'computer_20250124', name: 'computer', display_width_px: 800, display_height_px: 600 }], messages: [ + { role: 'assistant', content: [{ type: 'tool_use', id: 'tool_computer', name: 'computer', input: { action: 'screenshot' } }] }, + { role: 'user', content: [image(), document(), { type: 'tool_result', tool_use_id: 'tool_computer', content: [image(), image()] }] }, +] }); + +for (const provider of ['openai', 'anthropic']) { + const capture = (m, s, p, extra = {}) => provider === 'openai' + ? oa.captureOpenAIAttachments(m, s, p, { operation: 'responses', direction: 'input', ...extra }) + : an.captureAnthropicAttachments(m, s, p, extra); + const input = provider === 'openai' ? oaInput : anInput; + test(`${provider}: typed image/document/screenshots preserve bytes, causal positions and distinct repeated frames`, async () => { + const f = await fixture(); + try { + const id = identity(), body = input(), saved = JSON.stringify(body); + capture(f.manager, id, body); capture(f.manager, { ...id }, body); + await f.manager.flush(); + assert.equal(f.initialized.length, provider === 'openai' ? 3 : 4); + assert.equal(f.uploaded.length, f.initialized.length); + assert.deepEqual(f.uploaded.map(b => b.toString('base64')).sort(), (provider === 'openai' ? [PNG, PDF, PNG] : [PNG, PDF, PNG, PNG]).sort()); + assert.equal(new Set(f.initialized.map(r => r.occurrence_id)).size, f.initialized.length); + assert.ok(f.initialized.every(r => r.direction === 'input' && r.association.parent.span_id === id.spanId)); + assert.equal(f.initialized.filter(r => r.purpose === 'computer_screenshot').length, provider === 'openai' ? 1 : 2); + assert.ok(!JSON.stringify(f.initialized).includes('synthetic-private-filename')); + assert.equal(JSON.stringify(body), saved); + } finally { await f.close(); } + }); + for (const key of ['disabled', 'hideInputs', 'hideInputMessages', 'hideInputText', 'piiRedaction', 'hideInputImages', 'serverRedaction', 'serverMetadata']) { + test(`${provider}: ${key} denies protected sources without reading or uploading`, async () => { + const f = await fixture(key === 'disabled' ? { enabled: false } : { traceConfig: { [key]: true } }, + key === 'serverRedaction' ? { require_media_redaction: true } : key === 'serverMetadata' ? { require_metadata_removal: true } : {}); + try { + let read = false; + const source = provider === 'openai' ? { input: [{ type: 'computer_call_output', call_id: 'call_computer', output: { type: 'computer_screenshot', get image_url() { read = true; throw Error('synthetic-secret'); } } }] } + : { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_computer', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', get data() { read = true; throw Error('synthetic-secret'); } } }] }] }] }; + const before = f.calls.length; capture(f.manager, identity(), source); + await f.manager.flush(); assert.equal(read, false); assert.equal(f.calls.length, before); + if (key === 'disabled') assert.equal(f.calls.length, 0); + } finally { await f.close(); } + }); + } + for (const capturePolicy of [{ captureContent: false }, { recordInputs: false }, { muted: true }, { temporaryChat: true }]) { + test(`${provider}: integration restriction ${Object.keys(capturePolicy)[0]} wins`, async () => { + const f = await fixture(); try { capture(f.manager, identity(), input(), { capture: capturePolicy }); await f.manager.flush(); assert.equal(f.initialized.length, 0); } finally { await f.close(); } + }); + } + test(`${provider}: malformed and oversized inline data is bounded; foreign URLs/files are not retrieved`, async () => { + const f = await fixture(); + try { + for (const data of ['!', 'a===', 'AB==', 'AAAA'.repeat(100000), 'https://127.0.0.1:1/?synthetic-secret']) { + const body = provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_image', image_url: data }] }] } + : { messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data } }] }] }; + capture(f.manager, identity(), body); + } + capture(f.manager, identity(), provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_file', file_id: 'file_secret' }] }] } + : { messages: [{ role: 'user', content: [{ type: 'document', source: { type: 'file', file_id: 'file_secret' } }, null, { type: 'tool_result', tool_use_id: {}, content: [image()] }] }] }); + await f.manager.flush(); assert.equal(f.initialized.length, 0); + await assertNoOpenTickets(f.manager); + } finally { await f.close(); } + }); + test(`${provider}: a negotiated upload limit below the inline cap still captures small media exactly`, async () => { + const f = await fixture({}, {}, { max_upload_bytes: 100 * 1024 }); + try { + const id = identity(), captured = capture(f.manager, id, input()); + await f.manager.flush(); + assert.equal(captured.size, provider === 'openai' ? 3 : 4); + assert.equal(f.uploaded.length, captured.size); + assert.ok(f.initialized.every(r => r.declared_bytes <= 100 * 1024)); + // Above the negotiated limit nothing is decoded or uploaded, and the reservation is returned. + const big = Buffer.alloc(150 * 1024, 7).toString('base64'); + capture(f.manager, identity(), provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_image', image_url: `data:image/png;base64,${big}` }] }] } + : { messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: big } }] }] }); + await f.manager.flush(); assert.equal(f.uploaded.length, captured.size); + await assertNoOpenTickets(f.manager); + } finally { await f.close(); } + }); + test(`${provider}: a nearly used owned-byte budget still admits small media at its exact size`, async () => { + const f = await fixture({ maxOwnedBytes: 300 * 1024 }); + try { + const big = Buffer.alloc(200 * 1024, 7).toString('base64'); + const one = data => provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_image', image_url: `data:image/png;base64,${data}` }] }] } + : { messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data } }] }] }; + // Same tick: the 200 KiB part still owns its bytes, so a cap-sized reservation would not fit. + const first = capture(f.manager, identity(), one(big)), second = capture(f.manager, identity(), one(PNG)); + assert.equal(first.size, 1); assert.equal(second.size, 1); + await f.manager.flush(); + assert.deepEqual(f.uploaded.map(b => b.length).sort((a, b) => a - b), [Buffer.from(PNG, 'base64').length, 200 * 1024]); + await assertNoOpenTickets(f.manager); + } finally { await f.close(); } + }); + test(`${provider}: unsampled identity and queued policy tightening cannot upload`, async () => { + const f = await fixture(); try { + capture(f.manager, { ...identity(), traceFlags: 0 }, input()); assert.equal(f.manager.status().ownedBytes, 0); + capture(f.manager, identity(), input()); f.manager.updatePolicy({ traceConfig: { hideInputs: true } }); await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } + }); +} + +test('Inline parts are capped at exactly 256 KiB of decoded bytes for both providers', async () => { + const f = await fixture(); try { + const at = Buffer.alloc(256 * 1024, 7).toString('base64'), over = Buffer.alloc(256 * 1024 + 3, 7).toString('base64'); + for (const data of [at, over]) { + oa.captureOpenAIAttachments(f.manager, identity(), { input: [{ role: 'user', content: [{ type: 'input_image', image_url: `data:image/png;base64,${data}` }] }] }, { operation: 'responses', direction: 'input' }); + oa.captureOpenAIAttachments(f.manager, identity(), { data: [{ b64_json: data }] }, { operation: 'images', direction: 'output' }); + an.captureAnthropicAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data } }] }] }); + } + await f.manager.flush(); + assert.deepEqual(f.uploaded.map(b => b.length), [262144, 262144, 262144]); + } finally { await f.close(); } +}); + +test('OpenAI generated images obey output and PII restrictions', async () => { + for (const traceConfig of [{}, { hideOutputs: true }, { hideOutputMessages: true }, { hideOutputText: true }, { piiRedaction: true }]) { + const f = await fixture({ traceConfig }); try { + oa.captureOpenAIAttachments(f.manager, identity(), { data: [{ b64_json: PNG }] }, { operation: 'images', direction: 'output' }); + await f.manager.flush(); assert.equal(f.uploaded.length, Object.keys(traceConfig).length ? 0 : 1); + } finally { await f.close(); } + } +}); + +test('Telemetry projections replace only admitted media with occurrence URIs and never mutate requests', async () => { + const f = await fixture(); try { + const cases = [ + [oaInput(), (m, s, x) => oa.captureOpenAIAttachments(m, s, x, { operation: 'responses', direction: 'input' }), (x, c) => oa.projectOpenAIMedia(x, c, 'responses', 'input'), 3], + [anInput(), (m, s, x) => an.captureAnthropicAttachments(m, s, x), (x, c) => an.projectAnthropicMedia(x, c), 4], + ]; + for (const [body, capture, project, count] of cases) { + const before = JSON.stringify(body); + // Nothing admitted: the exact same payload object is returned, so existing telemetry is unchanged. + assert.equal(project(body, new Map()), body); + const captured = capture(f.manager, identity(), body); + assert.equal(captured.size, count); + const projected = JSON.stringify(project(body, captured)); + assert.equal(JSON.stringify(body), before); + for (const secret of [PNG, PDF]) assert.ok(!projected.includes(secret)); + for (const { occurrenceId } of captured.values()) assert.ok(projected.includes(`fi-attachment:${occurrenceId}`)); + } + await f.manager.flush(); + assert.equal(f.uploaded.length, 7); + } finally { await f.close(); } +}); + +test('Denied media keeps the existing telemetry payload untouched', async () => { + const f = await fixture({ traceConfig: { hideInputImages: true } }); try { + const body = oaInput(), captured = oa.captureOpenAIAttachments(f.manager, identity(), body, { operation: 'responses', direction: 'input' }); + // Only the PDF is admitted; both images stay exactly as the application sent them. + assert.deepEqual([...captured.values()].map(v => v.mimeType), ['application/pdf']); + const projected = oa.projectOpenAIMedia(body, captured, 'responses', 'input'); + assert.equal(projected.input[0].content[0], body.input[0].content[0]); + assert.equal(projected.input[1], body.input[1]); + assert.ok(projected.input[0].content[1].file_data.startsWith('fi-attachment:')); + } finally { await f.close(); } +}); + +test('Hostile payloads fail capture without throwing into the application', async () => { + const f = await fixture(); try { + const hostile = new Proxy({}, { get() { throw Error('synthetic-secret'); }, has() { throw Error('synthetic-secret'); } }); + for (const payload of [hostile, { input: [hostile] }, { messages: [hostile] }, { messages: [{ role: 'user', content: [hostile] }] }, null, 7, 'x']) { + assert.equal(oa.captureOpenAIAttachments(f.manager, identity(), payload, { operation: 'responses', direction: 'input' }).size, 0); + assert.equal(oa.captureOpenAIAttachments(f.manager, identity(), payload, { operation: 'chat', direction: 'input' }).size, 0); + assert.equal(an.captureAnthropicAttachments(f.manager, identity(), payload).size, 0); + assert.doesNotThrow(() => an.projectAnthropicMedia(payload, new Map([['input/0/0', { occurrenceId: 'x', mimeType: 'image/png' }]]))); + assert.doesNotThrow(() => oa.projectOpenAIMedia(payload, new Map([['responses/input/0/0', { occurrenceId: 'x', mimeType: 'image/png' }]]), 'responses', 'input')); + } + await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } +}); + +test('Capability descriptors name every unsupported source explicitly', () => { + for (const name of ['remote_url', 'provider_file', 'partial_image', 'images.generate.stream', 'images.edit', 'speech']) assert.ok(oa.openAIAttachmentCapabilities.unsupported.includes(name)); + // Streamed Responses output is captured from the consumed terminal event; partial images never are. + assert.ok(oa.openAIAttachmentCapabilities.operations.includes('responses.stream.output')); + assert.ok(!oa.openAIAttachmentCapabilities.unsupported.includes('streamed_output')); + for (const name of ['remote_url', 'provider_file', 'native_span_hookup', 'generated_image', 'streamed_output']) assert.ok(an.anthropicAttachmentCapabilities.unsupported.includes(name)); + assert.ok(Object.isFrozen(oa.openAIAttachmentCapabilities) && Object.isFrozen(an.anthropicAttachmentCapabilities)); +}); + +test('CJS/ESM observations share deduplication without creating a second capture', async () => { + const f = await fixture(); try { + const esm = await import(pathToFileURL(path.resolve(__dirname, '../dist/esm/attachments.js'))); + const s = identity(), first = oa.captureOpenAIAttachments(f.manager, s, oaInput(), { operation: 'responses', direction: 'input' }); + const second = esm.captureOpenAIAttachments(f.manager, { ...s }, oaInput(), { operation: 'responses', direction: 'input' }); + // The second observer reuses the first occurrence, so both projections point at one upload. + assert.deepEqual([...second.entries()], [...first.entries()]); + const anthropicESM = await import(pathToFileURL(path.resolve(__dirname, '../../traceai_anthropic/dist/esm/attachments.js'))); + const a = identity(), cjs = an.captureAnthropicAttachments(f.manager, a, anInput()), mjs = anthropicESM.captureAnthropicAttachments(f.manager, { ...a }, anInput()); + assert.deepEqual([...mjs.entries()], [...cjs.entries()]); + await f.manager.flush(); assert.equal(f.uploaded.length, 7); + } finally { await f.close(); } +}); + +test('Anthropic tool-result images are not labelled computer screenshots without a typed computer tool', async () => { + const f = await fixture(); try { + an.captureAnthropicAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_photo', content: [image()] }] }] }); + await f.manager.flush(); assert.equal(f.initialized[0].purpose, 'tool_result'); assert.equal(f.initialized[0].part_index, 0); + } finally { await f.close(); } +}); + +test('Anthropic error-tool media respects hideErrorDetails before source access', async () => { + const f = await fixture({ traceConfig: { hideErrorDetails: true } }); try { + let read = false; + an.captureAnthropicAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_error', is_error: true, + content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', get data() { read = true; return PNG; } } }] }] }] }); + await f.manager.flush(); assert.equal(read, false); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } +}); + +test('OpenAI image error bodies are not treated as generated media', async () => { + const f = await fixture(); try { + oa.captureOpenAIAttachments(f.manager, identity(), { error: { message: 'synthetic' }, data: [{ b64_json: PNG }] }, { operation: 'images', direction: 'output' }); + await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } +}); + +test('Local instrumentor privacy restrictions cannot be loosened by a permissive shared manager', async () => { + for (const traceConfig of [{ hideInputs: true }, { hideInputMessages: true }, { hideInputText: true }, { piiRedaction: true }]) { + const f = await fixture(); try { + assert.equal(oa.captureOpenAIAttachments(f.manager, identity(), oaInput(), { operation: 'responses', direction: 'input', traceConfig }).size, 0); + assert.equal(an.captureAnthropicAttachments(f.manager, identity(), anInput(), { traceConfig }).size, 0); + await f.manager.flush(); assert.equal(f.initialized.length, 0, JSON.stringify(traceConfig)); + } finally { await f.close(); } + } + for (const traceConfig of [{ hideOutputs: true }, { piiRedaction: true }]) { + const f = await fixture(); try { + oa.captureOpenAIAttachments(f.manager, identity(), { data: [{ b64_json: PNG }] }, { operation: 'images', direction: 'output', traceConfig }); + await f.manager.flush(); assert.equal(f.initialized.length, 0, JSON.stringify(traceConfig)); + } finally { await f.close(); } + } + // Image-only restriction: the PDF still uploads, no image (including screenshots) does. + const f = await fixture(); try { + oa.captureOpenAIAttachments(f.manager, identity(), oaInput(), { operation: 'responses', direction: 'input', traceConfig: { hideInputImages: true } }); + an.captureAnthropicAttachments(f.manager, identity(), anInput(), { traceConfig: { hideInputImages: true } }); + await f.manager.flush(); assert.deepEqual(f.initialized.map(r => r.mime_type), ['application/pdf', 'application/pdf']); + } finally { await f.close(); } +}); + +test('Local hideErrorDetails denies Anthropic error tool media even when the manager allows it', async () => { + const f = await fixture(); try { + an.captureAnthropicAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_error', is_error: true, content: [image()] }] }] }, + { traceConfig: { hideErrorDetails: true } }); + await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } +}); + +async function providerServer() { + const requests = []; + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const c of req) chunks.push(c); + const body = JSON.parse(Buffer.concat(chunks)); requests.push({ path: req.url, body }); + if (req.url.endsWith('/responses') && body.stream) { + // A streamed Responses call whose completed response carries a generated image. + const response = { id: 'resp_stream', object: 'response', status: 'completed', model: body.model, output: [{ type: 'image_generation_call', id: 'img_stream', status: 'completed', result: PNG }], usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } }; + res.setHeader('content-type', 'text/event-stream'); + res.write(`event: response.created\ndata: ${JSON.stringify({ type: 'response.created', sequence_number: 0, response: { ...response, status: 'in_progress', output: [] } })}\n\n`); + res.end(`event: response.completed\ndata: ${JSON.stringify({ type: 'response.completed', sequence_number: 1, response })}\n\n`); + return; + } + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('/chat/completions')) res.end(JSON.stringify({ id: 'chat_fixture', object: 'chat.completion', model: body.model, choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })); + else if (req.url.endsWith('/responses')) res.end(JSON.stringify({ id: 'resp_fixture', object: 'response', status: 'completed', model: body.model, output: [{ type: 'computer_call', id: 'computer_fixture', call_id: 'call_computer', status: 'completed', action: { type: 'screenshot' } }, { type: 'image_generation_call', id: 'img_fixture', status: 'completed', result: PNG }], usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } })); + else if (req.url.endsWith('/images/generations')) res.end(JSON.stringify({ created: 1, data: [{ b64_json: PNG }] })); + else res.end(JSON.stringify({ id: 'msg_fixture', type: 'message', role: 'assistant', model: body.model, content: [{ type: 'text', text: 'ok' }], stop_reason: 'end_turn', stop_sequence: null, usage: { input_tokens: 1, output_tokens: 1 } })); + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + return { requests, url: `http://127.0.0.1:${server.address().port}/v1`, async close() { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } }; +} + +test('Real SDKs preserve APIPromise/request/response behavior and upload only through the shared manager', async () => { + const sdkModule = require(process.env.SDK_OPENAI || 'openai'); + const anthropicModule = require(process.env.SDK_ANTHROPIC || '../../traceai_anthropic/node_modules/@anthropic-ai/sdk'); + const OpenAI = sdkModule.OpenAI || sdkModule.default || sdkModule; + const Anthropic = anthropicModule.Anthropic || anthropicModule.default || anthropicModule; + const native = 'openTelemetry' in Anthropic.prototype; + const f = await fixture(), p = await providerServer(); + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + const oi = new OpenAIInstrumentation({ instrumentationConfig: { enabled: false }, attachments: f.manager }); + const ai = new AnthropicInstrumentation({ instrumentationConfig: { enabled: false }, attachments: f.manager }); + oi.setTracerProvider(provider); ai.setTracerProvider(provider); + oi.manuallyInstrument(sdkModule); ai.manuallyInstrument(anthropicModule); + try { + const client = new OpenAI({ apiKey: 'synthetic-provider-key', baseURL: p.url, maxRetries: 0 }); + const body = { model: 'synthetic-model', messages: [{ role: 'user', content: [{ type: 'image_url', image_url: { url: `data:image/png;base64,${PNG}` } }] }] }; + const pending = client.chat.completions.create(body); assert.equal(typeof pending.withResponse, 'function'); + assert.equal((await pending.withResponse()).data.choices[0].message.content, 'ok'); + let expected = 1; + if (client.responses) { + const result = await client.responses.create({ model: 'synthetic-model', ...oaInput() }); + assert.equal(result.output[0].call_id, 'call_computer'); expected += 4; + } + const generated = await client.images.generate({ prompt: 'synthetic', model: 'synthetic-model', response_format: 'b64_json' }); + assert.equal(generated.data[0].b64_json, PNG); expected++; + const ac = new Anthropic({ apiKey: 'synthetic-provider-key', baseURL: p.url, maxRetries: 0 }); + const request = ac.messages.create({ model: 'synthetic-model', max_tokens: 10, ...anInput() }); + assert.equal(typeof request.withResponse, 'function'); assert.equal((await request.withResponse()).data.content[0].text, 'ok'); + if (!native) expected += 4; + await f.manager.flush(); await provider.forceFlush(); + assert.equal(f.uploaded.length, expected); + assert.ok(p.requests.some(r => JSON.stringify(r.body).includes(PNG))); + assert.ok(!JSON.stringify(exporter.getFinishedSpans().map(s => s.attributes)).includes(PNG)); + assert.ok(exporter.getFinishedSpans().some(s => s.attributes['fi.attachment.refs'])); + // Each admitted reference appears as an occurrence URI in its own span's telemetry. + for (const span of exporter.getFinishedSpans()) { + const refs = JSON.parse(span.attributes['fi.attachment.refs'] ?? '[]'), text = JSON.stringify(span.attributes); + for (const ref of refs.filter(r => r.direction === 'input')) assert.ok(text.includes(`fi-attachment:${ref.occurrence_id}`), span.name); + } + if (client.responses) { + // Streamed Responses: the stream is delivered unchanged; the generated image in the terminal event the + // application consumed uploads once and is projected out of that span's output telemetry. + const stream = await client.responses.create({ model: 'synthetic-model', input: 'synthetic', stream: true }); + const events = []; for await (const event of stream) { events.push(event.type); if (event.type === 'response.completed') assert.equal(event.response.output[0].result, PNG); } + assert.deepEqual(events, ['response.created', 'response.completed']); + expected++; + await f.manager.flush(); await provider.forceFlush(); assert.equal(f.uploaded.length, expected); + assert.equal(f.uploaded.at(-1).toString('base64'), PNG); + const streamed = exporter.getFinishedSpans().filter(s => s.name === 'OpenAI Responses').at(-1); + const refs = JSON.parse(streamed.attributes['fi.attachment.refs'] ?? '[]'); + assert.equal(refs.length, 1); assert.equal(refs[0].direction, 'output'); assert.equal(refs[0].purpose, 'model_output'); + assert.ok(String(streamed.attributes['output.value']).includes(`fi-attachment:${refs[0].occurrence_id}`)); + assert.ok(!JSON.stringify(streamed.attributes).includes(PNG)); + // A consumer that stops early never reaches the terminal event: nothing is captured. + const early = await client.responses.create({ model: 'synthetic-model', input: 'synthetic', stream: true }); + for await (const event of early) { if (event.type === 'response.created') break; } + await f.manager.flush(); assert.equal(f.uploaded.length, expected); + } + // A raw-response caller owns the body; instrumentation must not parse for media. + const raw = await client.images.generate({ prompt: 'synthetic', response_format: 'b64_json' }).asResponse(); + assert.equal(raw.bodyUsed, false); await raw.arrayBuffer(); + await f.manager.flush(); assert.equal(f.uploaded.length, expected); + // Requests still reach the provider unchanged after privacy tightening; media egress stops. + f.manager.updatePolicy({ traceConfig: { hideInputs: true, piiRedaction: true } }); + await client.chat.completions.create(body); + await ac.messages.create({ model: 'synthetic-model', max_tokens: 10, ...anInput() }); + await client.images.generate({ prompt: 'synthetic', response_format: 'b64_json' }); + await f.manager.flush(); assert.equal(f.uploaded.length, expected); + console.log(JSON.stringify({ nativeAnthropic: native, responsesAvailable: !!client.responses, uploads: expected, spans: exporter.getFinishedSpans().length })); + } finally { + oi.disable(); ai.disable(); await provider.shutdown(); await f.close(); await p.close(); + } +}); + +test('A configured but disabled manager leaves existing span telemetry byte-identical', async () => { + const sdkModule = require(process.env.SDK_OPENAI || 'openai'); + const anthropicModule = require(process.env.SDK_ANTHROPIC || '../../traceai_anthropic/node_modules/@anthropic-ai/sdk'); + const OpenAI = sdkModule.OpenAI || sdkModule.default || sdkModule; + const Anthropic = anthropicModule.Anthropic || anthropicModule.default || anthropicModule; + const native = 'openTelemetry' in Anthropic.prototype; + const p = await providerServer(), disabled = await fixture({ enabled: false }); + const run = async attachments => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + const oi = new OpenAIInstrumentation({ instrumentationConfig: { enabled: false }, attachments }); + const ai = new AnthropicInstrumentation({ instrumentationConfig: { enabled: false }, attachments }); + oi.setTracerProvider(provider); ai.setTracerProvider(provider); + oi.manuallyInstrument(sdkModule); ai.manuallyInstrument(anthropicModule); + try { + const client = new OpenAI({ apiKey: 'synthetic-provider-key', baseURL: p.url, maxRetries: 0 }); + await client.chat.completions.create({ model: 'synthetic-model', messages: [{ role: 'user', content: [{ type: 'image_url', image_url: { url: `data:image/png;base64,${PNG}` } }] }] }); + if (client.responses) await client.responses.create({ model: 'synthetic-model', ...oaInput() }); + if (!native) await new Anthropic({ apiKey: 'synthetic-provider-key', baseURL: p.url, maxRetries: 0 }).messages.create({ model: 'synthetic-model', max_tokens: 10, ...anInput() }); + await provider.forceFlush(); + const strip = ({ name, attributes }) => ({ name, attributes: Object.fromEntries(Object.entries(attributes).filter(([k]) => !/latency|duration|time/i.test(k))) }); + return exporter.getFinishedSpans().map(strip); + } finally { oi.disable(); ai.disable(); await provider.shutdown(); } + }; + try { + const baseline = await run(undefined), off = await run(disabled.manager); + assert.ok(baseline.length >= 1); + assert.deepEqual(off, baseline); + assert.equal(disabled.calls.length, 0); + } finally { await disabled.close(); await p.close(); } +}); + +test('OpenAI data URIs of a sibling image type are admitted under their exact MIME', async () => { + const f = await fixture(); try { + const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 1, 2, 3, 0xff, 0xd9]).toString('base64'); + const captured = oa.captureOpenAIAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [ + { type: 'image_url', image_url: { url: `data:image/jpeg;base64,${jpeg}` } }, + { type: 'image_url', image_url: { url: `data:image/gif;base64,${jpeg}` } }, + ] }] }, { operation: 'chat', direction: 'input' }); + await f.manager.flush(); + assert.deepEqual([...captured.values()].map(v => v.mimeType), ['image/jpeg']); + assert.deepEqual(f.initialized.map(r => r.mime_type), ['image/jpeg']); + assert.equal(f.uploaded[0].toString('base64'), jpeg); + await assertNoOpenTickets(f.manager); + } finally { await f.close(); } +}); + +test('No core degradation warning was raised by any fixture', () => { + assert.deepEqual(coreWarnings, []); +});