diff --git a/.github/workflows/auto-typescript-tests.yml b/.github/workflows/auto-typescript-tests.yml index ed29b9b42..bc932fc24 100644 --- a/.github/workflows/auto-typescript-tests.yml +++ b/.github/workflows/auto-typescript-tests.yml @@ -4,7 +4,7 @@ name: Zero-code agent (@traceai/auto) tests # integration tests load the built instrumentors, so the whole workspace is built. on: pull_request: - branches: [dev, main] + branches: [dev, main, feat/attachments-ts-core] paths: - 'typescript/packages/traceai_auto/**' - 'typescript/packages/fi-core/**' diff --git a/.github/workflows/openai-anthropic-attachments-tests.yml b/.github/workflows/openai-anthropic-attachments-tests.yml new file mode 100644 index 000000000..a8335241c --- /dev/null +++ b/.github/workflows/openai-anthropic-attachments-tests.yml @@ -0,0 +1,70 @@ +name: OpenAI and Anthropic TypeScript attachments +on: + pull_request: + paths: + - 'typescript/packages/traceai_openai/**' + - 'typescript/packages/traceai_anthropic/**' + - 'typescript/packages/fi-core/**' + - 'typescript/packages/fi-semantic-conventions/**' + - 'typescript/pnpm-lock.yaml' + - '.github/workflows/openai-anthropic-attachments-tests.yml' + workflow_dispatch: +permissions: + contents: read +jobs: + attachments: + name: OpenAI ${{ matrix.openai }} / Anthropic ${{ matrix.anthropic }} + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - { openai: '4.0.0', anthropic: '0.20.0' } + - { openai: '5.23.2', anthropic: '0.27.3' } + - { openai: '6.0.0', anthropic: '0.133.0' } + - { openai: '7.47.0', anthropic: '0.134.0' } + - { openai: 'latest', anthropic: 'latest' } + defaults: + run: + shell: bash + working-directory: typescript + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22.23.4' + - name: Install pinned workspace dependencies and real SDK fixtures + run: | + mkdir -p "$RUNNER_TEMP/corepack-bin" + corepack enable --install-directory "$RUNNER_TEMP/corepack-bin" pnpm + echo "$RUNNER_TEMP/corepack-bin" >> "$GITHUB_PATH" + corepack pnpm --filter @traceai/openai... --filter @traceai/anthropic... install --frozen-lockfile --ignore-scripts + npm install --prefix "$RUNNER_TEMP/sdks" --ignore-scripts --no-audit --no-fund \ + "openai-under-test@npm:openai@${{ matrix.openai }}" \ + "anthropic-under-test@npm:@anthropic-ai/sdk@${{ matrix.anthropic }}" + - name: Build CJS and ESM + run: | + pnpm run build:deps + pnpm --filter @traceai/openai run build + pnpm --filter @traceai/anthropic run build + - name: Network-isolated attachment adapters against real SDKs and loopback mocks + run: | + sudo unshare --net -- bash -c 'ip link set lo up; exec "$@"' bash \ + env PATH="$PATH" \ + SDK_OPENAI="$RUNNER_TEMP/sdks/node_modules/openai-under-test" \ + SDK_ANTHROPIC="$RUNNER_TEMP/sdks/node_modules/anthropic-under-test" \ + node --test packages/traceai_openai/test/attachments-providers.test.cjs + - name: Network-isolated package unit suites + if: matrix.openai == 'latest' + run: | + sudo unshare --net -- bash -c 'ip link set lo up; exec "$@"' bash \ + env PATH="$PATH" node packages/traceai_openai/node_modules/jest/bin/jest.js \ + --config packages/traceai_openai/jest.config.js --runInBand + sudo unshare --net -- bash -c 'ip link set lo up; exec "$@"' bash \ + env PATH="$PATH" node node_modules/jest/bin/jest.js \ + --config packages/traceai_anthropic/jest.config.js --runInBand + git diff --exit-code -- pnpm-lock.yaml diff --git a/typescript/packages/traceai_anthropic/src/attachments.ts b/typescript/packages/traceai_anthropic/src/attachments.ts new file mode 100644 index 000000000..cb72aee72 --- /dev/null +++ b/typescript/packages/traceai_anthropic/src/attachments.ts @@ -0,0 +1,168 @@ +import type { TraceConfigOptions } from "@traceai/fi-core"; +import type { AttachmentManager, AttachmentSpan, AttachmentCorrelation, AttachmentCapturePolicy, AttachmentOptions } from "@traceai/fi-core/attachments"; + +/** Native SDK span ownership is deliberately not overridden by this adapter. */ +export const anthropicAttachmentCapabilities = Object.freeze({ + schemaVersion: 1, + operations: ["messages.create.input", "tool_result.input"] as const, + sources: ["inline_base64"] as const, + /** Fail closed: no capture is attempted for these. */ + unsupported: ["remote_url", "provider_file", "native_span_hookup", "generated_image", "beta_messages_hookup", "streamed_output"] as const, + maxInlineBytes: 256 * 1024, +}); + +/** Admitted media positions for one observation: position key -> occurrence. */ +export type AnthropicCapturedMedia = ReadonlyMap>; +type Captured = Map>; +type RecordValue = Record; +const record = (value: unknown): RecordValue => value !== null && typeof value === "object" ? value as RecordValue : {}; +const MAX_PARTS = 256; +const MAX_SPANS = 1024; +const EMPTY: AnthropicCapturedMedia = new Map(); +const registryKey = Symbol.for("@traceai/anthropic/attachment-observations/v1"); +type Registry = WeakMap>; +const host = globalThis as typeof globalThis & { [registryKey]?: Registry }; +const observations = (host[registryKey] ??= new WeakMap()); +const TOOL_ID = /^[a-zA-Z0-9_-]{1,128}$/; + +const MAX_ENCODED = 4 * Math.ceil(anthropicAttachmentCapabilities.maxInlineBytes / 3); + +/** Exact decoded length from the encoded string's shape only (no decode); 0 when out of bounds. */ +function bound(encoded: unknown): number { + if (typeof encoded !== "string" || !encoded.length || encoded.length % 4 !== 0 || encoded.length > MAX_ENCODED) return 0; + return (encoded.length / 4) * 3 - (encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0); +} + +/** Strict decode after admission. The ticket's budget is a hard bound; padding and alphabet are strict. */ +function decode(encoded: unknown, maxBytes: number): Uint8Array | undefined { + // Bound allocation BEFORE decoding. + if (!bound(encoded) || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded as string)) return undefined; + const bytes = Buffer.from(encoded as string, "base64"); + return bytes.length && bytes.length <= maxBytes && bytes.length <= anthropicAttachmentCapabilities.maxInlineBytes && + bytes.toString("base64") === encoded ? bytes : undefined; +} + +/** Observe only Messages' typed image/document blocks and one level of tool_result content. + * Every part is preflighted (deny-wins, budget reserved) before its base64 is read or decoded; an admitted + * ticket that is not used is released at once. The manager owns policy, upload, lifecycle and references. A native integration + * must supply its actual emitted span/correlation, never an ambient guessed span. + * Returns the admitted positions so the caller can project them out of span telemetry. + */ +export function captureAnthropicAttachments(manager: AttachmentManager | undefined, + span: AttachmentSpan | AttachmentCorrelation, payload: unknown, + options: { traceConfig?: TraceConfigOptions; capture?: AttachmentCapturePolicy } = {}): AnthropicCapturedMedia { + if (!manager) return EMPTY; + const captured: Captured = new Map(); + try { + const correlation = "snapshot" in span ? span : manager.correlation(span); + const identity = correlation.snapshot(); + if (!identity?.sampled) return EMPTY; + const messages = record(payload).messages; + if (!Array.isArray(messages)) return EMPTY; + let spans = observations.get(manager); + if (!spans) { spans = new Map(); observations.set(manager, spans); } + const spanKey = `${identity.traceId}/${identity.spanId}`; + const config = options.traceConfig ?? {}; + const hidden = config.hideInputs || config.hideInputMessages || config.hideInputText || config.piiRedaction; + // Screenshot classification requires a typed computer tool AND an assistant tool_use naming it. + const computerNames = new Set(); + const tools = record(payload).tools; + if (Array.isArray(tools)) for (const value of tools.slice(0, MAX_PARTS)) { + const tool = record(value); + if (typeof tool.type === "string" && /^computer_\d{8}$/.test(tool.type) && typeof tool.name === "string") computerNames.add(tool.name); + } + const computerCalls = new Set(); + for (const value of messages.slice(0, MAX_PARTS)) { + const message = record(value); + if (message.role !== "assistant" || !Array.isArray(message.content)) continue; + for (const value of message.content.slice(0, MAX_PARTS)) { + const block = record(value); + if (block.type === "tool_use" && typeof block.id === "string" && typeof block.name === "string" && computerNames.has(block.name)) computerCalls.add(block.id); + } + } + let visited = 0; + messages.slice(0, MAX_PARTS).forEach((value, messageIndex) => { + const message = record(value); + if (!Array.isArray(message.content)) return; + // Nested positions are relative to their tool result, identified by toolCallId. + const visit = (value: unknown, index: number, position: string, toolCallId?: string, errorDetail = false) => { + if (++visited > MAX_PARTS) return; + const part = record(value); + if (part.type !== "image" && part.type !== "document") return; + const source = record(part.source); + if (source.type !== "base64") return; + const mime = source.media_type; + if (typeof mime !== "string" || !(part.type === "image" ? ["image/png", "image/jpeg", "image/webp"] : ["application/pdf"]).includes(mime)) return; + const positionKey = `input/${messageIndex}/${position}`; + const previous = spans!.get(spanKey)?.get(positionKey); + if (previous) { captured.set(positionKey, previous); return; } + const role: AttachmentOptions["role"] = toolCallId ? "tool" : message.role === "assistant" ? "assistant" : "user"; + const request = { mimeType: mime, direction: "input" as const, + purpose: (part.type === "document" ? "document" : toolCallId ? computerCalls.has(toolCallId) ? "computer_screenshot" : "tool_result" : "model_input") as AttachmentOptions["purpose"], + capture: { ...options.capture, modelInput: true, errorDetail: errorDetail || options.capture?.errorDetail, + ...(hidden || (part.type === "image" && config.hideInputImages) || (errorDetail && config.hideErrorDetails) ? { captureContent: false } : {}) } }; + // Preflight (deny-wins, budget reserved) before the base64 source is read or decoded (design section 1). + // The source is read at most once. + let data: unknown, dataRead = false; + const read = () => { if (!dataRead) { dataRead = true; data = source.data; } return data; }; + let ticket = manager.preflight(correlation, { ...request, maxBytes: anthropicAttachmentCapabilities.maxInlineBytes }); + // too_large (negotiated limit below the cap) and queue_full (owned budget nearly used) are evaluated after every + // privacy, correlation and sampling step, so sizing the source exactly and retrying once is safe here. + if (ticket.reason === "too_large" || ticket.reason === "queue_full") { + const exact = bound(read()); + if (exact > 0) ticket = manager.preflight(correlation, { ...request, maxBytes: exact }); + } + if (!ticket.admitted) return; + const bytes = decode(read(), ticket.maxBytes); + if (!bytes) { ticket.release(); return; } + const handle = ticket.attach({ kind: "Bytes", data: bytes }, { messageIndex, partIndex: index, role, ...(toolCallId ? { toolCallId } : {}) }); + if (handle.admission === "queued") { + const occurrence = Object.freeze({ occurrenceId: handle.occurrenceId, mimeType: mime }); + let positions = spans!.get(spanKey); + if (!positions) { + if (spans!.size >= MAX_SPANS) spans!.delete(spans!.keys().next().value!); + positions = new Map(); spans!.set(spanKey, positions); + } + if (positions.size < MAX_PARTS) positions.set(positionKey, occurrence); + captured.set(positionKey, occurrence); + } + }; + message.content.slice(0, MAX_PARTS).forEach((value, index) => { + const block = record(value); + if (block.type === "tool_result" && Array.isArray(block.content)) { + // Invalid tool IDs fail closed instead of losing causal identity. + if (typeof block.tool_use_id !== "string" || !TOOL_ID.test(block.tool_use_id)) return; + block.content.slice(0, MAX_PARTS).forEach((child, childIndex) => visit(child, childIndex, `${index}/${childIndex}`, block.tool_use_id as string, block.is_error === true)); + } else visit(value, index, String(index)); + }); + }); + } catch { /* No source-bearing exceptions or changes to the application's result. */ } + return captured; +} + +/** Telemetry-only projection. ONLY admitted positions are replaced with a `fi-attachment:` occurrence URI + * source, so uploaded bytes are not exported twice. Everything else is returned exactly as before, and the + * request passed to the SDK is never mutated. + */ +export function projectAnthropicMedia(payload: unknown, captured: AnthropicCapturedMedia): any { + if (!captured.size) return payload; + try { + const body = record(payload); + if (!Array.isArray(body.messages)) return payload; + const replace = (value: unknown, position: string): unknown => { + const occurrence = captured.get(position); + return occurrence ? { ...record(value), source: { type: "url", url: `fi-attachment:${occurrence.occurrenceId}` } } : value; + }; + return { ...body, messages: body.messages.map((value, messageIndex) => { + const message = record(value); + if (!Array.isArray(message.content)) return value; + return { ...message, content: message.content.map((value, index) => { + const block = record(value); + if (block.type === "tool_result" && Array.isArray(block.content)) { + return { ...block, content: block.content.map((child, childIndex) => replace(child, `input/${messageIndex}/${index}/${childIndex}`)) }; + } + return replace(value, `input/${messageIndex}/${index}`); + }) }; + }) }; + } catch { return payload; } +} diff --git a/typescript/packages/traceai_anthropic/src/index.ts b/typescript/packages/traceai_anthropic/src/index.ts index 378f38976..bc14a48fb 100644 --- a/typescript/packages/traceai_anthropic/src/index.ts +++ b/typescript/packages/traceai_anthropic/src/index.ts @@ -1,2 +1,3 @@ export * from './version'; -export * from './instrumentation'; \ No newline at end of file +export * from './instrumentation'; +export * from './attachments'; diff --git a/typescript/packages/traceai_anthropic/src/instrumentation.ts b/typescript/packages/traceai_anthropic/src/instrumentation.ts index 77d132430..91561e08c 100644 --- a/typescript/packages/traceai_anthropic/src/instrumentation.ts +++ b/typescript/packages/traceai_anthropic/src/instrumentation.ts @@ -1,4 +1,6 @@ import Anthropic from "@anthropic-ai/sdk"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import { captureAnthropicAttachments, projectAnthropicMedia } from "./attachments"; import { InstrumentationBase, InstrumentationConfig, @@ -334,16 +336,22 @@ function finishInstrumentationStream( export class AnthropicInstrumentation extends InstrumentationBase{ private fiTracer!: FITracer; private _traceConfig?: TraceConfigOptions; + private attachments?: AttachmentManager; constructor({ instrumentationConfig, traceConfig, + attachments, }: { instrumentationConfig?: InstrumentationConfig; traceConfig?: TraceConfigOptions; + /** The provider-owned shared attachment manager. */ + attachments?: AttachmentManager; } = {}) { super("@traceai/instrumentation-anthropic", VERSION,Object.assign({}, instrumentationConfig)); this._traceConfig = traceConfig; + this.attachments = attachments; + this.fiTracer = new FITracer({ tracer: this.tracer, traceConfig: this._traceConfig }); } public override enable(): void { @@ -404,21 +412,26 @@ export class AnthropicInstrumentation extends InstrumentationBase{ // Not async: the SDK returns an APIPromise whose withResponse(), asResponse() // and lazy body parsing callers (including messages.stream()) depend on. return function patchedMessagesCreate(this: Anthropic.Messages, ...args: Parameters) { - const params = args[0] as Anthropic.MessageCreateParams; - const { messages: _messages, system: _system, tools: _tools, ...invocationParameters } = params; + const originalParams = args[0] as Anthropic.MessageCreateParams; + const { messages: _messages, system: _system, tools: _tools, ...invocationParameters } = originalParams; + // Content attributes are set after capture so admitted media can be projected out first. const span = instrumentation.fiTracer.startSpan("Anthropic Messages Create", { kind: SpanKind.INTERNAL, attributes: { [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, - [SemanticConventions.LLM_MODEL_NAME]: params.model, + [SemanticConventions.LLM_MODEL_NAME]: originalParams.model, [SemanticConventions.LLM_PROVIDER]: LLMProvider.ANTHROPIC, [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", + }, + }); + const captured = captureAnthropicAttachments(instrumentation.attachments, span, originalParams, { traceConfig: instrumentation._traceConfig }); + const params = projectAnthropicMedia(originalParams, captured) as Anthropic.MessageCreateParams; + span.setAttributes({ [SemanticConventions.LLM_INVOCATION_PARAMETERS]: safelyJSONStringify(invocationParameters) ?? "", ...getAnthropicInputMessagesAttributes(params), ...getAnthropicToolsAttributes(params), [SemanticConventions.RAW_INPUT]: safelyJSONStringify(params) ?? "", - }, }); const execContext = getExecContext(span); diff --git a/typescript/packages/traceai_auto/docs/CONFIGURATION.md b/typescript/packages/traceai_auto/docs/CONFIGURATION.md index 9c99b8d2f..b6a71e20a 100644 --- a/typescript/packages/traceai_auto/docs/CONFIGURATION.md +++ b/typescript/packages/traceai_auto/docs/CONFIGURATION.md @@ -230,6 +230,35 @@ When error hiding is unset, any input/output value, message or text hide flag en Known-secret redaction is always active on traceAI-owned exports. The registry is bounded to 1024 values of at least eight characters. `registerSecret(value)` adds an exact string without coercing objects. Unknown free-text credentials, transformed secrets and credentials hidden in unparsed bodies may remain unknown; hide sensitive content instead of relying on inference. +## Attachments (images, files, audio) + +Attachments are **off** by default. Upgrading `@traceai/auto` captures no new media. New recordings are a separate switch, also off. + +| Environment | `init` option | Default and behavior | +|---|---|---| +| `FI_ATTACHMENTS_ENABLED` | `attachments.enabled` | `false`. A manager that reads the process environment (the default) also treats a value that is not true as a deny at every admission | +| `FI_ATTACHMENTS_ENDPOINT` | `attachments.endpoint` | Unset. The attachment control origin, independent of `baseUrl` and the OTLP collector; must be HTTPS | +| — | `attachments` | Options for the manager auto creates. Hide and PII flags from either `traceConfig` win; an attachment-level `false` cannot relax a global hide flag | +| — | `attachmentManager` | Borrow the application's manager. Auto never shuts down a borrowed manager and only adds restrictions to it | + +Auto uses ONE manager: `attachmentManager`, else the selected provider's `attachments`, else its own. It passes that manager to every instrumentor it enables as their `attachments` option, and `getAttachmentManager()` returns it. Every source is checked against the deny-wins policy (local hide/PII flags, server policy, sampling, budget) before it is read or decoded; storage stays private. + +Wire capture (`wire`) observes only allowlisted HTTPS protocol entries (`report().attachments.routes`): OpenAI speech, transcription and translation, image generation, edits and variations, chat completions, responses, and Deepgram speak. It never fetches URLs, opens files, clones, tees or reads a response body itself. + +| Source | Wire-only call | Under a native model span | +|---|---|---| +| JSON input images, files, computer screenshots (chat, responses) | Captured, from the wire request copy | By the instrumentor (`@traceai/openai`); the same position is never uploaded twice | +| JSON output images (images `b64_json`, responses `image_generation_call`) | Captured, from the wire response copy | By the instrumentor where it covers the operation; wire capture needs an explicit integration policy (`withAttachmentCapturePolicy`) | +| Speech audio the application read with `arrayBuffer()` / `blob()` | Captured | Needs an explicit integration policy | +| Speech audio the application read incrementally (reader, async iterator) | Captured: held in a bounded buffer until the wire span exists, then checked again | Needs an explicit integration policy; cancel, early return, pipe or tee upload nothing | +| FormData audio / image files | Transcription and translation audio captured: held (not read) until the wire span exists, then checked again. Image edits and variations have no wire span, so their files are not captured | Needs an explicit integration policy | + +A wire-only capture waits for the wire span, which exists only once the response completed. When the span is emitted, policy, the environment switch and sampling are checked again before anything is sent. If the call fails or is aborted (fetch throws or rejects, the connection drops, the request is aborted through its `AbortSignal`, or the response body fails while it is read), the held capture is dropped at once: the wire span is still recorded with an error status, but nothing is attached to it. The same happens when no span will come (the call was not traced, or its span was not delivered). A call that completed with an HTTP error status (4xx/5xx) did finish, so its held input is attached to that span, as with a native instrumentor span. A span that arrives after five minutes binds nothing. Bytes held this way are capped at half the owned budget, so a larger file is not captured (`queue_full`). + +Media in Anthropic `messages.create` input (inline base64 images and documents) is captured by `@traceai/anthropic` when auto enables it; `report().integrations[].media` reports `provider_adapter` for `openai` and `anthropic` when the installed instrumentor exposes its capability descriptor. + +An adapter that captures media itself marks its context with `withAttachmentOwnership(context)` (context key `traceai.auto.attachment_owner.v1`), and wire capture then skips that call. `report().attachments` shows the manager state, queue counters, routes, sources and unsupported sources; `report().integrations[].media` shows whether an integration captures media through a provider adapter. No row is a playback claim. + ## Bring your own OpenTelemetry destination With no complete FI key pair, `init({ exporter })`, `init({ spanProcessor })`, `init({ provider })`, or an existing application provider activates instrumentation **without creating an FI export route**. If neither a destination nor a complete pair exists, instrumentation is disabled. With a complete pair and a provider but no explicit exporter/processor, the agent attempts to attach its FI route to that provider. @@ -275,6 +304,9 @@ Standard `OTEL_SERVICE_NAME` and `OTEL_RESOURCE_ATTRIBUTES` preserve application | `forceFlush()` | Flush completed spans with a five-second deadline; does not consume unread responses | | `shutdown()` | Idempotent hook removal and bounded drain/shutdown; no supported reinitialization afterwards | | `registerSecret(value)` | Register an exact sensitive string of at least eight characters; ignore other values | +| `getAttachmentManager()` | The one attachment manager auto selected and passed to instrumentors, or `undefined` before `init` | +| `withAttachmentCapturePolicy(context, policy)` | Supply an integration's capture restrictions for wire media under its native span; inherited denials always win | +| `withAttachmentOwnership(context)` | Mark a call whose media an adapter captures itself, so wire capture does not upload it again | ## CLI diff --git a/typescript/packages/traceai_auto/scripts/postbuild.cjs b/typescript/packages/traceai_auto/scripts/postbuild.cjs index 347c39c71..f743f0e6f 100644 --- a/typescript/packages/traceai_auto/scripts/postbuild.cjs +++ b/typescript/packages/traceai_auto/scripts/postbuild.cjs @@ -2,7 +2,7 @@ const fs = require('node:fs'); for (const dir of ['dist/esm', 'dist/esnext']) { fs.writeFileSync(`${dir}/package.json`, '{"type":"module"}\n'); // A single CJS implementation keeps init idempotent across mixed ESM/CJS apps. - fs.writeFileSync(`${dir}/index.js`, 'export { init, shutdown, forceFlush, report, registerSecret } from "../src/index.js";\n'); + fs.writeFileSync(`${dir}/index.js`, 'export { init, shutdown, forceFlush, report, registerSecret, getAttachmentManager, withAttachmentOwnership, withAttachmentCapturePolicy } from "../src/index.js";\n'); fs.writeFileSync(`${dir}/register.js`, 'import "../src/register.js";\n'); for (const name of ['esbuild', 'vite', 'rollup', 'webpack', 'next']) { const named = name === 'next' ? 'withTraceAI' : name === 'webpack' ? 'traceAI, TraceAIPlugin' : 'traceAI'; diff --git a/typescript/packages/traceai_auto/src/attachments.ts b/typescript/packages/traceai_auto/src/attachments.ts new file mode 100644 index 000000000..04fb7298f --- /dev/null +++ b/typescript/packages/traceai_auto/src/attachments.ts @@ -0,0 +1,79 @@ +import { createNodeAttachmentManager, type NodeAttachmentManagerOptions } from "@traceai/fi-core/attachments/node"; +import type { AttachmentManager, AttachmentStatus } from "@traceai/fi-core/attachments"; +import type { Config, Options } from "./config"; +import { WIRE_MEDIA_ROUTES } from "./wire/attachments"; + +const restrictive = (traceConfig: Record | undefined) => Object.fromEntries(Object.entries(traceConfig ?? {}).filter(([key, value]) => + (key.startsWith("hide") || key === "piiRedaction") && value === true)); + +export function attachmentOptions(config: Config, options: Options): NodeAttachmentManagerOptions { + const explicit = options.attachments ?? {}; + return { ...explicit, environment: () => process.env, projectName: config.projectName, projectType: config.projectType, + // Only resolved FI credentials, never model/provider credentials. A neutral + // exporter does not imply an attachment destination or credential authority. + headers: explicit.headers ?? (config.apiKey && config.secretKey ? { "x-api-key": config.apiKey, "x-secret-key": config.secretKey } : {}), + // A restriction from either side wins; an attachment-level `false` cannot relax a global hide flag. + traceConfig: { ...explicit.traceConfig, ...config.traceConfig, ...restrictive(explicit.traceConfig as Record) } as NodeAttachmentManagerOptions["traceConfig"] }; +} +interface AttachmentProvider { attachments?: AttachmentManager; getDelegate?(): AttachmentProvider } +/** + * One manager per process: the application's (explicit option, then the provider's), otherwise the one auto's own + * provider created. Adapters and wire observation all receive this same manager. + */ +export function selectAttachmentManager(provider: AttachmentProvider, config: Config, options: Options, ownsProvider: boolean): { manager: AttachmentManager; owned: boolean } { + let existing: AttachmentManager | undefined; + try { existing = options.attachmentManager ?? provider?.attachments ?? provider?.getDelegate?.()?.attachments; } catch { existing = undefined; } + if (existing) { + // Borrowing never turns capture on and never loosens application policy: only auto's hide/PII flags are added. + // A borrowed manager is not permanently disabled here (updatePolicy is tighten-only). The env override + // (FI_ATTACHMENTS_ENABLED=false) is applied at every admission by a manager whose `environment` reads + // `process.env` (auto's own and the `createNodeAttachmentManager` default); a borrowed manager built with a custom + // `environment` sees only what that function returns. + try { existing.updatePolicy({ traceConfig: restrictive(attachmentOptions(config, options).traceConfig as Record) as NodeAttachmentManagerOptions["traceConfig"] }); } catch { /* fail closed below */ } + return { manager: existing, owned: ownsProvider && !options.attachmentManager }; + } + return { manager: createNodeAttachmentManager(attachmentOptions(config, options)), owned: true }; +} + +/** What auto can capture, independent of whether the manager is on. Nothing here is a playback claim. */ +export const AUTO_ATTACHMENT_SOURCES = Object.freeze([ + "wire FormData Blob (transcription audio, image edit image/mask) under a native model span with an explicit capture policy", + "wire-only FormData Blob (transcription and translation audio), held bounded until the wire span exists", + "wire JSON input images/files/computer screenshots (OpenAI chat, responses) via the @traceai/openai adapter", + "wire JSON output images (OpenAI images b64_json, responses image_generation_call) consumed by the application via json()", + "wire speech audio materialized by the application via arrayBuffer()/blob()", + "wire speech audio read incrementally by the application (reader or async iterator) under a native model span with an explicit capture policy", + "wire-only speech audio read incrementally by the application (reader or async iterator), held bounded until the wire span exists", +]); +export const AUTO_ATTACHMENT_UNSUPPORTED = Object.freeze([ + "wire-only image edits and variations (no wire span is emitted for these routes; held files are dropped at once)", + "wire-only FormData or incremental speech above the pre-bind budget (half the owned budget): queue_full", + "wire-only captures whose wire span is never emitted, or binds after the deadline: correlation_unavailable", + "native model spans without an explicit integration capture policy", + "raw multipart bodies, streamed request bodies, compressed responses consumed before the span is emitted", + "RemoteUrl / ProviderFile retrieval", + "piped or teed response bodies", + "WebSocket / RTC audio", + "new recordings", + "product playback qualification", +]); +export interface AttachmentReport { + state: AttachmentStatus["state"] | "not_initialized"; + status?: Omit; + transport: "shared_manager"; + routes: string[]; + sources: readonly string[]; + unsupported: readonly string[]; +} +export function attachmentReport(manager?: AttachmentManager): AttachmentReport { + let status: AttachmentStatus | undefined; + try { status = manager?.status(); } catch { status = undefined; } + const { state, ...counts } = status ?? { state: "not_initialized" as const }; + return { + state, ...(status ? { status: counts as Omit } : {}), + transport: "shared_manager", + routes: WIRE_MEDIA_ROUTES.map(route => `POST https://${route.host}${route.path} (${route.operation})`), + sources: AUTO_ATTACHMENT_SOURCES, + unsupported: AUTO_ATTACHMENT_UNSUPPORTED, + }; +} diff --git a/typescript/packages/traceai_auto/src/config.ts b/typescript/packages/traceai_auto/src/config.ts index cd216b331..59e475358 100644 --- a/typescript/packages/traceai_auto/src/config.ts +++ b/typescript/packages/traceai_auto/src/config.ts @@ -6,7 +6,14 @@ import type { TracerProvider } from "@opentelemetry/api"; import type { PrivacyOptions as TraceConfigOptions } from "./export-privacy"; import { compileOutgoingPolicy, type OutgoingPolicy, type OutgoingPropagationMode } from "./outgoing-policy"; +import type { NodeAttachmentManagerOptions } from "@traceai/fi-core/attachments/node"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; + export interface Options { + /** Default OFF. The control endpoint is independent of baseUrl/OTLP. */ + attachments?: NodeAttachmentManagerOptions; + /** Borrow the application's manager; auto never shuts down a borrowed manager. */ + attachmentManager?: AttachmentManager; /** Experimental node:http/node:https propagation; off until explicitly selected. */ propagateOutgoingAdapters?: boolean; /** default/all retain mandatory vendor-host, provider-port and redirect vetoes. */ diff --git a/typescript/packages/traceai_auto/src/index.ts b/typescript/packages/traceai_auto/src/index.ts index 7eea28f79..949ccdf05 100644 --- a/typescript/packages/traceai_auto/src/index.ts +++ b/typescript/packages/traceai_auto/src/index.ts @@ -10,6 +10,10 @@ import { installOutgoingPropagation } from "./outgoing-node"; import { enableLibraries, IntegrationStatus, inventory } from "./libraries"; import { FetchHook, patchFetch, trackCoverage } from "./wire/fetch"; import { captureSecrets, SpanResult } from "./wire/core"; +import type { Span } from "@opentelemetry/api"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import { attachmentOptions, attachmentReport, selectAttachmentManager } from "./attachments"; +export { withAttachmentOwnership, withAttachmentCapturePolicy } from "./wire/attachments"; import { withDeadline } from "./deadline"; import { privacyFilter } from "./export-privacy"; @@ -24,6 +28,7 @@ export type { Options } from "./config"; interface AgentState { config: Config; statuses: IntegrationStatus[]; active: boolean; provider?: any; ownsProvider?: boolean; spanDelivery?: AutoSpanDelivery; + attachments?: AttachmentManager; ownsAttachments?: boolean; ownsSpanExporter?: boolean; processor?: SpanProcessor; processorIsObserver?: boolean; convertedSpanDestination?: ConvertedSpanDestination; detach?: () => void; removeLibraries?: () => void; fetch?: FetchHook; @@ -90,8 +95,10 @@ function removeHooks(agent: AgentState): void { () => agent.removeOutgoing?.(), () => agent.fetch?.remove(), () => agent.removeObservation?.(), () => agent.removeLibraries?.(), ]) { try { remove(); } catch { /* A broken cleanup must not skip the others. */ } } } -export interface Report { active: boolean; config: Record; integrations: IntegrationStatus[]; collector: string } -/** Initialize once. No network calls or thrown startup errors. */ +export interface Report { active: boolean; config: Record; integrations: IntegrationStatus[]; collector: string; attachments: ReturnType } +/** The same manager passed to adapters; await ready() explicitly at startup if needed. */ +export function getAttachmentManager(): AttachmentManager | undefined { return sharedState()?.attachments; } +/** Initialize once. No awaited network calls or thrown startup errors; opted-in attachments bootstrap asynchronously. */ export function init(options: Options = {}): TracerProvider | undefined { const existing = sharedState(); if (existing) { warn("initialized", "Already initialized; keeping the existing provider and configuration."); return existing.active ? existing.provider : undefined; } @@ -142,10 +149,13 @@ export function init(options: Options = {}): TracerProvider | undefined { if (!agent.provider) { agent.provider = register({ projectName: config.projectName, projectType: config.projectType === "observe" ? ProjectType.OBSERVE : ProjectType.EXPERIMENT, setGlobalTracerProvider: false, verbose: false, endpoint: config.baseUrl, batch: true, spanProcessors: [processor], + traceConfig: config.traceConfig, attachments: options.attachmentManager ? { enabled: false } : attachmentOptions(config, options), sampler: new ParentBasedSampler({ root: new TraceIdRatioBasedSampler(config.sampleRate) }) }); agent.ownsProvider = true; if (!existing) agent.ownsGlobalProvider = trace.setGlobalTracerProvider(agent.provider); } + const attachmentRuntime = selectAttachmentManager(agent.provider, config, options, agent.ownsProvider === true); + agent.attachments = attachmentRuntime.manager; agent.ownsAttachments = attachmentRuntime.owned; if (agent.processorIsObserver) { agent.convertedSpanDestination = borrowConvertedSpanDestination(agent.provider, processor); const destination = agent.convertedSpanDestination; @@ -180,7 +190,7 @@ export function init(options: Options = {}): TracerProvider | undefined { const admissionPrivacy = privacyFilter(options.traceConfig, [config.apiKey, config.secretKey].filter((v): v is string => !!v)); const wirePipeline = (tracer as any)._spanProcessor; const wireDestinationReady = () => hasAutoSpanSink(wirePipeline, agent.processorIsObserver ? [processor] : []); - const emit = (result: SpanResult, parent: Context, start: number, end: number) => { + const emit = (result: SpanResult, parent: Context, start: number, end: number, bindMedia?: (span: Span) => void) => { const canDeliver = () => agent.spanDelivery?.canForward(wireDestinationReady) === true; // Credential discovery is independent of which semantic copy wins delivery. let secrets: ReturnType; @@ -210,6 +220,7 @@ export function init(options: Options = {}): TracerProvider | undefined { if (secrets === null) failedCaptures.add(span.getContextSpan()); span.setStatus({ code: result.ok ? SpanStatusCode.OK : SpanStatusCode.ERROR, message: result.description }); const recording = span.isRecording(); + try { bindMedia?.(span); } catch { /* Media never affects semantic delivery. */ } span.end(end); submitted = recording; } catch { /* Internal failures are not application failures. */ } @@ -219,14 +230,14 @@ export function init(options: Options = {}): TracerProvider | undefined { // Adapters stay off unless explicitly selected; destination policy is not activation. agent.removeOutgoing = installOutgoingPropagation(outgoingPolicyForConfig(config), createOutgoingPrivacy(config.traceConfig, credentialsForConfig(config)), agent.statuses, warn, config.propagateOutgoingAdapters); if (config.wireCapture && enabled("wire", config)) agent.removeObservation = observeTransports(); - agent.removeLibraries = enableLibraries(config, options, instrumentationProvider, agent.statuses, warn, processor); + agent.removeLibraries = enableLibraries(config, { ...options, attachmentManager: agent.attachments }, instrumentationProvider, agent.statuses, warn, processor); // Stay outside the installed mapping adapters: Genkit can shorten an SDK- // truncated value when it scrubs a local path or identity. Inspect the native // span first so that its detached export copy cannot erase that loss signal. const mappedEnd = processor.onEnd.bind(processor); processor.onEnd = span => { rememberSpanValueLimit(span); mappedEnd(span); }; if (config.wireCapture && enabled("wire", config) && typeof globalThis.fetch === "function") { - agent.fetch = patchFetch(config.wireMaxBodyBytes, emit, [config.apiKey, config.secretKey].filter((v): v is string => !!v)); agent.statuses.push({ name: "wire", status: "instrumented" }); + agent.fetch = patchFetch(config.wireMaxBodyBytes, emit, [config.apiKey, config.secretKey].filter((v): v is string => !!v), agent.attachments); agent.statuses.push({ name: "wire", status: "instrumented" }); } else agent.statuses.push({ name: "wire", status: "disabled" }); agent.active = true; agent.beforeExit = () => { void forceFlush(); }; @@ -244,7 +255,7 @@ export function init(options: Options = {}): TracerProvider | undefined { try { if (failed.propagator && (propagation as any)._getGlobalPropagator?.() === failed.propagator) propagation.disable(); } catch { /* Leave application globals alone. */ } try { if (failed.ownsGlobalProvider && existingProvider() === failed.provider) trace.disable(); } catch { /* Leave app globals alone. */ } try { if (failed.contextManager && (context as any)._getContextManager?.() === failed.contextManager) context.disable(); } catch { /* Fail open. */ } - void withDeadline(async () => { await failed.processor?.shutdown(); }).catch(() => {}); + void withDeadline(async () => { await Promise.all([failed.processor?.shutdown(), failed.ownsAttachments ? failed.attachments?.shutdown() : undefined]); }).catch(() => {}); // Configuration is terminal for this process, even after failed startup. // Hooks are rolled back, but init never installs a second provider or loader. } @@ -255,16 +266,16 @@ export function report(): Report { sharedState(); if (!state) { const config = resolveConfig(); - return { active: false, config: maskConfig(config), integrations: inventory(config), collector: "not_tested (agent not initialized)" }; + return { active: false, config: maskConfig(config), integrations: inventory(config), collector: "not_tested (agent not initialized)", attachments: attachmentReport() }; } - return { active: state.active, config: maskConfig(state.config), integrations: state.statuses.map(s => ({ ...s })), collector: state.collector }; + return { active: state.active, config: maskConfig(state.config), integrations: state.statuses.map(s => ({ ...s })), collector: state.collector, attachments: attachmentReport(state.attachments) }; } export async function forceFlush(): Promise { sharedState(); try { await withDeadline(async () => { await Promise.all([...(state?.fetch?.pending ?? []), state?.spanDelivery?.drain()]); - await state?.processor?.forceFlush(); + await Promise.all([state?.processor?.forceFlush(), state?.attachments?.flush(2500)]); }); } catch { if (state?.collector !== "failed") warn("flush", "Trace flush failed; the application will continue."); } } @@ -284,7 +295,7 @@ export async function shutdown(): Promise { await withDeadline(async () => { await Promise.all([...(agent.fetch?.pending ?? []), agent.spanDelivery?.drain()]); // Never start a late flush after the outer deadline race has returned. - if (Date.now() < drainDeadline) await agent.processor?.forceFlush(); + if (Date.now() < drainDeadline) await Promise.all([agent.processor?.forceFlush(), agent.attachments?.flush(remaining(drainDeadline))]); }, remaining(drainDeadline)); } catch { warn("shutdown", "Trace shutdown failed; the application will continue."); } finally { @@ -294,7 +305,7 @@ export async function shutdown(): Promise { agent.spanDelivery?.revoke(); agent.convertedSpanDestination?.revoke(); try { agent.detach?.(); } catch { /* Leave the application's processors intact. */ } - try { await withDeadline(async () => { await agent.processor?.shutdown(); }, Math.min(2500, remaining(deadline))); } + try { await withDeadline(async () => { await Promise.all([agent.processor?.shutdown(), agent.ownsAttachments ? agent.attachments?.shutdown(Math.max(1, remaining(deadline))) : undefined]); }, Math.min(2500, remaining(deadline))); } catch { warn("shutdown", "Trace shutdown failed; the application will continue."); } } })(); diff --git a/typescript/packages/traceai_auto/src/libraries.ts b/typescript/packages/traceai_auto/src/libraries.ts index c2b8618b8..4b3ebf466 100644 --- a/typescript/packages/traceai_auto/src/libraries.ts +++ b/typescript/packages/traceai_auto/src/libraries.ts @@ -13,7 +13,11 @@ import { Config, enabled, Options } from "./config"; export type Status = "instrumented" | "pending" | "not_installed" | "missing_instrumentor" | "disabled" | "manual" | "failed"; export interface Integration { name: string; package: string; targets: string[]; modules: string[]; className?: string; manual?: string; enable?: string } -export interface IntegrationStatus { name: string; package?: string; targets?: string[]; status: Status; reason?: string } +/** Media capture is tracked apart from span coverage: an SDK span hook does not imply media capture, and vice versa. */ +export type MediaStatus = "provider_adapter" | "unqualified"; +export interface IntegrationStatus { name: string; package?: string; targets?: string[]; status: Status; reason?: string; media?: MediaStatus } +/** Integrations whose instrumentor captures media through the shared manager it receives as `attachments` (#682). */ +const MEDIA_ADAPTERS = new Map([["openai", "openAIAttachmentCapabilities"], ["anthropic", "anthropicAttachmentCapabilities"]]); export const integrations: Integration[] = catalog as Integration[]; export function appRequire() { return createRequire(join(process.cwd(), "__traceai_auto__.cjs")); } function resolvable(name: string): boolean { @@ -22,7 +26,7 @@ function resolvable(name: string): boolean { } } export function inventory(config: Config): IntegrationStatus[] { - return integrations.map(i => ({ name: i.name, package: i.package, targets: i.targets, + return integrations.map(i => ({ name: i.name, package: i.package, targets: i.targets, media: MEDIA_ADAPTERS.has(i.name) ? "provider_adapter" : "unqualified", status: !enabled(i.name, config) ? "disabled" : ![...i.targets, ...i.modules].some(resolvable) ? "not_installed" : i.manual && !i.enable ? "manual" : !resolvable(i.package) && i.name !== "openai" ? "missing_instrumentor" : "pending", ...(i.manual ? { reason: i.manual } : {}) })); } @@ -109,12 +113,14 @@ export function enableLibraries(config: Config, options: Options, provider: Trac // Keep this require inside the enabled branch: disabled integrations register no hooks. const path = entry.enable && resolvable(`${entry.package}/auto`) ? `${entry.package}/auto` : entry.package; const exports = resolvable(path) ? appRequire()(path) : entry.name === "openai" ? require("@traceai/openai") : appRequire()(path); + // Version aware: an installed instrumentor without the capability descriptor captures no media. + if (status.media === "provider_adapter" && !exports?.[MEDIA_ADAPTERS.get(entry.name)!]) status.media = "unqualified"; if (entry.enable) { const enable = exports.enable ?? exports[entry.enable]; if (typeof enable !== "function") { status.status = "manual"; status.reason = "Installed instrumentor has no zero-argument enable entry; wire capture remains available"; continue; } // Enable entries receive the selected neutral destination explicitly. // Never synthesize FI credentials to unlock a framework integration. - const result = enable({ tracerProvider: provider, spanProcessor, traceConfig: options.traceConfig }); + const result = enable({ tracerProvider: provider, spanProcessor, traceConfig: options.traceConfig, attachments: options.attachmentManager }); if (result && typeof result.setTracerProvider === "function") result.setTracerProvider(provider); status.status = result === false || result === undefined ? "manual" : result?.status ?? "instrumented"; status.reason = result?.reason; @@ -123,7 +129,7 @@ export function enableLibraries(config: Config, options: Options, provider: Trac } const Instrumentor = exports[entry.className!]; if (typeof Instrumentor !== "function") throw new Error("Missing exported Instrumentation class"); - instance = new Instrumentor({ instrumentationConfig: { enabled: false }, traceConfig: options.traceConfig, tracerProvider: provider, exclusiveProcessor: false }); + instance = new Instrumentor({ instrumentationConfig: { enabled: false }, traceConfig: options.traceConfig, tracerProvider: provider, exclusiveProcessor: false, attachments: options.attachmentManager }); const known = (instance._modules ?? []).map((definition: any) => ({ definition, version: moduleVersion(definition.name) })).filter((item: any) => item.version); if (known.length && known.every(({ definition, version }: any) => !definition.supportedVersions.some((range: string) => satisfies(version, range, { includePrerelease: definition.includePrerelease })))) { instance.disable(); status.status = "failed"; status.reason = "Installed target version is outside the instrumentor's supported range"; diff --git a/typescript/packages/traceai_auto/src/wire/attachments.ts b/typescript/packages/traceai_auto/src/wire/attachments.ts new file mode 100644 index 000000000..19c794305 --- /dev/null +++ b/typescript/packages/traceai_auto/src/wire/attachments.ts @@ -0,0 +1,225 @@ +import { createContextKey, trace, type Context, type Span } from "@opentelemetry/api"; +import type { + AttachmentCapturePolicy, AttachmentCorrelation, AttachmentDeferredCorrelation, AttachmentManager, AttachmentPurpose, AttachmentStream, +} from "@traceai/fi-core/attachments"; +import type { FetchMediaObserver, FetchMediaSink } from "./fetch-runtime"; +import { header } from "./core"; + +const MEDIA_POLICY = createContextKey("traceai.auto.attachment_policy.v1"); +/** Supply the integration's capture restrictions; inherited denials always win. */ +export function withAttachmentCapturePolicy(parent: Context, policy: AttachmentCapturePolicy): Context { + const previous = parent.getValue(MEDIA_POLICY) as AttachmentCapturePolicy | undefined; + const merged = { ...policy }; + if (Object.values(policy).some(value => value !== undefined && typeof value !== "boolean")) merged.captureContent = false; + for (const key of ["captureContent", "recordInputs", "recordOutputs", "requested"] as const) { + if (previous?.[key] === false) merged[key] = false; + else if (merged[key] === undefined) merged[key] = previous?.[key]; + } + for (const key of ["muted", "temporaryChat", "modelInput", "containsInput", "containsOutput", "errorDetail", "newRecording"] as const) { + if (previous?.[key]) merged[key] = true; + } + return parent.setValue(MEDIA_POLICY, Object.freeze(merged)); +} + +// `createContextKey` is `Symbol.for`, so adapters in other packages (for example the ElevenLabs +// instrumentation) can set the same marker without importing auto. +const MEDIA_OWNER = createContextKey("traceai.auto.attachment_owner.v1"); +/** An adapter that captures media owns it independently of semantic span coverage. */ +export function withAttachmentOwnership(parent: Context): Context { return parent.setValue(MEDIA_OWNER, true); } + +const IMAGE_MIMES = new Set(["image/png", "image/jpeg", "image/webp"]); +const AUDIO_MIMES = new Set(["audio/wav", "audio/ogg", "audio/mpeg", "audio/mp4", "audio/webm", "audio/flac"]); +/** Consumed incremental audio is reserved up front; an object larger than this is discarded whole (too_large). */ +export const WIRE_STREAM_MAX_BYTES = 4 * 1024 * 1024; +const WIRE_STREAM_FALLBACK_BYTES = 1024 * 1024; +/** Request JSON is parsed only from the wire path's own bounded copy, and only up to this size. */ +const MAX_JSON_BYTES = 4 * 1024 * 1024; +/** + * How long a wire-only capture may wait for its wire span, which exists only once the response completed. The core + * maximum, because a transcription of a long recording can take minutes; every path on which no span will be emitted + * cancels the target at once (`settle`), so this bound is only the backstop (for example a hook removed mid-flight). + */ +export const WIRE_DEFERRED_BIND_TIMEOUT_MS = 300000; + +type Operation = "speech" | "transcription" | "image_edit" | "image_generation" | "chat" | "responses"; +/** Allowlisted protocol entries: exact HTTPS origin and path. A compatible path on another host is not authority to capture. */ +export const WIRE_MEDIA_ROUTES: ReadonlyArray> = Object.freeze([ + { host: "api.openai.com", path: "/v1/audio/speech", operation: "speech" }, + { host: "api.openai.com", path: "/v1/audio/transcriptions", operation: "transcription" }, + { host: "api.openai.com", path: "/v1/audio/translations", operation: "transcription" }, + { host: "api.openai.com", path: "/v1/images/edits", operation: "image_edit" }, + { host: "api.openai.com", path: "/v1/images/variations", operation: "image_edit" }, + { host: "api.openai.com", path: "/v1/images/generations", operation: "image_generation" }, + { host: "api.openai.com", path: "/v1/chat/completions", operation: "chat" }, + { host: "api.openai.com", path: "/v1/responses", operation: "responses" }, + { host: "api.deepgram.com", path: "/v1/speak", operation: "speech" }, +].map(route => Object.freeze(route)) as { host: string; path: string; operation: Operation }[]); +function operation(method: string, url: string): Operation | undefined { + try { + const u = new URL(url); + if (method.toUpperCase() !== "POST" || u.protocol !== "https:" || u.username || u.password || (u.port && u.port !== "443")) return; + return WIRE_MEDIA_ROUTES.find(route => route.host === u.hostname && route.path === u.pathname)?.operation; + } catch { /* Unknown destinations never acquire media ownership. */ } +} +type OpenAIAdapter = typeof import("@traceai/openai").captureOpenAIAttachments; +let openAIAdapter: OpenAIAdapter | null | undefined; +/** Loaded only when attachments are active, so a disabled auto loads no extra instrumentation module. */ +function captureOpenAIAttachments(...args: Parameters): void { + if (openAIAdapter === undefined) { + try { openAIAdapter = (require("@traceai/openai") as typeof import("@traceai/openai")).captureOpenAIAttachments ?? null; } + catch { openAIAdapter = null; } + } + // Without the shared provider adapter (an older @traceai/openai), JSON media is not captured: no private parser. + if (typeof openAIAdapter === "function") openAIAdapter(...args); +} +function mediaType(value: string): string { return value.split(";")[0].trim().toLowerCase(); } +function modelSpan(parent: Context): Span | undefined { + const span = trace.getSpan(parent); + // Do not relink wire media to an unrelated ambient HTTP/tool/root span. + const readable = span as (Span & { attributes?: Record; span?: { attributes?: Record } }) | undefined; + const attributes = readable?.attributes ?? readable?.span?.attributes; + const kind = attributes?.["gen_ai.span.kind"] ?? attributes?.["fi.span.kind"]; + return span?.isRecording() && ["LLM", "EMBEDDING"].includes(String(kind)) ? span : undefined; +} +/** True when the manager could admit anything at all. A disabled (default) manager makes auto behave exactly as before. */ +export function attachmentsActive(manager: AttachmentManager | undefined): manager is AttachmentManager { + try { return !!manager && manager.status().state !== "disabled"; } catch { return false; } +} +function parseJson(bytes: Uint8Array | undefined): unknown { + if (!bytes || !bytes.byteLength || bytes.byteLength > MAX_JSON_BYTES) return undefined; + try { return JSON.parse(new TextDecoder().decode(bytes)); } catch { return undefined; } +} +function imageFormat(value: unknown): "png" | "jpeg" | "webp" | undefined { + return value === "png" || value === "jpeg" || value === "webp" ? value : undefined; +} + +/** + * Wire media observation for allowlisted protocol entries (A03/A04). + * + * - Capture happens only against a REAL span: a recording native model span whose integration supplied an explicit + * capture policy, or the wire span itself once it is emitted (`bind`). The ambient parent is never used. + * - Every source passes `manager.preflight` (the full deny-wins lattice plus budget) BEFORE it is read, copied or + * decoded. JSON media goes through the OpenAI provider adapter (#682), so an SDK span and a wire observation of the + * same span and position share one occurrence. + * - No reader, clone, tee, filesystem, URL retrieval or recorder: only values and chunks the application consumed. + * - Wire-only FormData files and wire-only incremental speech reads are admitted against a deferred target before the + * wire span exists (the core holds bounded bytes and reads or sends nothing). `bind` binds the wire span, where policy + * and sampling are evaluated again; a span that never comes (`settle`), a late bind or a timeout uploads nothing. + * JSON media never uses the deferred target: it is captured at `bind`, from the wire path's own bounded copies. + */ +export function wireAttachmentObserver(manager: AttachmentManager, parent: Context, method: string, url: string): FetchMediaObserver | undefined { + if (parent.getValue(MEDIA_OWNER) || manager.status().state !== "ready") return; + const op = operation(method, url); + if (!op) return; + const native = modelSpan(parent); + const capture = parent.getValue(MEDIA_POLICY) as AttachmentCapturePolicy | undefined; + // An external native instrumentor can have stricter per-call privacy than auto. + // It must supply its policy, not silently inherit an unrelated manager opt-in. + if (native && !capture) return; + let correlation: AttachmentCorrelation | undefined = native ? manager.correlation(native) : undefined; + let outputObserved = false, inputObserved = false, format: ReturnType; + // Wire-only: one deferred target per call, created only when a FormData file or an incremental read needs it. + let deferred: AttachmentDeferredCorrelation | undefined, settled = false; + /** The capture target now: the native or bound wire span, else (until the wire span exists or never will) the deferred one. */ + const target = (): AttachmentCorrelation | undefined => { + if (correlation || settled) return correlation; + try { deferred ??= manager.deferredCorrelation({ bindTimeoutMs: WIRE_DEFERRED_BIND_TIMEOUT_MS }); } catch { return undefined; } + return deferred; + }; + + /** Preflight at the exact size, then attach. The source is touched only after admission. */ + const attachExact = (size: number, mimeType: string, direction: "input" | "output", purpose: AttachmentPurpose, + source: () => { kind: "Bytes"; data: Uint8Array } | { kind: "Blob"; data: Blob }, sourceKind: "Bytes" | "Blob") => { + if (!(size > 0)) return; + const into = target(); + if (!into) return; + const ticket = manager.preflight(into, { mimeType, direction, purpose, maxBytes: size, sourceKind, capture: { ...capture, ...(direction === "input" ? { modelInput: true } : {}) } }); + if (!ticket.admitted) return; + try { ticket.attach(source() as never); } catch { ticket.release(); } + }; + const captureJsonInput = (body: unknown) => { + if (inputObserved || !correlation || !body || typeof body !== "object") return; + inputObserved = true; + format = imageFormat((body as Record).output_format); + if (op === "chat" || op === "responses") captureOpenAIAttachments(manager, correlation, body, { operation: op, direction: "input", capture }); + }; + const jsonOutput = (value: unknown) => { + if (outputObserved || !correlation || !value || typeof value !== "object" || !(op === "image_generation" || op === "image_edit" || op === "responses")) return; + outputObserved = true; + // Typed provider adapter only (#682): no recursive scraping, no URL retrieval, decode after admission. + captureOpenAIAttachments(manager, correlation, value, { operation: op === "responses" ? "responses" : "images", direction: "output", capture, imageFormat: format }); + }; + const formData = (body: FormData) => { + let entries = 0; + for (const [name, value] of body.entries()) { + if (++entries > 64) break; + if (typeof value === "string") { if (name === "output_format") format = imageFormat(value); continue; } + const type = mediaType(value.type); + if (op === "transcription" && name === "file" && AUDIO_MIMES.has(type)) { + attachExact(value.size, type, "input", "model_input", () => ({ kind: "Blob", data: value }), "Blob"); + } else if (op === "image_edit" && ["image", "image[]", "mask"].includes(name) && IMAGE_MIMES.has(type)) { + attachExact(value.size, type, "input", "model_input", () => ({ kind: "Blob", data: value }), "Blob"); + } + } + }; + return { + bind(span, request, response) { + // The emitter supplies its actual identity before ending, never a later ambient span. + if (correlation || settled) return; + settled = true; + // Pending wire-only captures are re-evaluated against this span now (policy, environment, sampling). `false` means + // late, cancelled or an invalid span identity: cancel (a no-op once cancelled) so nothing waits for the deadline. + try { if (deferred && !deferred.bind(span)) deferred.cancel(); } catch { /* never throws; fail closed */ } + correlation = manager.correlation(span); + // JSON uses the wire path's own bounded copies; nothing was retained for media before this point. + if (op === "chat" || op === "responses" || op === "image_generation") captureJsonInput(parseJson(request?.body)); + // Output JSON, however the application consumed it (json(), text(), arrayBuffer()); the same span and + // position captured again from json() below resolves to the first occurrence in the adapter's registry. + if (response && (response.status ?? 0) >= 200 && (response.status ?? 0) < 300 && !response.truncated && mediaType(header(response.headers, "content-type")) === "application/json") jsonOutput(parseJson(response.body)); + }, + settle() { + // No wire span will be bound (unmatched call; a thrown, rejected or aborted call, including a response body that + // failed or was aborted while read, whose error wire span binds nothing; delivery gate; build failure): end now. + if (correlation || settled) return; + settled = true; + try { deferred?.cancel(); } catch { /* never throws */ } + }, + request(body) { + // FormData files are admitted before the span exists (deferred target when wire-only); JSON waits for `bind`. + if (body instanceof FormData && (op === "transcription" || op === "image_edit")) formData(body); + else if (!correlation) return; + else if (typeof body === "string" && (op === "chat" || op === "responses" || op === "image_generation") && body.length <= MAX_JSON_BYTES) { + try { captureJsonInput(JSON.parse(body)); } catch { /* Not JSON: nothing to capture. */ } + } + }, + consumed(mimeType, successful, contentLength): FetchMediaSink | undefined { + // Incremental reads need the target before the first chunk: the native span, else the deferred wire target (A04). + const type = mediaType(mimeType); + if (outputObserved || !successful || op !== "speech" || !AUDIO_MIMES.has(type)) return; + const into = target(); + if (!into) return; + const open = (maxBytes: number) => manager.startAttachmentStream(into, { mimeType: type, direction: "output", purpose: "model_output", maxBytes, capture }); + const exact = contentLength !== undefined && contentLength > 0 && contentLength <= WIRE_STREAM_MAX_BYTES ? contentLength : undefined; + let stream: AttachmentStream = open(exact ?? WIRE_STREAM_MAX_BYTES); + // The negotiated upload limit or the owned budget can be below the default reservation. + if (!exact && (stream.reason === "too_large" || stream.reason === "queue_full")) stream = open(WIRE_STREAM_FALLBACK_BYTES); + if (stream.reason !== "queued") return; + outputObserved = true; + return { write: chunk => stream.tryWrite(chunk), end: () => { stream.finish(); }, abort: () => stream.abort("incomplete") }; + }, + materialized(value, methodName, mimeType, successful) { + if (outputObserved || !successful) return; + const type = mediaType(mimeType); + if (op === "speech" && AUDIO_MIMES.has(type)) { + if (methodName === "blob" && value instanceof Blob) { + outputObserved = true; + attachExact(value.size, type, "output", "model_output", () => ({ kind: "Blob", data: value }), "Blob"); + } else if (methodName === "arrayBuffer" && value instanceof ArrayBuffer) { + outputObserved = true; + // The view (and the core's synchronous copy) exists only after admission. + attachExact(value.byteLength, type, "output", "model_output", () => ({ kind: "Bytes", data: new Uint8Array(value) }), "Bytes"); + } + } else if (correlation && methodName === "json" && type === "application/json") jsonOutput(value); + }, + }; +} diff --git a/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts b/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts index 0f80b1d57..ada04d071 100644 --- a/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts +++ b/typescript/packages/traceai_auto/src/wire/fetch-runtime.ts @@ -2,7 +2,21 @@ import { build, checkRequest, header, isBinary, match, RequestData, ResponseData import { encodeBytes, byteLength, concatBytes } from "./bytes"; import { BodyDecoder, decodeResponse, responseEncoding } from "./encoding"; import { streamingMilestones } from "./stream-timing"; +import type { Span } from "@opentelemetry/api"; +/** Receives only chunks the application already consumed; never pulls. */ +export interface FetchMediaSink { write(chunk: Uint8Array): boolean; end(): void; abort(): void } +export interface FetchMediaObserver { + /** The wire span, before it ends, with the wire path's own bounded request and response copies. */ + bind(span: Span, request?: RequestData, response?: ResponseData): void; + /** No wire span will be bound for this call: end anything held for it now (never waits for a timeout). */ + settle?(): void; + request(body: BodyInit | null | undefined): void; + /** The application started incremental consumption of a response body (reader or async iterator). */ + consumed?(mimeType: string, successful: boolean, contentLength: number | undefined): FetchMediaSink | undefined; + materialized(value: unknown, method: "arrayBuffer" | "blob" | "json", mimeType: string, successful: boolean): void; +} export interface FetchContext { + media?(parent: T, method: string, url: string): FetchMediaObserver | undefined; active(): T; skip(parent: T): boolean; run(parent: T, fn: () => R): R; observe?(promise: Promise, ok: (value: R) => void, fail: (error: unknown) => void): void; observeRequest?(request: RequestData): void; @@ -91,11 +105,63 @@ function observeMethod(object: any, name: PropertyKey, after: (result: any, args } catch { /* Frozen/custom runtime objects stay untouched. */ } } export interface FetchHook { remove(): void; pending: Set> } -export function createFetchHook(max: number, emit: (result: SpanResult, parent: T, start: number, end: number) => void, scope: FetchContext, secrets: string[] = [], decodeBody?: BodyDecoder): FetchHook { +export function createFetchHook(max: number, emit: (result: SpanResult, parent: T, start: number, end: number, bindMedia?: (span: Span) => void) => void, scope: FetchContext, secrets: string[] = [], decodeBody?: BodyDecoder): FetchHook { const original = globalThis.fetch; const observe = scope.observe ?? tap; const pending = new Set>(); let active = true; + /** + * A04: forward chunks the application reads from THIS response body (reader `read()` or async-iterator `next()`) + * to the media sink, after the application's own promise fulfils. No read, clone, tee or pipe is ever issued here. + * The sink opens only when the application starts consuming; piping, teeing or a clone of the body is unsupported + * and aborts the object, so nothing partial is uploaded as if complete. + */ + function observeConsumption(response: Response, observer: FetchMediaObserver, mimeType: string, responseHeaders: Record): void { + let stream: ReadableStream | null; + try { stream = response.body; } catch { return; } + if (!stream) return; + const body = stream; + let sink: FetchMediaSink | undefined, state: "idle" | "open" | "closed" = "idle"; + const length = responseHeaders["content-length"]; + const declared = length && /^\d+$/.test(length) && !header(responseHeaders, "content-encoding") ? Number(length) : undefined; + const open = () => { + if (state !== "idle" || !active) return; + try { sink = observer.consumed!(mimeType, response.ok, declared); } catch { sink = undefined; } + state = sink ? "open" : "closed"; + }; + const close = (complete: boolean) => { + if (state !== "open") { state = "closed"; return; } + state = "closed"; + try { if (complete && active) sink!.end(); else sink!.abort(); } catch { /* Media never reaches the application. */ } + }; + const read = (item: ReadableStreamReadResult) => { + if (state !== "open") return; + if (item.done) { close(true); return; } + if (!(item.value instanceof Uint8Array) || !active) { close(false); return; } + // The core copies synchronously, so a reused BYOB buffer cannot change uploaded bytes. + try { if (!sink!.write(item.value)) close(false); } catch { close(false); } + }; + const readers = new WeakSet(); + observeMethod(body, "getReader", reader => { + if (readers.has(reader)) return; readers.add(reader); + open(); + observeMethod(reader, "read", promise => observe(promise, read, () => close(false))); + observeMethod(reader, "cancel", () => close(false)); + observeMethod(reader, "releaseLock", () => close(false)); + }); + const iterator = (it: any) => { + if (readers.has(it)) return; readers.add(it); + open(); + observeMethod(it, "next", promise => observe(promise, read, () => close(false))); + // An early return (break) leaves the object incomplete, cancelled or not. + observeMethod(it, "return", () => close(false)); + }; + observeMethod(body, "values", it => iterator(it)); + observeMethod(body, Symbol.asyncIterator, it => iterator(it)); + observeMethod(body, "cancel", () => close(false)); + for (const name of ["tee", "pipeTo", "pipeThrough"]) observeMethod(body, name, () => close(false)); + observeMethod(response, "clone", () => close(false)); + } const patched = new Proxy(original, { apply(target, receiver, argArray: [RequestInfo | URL, RequestInit?]) { if (!active) return Reflect.apply(target, receiver, argArray); @@ -114,11 +180,13 @@ export function createFetchHook(max: number, emit: (result: SpanResult, paren try { skip = scope.skip(parent); } catch { /* Broken context still observes headers/URL. */ } if (skip) return passThrough(); const capture = () => { try { return scope.capture?.(parent) !== false; } catch { return false; } }; + let media: FetchMediaObserver | undefined; let req: RequestData | undefined, found: ReturnType, initial: ReturnType; const inspect = () => { const [input, init] = argArray, info = head(); observeRequest(info); // Every route, not just catalog-matched AI calls. found = match(info.method, info.url); + try { media = scope.media?.(parent, info.method, info.url); media?.request(init?.body); } catch { /* Independent, fail-open media admission. */ } // Covered model operations observe URL/headers only, as before. Selection // is delayed until catalog match; unsupported calls cannot choose a route. if (found && capture()) { @@ -142,10 +210,31 @@ export function createFetchHook(max: number, emit: (result: SpanResult, paren const span = found && req && initial && build(found, req, { headers: {}, size: 0, error: hostError instanceof Error ? hostError : new Error("Transport failure") }, initial); if (span) emit(span, parent, start, errorAt); } catch { /* Preserve the host error identity. */ } + finally { try { media?.settle?.(); } catch { /* Media never reaches the application. */ } } throw hostError; } - try { inspect(); } catch { return result; } - if (!found || !req || !initial) return result; + const settle = () => { try { media?.settle?.(); } catch { /* Media never reaches the application. */ } }; + try { inspect(); } catch { settle(); return result; } + if (media) { + const observer = media, observed = new WeakSet(); + const decorate = (response: Response) => { + if (observed.has(response)) return; + observed.add(response); + const responseHeaders = headersOf(response.headers); + const mimeType = header(responseHeaders, "content-type"); + for (const method of ["arrayBuffer", "blob", "json"] as const) { + observeMethod(response, method, promise => observe(promise, value => { + if (active) observer.materialized(value, method, mimeType, response.ok); + }, () => {})); + } + if (observer.consumed) observeConsumption(response, observer, mimeType, responseHeaders); + // Only follow clones explicitly created by the application. Never call clone ourselves. + observeMethod(response, "clone", clone => decorate(clone)); + }; + observe(result, decorate, () => {}); + } + // No wire span for this call: a deferred wire-only capture can never bind (a native target is unaffected). + if (!found || !req || !initial) { settle(); return result; } const matched = found, request = req, parsed = initial; let resolve!: () => void; const done = new Promise(r => { resolve = r; }); @@ -154,8 +243,13 @@ export function createFetchHook(max: number, emit: (result: SpanResult, paren if (finished) return; finished = true; const complete = () => { if (audio) { response.audioOutputBytes = audio.bytes; response.body = undefined; } - try { const span = build(matched, request, response, parsed); if (span) emit(span, parent, start, end); } catch { /* Capture must not reach application code. */ } - finally { resolve(); pending.delete(done); } + // A transport failure or abort (fetch rejected, or the response body failed or was aborted while read) still + // emits an error wire span, but nothing held is bound to it, as on the synchronous-throw path: the held + // capture is cancelled below. A completed HTTP error response (4xx/5xx) is a finished call and binds. + const bindMedia = media && !response.error ? (span: Span) => media!.bind(span, request, response) : undefined; + try { const span = build(matched, request, response, parsed); if (span) emit(span, parent, start, end, bindMedia); } catch { /* Capture must not reach application code. */ } + // `emit` binds only when it starts the wire span; any other outcome cancels what was held for it. + finally { settle(); resolve(); pending.delete(done); } }; // Only completed responses awaiting bounded decoding are flushable, never unread streams. // Compressed JSON/SSE audio must be decoded before its base64 length scanner. diff --git a/typescript/packages/traceai_auto/src/wire/fetch.ts b/typescript/packages/traceai_auto/src/wire/fetch.ts index 95f2597e2..578af28e3 100644 --- a/typescript/packages/traceai_auto/src/wire/fetch.ts +++ b/typescript/packages/traceai_auto/src/wire/fetch.ts @@ -1,7 +1,9 @@ -import { context, createContextKey, trace, Context } from "@opentelemetry/api"; +import { context, createContextKey, trace, Context, type Span } from "@opentelemetry/api"; import { autoModelResponsibility, autoModelWireAllowed, hasAutoModelCoverage } from "@traceai/fi-core"; import { isTracingSuppressed } from "@opentelemetry/core"; import { createFetchHook } from "./fetch-runtime"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import { attachmentsActive, wireAttachmentObserver } from "./attachments"; import { decodeNodeBody } from "./encoding-node"; import { observePromise, preparePromiseObservation } from "./promise-node"; import type { SpanResult } from "./core"; @@ -35,9 +37,13 @@ function wireAllowed(ctx: Context, phase: "guard" | "inspect" | "request"): bool export function shouldSkip(ctx: Context, consultOperation = true): boolean { return isTracingSuppressed(ctx) || !!ctx.getValue(IN_FLIGHT) || !wireAllowed(ctx, consultOperation ? "inspect" : "guard"); } -export function patchFetch(max: number, emit: (result: SpanResult, parent: Context, start: number, end: number) => void, secrets: string[] = []) { +export function patchFetch(max: number, emit: (result: SpanResult, parent: Context, start: number, end: number, bindMedia?: (span: Span) => void) => void, secrets: string[] = [], attachments?: AttachmentManager) { preparePromiseObservation(); - return createFetchHook(max, emit, { active: () => context.active(), skip: parent => shouldSkip(parent, false), + // With attachments OFF (the default) both decisions are exactly the pre-attachment ones. When ON, a call that + // native coverage keeps out of wire spans still reaches media observation; span capture keeps its own gate below. + return createFetchHook(max, emit, { active: () => context.active(), + skip: parent => attachmentsActive(attachments) ? isTracingSuppressed(parent) || !!parent.getValue(IN_FLIGHT) : shouldSkip(parent, false), + media: attachments ? (parent, method, url) => attachmentsActive(attachments) ? wireAttachmentObserver(attachments, parent, method, url) : undefined : undefined, // Same context decision, now after supported URL matching. Only the emit // completion boundary can commit wire; credential observation stays separate. capture: parent => wireAllowed(parent, "request"), observe: observePromise, observeRequest: observeOutgoing, diff --git a/typescript/packages/traceai_auto/test/attachment-adapters.test.ts b/typescript/packages/traceai_auto/test/attachment-adapters.test.ts new file mode 100644 index 000000000..0be17ab23 --- /dev/null +++ b/typescript/packages/traceai_auto/test/attachment-adapters.test.ts @@ -0,0 +1,305 @@ +import { spawnSync } from "node:child_process"; +import { resolve } from "node:path"; + +// A02: auto hands the ONE selected manager to the instrumentors it enables, as their `attachments` option. +// A real OpenAI SDK call (loopback only, synthetic image) must upload exactly once, from the native adapter, bound to +// the native LLM span; the wire layer must not upload it again. Runs in a child process because init is process-global. +const root = resolve(__dirname, ".."); +function run(withManager: boolean) { + const code = ` + const assert = require('node:assert/strict'); + const http = require('node:http'); + const { randomUUID } = require('node:crypto'); + const { InMemorySpanExporter } = require('@opentelemetry/sdk-trace-base'); + const { AttachmentManager, createAttachmentHttpTransport } = require('@traceai/fi-core/attachments'); + const auto = require('./dist/src/index.js'); + const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII='; + (async () => { + const initialized = [], uploaded = []; + const scope = { organization_id: randomUUID(), workspace_id: randomUUID(), project_id: randomUUID() }; + const expiry = () => new Date(Date.now() + 3600000).toISOString(); + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const c of req) chunks.push(c); const bytes = Buffer.concat(chunks); + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('attachment-sessions')) { + res.end(JSON.stringify({ schema_version: 1, scope, capture_session_id: JSON.parse(bytes).capture_session_id, + policy: { version: 'fixture-v1', enabled: true, allowed_sources: ['Bytes', 'Blob'], allowed_modalities: ['image', 'audio'], require_media_redaction: false, + require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false }, + limits: { max_upload_bytes: 1048576, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, + active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, + upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }, + expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); + } else if (req.url.endsWith('/uploads')) { + const body = JSON.parse(bytes); initialized.push(body); res.statusCode = 201; + res.end(JSON.stringify({ upload_id: body.attachment_id, attachment_id: body.attachment_id, occurrence_id: body.occurrence_id, state: 'pending_upload', + reserved_bytes: body.declared_bytes, expires_at: expiry(), transfer_mode: 'authenticated_stream', write_capability: 'synthetic-upload-only' })); + } else if (req.url.endsWith('/content')) { + uploaded.push(bytes); res.end(JSON.stringify({ upload_id: req.url.split('/').at(-2), state: 'pending_upload', reason: 'queued' })); + } else if (req.url.includes('/tracer/')) { + res.statusCode = 202; res.end(JSON.stringify({ upload_id: req.url.split('/').at(-2), state: 'validating', reason: 'binding_pending' })); + } else { + // The synthetic model API. + res.end(JSON.stringify({ id: 'chatcmpl-synthetic', object: 'chat.completion', created: 1, model: 'gpt-4o', + choices: [{ index: 0, finish_reason: 'stop', message: { role: 'assistant', content: 'synthetic' } }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })); + } + }); + await new Promise(r => server.listen(0, '127.0.0.1', r)); + const origin = 'http://127.0.0.1:' + server.address().port; + const client = async req => { + const body = req.body instanceof Blob ? Buffer.from(await req.body.arrayBuffer()) : req.body; + return new Promise((resolve, reject) => { + const out = http.request(origin + new URL(req.url).pathname, { method: req.method, headers: req.headers, signal: req.signal }, response => { + const c = []; response.on('data', d => c.push(d)); response.on('end', () => resolve({ status: response.statusCode, headers: {}, body: Buffer.concat(c) })); + }); out.on('error', reject); out.end(body); + }); + }; + const manager = new AttachmentManager({ enabled: true, projectName: 'synthetic', projectType: 'observe', environment: () => ({}), + transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic' }, client }) }); + assert.equal(await manager.ready(), true); + const exporter = new InMemorySpanExporter(); + const options = { exporter, instrumentations: ['openai', 'wire'], configFile: '/nonexistent', apiKey: '', secretKey: '' }; + if (${withManager}) options.attachmentManager = manager; + auto.init(options); + assert.equal(auto.getAttachmentManager() === manager, ${withManager}); + const OpenAI = require('openai').default ?? require('openai').OpenAI; + const sdk = new OpenAI({ apiKey: 'synthetic-provider-key', baseURL: origin + '/v1' }); + await sdk.chat.completions.create({ model: 'gpt-4o', messages: [{ role: 'user', content: [{ type: 'text', text: 'synthetic' }, { type: 'image_url', image_url: { url: 'data:image/png;base64,' + PNG } }] }] }); + await auto.forceFlush(); await manager.flush(2000); + const openai = auto.report().integrations.find(row => row.name === 'openai'); + assert.equal(openai.status, 'instrumented'); assert.equal(openai.media, 'provider_adapter'); + const spans = exporter.getFinishedSpans().filter(s => s.attributes['gen_ai.span.kind'] === 'LLM' || s.attributes['fi.span.kind'] === 'LLM'); + process.stdout.write(JSON.stringify({ uploads: uploaded.map(b => b.toString('base64')), parents: initialized.map(r => r.association.parent.span_id), + purposes: initialized.map(r => r.purpose), llmSpans: spans.map(s => s.spanContext().spanId) })); + await auto.shutdown(); await manager.shutdown(); server.closeAllConnections(); server.close(); + })().catch(error => { process.stderr.write(String(error && error.stack || error)); process.exitCode = 1; }); + `; + return spawnSync(process.execPath, ["-e", code], { + cwd: root, encoding: "utf8", timeout: 30000, + env: { ...process.env, FI_ATTACHMENTS_ENABLED: "", FI_API_KEY: "", FI_SECRET_KEY: "", FI_CONFIG_FILE: "/nonexistent", FI_AUTO_DEBUG: "false" }, + }); +} +const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + +test("the selected manager reaches the OpenAI instrumentor as `attachments`: one upload, bound to the native span", () => { + const result = run(true); + expect(result.stderr).toBe(""); expect(result.status).toBe(0); + const out = JSON.parse(result.stdout); + expect(out.uploads).toEqual([png]); + expect(out.purposes).toEqual(["model_input"]); + expect(out.llmSpans).toHaveLength(1); expect(out.parents).toEqual(out.llmSpans); +}); +test("without an enabled manager (the default) the same call uploads nothing", () => { + const result = run(false); + expect(result.stderr).toBe(""); expect(result.status).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ uploads: [] }); +}); + +// Shared loopback double for the attachment control plane (synthetic credentials and scope only). +const controlServer = ` + const http = require('node:http'); + const { randomUUID } = require('node:crypto'); + const initialized = [], uploaded = [], transfers = []; + const scope = { organization_id: randomUUID(), workspace_id: randomUUID(), project_id: randomUUID() }; + const expiry = () => new Date(Date.now() + 3600000).toISOString(); + function control(req, bytes, res) { + transfers.push(req.url); res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('attachment-sessions')) { + res.end(JSON.stringify({ schema_version: 1, scope, capture_session_id: JSON.parse(bytes).capture_session_id, + policy: { version: 'fixture-v1', enabled: true, allowed_sources: ['Bytes', 'Blob'], allowed_modalities: ['image', 'audio', 'document'], require_media_redaction: false, + require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false }, + limits: { max_upload_bytes: 1048576, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, + active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, + upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }, + expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); + } else if (req.url.endsWith('/uploads')) { + const body = JSON.parse(bytes); initialized.push(body); res.statusCode = 201; + res.end(JSON.stringify({ upload_id: body.attachment_id, attachment_id: body.attachment_id, occurrence_id: body.occurrence_id, state: 'pending_upload', + reserved_bytes: body.declared_bytes, expires_at: expiry(), transfer_mode: 'authenticated_stream', write_capability: 'synthetic-upload-only' })); + } else if (req.url.endsWith('/content')) { + uploaded.push(bytes); res.end(JSON.stringify({ upload_id: req.url.split('/').at(-2), state: 'pending_upload', reason: 'queued' })); + } else { res.statusCode = 202; res.end(JSON.stringify({ upload_id: req.url.split('/').at(-2), state: 'validating', reason: 'binding_pending' })); } + } + function controlClient(origin) { + return async req => { + const body = req.body instanceof Blob ? Buffer.from(await req.body.arrayBuffer()) : req.body; + return new Promise((resolve, reject) => { + const out = http.request(origin + new URL(req.url).pathname, { method: req.method, headers: { ...req.headers, 'x-attachment-control': '1' }, signal: req.signal }, response => { + const c = []; response.on('data', d => c.push(d)); response.on('end', () => resolve({ status: response.statusCode, headers: {}, body: Buffer.concat(c) })); + }); out.on('error', reject); out.end(body); + }); + }; + } +`; +function wavBase64(samples = 80) { + const b = Buffer.alloc(44 + samples * 2); b.write("RIFF"); b.writeUInt32LE(b.length - 8, 4); b.write("WAVEfmt ", 8); + b.writeUInt32LE(16, 16); b.writeUInt16LE(1, 20); b.writeUInt16LE(1, 22); b.writeUInt32LE(8000, 24); + b.writeUInt32LE(16000, 28); b.writeUInt16LE(2, 32); b.writeUInt16LE(16, 34); b.write("data", 36); b.writeUInt32LE(samples * 2, 40); + for (let i = 0; i < samples; i++) b.writeInt16LE(Math.round(8000 * Math.sin(i / 3)), 44 + i * 2); + return b.toString("base64"); +} + +// auto.init() with an ENABLED manager against allowlisted HTTPS hosts, wire only (no native instrumentor). The +// application's fetch is redirected to loopback BEFORE init, so auto's wire hook sees the real provider URLs. +function runWire(env: Record = {}) { + const code = ` + const assert = require('node:assert/strict'); + const { InMemorySpanExporter } = require('@opentelemetry/sdk-trace-base'); + const { AttachmentManager, createAttachmentHttpTransport } = require('@traceai/fi-core/attachments'); + const auto = require('./dist/src/index.js'); + ${controlServer} + const WAV = Buffer.from('${wavBase64()}', 'base64'); + (async () => { + let providerCalls = 0; + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const c of req) chunks.push(c); const bytes = Buffer.concat(chunks); + if (req.headers['x-attachment-control']) { control(req, bytes, res); return; } + providerCalls++; + const path = req.url.split('?')[0]; + if (path === '/v1/audio/transcriptions') { res.setHeader('content-type', 'application/json'); res.end('{"text":"synthetic"}'); return; } + // Speech: several spaced chunks, so incremental readers see several reads. + res.writeHead(200, { 'content-type': 'audio/wav' }); + const size = Math.ceil(WAV.length / 4); + for (let i = 0; i < WAV.length; i += size) { res.write(WAV.subarray(i, i + size)); await new Promise(r => setTimeout(r, 15)); } + res.end(); + }); + await new Promise(r => server.listen(0, '127.0.0.1', r)); + const origin = 'http://127.0.0.1:' + server.address().port; + const nativeFetch = globalThis.fetch; + globalThis.fetch = (input, init) => { const u = new URL(input instanceof Request ? input.url : String(input)); return nativeFetch(origin + u.pathname + u.search, init); }; + const manager = new AttachmentManager({ enabled: true, projectName: 'synthetic', projectType: 'observe', environment: () => process.env, + transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic' }, client: controlClient(origin) }) }); + const ready = await manager.ready(); + const exporter = new InMemorySpanExporter(); + auto.init({ exporter, instrumentations: ['wire'], configFile: '/nonexistent', apiKey: '', secretKey: '', attachmentManager: manager }); + assert.equal(auto.getAttachmentManager(), manager); + // The kill switch flipped after bootstrap: a process-env manager must deny at admission. + if (process.env.TRACEAI_TEST_KILL_SWITCH === '1') process.env.FI_ATTACHMENTS_ENABLED = 'false'; + const json = { 'content-type': 'application/json' }; + // 1. Speech the application materialized. + const speech = await fetch('https://api.openai.com/v1/audio/speech', { method: 'POST', headers: json, body: JSON.stringify({ model: 'tts-1', input: 'synthetic', voice: 'alloy' }) }); + assert.deepEqual(Buffer.from(await speech.arrayBuffer()), WAV); + // 2. Wire-only FormData (the OpenAI SDK transcription route on openai 5.x+). + const form = new FormData(); form.set('model', 'whisper-1'); form.set('file', new Blob([WAV], { type: 'audio/wav' }), 'synthetic.wav'); + assert.deepEqual(await (await fetch('https://api.openai.com/v1/audio/transcriptions', { method: 'POST', body: form })).json(), { text: 'synthetic' }); + // 3. Wire-only incremental read (the Deepgram SDK speak route). + const speak = await fetch('https://api.deepgram.com/v1/speak?model=aura-asteria-en', { method: 'POST', headers: json, body: JSON.stringify({ text: 'synthetic' }) }); + const reader = speak.body.getReader(), parts = []; + for (;;) { const x = await reader.read(); if (x.done) break; parts.push(Buffer.from(x.value)); } + assert.deepEqual(Buffer.concat(parts), WAV); + await auto.forceFlush(); await manager.flush(2000); + const report = auto.report().attachments; + const spans = exporter.getFinishedSpans(); + process.stdout.write(JSON.stringify({ ready, state: report.state, providerCalls, uploads: uploaded.map(b => b.toString('base64')), + initialized: initialized.map(r => ({ parent: r.association.parent.span_id, direction: r.direction, purpose: r.purpose, mime: r.mime_type, source: r.source_kind })), + spans: spans.map(s => s.spanContext().spanId), controlCalls: transfers.filter(p => !p.endsWith('attachment-sessions')).length, + status: manager.status() })); + await auto.shutdown(); await manager.shutdown(); server.closeAllConnections(); server.close(); + })().catch(error => { process.stderr.write(String(error && error.stack || error)); process.exitCode = 1; }); + `; + // This manager reads process.env at each admission, so the switch must be UNSET (an empty value is a deny). + const childEnv: Record = { ...process.env, FI_API_KEY: "", FI_SECRET_KEY: "", FI_CONFIG_FILE: "/nonexistent", FI_AUTO_DEBUG: "false", ...env }; + delete childEnv.FI_ATTACHMENTS_ENABLED; + return spawnSync(process.execPath, ["-e", code], { cwd: root, encoding: "utf8", timeout: 30000, env: childEnv as NodeJS.ProcessEnv }); +} +test("auto.init + enabled manager + allowlisted hosts: speech, wire-only FormData and wire-only reads each upload once, bound to their wire span", () => { + const result = runWire(); + expect(result.stderr).toBe(""); expect(result.status).toBe(0); + const out = JSON.parse(result.stdout); + expect(out).toMatchObject({ ready: true, state: "ready", providerCalls: 3 }); + expect(out.uploads).toEqual([wavBase64(), wavBase64(), wavBase64()]); + expect(out.spans).toHaveLength(3); + // Each upload is bound to a distinct wire span that auto actually exported. + expect(new Set(out.initialized.map((r: any) => r.parent))).toEqual(new Set(out.spans)); + expect(out.initialized.map((r: any) => `${r.direction}/${r.purpose}/${r.mime}`).sort()).toEqual(["input/model_input/audio/wav", "output/model_output/audio/wav", "output/model_output/audio/wav"]); + expect(out.status).toMatchObject({ openTickets: 0, openStreams: 0, ownedBytes: 0 }); +}); +test("the same calls with FI_ATTACHMENTS_ENABLED=false set after bootstrap send nothing (kill switch at admission)", () => { + const result = runWire({ TRACEAI_TEST_KILL_SWITCH: "1" }); + expect(result.stderr).toBe(""); expect(result.status).toBe(0); + const out = JSON.parse(result.stdout); + // Provider traffic and wire spans are unchanged; no attachment control traffic beyond the bootstrap session. + expect(out).toMatchObject({ ready: true, providerCalls: 3, uploads: [], initialized: [], controlCalls: 0 }); + expect(out.spans).toHaveLength(3); + expect(out.status).toMatchObject({ openTickets: 0, openStreams: 0, ownedBytes: 0 }); +}); + +// Anthropic: auto enables the INSTALLED @traceai/anthropic (resolved from the application directory, like a real app) +// and hands it the same manager. The application directory links only the SDK and the instrumentor. +function anthropicApp(): string { + const { mkdtempSync, mkdirSync, symlinkSync, readFileSync } = require("node:fs") as typeof import("node:fs"); + const { tmpdir } = require("node:os") as typeof import("node:os"); + const { dirname, join } = require("node:path") as typeof import("node:path"); + const app = mkdtempSync(join(tmpdir(), "traceai-auto-anthropic-")); + const packageRoot = (name: string, from: string) => { + let dir = dirname(require.resolve(name, { paths: [from] })); + for (;;) { + try { if (JSON.parse(readFileSync(join(dir, "package.json"), "utf8")).name === name) return dir; } catch { /* ascend */ } + const parent = dirname(dir); if (parent === dir) throw new Error(`Cannot locate ${name}`); dir = parent; + } + }; + for (const [name, target] of [["@anthropic-ai/sdk", packageRoot("@anthropic-ai/sdk", root)], ["@traceai/anthropic", resolve(root, "../traceai_anthropic")]]) { + mkdirSync(join(app, "node_modules", dirname(name)), { recursive: true }); + symlinkSync(target, join(app, "node_modules", name), "dir"); + } + return app; +} +function runAnthropic(app: string) { + const fiCore = require.resolve("@traceai/fi-core/attachments", { paths: [root] }); + const sdkTrace = require.resolve("@opentelemetry/sdk-trace-base", { paths: [root] }); + const code = ` + const assert = require('node:assert/strict'); + const { InMemorySpanExporter } = require(${JSON.stringify(sdkTrace)}); + const { AttachmentManager, createAttachmentHttpTransport } = require(${JSON.stringify(fiCore)}); + const auto = require(${JSON.stringify(resolve(root, "dist/src/index.js"))}); + ${controlServer} + const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII='; + (async () => { + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const c of req) chunks.push(c); const bytes = Buffer.concat(chunks); + if (req.headers['x-attachment-control']) { control(req, bytes, res); return; } + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({ id: 'msg_synthetic', type: 'message', role: 'assistant', model: 'claude-synthetic', stop_reason: 'end_turn', stop_sequence: null, + content: [{ type: 'text', text: 'synthetic' }], usage: { input_tokens: 1, output_tokens: 1 } })); + }); + await new Promise(r => server.listen(0, '127.0.0.1', r)); + const origin = 'http://127.0.0.1:' + server.address().port; + const manager = new AttachmentManager({ enabled: true, projectName: 'synthetic', projectType: 'observe', environment: () => ({}), + transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic' }, client: controlClient(origin) }) }); + assert.equal(await manager.ready(), true); + const exporter = new InMemorySpanExporter(); + auto.init({ exporter, instrumentations: ['anthropic', 'wire'], configFile: '/nonexistent', apiKey: '', secretKey: '', attachmentManager: manager }); + const Anthropic = require('@anthropic-ai/sdk').default ?? require('@anthropic-ai/sdk').Anthropic; + const sdk = new Anthropic({ apiKey: 'synthetic-provider-key', baseURL: origin }); + await sdk.messages.create({ model: 'claude-synthetic', max_tokens: 8, messages: [{ role: 'user', content: [ + { type: 'text', text: 'synthetic' }, { type: 'image', source: { type: 'base64', media_type: 'image/png', data: PNG } }] }] }); + await auto.forceFlush(); await manager.flush(2000); + const row = auto.report().integrations.find(r => r.name === 'anthropic'); + const spans = exporter.getFinishedSpans().filter(s => s.attributes['gen_ai.span.kind'] === 'LLM' || s.attributes['fi.span.kind'] === 'LLM'); + process.stdout.write(JSON.stringify({ status: row && row.status, media: row && row.media, uploads: uploaded.map(b => b.toString('base64')), + parents: initialized.map(r => r.association.parent.span_id), purposes: initialized.map(r => r.purpose), llmSpans: spans.map(s => s.spanContext().spanId) })); + await auto.shutdown(); await manager.shutdown(); server.closeAllConnections(); server.close(); + })().catch(error => { process.stderr.write(String(error && error.stack || error)); process.exitCode = 1; }); + `; + return spawnSync(process.execPath, ["-e", code], { + cwd: app, encoding: "utf8", timeout: 30000, + env: { ...process.env, FI_ATTACHMENTS_ENABLED: "", FI_API_KEY: "", FI_SECRET_KEY: "", FI_CONFIG_FILE: "/nonexistent", FI_AUTO_DEBUG: "false" }, + }); +} +test("anthropic: report().integrations[].media is provider_adapter, and the installed instrumentor uploads the image once, bound to its span", () => { + const app = anthropicApp(); + try { + const result = runAnthropic(app); + expect(result.stderr).toBe(""); expect(result.status).toBe(0); + const out = JSON.parse(result.stdout); + expect(out).toMatchObject({ status: "instrumented", media: "provider_adapter", uploads: [png], purposes: ["model_input"] }); + expect(out.llmSpans).toHaveLength(1); expect(out.parents).toEqual(out.llmSpans); + } finally { require("node:fs").rmSync(app, { recursive: true, force: true }); } +}); +test("inventory reports media capture per integration, apart from span status", () => { + const { inventory } = require("../src/libraries") as typeof import("../src/libraries"); + const { resolveConfig } = require("../src/config") as typeof import("../src/config"); + const rows = inventory(resolveConfig({ configFile: "/nonexistent" }, {})); + const media = (name: string) => rows.find(row => row.name === name)?.media; + expect(media("openai")).toBe("provider_adapter"); expect(media("anthropic")).toBe("provider_adapter"); + expect(rows.filter(row => row.media === "provider_adapter").map(row => row.name).sort()).toEqual(["anthropic", "openai"]); +}); diff --git a/typescript/packages/traceai_auto/test/attachment-lifecycle.test.ts b/typescript/packages/traceai_auto/test/attachment-lifecycle.test.ts new file mode 100644 index 000000000..a8fcefab1 --- /dev/null +++ b/typescript/packages/traceai_auto/test/attachment-lifecycle.test.ts @@ -0,0 +1,44 @@ +import { spawnSync } from "node:child_process"; +import { resolve } from "node:path"; + +const root = resolve(__dirname, ".."); +function run(mode: string, preload?: string) { + const code = ` + const assert = require('node:assert/strict'); + const { BasicTracerProvider, InMemorySpanExporter } = require('@opentelemetry/sdk-trace-base'); + const { AttachmentManager } = require('@traceai/fi-core/attachments'); + const auto = require('./dist/src/index.js'); + (async () => { + const manager = new AttachmentManager({ enabled: false, environment: () => ({}) }); + const host = new BasicTracerProvider(); + if (${JSON.stringify(mode)} === 'host') host.attachments = manager; + const options = { exporter: new InMemorySpanExporter(), instrumentations: ['wire'], configFile: '/nonexistent' }; + if (${JSON.stringify(mode)} === 'borrowed') options.attachmentManager = manager; + if (${JSON.stringify(mode)} === 'host') options.provider = host; + let network = 0; + globalThis.fetch = () => { network++; throw new Error('unexpected network'); }; + const provider = auto.init(options); + assert.ok(provider); + const selected = auto.getAttachmentManager(); + assert.ok(selected); + if (['host', 'borrowed'].includes(${JSON.stringify(mode)})) assert.equal(selected, manager); + else assert.equal(selected, provider.attachments); + assert.equal(await selected.ready(100), false); + assert.equal(selected.status().state, 'disabled'); + assert.equal(auto.report().attachments.state, 'disabled'); assert.equal(auto.report().attachments.transport, 'shared_manager'); + await auto.forceFlush(); + await auto.shutdown(); await auto.shutdown(); + assert.equal(selected.status().state, 'disabled'); + assert.equal(network, 0); + await manager.shutdown(); await host.shutdown(); + process.stdout.write('LIFECYCLE_OK'); + })().catch(() => { process.stderr.write('LIFECYCLE_FAILED'); process.exitCode = 1; }); + `; + return spawnSync(process.execPath, [...(preload ? ['--require', preload] : []), '-e', code], { + cwd: root, encoding: 'utf8', timeout: 15000, + env: { ...process.env, FI_ATTACHMENTS_ENABLED: 'false', FI_API_KEY: '', FI_SECRET_KEY: '', FI_CONFIG_FILE: '/nonexistent', FI_AUTO_DEBUG: 'false' }, + }); +} +test.each(['owned', 'host', 'borrowed'])('%s manager uses one selected runtime and preserves lifecycle ownership', mode => { + const result = run(mode); expect(result.error).toBeUndefined(); expect(result.status).toBe(0); expect(result.stdout).toContain('LIFECYCLE_OK'); +}); diff --git a/typescript/packages/traceai_auto/test/attachment-owner.test.ts b/typescript/packages/traceai_auto/test/attachment-owner.test.ts new file mode 100644 index 000000000..9d8faf872 --- /dev/null +++ b/typescript/packages/traceai_auto/test/attachment-owner.test.ts @@ -0,0 +1,203 @@ +/* + * Owner mark vs the wire layer, for one call seen by two layers (an adapter span around a provider SDK call, and auto's + * wire capture of the same HTTP call). The adapter that captures claims the call with the shared context key + * Symbol.for("traceai.auto.attachment_owner.v1"); the wire layer must then upload nothing for that call, including + * wire-only incremental speech reads that would otherwise capture through a deferred target. + * + * The provider-media claim (@traceai/provider-media) is stubbed here through the public Symbol key, so this file + * needs no adapter package: the stub claims exactly as an adapter does (around the SDK call only; the application + * reads the body later, outside that context) and uploads its own copy, bound to its own span. + * Provider and attachment traffic both terminate on loopback. Synthetic data only. + */ +import { createServer, request as httpRequest, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import { AddressInfo } from "node:net"; +import { randomUUID } from "node:crypto"; +import { context, trace, ROOT_CONTEXT, type Span } from "@opentelemetry/api"; +import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; +import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; +import { AttachmentManager, createAttachmentHttpTransport, type AttachmentHttpClient } from "@traceai/fi-core/attachments"; +import { patchFetch } from "../src/wire/fetch"; +import { withAttachmentCapturePolicy } from "../src/wire/attachments"; + +const OWNER = Symbol.for("traceai.auto.attachment_owner.v1"); +function wav(samples: number, step: number) { + const b = Buffer.alloc(44 + samples * 2); b.write("RIFF"); b.writeUInt32LE(b.length - 8, 4); b.write("WAVEfmt ", 8); + b.writeUInt32LE(16, 16); b.writeUInt16LE(1, 20); b.writeUInt16LE(1, 22); b.writeUInt32LE(8000, 24); + b.writeUInt32LE(16000, 28); b.writeUInt16LE(2, 32); b.writeUInt16LE(16, 34); b.write("data", 36); b.writeUInt32LE(samples * 2, 40); + for (let i = 0; i < samples; i++) b.writeInt16LE(Math.round(8000 * Math.sin(i / step)), 44 + i * 2); + return b; +} +// Distinguishable payloads per provider route. +const openaiAudio = wav(80, 3), deepgramAudio = wav(96, 5); +const OPENAI_SPEECH = "https://api.openai.com/v1/audio/speech", DEEPGRAM_SPEAK = "https://api.deepgram.com/v1/speak?model=aura-asteria-en"; + +let server: Server, origin: string, manager: AttachmentManager, hook: ReturnType | undefined; +let provider: BasicTracerProvider, exporter: InMemorySpanExporter; +let uploads: Array<{ data: Buffer; request: any }>, pending: Map, wireSpans: string[]; +const nativeFetch = globalThis.fetch; +const serverPolicy = { version: "fixture-v1", enabled: true, allowed_sources: ["Bytes", "Blob"], allowed_modalities: ["image", "audio", "document", "file"], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false }; +const limits = { max_upload_bytes: 20971520, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, + active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, + upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }; +const scope = { organization_id: randomUUID(), workspace_id: randomUUID(), project_id: randomUUID() }; +const expiry = () => new Date(Date.now() + 3600000).toISOString(); +function control(req: IncomingMessage, body: Buffer, res: ServerResponse) { + res.setHeader("content-type", "application/json"); + if (req.url!.endsWith("attachment-sessions")) { + const parsed = JSON.parse(body.toString()); + res.end(JSON.stringify({ schema_version: 1, scope, capture_session_id: parsed.capture_session_id, policy: serverPolicy, limits, expires_at: expiry(), transfer_modes: ["authenticated_stream"] })); + } else if (req.url!.endsWith("/uploads")) { + const parsed = JSON.parse(body.toString()); pending.set(parsed.attachment_id, parsed); res.statusCode = 201; + res.end(JSON.stringify({ upload_id: parsed.attachment_id, attachment_id: parsed.attachment_id, occurrence_id: parsed.occurrence_id, state: "pending_upload", + reserved_bytes: parsed.declared_bytes, expires_at: expiry(), transfer_mode: "authenticated_stream", write_capability: "synthetic-upload-only" })); + } else if (req.url!.endsWith("/content")) { + const id = req.url!.split("/").at(-2)!; uploads.push({ data: body, request: pending.get(id) }); + res.end(JSON.stringify({ upload_id: id, state: "pending_upload", reason: "queued" })); + } else { res.statusCode = 202; res.end(JSON.stringify({ upload_id: req.url!.split("/").at(-2), state: "validating", reason: "binding_pending" })); } +} +const client: AttachmentHttpClient = async req => { + const body = req.body instanceof Blob ? Buffer.from(await req.body.arrayBuffer()) : req.body; + return new Promise((resolve, reject) => { + const outgoing = httpRequest(`${origin}${new URL(req.url).pathname}`, { method: req.method, headers: { ...req.headers, "x-attachment-control": "1" }, signal: req.signal }, response => { + const chunks: Buffer[] = []; response.on("data", c => chunks.push(c)); response.on("end", () => resolve({ status: response.statusCode!, headers: {}, body: Buffer.concat(chunks) } as any)); + }); outgoing.on("error", reject); outgoing.end(body as any); + }); +}; + +beforeAll(() => context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable())); +afterAll(() => context.disable()); +beforeEach(async () => { + uploads = []; pending = new Map(); wireSpans = []; + server = createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", c => chunks.push(c)); + req.on("end", () => { + const body = Buffer.concat(chunks); + if (req.headers["x-attachment-control"]) { control(req, body, res); return; } + if (req.url!.startsWith("/v1/audio/transcriptions")) { res.writeHead(200, { "content-type": "application/json" }); res.end('{"text":"synthetic"}'); return; } + const data = req.url!.startsWith("/v1/speak") ? deepgramAudio : openaiAudio; + res.writeHead(200, { "content-type": "audio/wav" }); + // Several spaced network chunks, so incremental readers see several reads. + void (async () => { for (let i = 0; i < data.length; i += 24) { res.write(data.subarray(i, i + 24)); await new Promise(r => setTimeout(r, 10)); } res.end(); })(); + }); + }); + await new Promise(done => server.listen(0, "127.0.0.1", done)); origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + exporter = new InMemorySpanExporter(); provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + manager = new AttachmentManager({ enabled: true, projectName: "attachment-owner-test", projectType: "observe", environment: () => ({}), + transport: createAttachmentHttpTransport({ endpoint: "https://attachments.example", headers: { authorization: "Bearer synthetic" }, client }) }); + expect(await manager.ready()).toBe(true); + // Auto's real fetch hook over a loopback redirect; the emitter binds media to the actual wire span. + globalThis.fetch = (input: RequestInfo | URL, init?: RequestInit) => { + const u = new URL(input instanceof Request ? input.url : String(input)); + return nativeFetch(`${origin}${u.pathname}${u.search}`, init); + }; + hook = patchFetch(4194304, (result, parent, start, end, bindMedia) => { + const span = provider.getTracer("wire").startSpan(result.name, { startTime: start, attributes: result.attributes }, parent); + wireSpans.push(span.spanContext().spanId); bindMedia?.(span); span.end(end); + }, [], manager); +}); +afterEach(async () => { + hook?.remove(); hook = undefined; globalThis.fetch = nativeFetch; + await manager.shutdown(1000); await provider.shutdown(); + server.closeAllConnections(); await new Promise(done => server.close(() => done())); +}); + +/** + * A provider adapter (stand-in for @traceai/provider-media): its own span, the owner claim around the provider call + * only, and its own upload of the bytes the application read. Returns what the call returned. + */ +function adapter() { + const span: Span = provider.getTracer("adapter").startSpan("Adapter Audio speech", { attributes: { "fi.span.kind": "LLM" } }); + return { + span, + claim: (fn: () => T): T => context.with(trace.setSpan(context.active(), span).setValue(OWNER, true), fn), + capture(data: Buffer) { + const ticket = manager.preflight(manager.correlation(span), { mimeType: "audio/wav", direction: "output", purpose: "model_output", maxBytes: data.length, sourceKind: "Bytes" }); + if (ticket.admitted) ticket.attach({ kind: "Bytes", data: new Uint8Array(data) }); + span.end(); + }, + }; +} +const post = (url: string) => fetch(url, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ text: "synthetic", input: "synthetic" }) }); +async function read(response: Response, mode: "reader" | "iterator" | "arrayBuffer"): Promise { + if (mode === "arrayBuffer") return Buffer.from(await response.arrayBuffer()); + const chunks: Buffer[] = []; + if (mode === "iterator") { for await (const chunk of response.body as any) chunks.push(Buffer.from(chunk)); return Buffer.concat(chunks); } + const reader = response.body!.getReader(); + for (;;) { const x = await reader.read(); if (x.done) return Buffer.concat(chunks); chunks.push(Buffer.from(x.value)); } +} +const parentOf = (data: Buffer) => uploads.find(u => Buffer.compare(u.data, data) === 0)?.request.association.parent.span_id; +/** "policy": an outer integration supplied an explicit capture policy, so without the claim the wire layer would also capture against the adapter's model span. */ +const outerContext = (p: "policy" | "no-policy") => p === "policy" ? withAttachmentCapturePolicy(ROOT_CONTEXT, { captureContent: true }) : ROOT_CONTEXT; +const settle = () => manager.flush(2000); +const idle = () => expect(manager.status()).toMatchObject({ openTickets: 0, openStreams: 0, ownedBytes: 0 }); + +describe("one call, two layers: the owner mark keeps wire capture out", () => { + test.each([[OPENAI_SPEECH, openaiAudio], [DEEPGRAM_SPEAK, deepgramAudio]] as const)("positive control: unowned wire-only %s read incrementally uploads once, bound to the wire span", async (url, data) => { + expect(await read(await post(url), "reader")).toEqual(data); + await settle(); expect(uploads.map(u => u.data)).toEqual([data]); + expect(wireSpans).toContain(parentOf(data)); idle(); + }); + + for (const [route, url, data] of [["OpenAI speech", OPENAI_SPEECH, openaiAudio], ["Deepgram speak", DEEPGRAM_SPEAK, deepgramAudio]] as const) { + for (const mode of ["reader", "iterator", "arrayBuffer"] as const) for (const p of ["policy", "no-policy"] as const) { + test(`adapter-owned ${route} (${mode}, ${p}, body read after the claim scope ended): exactly one upload, the adapter's`, async () => { + await context.with(outerContext(p), async () => { + const a = adapter(); + const response = await a.claim(() => post(url)); + const bytes = await read(response, mode); expect(bytes).toEqual(data); + a.capture(bytes); + await settle(); + expect(uploads.map(u => u.data)).toEqual([data]); + expect(parentOf(data)).toBe(a.span.spanContext().spanId); + expect(wireSpans).not.toContain(parentOf(data)); idle(); + }); + }); + } + test(`an outer owner alone (the ElevenLabs marker, no adapter upload) on ${route}: nothing uploads, nothing is held`, async () => { + const response = await context.with(ROOT_CONTEXT.setValue(OWNER, true), () => post(url)); + expect(await read(response, "reader")).toEqual(data); + await settle(); expect(uploads).toHaveLength(0); idle(); + }); + } + + test.each(["policy", "no-policy"] as const)("adapter-owned transcription FormData (%s): the file path is skipped too", async p => { + await context.with(outerContext(p), async () => { + const a = adapter(); + const body = new FormData(); body.set("model", "whisper-1"); body.set("file", new Blob([openaiAudio], { type: "audio/wav" }), "synthetic.wav"); + await (await a.claim(() => fetch("https://api.openai.com/v1/audio/transcriptions", { method: "POST", body }))).json(); + a.capture(openaiAudio); + await settle(); expect(uploads.map(u => u.data)).toEqual([openaiAudio]); + expect(parentOf(openaiAudio)).toBe(a.span.spanContext().spanId); idle(); + }); + }); + + test.each(["policy", "no-policy"] as const)("concurrent (%s): an owned OpenAI read and an unowned Deepgram read upload once each; the claim does not leak", async p => { + await context.with(outerContext(p), async () => { + const a = adapter(); + const [owned, unowned] = await Promise.all([ + (async () => { const bytes = await read(await a.claim(() => post(OPENAI_SPEECH)), "reader"); a.capture(bytes); return bytes; })(), + (async () => read(await post(DEEPGRAM_SPEAK), "reader"))(), + ]); + expect(owned).toEqual(openaiAudio); expect(unowned).toEqual(deepgramAudio); + await settle(); + expect(uploads).toHaveLength(2); + expect(parentOf(openaiAudio)).toBe(a.span.spanContext().spanId); + expect(wireSpans).toContain(parentOf(deepgramAudio)); idle(); + }); + }); + + test.each(["policy", "no-policy"] as const)("real OpenAI SDK (%s): the claim around speech.create covers the later incremental read", async p => { + const mod = require(process.env.ATT_OPENAI_MODULE || "openai"); const OpenAI = mod.default ?? mod.OpenAI; + const sdk = new OpenAI({ apiKey: "synthetic-provider-key", fetch: (...args: Parameters) => globalThis.fetch(...args) }); + await context.with(outerContext(p), async () => { + const a = adapter(); + const response = await a.claim(() => sdk.audio.speech.create({ model: "tts-1", voice: "alloy", input: "synthetic", response_format: "wav" })); + const bytes = await read(response, "reader"); expect(bytes).toEqual(openaiAudio); + a.capture(bytes); + await settle(); expect(uploads.map(u => u.data)).toEqual([openaiAudio]); + expect(parentOf(openaiAudio)).toBe(a.span.spanContext().spanId); idle(); + }); + }); +}); diff --git a/typescript/packages/traceai_auto/test/attachments.test.ts b/typescript/packages/traceai_auto/test/attachments.test.ts new file mode 100644 index 000000000..205cd70a7 --- /dev/null +++ b/typescript/packages/traceai_auto/test/attachments.test.ts @@ -0,0 +1,538 @@ +import { createServer, request as httpRequest, type Server } from "node:http"; +import { AddressInfo } from "node:net"; +import { randomUUID } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { context, trace, ROOT_CONTEXT, INVALID_SPAN_CONTEXT } from "@opentelemetry/api"; +import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; +import { AlwaysOffSampler, BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; +import { AttachmentManager, createAttachmentHttpTransport, type AttachmentHttpClient, type AttachmentManagerOptions } from "@traceai/fi-core/attachments"; +import { captureOpenAIAttachments } from "@traceai/openai"; +import { patchFetch } from "../src/wire/fetch"; +import { withAttachmentOwnership, withAttachmentCapturePolicy } from "../src/wire/attachments"; +import { attachmentOptions, selectAttachmentManager, attachmentReport } from "../src/attachments"; +import { resolveConfig } from "../src/config"; + +// Synthetic media/credentials only. Provider and attachment traffic both terminate on loopback. +const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64"); +function wav(samples = 80) { + const b = Buffer.alloc(44 + samples * 2); b.write("RIFF"); b.writeUInt32LE(b.length - 8, 4); b.write("WAVEfmt ", 8); + b.writeUInt32LE(16, 16); b.writeUInt16LE(1, 20); b.writeUInt16LE(1, 22); b.writeUInt32LE(8000, 24); + b.writeUInt32LE(16000, 28); b.writeUInt16LE(2, 32); b.writeUInt16LE(16, 34); b.write("data", 36); b.writeUInt32LE(samples * 2, 40); + for (let i = 0; i < samples; i++) b.writeInt16LE(Math.round(8000 * Math.sin(i / 3)), 44 + i * 2); + return b; +} +const audio = wav(); +let original: typeof fetch, server: Server, origin: string, manager: AttachmentManager, hook: ReturnType | undefined; +let uploads: Buffer[], requests: any[], transfers: string[], providerBodies: Buffer[], providerRequests: number; +let provider: BasicTracerProvider, exporter: InMemorySpanExporter; +let responseStatus: number, responseData: Buffer, responseType: string, responseChunks: number; +/** `reset` destroys the socket once the request body arrived; `hang` never answers; `partial` sends half the body, then destroys. */ +let responseMode: "normal" | "reset" | "hang" | "partial"; +const nativeFetch = globalThis.fetch; +const serverPolicy = { version: "fixture-v1", enabled: true, allowed_sources: ["Bytes", "Blob"], allowed_modalities: ["image", "audio", "document", "file"], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false }; +const limits = { max_upload_bytes: 20971520, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, + active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, + upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }; +let policy: Record = serverPolicy; +const scope = { organization_id: randomUUID(), workspace_id: randomUUID(), project_id: randomUUID() }; +// Sessions of one hour or more: the core renews within max(60 s, 20% of lifetime) of expiry. +const expiry = () => new Date(Date.now() + 3600000).toISOString(); +function control(req: import("node:http").IncomingMessage, body: Buffer, res: import("node:http").ServerResponse) { + transfers.push(req.url!); res.setHeader("content-type", "application/json"); + if (req.url!.endsWith("attachment-sessions")) { + const parsed = JSON.parse(body.toString()); + res.end(JSON.stringify({ schema_version: 1, scope, capture_session_id: parsed.capture_session_id, policy, limits, expires_at: expiry(), transfer_modes: ["authenticated_stream"] })); + } else if (req.url!.endsWith("/uploads")) { + const parsed = JSON.parse(body.toString()); requests.push(parsed); res.statusCode = 201; + res.end(JSON.stringify({ upload_id: parsed.attachment_id, attachment_id: parsed.attachment_id, occurrence_id: parsed.occurrence_id, state: "pending_upload", + reserved_bytes: parsed.declared_bytes, expires_at: expiry(), transfer_mode: "authenticated_stream", write_capability: "synthetic-upload-only" })); + } else if (req.url!.endsWith("/content")) { + uploads.push(body); res.end(JSON.stringify({ upload_id: req.url!.split("/").at(-2), state: "pending_upload", reason: "queued" })); + } else { res.statusCode = 202; res.end(JSON.stringify({ upload_id: req.url!.split("/").at(-2), state: "validating", reason: "binding_pending" })); } +} +const client: AttachmentHttpClient = async req => { + // Content PUTs may carry the application's own Blob; the double reads it only when the core sends it. + const body = req.body instanceof Blob ? Buffer.from(await req.body.arrayBuffer()) : req.body; + return new Promise((resolve, reject) => { + const outgoing = httpRequest(`${origin}${new URL(req.url).pathname}`, { method: req.method, headers: { ...req.headers, "x-attachment-control": "1" }, signal: req.signal }, response => { + const chunks: Buffer[] = []; response.on("data", c => chunks.push(c)); response.on("end", () => resolve({ status: response.statusCode!, headers: {}, body: Buffer.concat(chunks) } as any)); + }); outgoing.on("error", reject); outgoing.end(body as any); + }); +}; +async function createManager(options: AttachmentManagerOptions = {}, serverOverrides: Record = {}) { + policy = { ...serverPolicy, ...serverOverrides }; + const result = new AttachmentManager({ enabled: true, projectName: "attachment-test", projectType: "observe", environment: () => ({}), + transport: createAttachmentHttpTransport({ endpoint: "https://attachments.example", headers: { authorization: "Bearer synthetic" }, client }), ...options }); + expect(await result.ready()).toBe(options.enabled !== false); + expect(result.status().state).toBe(options.enabled === false ? "disabled" : "ready"); + return result; +} +/** + * `emit: "none"` models an emitter that never starts the wire span (for example its delivery gate refused it); + * `"invalid"` binds a span with an invalid identity, which a deferred target refuses to bind. + */ +function install(target = manager, emit: "span" | "none" | "invalid" = "span") { + globalThis.fetch = (input: RequestInfo | URL, init?: RequestInit) => { + const u = new URL(input instanceof Request ? input.url : String(input)); + return nativeFetch(`${origin}${u.pathname}${u.search}`, init); + }; + hook = patchFetch(4194304, (result, parent, start, end, bindMedia) => { + if (emit === "none") return; + const span = emit === "invalid" ? trace.wrapSpanContext(INVALID_SPAN_CONTEXT) : provider.getTracer("wire-test").startSpan(result.name, { startTime: start, attributes: result.attributes }, parent); + bindMedia?.(span); span.end(end); + }, [], target); +} +const transcription = (signal?: AbortSignal) => { + const body = new FormData(); body.set("model", "whisper-1"); body.set("file", new Blob([audio], { type: "audio/wav" }), "PRIVATE_CANARY_FILENAME.wav"); + return fetch("https://api.openai.com/v1/audio/transcriptions", { method: "POST", body, signal }); +}; +/** Polls a condition well inside the 300 s deferred deadline, so a pass proves the call's own exit released it. */ +async function eventually(condition: () => boolean, ms = 2000) { + for (const until = Date.now() + ms; !condition() && Date.now() < until;) await new Promise(r => setTimeout(r, 5)); + expect(condition()).toBe(true); +} +async function readAll(response: Response, mode: "reader" | "iterator" = "reader"): Promise { + const chunks: Buffer[] = []; + if (mode === "iterator") { for await (const chunk of response.body as any) chunks.push(Buffer.from(chunk)); } + else { const reader = response.body!.getReader(); for (;;) { const x = await reader.read(); if (x.done) break; chunks.push(Buffer.from(x.value)); } } + return Buffer.concat(chunks); +} +let nativeSpanId = ""; +function inNative(fn: () => T, kind = "LLM", policyContext = true): T { + const span = provider.getTracer("native-test").startSpan("native", { attributes: { "gen_ai.span.kind": kind } }); + nativeSpanId = span.spanContext().spanId; + const parent = trace.setSpan(ROOT_CONTEXT, span); + const result = context.with(policyContext ? withAttachmentCapturePolicy(parent, { captureContent: true }) : parent, fn); + // Promise work retains this exact request context; do not consult later ambient spans. + if (result instanceof Promise) return result.finally(() => span.end()) as T; + span.end(); return result; +} +async function settle() { await manager.flush(2000); } +const speech = (route = "api.openai.com/v1/audio/speech") => fetch(`https://${route}`, { method: "POST", body: JSON.stringify({ model: "tts-1", input: "synthetic", voice: "alloy" }) }); +beforeAll(() => context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable())); +afterAll(() => context.disable()); +beforeEach(async () => { + original = globalThis.fetch; uploads = []; requests = []; transfers = []; providerBodies = []; providerRequests = 0; + responseStatus = 200; responseData = audio; responseType = "audio/wav"; responseChunks = 1; responseMode = "normal"; + server = createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", chunk => chunks.push(chunk)); + req.on("end", () => { + const body = Buffer.concat(chunks); + if (req.headers["x-attachment-control"]) { control(req, body, res); return; } + providerRequests++; providerBodies.push(body); + if (responseMode === "reset") { req.socket.destroy(); return; } + if (responseMode === "hang") return; + res.writeHead(responseStatus, { "content-type": responseType }); + if (responseMode === "partial") { res.write(responseData.subarray(0, Math.ceil(responseData.length / 2)), () => setTimeout(() => req.socket.destroy(), 15)); return; } + // Several network chunks, spaced so loopback does not coalesce them: incremental readers see several reads. + const size = Math.ceil(responseData.length / responseChunks), data = responseData; + void (async () => { + for (let i = 0; i < data.length; i += size) { res.write(data.subarray(i, i + size)); if (responseChunks > 1) await new Promise(r => setTimeout(r, 15)); } + res.end(); + })(); + }); + }); + await new Promise(done => server.listen(0, "127.0.0.1", done)); origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + exporter = new InMemorySpanExporter(); provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + manager = await createManager(); +}); +afterEach(async () => { + hook?.remove(); hook = undefined; globalThis.fetch = original; + await manager.shutdown(1000); await provider.shutdown(); + server.closeAllConnections(); await new Promise(done => server.close(() => done())); +}); +const sessionsOnly = () => transfers.filter(path => !path.endsWith("attachment-sessions")); + +describe("A04 speech output", () => { + test.each(["arrayBuffer", "blob"] as const)("wire-only: application-materialized %s uploads once, bound to the actual wire span", async method => { + install(); const response = await speech(); + const result = await response[method](); + expect(Buffer.from(result instanceof Blob ? await result.arrayBuffer() : result)).toEqual(audio); + await settle(); expect(uploads).toEqual([audio]); + const span = exporter.getFinishedSpans()[0]; + expect(requests[0].association.parent.span_id).toBe(span.spanContext().spanId); + expect(requests[0]).toMatchObject({ direction: "output", purpose: "model_output", mime_type: "audio/wav", declared_bytes: audio.length }); + // The body is consumed through a reader the runtime opens, so the bytes arrive through a deferred target that binds + // inside the emitter, before the wire span ends: the reference is projected onto that span, once. + const refs = JSON.parse(String(span.attributes["fi.attachment.refs"])); + expect(refs).toEqual([expect.objectContaining({ occurrence_id: requests[0].occurrence_id, modality: "audio", direction: "output" })]); + expect(manager.status()).toMatchObject({ openTickets: 0, openStreams: 0, ownedBytes: 0 }); + }); + test("native span + explicit policy: arrayBuffer is captured once from consumed chunks, with no second span", async () => { + responseChunks = 4; install(); + await inNative(async () => { await (await speech()).arrayBuffer(); }); + await settle(); expect(uploads).toEqual([audio]); expect(exporter.getFinishedSpans()).toHaveLength(1); + expect(requests[0].association.parent.span_id).toBe(nativeSpanId); + const refs = JSON.parse(String(exporter.getFinishedSpans()[0].attributes["fi.attachment.refs"])); + expect(refs).toEqual([expect.objectContaining({ occurrence_id: requests[0].occurrence_id, modality: "audio", direction: "output" })]); + }); + test("T-CS-01 native span: application reader consumption is observed with no extra read, clone or tee", async () => { + responseChunks = 5; install(); + const tee = jest.spyOn(ReadableStream.prototype, "tee"), clone = jest.spyOn(Response.prototype, "clone"); + const reads = jest.spyOn(ReadableStreamDefaultReader.prototype, "read"); + let appReads = 0, responseBody: unknown, appReader: unknown; + await inNative(async () => { + const response = await speech(); responseBody = response.body; const reader = response.body!.getReader(); appReader = reader; const chunks: Buffer[] = []; + for (;;) { appReads++; const x = await reader.read(); if (x.done) break; chunks.push(Buffer.from(x.value)); } + expect(Buffer.concat(chunks)).toEqual(audio); + }); + // Only this reader's calls: undici reads its own request-body stream internally. + const underlying = reads.mock.contexts.filter(receiver => receiver === appReader).length; + await settle(); + // undici itself tees the REQUEST body (cloneRequest); the response body must never be teed or cloned. + expect(tee.mock.contexts).not.toContain(responseBody); expect(clone).not.toHaveBeenCalled(); tee.mockRestore(); clone.mockRestore(); reads.mockRestore(); + expect(appReads).toBeGreaterThan(2); expect(underlying).toBe(appReads); expect(uploads).toEqual([audio]); + }); + test("native span: async-iterator consumption uploads once; a break before the end uploads nothing", async () => { + responseChunks = 4; install(); + await inNative(async () => { + const chunks: Buffer[] = []; for await (const chunk of (await speech()).body as any) chunks.push(Buffer.from(chunk)); + expect(Buffer.concat(chunks)).toEqual(audio); + }); + await settle(); expect(uploads).toEqual([audio]); + uploads = []; + await inNative(async () => { for await (const _chunk of (await speech()).body as any) break; }); + await settle(); expect(uploads).toHaveLength(0); expect(manager.status()).toMatchObject({ openStreams: 0, ownedBytes: 0 }); + }); + test("adapter media ownership is independent and prevents duplicate uploads", async () => { + install(); await inNative(() => context.with(withAttachmentOwnership(context.active()), async () => { await (await speech()).arrayBuffer(); })); + await settle(); expect(uploads).toHaveLength(0); expect(exporter.getFinishedSpans()).toHaveLength(1); + }); + test("the ElevenLabs marker (Symbol.for key, no auto import) is honoured", async () => { + install(); await inNative(() => context.with(context.active().setValue(Symbol.for("traceai.auto.attachment_owner.v1"), true), async () => { await (await speech()).arrayBuffer(); })); + await settle(); expect(uploads).toHaveLength(0); + }); + test.each(["api.openai.com/v1/audio/speech", "api.deepgram.com/v1/speak"])("recognized %s: cloned application reads upload once and stay byte-identical", async route => { + install(); await inNative(async () => { + const response = await speech(route); const copy = response.clone(); + const [one, two] = await Promise.all([response.arrayBuffer(), copy.blob()]); + expect(Buffer.from(one)).toEqual(audio); expect(Buffer.from(await two.arrayBuffer())).toEqual(audio); + }); await settle(); expect(uploads).toEqual([audio]); + }); + test("unread, cancelled and cancelled-midway bodies cause no eager read and upload nothing", async () => { + responseChunks = 4; install(); + await inNative(async () => { + const unread = await speech(); expect(unread.bodyUsed).toBe(false); await unread.body!.cancel(); + const partial = await speech(); const reader = partial.body!.getReader(); await reader.read(); await reader.cancel(); + }); + // Wire-only (deferred target): a reader cancelled midway uploads nothing, and nothing stays held. + const wireOnly = await speech(); const reader = wireOnly.body!.getReader(); await reader.read(); await reader.cancel(); + await settle(); expect(uploads).toHaveLength(0); expect(manager.status()).toMatchObject({ openStreams: 0, openTickets: 0, ownedBytes: 0 }); + }); + test.each([ + ["api.openai.com/v1/audio/speech", { model: "tts-1", input: "synthetic", voice: "alloy" }, "reader"], + ["api.openai.com/v1/audio/speech", { model: "tts-1", input: "synthetic", voice: "alloy" }, "iterator"], + ["api.deepgram.com/v1/speak", { text: "synthetic" }, "reader"], + ] as const)("wire-only %s (%j, %s): incremental reads upload once, bound to the wire span through a deferred target", async (route, body, mode) => { + responseChunks = 4; install(); + const response = await fetch(`https://${route}`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); + expect(await readAll(response, mode)).toEqual(audio); + await settle(); expect(uploads).toEqual([audio]); + const spans = exporter.getFinishedSpans(); expect(spans).toHaveLength(1); + expect(requests).toHaveLength(1); + expect(requests[0].association.parent.span_id).toBe(spans[0].spanContext().spanId); + expect(requests[0]).toMatchObject({ direction: "output", purpose: "model_output", mime_type: "audio/wav", declared_bytes: audio.length }); + expect(manager.status()).toMatchObject({ openStreams: 0, openTickets: 0, ownedBytes: 0 }); + }); + test.each(["none", "invalid"] as const)("wire-only captures whose wire span is never bound (emit: %s) are dropped when the call completes, not after the deadline", async emit => { + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); install(manager, emit); + await (await transcription()).json(); + responseType = "audio/wav"; responseData = audio; responseChunks = 4; + expect(await readAll(await speech())).toEqual(audio); + await new Promise(done => setImmediate(done)); + expect(manager.status()).toMatchObject({ openStreams: 0, openTickets: 0, ownedBytes: 0 }); + await settle(); expect(uploads).toHaveLength(0); expect(sessionsOnly()).toEqual([]); expect(exporter.getFinishedSpans()).toHaveLength(0); + }); + test("piped bodies are not captured", async () => { + install(); await inNative(async () => { + const response = await speech(); const chunks: Uint8Array[] = []; + await response.body!.pipeTo(new WritableStream({ write(chunk) { chunks.push(chunk); } })); + expect(Buffer.concat(chunks)).toEqual(audio); + }); await settle(); expect(uploads).toHaveLength(0); + }); + test("an object above the stream cap is discarded whole (too_large), never truncated", async () => { + responseData = Buffer.concat([audio, Buffer.alloc(4 * 1024 * 1024)]); responseChunks = 8; install(); + await inNative(async () => { const reader = (await speech()).body!.getReader(); while (!(await reader.read()).done) { /* app reads */ } }); + await settle(); expect(uploads).toHaveLength(0); expect(manager.status().ownedBytes).toBe(0); + }); +}); + +describe("A03 FormData and JSON media", () => { + test("native: transcription FormData keeps body, boundary and filename on the provider path only", async () => { + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); + const body = new FormData(); body.set("model", "whisper-1"); body.set("file", new Blob([audio], { type: "audio/wav" }), "PRIVATE_CANARY_FILENAME.wav"); + install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/audio/transcriptions", { method: "POST", body })).json(); }); + await settle(); expect(uploads).toEqual([audio]); expect(providerBodies[0].includes(audio)).toBe(true); expect(providerBodies[0].toString()).toContain("PRIVATE_CANARY_FILENAME.wav"); + expect(requests[0]).toMatchObject({ direction: "input", purpose: "model_input", source_kind: "Blob" }); + expect(JSON.stringify(requests)).not.toContain("PRIVATE_CANARY"); expect(JSON.stringify(exporter.getFinishedSpans().map(s => s.attributes))).not.toContain("PRIVATE_CANARY"); + expect(body.get("file")).toBeInstanceOf(Blob); + }); + test("native: image edits accept only typed allowlisted FormData fields", async () => { + responseType = "application/json"; responseData = Buffer.from('{"data":[]}'); + const body = new FormData(); body.set("image", new Blob([png], { type: "image/png" })); body.set("untrusted_file", new Blob([png], { type: "image/png" })); + install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body })).json(); }); + await settle(); expect(uploads).toEqual([png]); + }); + test("wire-only transcription FormData is held (unread) until the wire span exists, then uploads once bound to it", async () => { + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); + install(); const pending = transcription(); + // Admitted before the span exists: bytes are reserved, nothing is sent until bind. + expect(manager.status().ownedBytes).toBe(audio.length); expect(sessionsOnly()).toEqual([]); + expect(await (await pending).json()).toEqual({ text: "synthetic" }); + await settle(); expect(uploads).toEqual([audio]); expect(providerBodies[0].includes(audio)).toBe(true); + const spans = exporter.getFinishedSpans(); expect(spans).toHaveLength(1); + expect(requests[0]).toMatchObject({ direction: "input", purpose: "model_input", source_kind: "Blob", mime_type: "audio/wav" }); + expect(requests[0].association.parent.span_id).toBe(spans[0].spanContext().spanId); + expect(JSON.stringify(requests)).not.toContain("PRIVATE_CANARY"); + expect(manager.status()).toMatchObject({ openTickets: 0, ownedBytes: 0 }); + }); + test.each(["reset", "abort", "partial"] as const)("wire-only transcription whose call fails after the send (%s) drops the held audio at once: the error wire span binds nothing", async mode => { + // reset: the connection drops, fetch rejects. abort: the application aborts after the send, fetch rejects (AbortError). + // partial: the response body fails while the application reads it. In each case the provider received the audio. + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic transcription"}'); + responseMode = mode === "abort" ? "hang" : mode; + install(); const controller = new AbortController(); const pending = transcription(controller.signal); + expect(manager.status().ownedBytes).toBe(audio.length); + if (mode === "abort") { await eventually(() => providerRequests === 1); controller.abort(); } + const failure = await pending.then(response => response.json()).then(() => undefined, (error: unknown) => error); + // Undici's errors come from Node's realm (not jest's), so check the shape, not `instanceof Error`. + expect((failure as Error | undefined)?.name).toBe(mode === "abort" ? "AbortError" : "TypeError"); + await eventually(() => manager.status().ownedBytes === 0); + expect(manager.status()).toMatchObject({ openTickets: 0, openStreams: 0, ownedBytes: 0 }); + expect(providerRequests).toBe(1); expect(providerBodies[0].includes(audio)).toBe(true); + // The error wire span is still emitted; nothing held was bound to it. + await eventually(() => exporter.getFinishedSpans().length === 1); + await settle(); expect(uploads).toHaveLength(0); expect(requests).toHaveLength(0); expect(sessionsOnly()).toEqual([]); + }); + test("wire-only transcription answered with an HTTP error (completed 400) is a finished call: the held audio binds to its wire span", async () => { + responseStatus = 400; responseType = "application/json"; responseData = Buffer.from('{"error":{"message":"synthetic","type":"invalid_request_error"}}'); + install(); const response = await transcription(); expect(response.status).toBe(400); await response.json(); + await settle(); expect(uploads).toEqual([audio]); + const spans = exporter.getFinishedSpans(); expect(spans).toHaveLength(1); + expect(requests[0].association.parent.span_id).toBe(spans[0].spanContext().spanId); + expect(manager.status()).toMatchObject({ openTickets: 0, ownedBytes: 0 }); + }); + test("wire-only image edits have no wire span: the held file is dropped when the call completes and nothing uploads", async () => { + responseType = "application/json"; responseData = Buffer.from('{"data":[]}'); + const body = new FormData(); body.set("image", new Blob([png], { type: "image/png" })); + install(); await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body })).json(); + expect(manager.status()).toMatchObject({ openTickets: 0, ownedBytes: 0 }); + await settle(); expect(uploads).toHaveLength(0); expect(sessionsOnly()).toEqual([]); + }); + test("a wire span that is not sampled uploads nothing held for it (re-checked at bind)", async () => { + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); + await provider.shutdown(); exporter = new InMemorySpanExporter(); + provider = new BasicTracerProvider({ sampler: new AlwaysOffSampler(), spanProcessors: [new SimpleSpanProcessor(exporter)] }); + install(); await (await transcription()).json(); + responseType = "audio/wav"; responseData = audio; responseChunks = 4; expect(await readAll(await speech())).toEqual(audio); + await settle(); expect(uploads).toHaveLength(0); expect(sessionsOnly()).toEqual([]); + expect(manager.status()).toMatchObject({ openStreams: 0, openTickets: 0, ownedBytes: 0 }); + }); + test("wire-only image JSON output uploads decoded bytes through the provider adapter, never URLs", async () => { + responseType = "application/json"; responseData = Buffer.from(JSON.stringify({ data: [{ b64_json: png.toString("base64") }, { url: "https://untrusted.invalid/private?token=CANARY" }] })); + install(); const body = await (await fetch("https://api.openai.com/v1/images/generations", { method: "POST", body: JSON.stringify({ model: "gpt-image-1", prompt: "synthetic" }) })).json(); + expect(body.data[0].b64_json).toBe(png.toString("base64")); await settle(); expect(uploads).toEqual([png]); + expect(providerRequests).toBe(1); + expect(requests[0]).toMatchObject({ direction: "output", purpose: "model_output", mime_type: "image/png", message_index: 0, part_index: 0 }); + expect(JSON.stringify(requests)).not.toContain("CANARY"); expect(JSON.stringify(exporter.getFinishedSpans().map(s => s.attributes))).not.toContain(png.toString("base64")); + }); + test("wire-only chat and responses JSON input: images and computer screenshots via the provider adapter", async () => { + responseType = "application/json"; responseData = Buffer.from('{"id":"synthetic","object":"chat.completion","choices":[]}'); + const uri = `data:image/png;base64,${png.toString("base64")}`; + install(); + await (await fetch("https://api.openai.com/v1/chat/completions", { method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-4o", messages: [{ role: "user", content: [{ type: "text", text: "synthetic" }, { type: "image_url", image_url: { url: uri } }] }] }) })).json(); + responseData = Buffer.from('{"id":"synthetic","object":"response","output":[]}'); + await (await fetch("https://api.openai.com/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "computer-use-preview", input: [{ type: "computer_call_output", call_id: "call_synthetic", output: { type: "computer_screenshot", image_url: uri } }] }) })).json(); + await settle(); expect(uploads).toEqual([png, png]); + expect(requests.map(r => r.purpose)).toEqual(["model_input", "computer_screenshot"]); + expect(requests[1].tool_call_id).toBe("call_synthetic"); + const spans = exporter.getFinishedSpans(); + expect(requests.map(r => r.association.parent.span_id)).toEqual(spans.map(s => s.spanContext().spanId)); + }); + test("a provider adapter capture on the same span and position is not uploaded again by wire (#682 registry)", async () => { + responseType = "application/json"; responseData = Buffer.from('{"id":"synthetic","object":"chat.completion","choices":[]}'); + const payload = { model: "gpt-4o", messages: [{ role: "user", content: [{ type: "image_url", image_url: { url: `data:image/png;base64,${png.toString("base64")}` } }] }] }; + install(); await inNative(async () => { + captureOpenAIAttachments(manager, trace.getSpan(context.active())!, payload, { operation: "chat", direction: "input" }); + await (await fetch("https://api.openai.com/v1/chat/completions", { method: "POST", body: JSON.stringify(payload) })).json(); + }); + await settle(); expect(uploads).toEqual([png]); + }); +}); + +describe("privacy negative controls", () => { + test.each([ + ["disabled", { enabled: false }], ["hide inputs", { traceConfig: { hideInputs: true } }], ["hide text", { traceConfig: { hideInputText: true } }], + ["hide images", { traceConfig: { hideInputImages: true } }], ["PII", { traceConfig: { piiRedaction: true } }], + ] as [string, AttachmentManagerOptions][])("%s prevents input image transfer at the network boundary", async (_name, options) => { + await manager.shutdown(); manager = await createManager(options); transfers = []; + responseType = "application/json"; responseData = Buffer.from('{"data":[]}'); + const body = new FormData(); body.set("image", new Blob([png], { type: "image/png" }), "PRIVATE_CANARY.png"); + install(); await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body })).json(); }); + await (await fetch("https://api.openai.com/v1/chat/completions", { method: "POST", + body: JSON.stringify({ messages: [{ role: "user", content: [{ type: "image_url", image_url: { url: `data:image/png;base64,${png.toString("base64")}` } }] }] }) })).json(); + // Wire-only FormData audio through the deferred target: denied at admission, nothing held (image hiding is image-only). + if (!options.traceConfig?.hideInputImages) { responseData = Buffer.from('{"text":"synthetic"}'); await (await transcription()).json(); } + await settle(); expect(sessionsOnly()).toEqual([]); expect(manager.status()).toMatchObject({ openTickets: 0, ownedBytes: 0 }); + }); + test.each(["hideInputs", "hideInputMessages", "piiRedaction"] as const)("%s set after a wire-only file was held denies it at bind", async flag => { + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); + install(); const pending = transcription(); + expect(manager.status().ownedBytes).toBe(audio.length); + manager.updatePolicy({ traceConfig: { [flag]: true } }); transfers = []; + await (await pending).json(); + await settle(); expect(sessionsOnly()).toEqual([]); expect(uploads).toHaveLength(0); expect(manager.status()).toMatchObject({ openTickets: 0, ownedBytes: 0 }); + }); + test.each(["hideOutputs", "hideOutputMessages", "hideOutputText", "piiRedaction"] as const)("%s prevents output speech and image transfer", async flag => { + manager.updatePolicy({ traceConfig: { [flag]: true } }); transfers = []; + install(); await inNative(async () => { await (await speech()).arrayBuffer(); }); + await (await speech()).arrayBuffer(); + responseChunks = 4; await readAll(await speech()); responseChunks = 1; + responseType = "application/json"; responseData = Buffer.from(JSON.stringify({ data: [{ b64_json: png.toString("base64") }] })); + await (await fetch("https://api.openai.com/v1/images/generations", { method: "POST", body: "{}" })).json(); + await settle(); expect(sessionsOnly()).toEqual([]); + }); + test("T-PF-09 server-required redaction denies before any base64 decode", async () => { + await manager.shutdown(); manager = await createManager({}, { require_media_redaction: true }); transfers = []; + responseType = "application/json"; responseData = Buffer.from(JSON.stringify({ data: [{ b64_json: png.toString("base64") }] })); + install(); const decode = jest.spyOn(Buffer, "from"); + await (await fetch("https://api.openai.com/v1/images/generations", { method: "POST", body: "{}" })).json(); + await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/generations", { method: "POST", body: "{}" })).json(); }); + const calls = decode.mock.calls.filter(args => args[0] === png.toString("base64") && (args as unknown[])[1] === "base64"); decode.mockRestore(); + await settle(); expect(calls).toHaveLength(0); expect(sessionsOnly()).toEqual([]); + }); + test("unknown host, non-audio MIME and error bodies do not upload", async () => { + install(); await inNative(async () => { + await (await speech("unknown.invalid/v1/audio/speech")).arrayBuffer(); + // A lookalike host or port under a native span: only the media allowlist stands between it and capture. + await (await speech("api.openai.com.evil.invalid/v1/audio/speech")).arrayBuffer(); + await (await speech("api.openai.com:8443/v1/audio/speech")).arrayBuffer(); + responseType = "application/octet-stream"; await (await speech()).arrayBuffer(); + responseType = "audio/wav"; responseStatus = 403; await (await speech()).arrayBuffer(); + }); + responseStatus = 200; await (await speech("api.openai.com.evil.invalid/v1/audio/speech")).arrayBuffer(); + await settle(); expect(uploads).toHaveLength(0); + }); + test("oversized media, raw multipart, and ended/unrelated native contexts fail closed", async () => { + responseType = "application/json"; responseData = Buffer.from('{"data":[]}'); + const body = new FormData(); body.set("image", new Blob([new Uint8Array(20 * 1024 * 1024 + 1)], { type: "image/png" })); + install(); await inNative(async () => { + await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body })).json(); + await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", headers: { "content-type": "multipart/form-data; boundary=x" }, body: "--x\r\nPRIVATE_CANARY" })).json(); + }); + const small = new FormData(); small.set("image", new Blob([png], { type: "image/png" })); + await inNative(async () => { await (await fetch("https://api.openai.com/v1/images/edits", { method: "POST", body: small })).json(); }, "TOOL"); + await settle(); expect(uploads).toHaveLength(0); expect(manager.status()).toMatchObject({ openTickets: 0, ownedBytes: 0 }); + }); + test("policy tightening between request and application consumption prevents capture", async () => { + install(); await inNative(async () => { const response = await speech(); manager.updatePolicy({ traceConfig: { hideOutputs: true } }); await response.arrayBuffer(); }); + await settle(); expect(uploads).toHaveLength(0); + }); + test("hook removal makes retained response wrappers inert", async () => { + install(); await inNative(async () => { const response = await speech(); hook!.remove(); await response.arrayBuffer(); }); + await settle(); expect(uploads).toHaveLength(0); + }); + test("native capture is denied without an explicit integration policy", async () => { + install(); await inNative(async () => { await (await speech()).arrayBuffer(); }, "LLM", false); + await settle(); expect(uploads).toHaveLength(0); + }); + test.each([{ captureContent: false }, { recordOutputs: false }, { muted: true }, { temporaryChat: true }])("integration restrictions cannot be loosened by child scopes: %j", async restriction => { + install(); await inNative(() => context.with(withAttachmentCapturePolicy(withAttachmentCapturePolicy(context.active(), restriction), { captureContent: true, recordOutputs: true, muted: false, temporaryChat: false }), async () => { + await (await speech()).arrayBuffer(); + })); await settle(); expect(uploads).toHaveLength(0); + }); + test("a disabled manager installs no media observer: unchanged provider traffic, no control traffic", async () => { + const disabled = new AttachmentManager({ enabled: false, environment: () => ({}) }); + transfers = []; install(disabled); await inNative(async () => { await (await speech()).arrayBuffer(); }); + await (await speech()).arrayBuffer(); + await settle(); expect(transfers).toEqual([]); expect(providerRequests).toBe(2); + }); +}); + +describe("A02 ownership and configuration", () => { + test("borrowed manager identity survives auto setup; auto adds only restrictions", async () => { + const config = resolveConfig({ traceConfig: { hideOutputs: true }, projectName: "auto-project", configFile: "/nonexistent" }, {}); + const selected = selectAttachmentManager({ attachments: manager }, config, {}, false); + expect(selected.manager).toBe(manager); expect(selected.owned).toBe(false); + expect(attachmentOptions(config, { attachments: { projectName: "wrong-project", traceConfig: { hideOutputs: false } } })).toMatchObject({ projectName: "auto-project", traceConfig: { hideOutputs: true } }); + install(); await inNative(async () => { await (await speech()).arrayBuffer(); }); + await settle(); expect(uploads).toHaveLength(0); + }); + test("FI_ATTACHMENTS_ENABLED=false denies at admission on a process-env manager, without permanently disabling a borrowed one", async () => { + const before = process.env.FI_ATTACHMENTS_ENABLED; + delete process.env.FI_ATTACHMENTS_ENABLED; + try { + // A manager whose `environment` reads process.env, as auto's own and the Node factory's default do. + await manager.shutdown(); manager = await createManager({ environment: () => process.env }); + process.env.FI_ATTACHMENTS_ENABLED = "false"; + selectAttachmentManager({ attachments: manager }, resolveConfig({ configFile: "/nonexistent" }, {}), {}, false); + transfers = []; install(); + // Kill switch at admission: materialized, incremental and FormData wire captures, native and wire-only, all denied. + await inNative(async () => { await (await speech()).arrayBuffer(); }); + await (await speech()).arrayBuffer(); + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); await (await transcription()).json(); + await settle(); expect(sessionsOnly()).toEqual([]); expect(uploads).toHaveLength(0); + expect(manager.status()).toMatchObject({ openTickets: 0, openStreams: 0, ownedBytes: 0 }); + // The switch is read at each admission: lifting it resumes capture on the same borrowed manager. + delete process.env.FI_ATTACHMENTS_ENABLED; + responseType = "audio/wav"; responseData = audio; + await inNative(async () => { await (await speech()).arrayBuffer(); }); + await settle(); expect(uploads).toEqual([audio]); + } finally { if (before === undefined) delete process.env.FI_ATTACHMENTS_ENABLED; else process.env.FI_ATTACHMENTS_ENABLED = before; } + }); + test("status report is policy and source aware and claims no playback", () => { + const report = attachmentReport(manager); + expect(report.state).toBe("ready"); expect(report.status).toMatchObject({ openTickets: 0 }); + expect(report.routes).toContain("POST https://api.openai.com/v1/audio/speech (speech)"); + expect(report.unsupported).toEqual(expect.arrayContaining(["WebSocket / RTC audio", "product playback qualification"])); + expect(report.unsupported.join("\n")).not.toMatch(/deferred-target preflight/); + expect(report.sources).toEqual(expect.arrayContaining([expect.stringContaining("wire-only FormData Blob"), expect.stringContaining("wire-only speech audio read incrementally")])); + expect(attachmentReport().state).toBe("not_initialized"); + }); +}); + +describe("real SDKs against loopback", () => { + const sdk = () => require(process.env.ATT_OPENAI_MODULE || "openai"); + // The package's exports map hides package.json; walk up from the resolved entry instead. + const sdkVersion = (): string => { + let dir = dirname(require.resolve(process.env.ATT_OPENAI_MODULE || "openai")); + for (let i = 0; i < 8; i++, dir = dirname(dir)) { + try { const pkg = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")); if (pkg.name === "openai") return pkg.version; } catch { /* keep walking */ } + } + return "0.0.0"; + }; + test("OpenAI SDK speech (wire-only) preserves WAV bytes and correlation", async () => { + const OpenAI = sdk().default ?? sdk().OpenAI; + install(); const client = new OpenAI({ apiKey: "synthetic-provider-key", fetch: (...args: Parameters) => globalThis.fetch(...args) }); + const response = await client.audio.speech.create({ model: "tts-1", voice: "alloy", input: "synthetic", response_format: "wav" }); + expect(Buffer.from(await response.arrayBuffer())).toEqual(audio); + await settle(); expect(uploads).toEqual([audio]); expect(exporter.getFinishedSpans()).toHaveLength(1); + expect(requests[0].association.parent.span_id).toBe(exporter.getFinishedSpans()[0].spanContext().spanId); + }); + test("OpenAI SDK image generation (wire-only) uploads the decoded PNG", async () => { + const OpenAI = sdk().default ?? sdk().OpenAI; + responseType = "application/json"; responseData = Buffer.from(JSON.stringify({ created: 1, data: [{ b64_json: png.toString("base64") }] })); + install(); const client = new OpenAI({ apiKey: "synthetic-provider-key", fetch: (...args: Parameters) => globalThis.fetch(...args) }); + const result = await client.images.generate({ model: "gpt-image-1", prompt: "synthetic" }); + expect(result.data[0].b64_json).toBe(png.toString("base64")); + await settle(); expect(uploads).toEqual([png]); + }); + test("OpenAI SDK transcription (wire-only) keeps the multipart request intact and uploads the file once, bound to the wire span", async () => { + const mod = sdk(); + // openai 4.x sends multipart as a Node stream, which a WHATWG fetch rejects without `duplex` (SDK limitation, not auto). + if (Number(sdkVersion().split(".")[0]) < 5) return; const OpenAI = mod.default ?? mod.OpenAI; const toFile = mod.toFile; + responseType = "application/json"; responseData = Buffer.from('{"text":"synthetic"}'); + install(); const client = new OpenAI({ apiKey: "synthetic-provider-key", fetch: (...args: Parameters) => globalThis.fetch(...args) }); + const result = await client.audio.transcriptions.create({ model: "whisper-1", file: await toFile(audio, "synthetic.wav", { type: "audio/wav" }) }); + expect(result.text).toBe("synthetic"); expect(providerBodies[0].includes(audio)).toBe(true); + await settle(); expect(uploads).toEqual([audio]); + const spans = exporter.getFinishedSpans(); expect(spans).toHaveLength(1); + expect(requests[0]).toMatchObject({ direction: "input", purpose: "model_input", source_kind: "Blob" }); + expect(requests[0].association.parent.span_id).toBe(spans[0].spanContext().spanId); + }); +}); diff --git a/typescript/packages/traceai_openai/src/attachments.ts b/typescript/packages/traceai_openai/src/attachments.ts new file mode 100644 index 000000000..b185f6294 --- /dev/null +++ b/typescript/packages/traceai_openai/src/attachments.ts @@ -0,0 +1,239 @@ +import type { TraceConfigOptions } from "@traceai/fi-core"; +import type { AttachmentManager, AttachmentOptions, AttachmentSpan, AttachmentCorrelation, AttachmentCapturePolicy } from "@traceai/fi-core/attachments"; + +/** Declarative handoff to auto/wire owners; this is not a native/wire hookup. */ +export const openAIAttachmentCapabilities = Object.freeze({ + schemaVersion: 1, + operations: ["chat.completions.create.input", "responses.create.input", "responses.create.output", "responses.stream.output", "images.generate.output"] as const, + sources: ["inline_base64"] as const, + /** Fail closed: no capture is attempted for these. Streamed Responses output is captured only from the + * terminal `response.completed` event the application consumed; partial (preview) images never are. */ + unsupported: ["remote_url", "provider_file", "partial_image", "images.generate.stream", "images.edit", "images.createVariation", "speech"] as const, + maxInlineBytes: 256 * 1024, +}); + +/** Admitted media positions for one observation: position key -> occurrence. */ +export type CapturedMedia = ReadonlyMap>; +type Captured = Map>; +type RecordValue = Record; +const record = (value: unknown): RecordValue => value !== null && typeof value === "object" ? value as RecordValue : {}; +const list = (value: unknown): unknown[] => Array.isArray(value) ? value : []; +const MAX_PARTS = 256; +const MAX_SPANS = 1024; +const EMPTY: CapturedMedia = new Map(); +const IMAGES = ["image/png", "image/jpeg", "image/webp"]; +const FILES = ["application/pdf", "application/octet-stream"]; +// Shared across CJS/ESM copies. Retains only admitted span IDs, positions and occurrence IDs, never sources. +const registryKey = Symbol.for("@traceai/openai/attachment-observations/v1"); +type Registry = WeakMap>; +const host = globalThis as typeof globalThis & { [registryKey]?: Registry }; +const observations = (host[registryKey] ??= new WeakMap()); + +export interface OpenAIAttachmentCaptureOptions { + operation: "chat" | "responses" | "images"; + direction: "input" | "output"; + traceConfig?: TraceConfigOptions; + capture?: AttachmentCapturePolicy; + /** The Images request's typed output_format; never inferred from filenames. */ + imageFormat?: "png" | "jpeg" | "webp"; +} + +/** The local instrumentor's restrictions; the shared manager applies its own policy as well (deny wins). */ +function restrictions(options: OpenAIAttachmentCaptureOptions): AttachmentCapturePolicy { + const config = options.traceConfig ?? {}; + const input = options.direction === "input"; + const hidden = input + ? config.hideInputs || config.hideInputMessages || config.hideInputText + : config.hideOutputs || config.hideOutputMessages || config.hideOutputText; + return { ...options.capture, + ...(hidden || config.piiRedaction ? { captureContent: false } : {}), + ...(input ? { modelInput: true } : {}), + }; +} + +type Parsed = { mimeType: string; encoded: string; bound: number }; +const MAX_ENCODED = 4 * Math.ceil(openAIAttachmentCapabilities.maxInlineBytes / 3); + +/** Structural parse only: no decode or allocation beyond the string the application already holds. + * Returns the candidate MIME named by a data URI (or the typed MIME for raw base64) and the exact decoded length. + */ +function parse(value: unknown, mimeTypes: string[], raw: boolean): Parsed | undefined { + if (typeof value !== "string") return undefined; + let mimeType = mimeTypes[0], start = 0; + if (!raw) { + const comma = value.indexOf(","); + const header = comma > 5 && comma < 128 ? value.slice(0, comma) : ""; + const match = /^data:([a-z]+\/[a-z0-9.+-]+);base64$/.exec(header); + if (!match || !mimeTypes.includes(match[1])) return undefined; + mimeType = match[1]; start = comma + 1; + } + const length = value.length - start; + // Bound allocation BEFORE slicing/decoding. + if (length <= 0 || length > MAX_ENCODED || length % 4 !== 0) return undefined; + const encoded = value.slice(start); + const padding = encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0; + return { mimeType, encoded, bound: (length / 4) * 3 - padding }; +} + +/** Strict decode after admission. Padding and alphabet are strict; the ticket's budget is a hard bound. */ +function decode(encoded: string, maxBytes: number): Uint8Array | undefined { + if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return undefined; + const bytes = Buffer.from(encoded, "base64"); + if (!bytes.length || bytes.length > maxBytes || bytes.length > openAIAttachmentCapabilities.maxInlineBytes || bytes.toString("base64") !== encoded) return undefined; + return bytes; +} + +const key = (options: Pick, messageIndex: number, partIndex: number) => + `${options.operation}/${options.direction}/${messageIndex}/${partIndex}`; + +/** Typed, synchronous observation only. No provider downloads or stream consumption. + * Pass the SAME emitted identity to SDK and wire observations: a second observation of a position + * returns the first occurrence instead of uploading again. + * Denied sources are never read: every part is preflighted (deny-wins, budget reserved) before its value is + * read or decoded, and an admitted ticket that is not used is released at once. + * Returns the admitted positions so the caller can project them out of span telemetry. + */ +export function captureOpenAIAttachments(manager: AttachmentManager | undefined, + span: AttachmentSpan | AttachmentCorrelation, payload: unknown, options: OpenAIAttachmentCaptureOptions): CapturedMedia { + if (!manager) return EMPTY; + const captured: Captured = new Map(); + try { + const correlation = "snapshot" in span ? span : manager.correlation(span); + const identity = correlation.snapshot(); + if (!identity?.sampled) return EMPTY; + let spans = observations.get(manager); + if (!spans) { spans = new Map(); observations.set(manager, spans); } + const spanKey = `${identity.traceId}/${identity.spanId}`; + const capture = restrictions(options); + let visited = 0; + const attach = (read: () => unknown, mimeTypes: string[], messageIndex: number, partIndex: number, + purpose: AttachmentOptions["purpose"], toolCallId?: string, raw = false, role: AttachmentOptions["role"] = "user") => { + if (++visited > MAX_PARTS) return; + const position = key(options, messageIndex, partIndex); + const previous = spans!.get(spanKey)?.get(position); + if (previous) { captured.set(position, previous); return; } + // The source is read at most once, and only after a preflight admitted this modality (design section 1). + let source: Parsed | undefined, sourceRead = false; + const parsed = () => { if (!sourceRead) { sourceRead = true; source = parse(read(), mimeTypes, raw); } return source; }; + const preflight = (mimeType: string) => { + const request = { mimeType, direction: options.direction, purpose, capture: { + ...capture, + ...(options.direction === "input" && mimeType.startsWith("image/") && options.traceConfig?.hideInputImages ? { captureContent: false } : {}), + } }; + let ticket = manager.preflight(correlation, { ...request, maxBytes: openAIAttachmentCapabilities.maxInlineBytes }); + // too_large (negotiated limit below the cap) and queue_full (owned budget nearly used) are evaluated after every + // privacy, correlation and sampling step, so sizing the source exactly and retrying once is safe here. + if (ticket.reason === "too_large" || ticket.reason === "queue_full") { + const bound = parsed()?.bound ?? 0; + if (bound > 0) ticket = manager.preflight(correlation, { ...request, maxBytes: bound }); + } + return ticket; + }; + let ticket = preflight(mimeTypes[0]); + if (!ticket.admitted) return; + const found = parsed(); + if (!found) { ticket.release(); return; } + // A data URI names a sibling MIME of the same modality: re-admit under the exact type before decoding. + if (found.mimeType !== mimeTypes[0]) { + ticket.release(); + ticket = preflight(found.mimeType); + if (!ticket.admitted) return; + } + const bytes = decode(found.encoded, ticket.maxBytes); + if (!bytes) { ticket.release(); return; } + const handle = ticket.attach({ kind: "Bytes", data: bytes }, { messageIndex, partIndex, role, ...(toolCallId ? { toolCallId } : {}) }); + if (handle.admission !== "queued") return; + const occurrence = Object.freeze({ occurrenceId: handle.occurrenceId, mimeType: found.mimeType }); + let positions = spans!.get(spanKey); + if (!positions) { + if (spans!.size >= MAX_SPANS) spans!.delete(spans!.keys().next().value!); + positions = new Map(); spans!.set(spanKey, positions); + } + if (positions.size < MAX_PARTS) positions.set(position, occurrence); + captured.set(position, occurrence); + }; + const toolCallId = (value: unknown) => typeof value === "string" && /^[a-zA-Z0-9_-]{1,128}$/.test(value) ? value : undefined; + const parts = (content: unknown, messageIndex: number, role: AttachmentOptions["role"]) => { + list(content).slice(0, MAX_PARTS).forEach((value, partIndex) => { + const part = record(value); + if (part.type === "image_url") attach(() => record(part.image_url).url, IMAGES, messageIndex, partIndex, "model_input", undefined, false, role); + else if (part.type === "input_image") attach(() => part.image_url, IMAGES, messageIndex, partIndex, "model_input", undefined, false, role); + else if (part.type === "input_file") attach(() => part.file_data, FILES, messageIndex, partIndex, "document", undefined, false, role); + else if (part.type === "file") attach(() => record(part.file).file_data, FILES, messageIndex, partIndex, "document", undefined, false, role); + }); + }; + const body = record(payload); + if (options.direction === "output" && body.error) return EMPTY; + if (options.direction === "input") { + const messages = options.operation === "chat" ? body.messages : options.operation === "responses" ? body.input : undefined; + list(messages).slice(0, MAX_PARTS).forEach((value, messageIndex) => { + const message = record(value); + if (message.type === "computer_call_output") { + const output = record(message.output); + // A screenshot is model-bound input. Its call id keeps the causal link to the computer_call. + if (output.type === "computer_screenshot") attach(() => output.image_url, IMAGES, messageIndex, 0, "computer_screenshot", + toolCallId(message.call_id), false, "tool"); + } else if (!message.type || message.type === "message") { + const role = ["user", "assistant", "tool", "system"].includes(String(message.role)) ? message.role as AttachmentOptions["role"] : "unknown"; + parts(message.content, messageIndex, role); + } + }); + } else if (options.operation === "images") { + // The Images API defaults to PNG. The typed request/response format wins where provided. + const format = body.output_format ?? options.imageFormat; + const mime = format === "jpeg" ? "image/jpeg" : format === "webp" ? "image/webp" : "image/png"; + list(body.data).slice(0, MAX_PARTS).forEach((value, index) => { + const item = record(value); + attach(() => item.b64_json, [mime], index, 0, "model_output", undefined, true, "assistant"); + }); + } else if (options.operation === "responses") { + list(body.output).slice(0, MAX_PARTS).forEach((value, index) => { + const item = record(value); + if (item.type === "image_generation_call" && item.status === "completed") { + const mime = item.output_format === "jpeg" ? "image/jpeg" : item.output_format === "webp" ? "image/webp" : "image/png"; + attach(() => item.result, [mime], index, 0, "model_output", undefined, true, "assistant"); + } + }); + } + } catch { /* Instrumentation must not throw or log source-bearing errors. */ } + return captured; +} + +const uri = (occurrence: { occurrenceId: string }) => `fi-attachment:${occurrence.occurrenceId}`; + +/** Telemetry-only projection. ONLY admitted positions are replaced with their `fi-attachment:` occurrence + * URI, so uploaded bytes are not exported twice. Everything else is returned exactly as before, and the + * request passed to the SDK is never mutated. + */ +export function projectOpenAIMedia(payload: unknown, captured: CapturedMedia, + operation: "chat" | "responses", direction: "input" | "output"): any { + if (!captured.size) return payload; + try { + const body = record(payload); + const field = direction === "input" ? operation === "chat" ? "messages" : "input" : "output"; + if (!Array.isArray(body[field])) return payload; + const at = (messageIndex: number, partIndex: number) => captured.get(key({ operation, direction }, messageIndex, partIndex)); + return { ...body, [field]: list(body[field]).map((value, messageIndex) => { + const message = record(value); + if (message.type === "computer_call_output") { + const occurrence = at(messageIndex, 0); + return occurrence ? { ...message, output: { ...record(message.output), image_url: uri(occurrence) } } : value; + } + if (message.type === "image_generation_call") { + const occurrence = at(messageIndex, 0); + return occurrence ? { ...message, result: uri(occurrence) } : value; + } + if (!Array.isArray(message.content)) return value; + return { ...message, content: message.content.map((value, partIndex) => { + const occurrence = at(messageIndex, partIndex); + if (!occurrence) return value; + const part = record(value); + if (part.type === "image_url") return { ...part, image_url: { ...record(part.image_url), url: uri(occurrence) } }; + if (part.type === "input_image") return { ...part, image_url: uri(occurrence) }; + if (part.type === "input_file") return { ...part, file_data: uri(occurrence) }; + if (part.type === "file") return { ...part, file: { ...record(part.file), file_data: uri(occurrence) } }; + return value; + }) }; + }) }; + } catch { return payload; } +} diff --git a/typescript/packages/traceai_openai/src/index.ts b/typescript/packages/traceai_openai/src/index.ts index a38448910..e886a4c31 100644 --- a/typescript/packages/traceai_openai/src/index.ts +++ b/typescript/packages/traceai_openai/src/index.ts @@ -1,2 +1,3 @@ export * from './instrumentation'; -export * from './version'; \ No newline at end of file +export * from './version'; +export * from './attachments'; diff --git a/typescript/packages/traceai_openai/src/instrumentation.ts b/typescript/packages/traceai_openai/src/instrumentation.ts index a04bc72d0..d2d05e7a4 100644 --- a/typescript/packages/traceai_openai/src/instrumentation.ts +++ b/typescript/packages/traceai_openai/src/instrumentation.ts @@ -1,4 +1,6 @@ import openai from "openai"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import { captureOpenAIAttachments, projectOpenAIMedia } from "./attachments"; import { InstrumentationBase, InstrumentationConfig, @@ -107,6 +109,7 @@ function getExecContext(span: Span) { export class OpenAIInstrumentation extends InstrumentationBase{ private fiTracer!: FITracer; private _traceConfig?: TraceConfigOptions; + private attachments?: AttachmentManager; // Lazily initialized: InstrumentationBase can patch during super(). // A Set is needed so disable() also undoes every manually patched ESM SDK. private patchedModules?: Set; @@ -114,6 +117,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ constructor({ instrumentationConfig, traceConfig, + attachments, }: { /** * The config for the instrumentation @@ -125,6 +129,8 @@ export class OpenAIInstrumentation extends InstrumentationBase{ * @see {@link TraceConfigOptions} */ traceConfig?: TraceConfigOptions; + /** The shared manager from register(...).attachments; never a per-adapter uploader. */ + attachments?: AttachmentManager; } = {}) { super( "@traceai/fi-instrumentation-openai", @@ -132,6 +138,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ Object.assign({}, instrumentationConfig), ); this._traceConfig = traceConfig; + this.attachments = attachments; // The base constructor can call enable() before our trace options exist. this.fiTracer = new FITracer({ tracer: this.tracer, traceConfig: this._traceConfig }); } @@ -223,40 +230,34 @@ export class OpenAIInstrumentation extends InstrumentationBase{ this: unknown, ...args: Parameters ) { - const body = args[0]; - // Prediction text is input content, captured only in maskable payloads. - const { messages: _messages, prediction: _prediction, ...invocationParameters } = body; - + const originalBody = args[0]; // diag.debug("@traceai/openai: ChatCompletion patch CALLED. Starting span..."); - - // --- ADD LOGS FOR TRACER AND CONTEXT --- - // const activeContextForSuppressionCheck = context.active(); - // const isSuppressed = isTracingSuppressed(activeContextForSuppressionCheck); - // diag.debug(`@traceai/openai: Is tracing suppressed? ${isSuppressed}`); - // diag.debug(`@traceai/openai: this.tracer type: ${instrumentation.tracer?.constructor?.name}`); // Accessing the base tracer - // diag.debug(`@traceai/openai: this.fiTracer type: ${instrumentation.fiTracer?.constructor?.name}`); - // --- END ADDED LOGS --- - const span = instrumentation.fiTracer.startSpan( - `OpenAI Chat Completions`, - { - kind: SpanKind.INTERNAL, - attributes: { - [SemanticConventions.FI_SPAN_KIND]: - FISpanKind.LLM, - [SemanticConventions.LLM_MODEL_NAME]: body.model, + // Content attributes are set after capture so admitted media can be projected out first. + const span = instrumentation.fiTracer.startSpan("OpenAI Chat Completions", { + kind: SpanKind.INTERNAL, + attributes: { + [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, + [SemanticConventions.LLM_MODEL_NAME]: originalBody.model, + [SemanticConventions.LLM_PROVIDER]: providerForResource(this), + [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", + }, + }); + const captured = captureOpenAIAttachments(instrumentation.attachments, span, originalBody, { + operation: "chat", direction: "input", traceConfig: instrumentation._traceConfig, + }); + const body = projectOpenAIMedia(originalBody, captured, "chat", "input") as typeof originalBody; + // Prediction text is input content, captured only in maskable payloads. + const { messages: _messages, prediction: _prediction, ...invocationParameters } = body; + span.setAttributes({ [SemanticConventions.INPUT_VALUE]: JSON.stringify(body), [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON, [SemanticConventions.LLM_INVOCATION_PARAMETERS]: JSON.stringify(invocationParameters), - [SemanticConventions.LLM_PROVIDER]: providerForResource(this), - [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", ...getLLMInputMessagesAttributes(body), ...getLLMToolsJSONSchema(body), [SemanticConventions.RAW_INPUT]: safelyJSONStringify(body) ?? "", - }, - }, - ); + }); // --- ADD LOG --- // diag.debug(`@traceai/openai: ChatCompletion span STARTED: ${span.spanContext().spanId}`); @@ -517,39 +518,42 @@ export class OpenAIInstrumentation extends InstrumentationBase{ const body = args[0] as ResponseCreateParamsBase & { prompt?: { id: string; version?: string; variables?: Record }; }; + // Content attributes are set after capture so admitted media can be projected out first. + const span = instrumentation.fiTracer.startSpan("OpenAI Responses", { + kind: SpanKind.INTERNAL, + attributes: { + [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, + [SemanticConventions.LLM_MODEL_NAME]: body.model, + [SemanticConventions.LLM_PROVIDER]: providerForResource(this), + [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", + }, + }); + const captured = captureOpenAIAttachments(instrumentation.attachments, span, body, { + operation: "responses", direction: "input", traceConfig: instrumentation._traceConfig, + }); // Telemetry-only copy: MCP credentials must never reach any exported // duplicate, but the original SDK request must remain untouched. + const projectedBody = projectOpenAIMedia(body, captured, "responses", "input"); const telemetryBody = { - ...body, + ...projectedBody, ...(body.tools && { tools: sanitizeToolDefinitions(body.tools) }), - ...(body.input && { input: sanitizeResponsesInput(body.input) }), + ...(projectedBody.input && { input: sanitizeResponsesInput(projectedBody.input) }), }; const { input: _messages, instructions: _instructions, prompt, ...parameters } = telemetryBody; const invocationParameters = { ...parameters, ...(prompt && { prompt: { id: prompt.id, version: prompt.version } }), }; - const span = instrumentation.fiTracer.startSpan( - `OpenAI Responses`, - { - kind: SpanKind.INTERNAL, - attributes: { - [SemanticConventions.FI_SPAN_KIND]: - FISpanKind.LLM, - [SemanticConventions.LLM_MODEL_NAME]: body.model, + span.setAttributes({ [SemanticConventions.INPUT_VALUE]: JSON.stringify(telemetryBody), [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON, [SemanticConventions.LLM_INVOCATION_PARAMETERS]: JSON.stringify(invocationParameters), [SemanticConventions.PROMPT_TEMPLATE_VARIABLES]: safelyJSONStringify(prompt?.variables) ?? undefined, - [SemanticConventions.LLM_PROVIDER]: providerForResource(this), - [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", - ...getResponsesInputMessagesAttributes(body), + ...getResponsesInputMessagesAttributes(telemetryBody), ...getLLMToolsJSONSchema(body), [SemanticConventions.RAW_INPUT]: safelyJSONStringify(telemetryBody) ?? "", - }, - }, - ); + }); const contextSpan = span.getContextSpan(); const execContext = getExecContext(contextSpan); const requestStartedAt = performance.now(); @@ -581,16 +585,22 @@ export class OpenAIInstrumentation extends InstrumentationBase{ if (endSpan) span.end(); return; } + // On the streamed path (endSpan false) `result` is the terminal response the application consumed + // from its own iterator, observed before the span ends. Partial (preview) images are never captured. + const outputMedia = captureOpenAIAttachments(instrumentation.attachments, span, result, { + operation: "responses", direction: "output", traceConfig: instrumentation._traceConfig, + }); + const projected = projectOpenAIMedia(result, outputMedia, "responses", "output") as ResponseType; const output = { id: result.id, object: result.object, status: result.status, - model: result.model, output: result.output, usage: result.usage, + model: result.model, output: projected.output, usage: result.usage, }; span.setAttributes({ [SemanticConventions.OUTPUT_VALUE]: JSON.stringify(output), [SemanticConventions.OUTPUT_MIME_TYPE]: MimeType.JSON, [SemanticConventions.GEN_AI_RESPONSE_MODEL]: result.model, [SemanticConventions.GEN_AI_RESPONSE_ID]: result.id, - ...getResponsesOutputMessagesAttributes(result), + ...getResponsesOutputMessagesAttributes(projected), ...getResponsesUsageAttributes(result), [SemanticConventions.RAW_OUTPUT]: safelyJSONStringify(output) ?? "", }); @@ -628,6 +638,46 @@ export class OpenAIInstrumentation extends InstrumentationBase{ ); } + // images.generate: generated b64 outputs. Streamed partial images, edits and variations stay unsupported. + if (typeof module.OpenAI.Images?.prototype?.generate === "function") { + type ImagesGenerateType = typeof module.OpenAI.Images.prototype.generate; + this._wrap(module.OpenAI.Images.prototype, "generate", (original: ImagesGenerateType): any => { + return function patchedImagesGenerate(this: unknown, ...args: Parameters) { + const request = (args[0] ?? {}) as { model?: string; output_format?: "png" | "jpeg" | "webp" }; + const span = instrumentation.fiTracer.startSpan("OpenAI Images", { + kind: SpanKind.INTERNAL, + attributes: { + [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, + [SemanticConventions.LLM_PROVIDER]: providerForResource(this), + [SemanticConventions.GEN_AI_OPERATION_NAME]: "image_generation", + ...(typeof request.model === "string" ? { [SemanticConventions.LLM_MODEL_NAME]: request.model } : {}), + [SemanticConventions.INPUT_VALUE]: safelyJSONStringify(args[0]) ?? "", + [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON, + }, + }); + const contextSpan = span.getContextSpan(); + const execContext = getExecContext(contextSpan); + const execPromise = safeExecuteInTheMiddle>( + () => context.with(trace.setSpan(execContext, contextSpan), () => original.apply(this, args)), + (error) => { if (error) endSpanWithError(span, error); }, + ); + const wrappedPromise = observeAPIPromise(execPromise, (result) => { + if (!span.isRecording()) return result; + if (!isStream(result)) { + captureOpenAIAttachments(instrumentation.attachments, span, result, { + operation: "images", direction: "output", traceConfig: instrumentation._traceConfig, + imageFormat: request.output_format, + }); + } + span.setStatus({ code: SpanStatusCode.OK }); + span.end(); + return result; + }, (error) => endSpanWithError(span, error), (response) => endRawResponseSpan(span, response)); + return context.bind(execContext, wrappedPromise); + }; + }); + } + // Keep the legacy diagnostic property, but never use it as the guard. // Immutable class exports still work via the identity map. try { (sdk as typeof openai & { fiPatched?: boolean }).fiPatched = true; } catch { /* immutable SDK */ } @@ -647,6 +697,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ this._unwrap(sdk.Completions.prototype, "create"); this._unwrap(sdk.Embeddings.prototype, "create"); if (sdk.Responses) this._unwrap(sdk.Responses.prototype, "create"); + if (sdk.Images) this._unwrap(sdk.Images.prototype, "generate"); try { (sdk as typeof openai & { fiPatched?: boolean }).fiPatched = false; } catch { /* immutable SDK */ } patchRegistry.owners.delete(sdk); // An automatic require hook may use a different namespace for this class. @@ -983,7 +1034,7 @@ function serializeChatCompletionInputMessage( if (part.type === "text") { return { type: "text", text: part.text }; } else if (part.type === "image_url") { - return { type: "image_url", image_url: { url: part.image_url.url } }; + return part.image_url ? { type: "image_url", image_url: { url: part.image_url.url } } : { type: "image_url" }; } return { type: part.type }; }); diff --git a/typescript/packages/traceai_openai/test/attachments-providers.test.cjs b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs new file mode 100644 index 000000000..fbb05607c --- /dev/null +++ b/typescript/packages/traceai_openai/test/attachments-providers.test.cjs @@ -0,0 +1,465 @@ +/* Cloud component qualification: real SDKs, synthetic media, loopback-only APIs. */ +const { test, after } = require('node:test'); +const assert = require('node:assert/strict'); +const http = require('node:http'); +const { once } = require('node:events'); +const { randomUUID } = require('node:crypto'); +const { pathToFileURL } = require('node:url'); +const path = require('node:path'); +const { AttachmentManager, createAttachmentHttpTransport } = require('@traceai/fi-core/attachments'); +const { BasicTracerProvider, SimpleSpanProcessor, InMemorySpanExporter } = require('@opentelemetry/sdk-trace-base'); +const { OpenAIInstrumentation } = require('../dist/src/instrumentation.js'); +const { AnthropicInstrumentation } = require('../../traceai_anthropic/dist/src/instrumentation.js'); +const oa = require('../dist/src/attachments.js'); +const an = require('../../traceai_anthropic/dist/src/attachments.js'); +const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aD1sAAAAASUVORK5CYII='; +const PDF = Buffer.from('%PDF-1.7\n% synthetic document\n%%EOF').toString('base64'); +const image = () => ({ type: 'image', source: { type: 'base64', media_type: 'image/png', data: PNG } }); +const document = () => ({ type: 'document', title: 'synthetic-private-filename', source: { type: 'base64', media_type: 'application/pdf', data: PDF } }); +const identity = () => ({ traceId: randomUUID().replace(/-/g, ''), spanId: randomUUID().replace(/-/g, '').slice(0, 16), traceFlags: 1 }); +const keepalive = setInterval(() => {}, 1000); +after(() => clearInterval(keepalive)); +// The core reports a silently degraded fixture (legacy transport, rejected F01 shape) as a process warning. +const coreWarnings = []; +process.on('warning', warning => { if (/^FI_ATTACHMENTS_/.test(warning.code ?? '')) coreWarnings.push(warning.code); }); +// Preflight tickets that are not used must be released at once, never left to the TTL sweep. +const assertNoOpenTickets = async manager => { + assert.equal(manager.status().openTickets, 0, 'open tickets'); await manager.flush(); assert.equal(manager.status().ownedBytes, 0, 'owned bytes'); +}; + +async function fixture(config = {}, serverPolicy = {}, serverLimits = {}) { + const calls = [], initialized = [], uploaded = []; + const scope = { organization_id: randomUUID(), workspace_id: randomUUID(), project_id: randomUUID() }; + // Sessions of one hour or more: the core renews within max(60 s, 20% of lifetime) of expiry. + const expiry = () => new Date(Date.now() + 3600000).toISOString(); + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const c of req) chunks.push(c); + const bytes = Buffer.concat(chunks); calls.push(req.url); + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('attachment-sessions')) { + const body = JSON.parse(bytes); + res.end(JSON.stringify({ schema_version: 1, scope, capture_session_id: body.capture_session_id, + policy: { version: 'fixture-v1', enabled: true, allowed_sources: ['Bytes'], allowed_modalities: ['image', 'document', 'file'], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, + record_audio: false, record_video: false, ...serverPolicy }, + limits: { max_upload_bytes: 1024 * 1024, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, + recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, + initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, + transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5, ...serverLimits }, expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); + } else if (req.url.endsWith('/uploads')) { + const body = JSON.parse(bytes); initialized.push(body); + res.statusCode = 201; + res.end(JSON.stringify({ upload_id: body.attachment_id, attachment_id: body.attachment_id, occurrence_id: body.occurrence_id, + state: 'pending_upload', reserved_bytes: body.declared_bytes, expires_at: expiry(), transfer_mode: 'authenticated_stream', write_capability: 'synthetic-upload-only' })); + } else if (req.url.endsWith('/content')) { + assert.equal(req.headers.authorization, 'Bearer synthetic-upload-only'); uploaded.push(bytes); + res.end(JSON.stringify({ upload_id: req.url.split('/').at(-2), state: 'pending_upload', reason: 'queued' })); + } else { + res.statusCode = 202; res.end(JSON.stringify({ upload_id: req.url.split('/').at(-2), state: 'validating', reason: 'binding_pending' })); + } + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const client = req => new Promise((resolve, reject) => { + const request = http.request(`http://127.0.0.1:${server.address().port}${new URL(req.url).pathname}`, { method: req.method, headers: req.headers, signal: req.signal }, response => { + const chunks = []; response.on('data', c => chunks.push(c)); response.on('end', () => resolve({ status: response.statusCode, body: Buffer.concat(chunks) })); + }); request.on('error', reject); request.end(req.body); + }); + const manager = new AttachmentManager({ enabled: true, projectName: 'synthetic', projectType: 'observe', + transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic' }, client }), ...config }); + assert.equal(await manager.ready(), config.enabled !== false); + return { manager, calls, initialized, uploaded, async close() { await manager.shutdown(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } }; +} +const oaInput = () => ({ input: [ + { role: 'user', content: [{ type: 'input_image', image_url: `data:image/png;base64,${PNG}` }, { type: 'input_file', filename: 'synthetic-private-filename', file_data: `data:application/pdf;base64,${PDF}` }] }, + { type: 'computer_call_output', call_id: 'call_computer', output: { type: 'computer_screenshot', image_url: `data:image/png;base64,${PNG}` } }, +] }); +const anInput = () => ({ tools: [{ type: 'computer_20250124', name: 'computer', display_width_px: 800, display_height_px: 600 }], messages: [ + { role: 'assistant', content: [{ type: 'tool_use', id: 'tool_computer', name: 'computer', input: { action: 'screenshot' } }] }, + { role: 'user', content: [image(), document(), { type: 'tool_result', tool_use_id: 'tool_computer', content: [image(), image()] }] }, +] }); + +for (const provider of ['openai', 'anthropic']) { + const capture = (m, s, p, extra = {}) => provider === 'openai' + ? oa.captureOpenAIAttachments(m, s, p, { operation: 'responses', direction: 'input', ...extra }) + : an.captureAnthropicAttachments(m, s, p, extra); + const input = provider === 'openai' ? oaInput : anInput; + test(`${provider}: typed image/document/screenshots preserve bytes, causal positions and distinct repeated frames`, async () => { + const f = await fixture(); + try { + const id = identity(), body = input(), saved = JSON.stringify(body); + capture(f.manager, id, body); capture(f.manager, { ...id }, body); + await f.manager.flush(); + assert.equal(f.initialized.length, provider === 'openai' ? 3 : 4); + assert.equal(f.uploaded.length, f.initialized.length); + assert.deepEqual(f.uploaded.map(b => b.toString('base64')).sort(), (provider === 'openai' ? [PNG, PDF, PNG] : [PNG, PDF, PNG, PNG]).sort()); + assert.equal(new Set(f.initialized.map(r => r.occurrence_id)).size, f.initialized.length); + assert.ok(f.initialized.every(r => r.direction === 'input' && r.association.parent.span_id === id.spanId)); + assert.equal(f.initialized.filter(r => r.purpose === 'computer_screenshot').length, provider === 'openai' ? 1 : 2); + assert.ok(!JSON.stringify(f.initialized).includes('synthetic-private-filename')); + assert.equal(JSON.stringify(body), saved); + } finally { await f.close(); } + }); + for (const key of ['disabled', 'hideInputs', 'hideInputMessages', 'hideInputText', 'piiRedaction', 'hideInputImages', 'serverRedaction', 'serverMetadata']) { + test(`${provider}: ${key} denies protected sources without reading or uploading`, async () => { + const f = await fixture(key === 'disabled' ? { enabled: false } : { traceConfig: { [key]: true } }, + key === 'serverRedaction' ? { require_media_redaction: true } : key === 'serverMetadata' ? { require_metadata_removal: true } : {}); + try { + let read = false; + const source = provider === 'openai' ? { input: [{ type: 'computer_call_output', call_id: 'call_computer', output: { type: 'computer_screenshot', get image_url() { read = true; throw Error('synthetic-secret'); } } }] } + : { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_computer', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', get data() { read = true; throw Error('synthetic-secret'); } } }] }] }] }; + const before = f.calls.length; capture(f.manager, identity(), source); + await f.manager.flush(); assert.equal(read, false); assert.equal(f.calls.length, before); + if (key === 'disabled') assert.equal(f.calls.length, 0); + } finally { await f.close(); } + }); + } + for (const capturePolicy of [{ captureContent: false }, { recordInputs: false }, { muted: true }, { temporaryChat: true }]) { + test(`${provider}: integration restriction ${Object.keys(capturePolicy)[0]} wins`, async () => { + const f = await fixture(); try { capture(f.manager, identity(), input(), { capture: capturePolicy }); await f.manager.flush(); assert.equal(f.initialized.length, 0); } finally { await f.close(); } + }); + } + test(`${provider}: malformed and oversized inline data is bounded; foreign URLs/files are not retrieved`, async () => { + const f = await fixture(); + try { + for (const data of ['!', 'a===', 'AB==', 'AAAA'.repeat(100000), 'https://127.0.0.1:1/?synthetic-secret']) { + const body = provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_image', image_url: data }] }] } + : { messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data } }] }] }; + capture(f.manager, identity(), body); + } + capture(f.manager, identity(), provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_file', file_id: 'file_secret' }] }] } + : { messages: [{ role: 'user', content: [{ type: 'document', source: { type: 'file', file_id: 'file_secret' } }, null, { type: 'tool_result', tool_use_id: {}, content: [image()] }] }] }); + await f.manager.flush(); assert.equal(f.initialized.length, 0); + await assertNoOpenTickets(f.manager); + } finally { await f.close(); } + }); + test(`${provider}: a negotiated upload limit below the inline cap still captures small media exactly`, async () => { + const f = await fixture({}, {}, { max_upload_bytes: 100 * 1024 }); + try { + const id = identity(), captured = capture(f.manager, id, input()); + await f.manager.flush(); + assert.equal(captured.size, provider === 'openai' ? 3 : 4); + assert.equal(f.uploaded.length, captured.size); + assert.ok(f.initialized.every(r => r.declared_bytes <= 100 * 1024)); + // Above the negotiated limit nothing is decoded or uploaded, and the reservation is returned. + const big = Buffer.alloc(150 * 1024, 7).toString('base64'); + capture(f.manager, identity(), provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_image', image_url: `data:image/png;base64,${big}` }] }] } + : { messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: big } }] }] }); + await f.manager.flush(); assert.equal(f.uploaded.length, captured.size); + await assertNoOpenTickets(f.manager); + } finally { await f.close(); } + }); + test(`${provider}: a nearly used owned-byte budget still admits small media at its exact size`, async () => { + const f = await fixture({ maxOwnedBytes: 300 * 1024 }); + try { + const big = Buffer.alloc(200 * 1024, 7).toString('base64'); + const one = data => provider === 'openai' ? { input: [{ role: 'user', content: [{ type: 'input_image', image_url: `data:image/png;base64,${data}` }] }] } + : { messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data } }] }] }; + // Same tick: the 200 KiB part still owns its bytes, so a cap-sized reservation would not fit. + const first = capture(f.manager, identity(), one(big)), second = capture(f.manager, identity(), one(PNG)); + assert.equal(first.size, 1); assert.equal(second.size, 1); + await f.manager.flush(); + assert.deepEqual(f.uploaded.map(b => b.length).sort((a, b) => a - b), [Buffer.from(PNG, 'base64').length, 200 * 1024]); + await assertNoOpenTickets(f.manager); + } finally { await f.close(); } + }); + test(`${provider}: unsampled identity and queued policy tightening cannot upload`, async () => { + const f = await fixture(); try { + capture(f.manager, { ...identity(), traceFlags: 0 }, input()); assert.equal(f.manager.status().ownedBytes, 0); + capture(f.manager, identity(), input()); f.manager.updatePolicy({ traceConfig: { hideInputs: true } }); await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } + }); +} + +test('Inline parts are capped at exactly 256 KiB of decoded bytes for both providers', async () => { + const f = await fixture(); try { + const at = Buffer.alloc(256 * 1024, 7).toString('base64'), over = Buffer.alloc(256 * 1024 + 3, 7).toString('base64'); + for (const data of [at, over]) { + oa.captureOpenAIAttachments(f.manager, identity(), { input: [{ role: 'user', content: [{ type: 'input_image', image_url: `data:image/png;base64,${data}` }] }] }, { operation: 'responses', direction: 'input' }); + oa.captureOpenAIAttachments(f.manager, identity(), { data: [{ b64_json: data }] }, { operation: 'images', direction: 'output' }); + an.captureAnthropicAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data } }] }] }); + } + await f.manager.flush(); + assert.deepEqual(f.uploaded.map(b => b.length), [262144, 262144, 262144]); + } finally { await f.close(); } +}); + +test('OpenAI generated images obey output and PII restrictions', async () => { + for (const traceConfig of [{}, { hideOutputs: true }, { hideOutputMessages: true }, { hideOutputText: true }, { piiRedaction: true }]) { + const f = await fixture({ traceConfig }); try { + oa.captureOpenAIAttachments(f.manager, identity(), { data: [{ b64_json: PNG }] }, { operation: 'images', direction: 'output' }); + await f.manager.flush(); assert.equal(f.uploaded.length, Object.keys(traceConfig).length ? 0 : 1); + } finally { await f.close(); } + } +}); + +test('Telemetry projections replace only admitted media with occurrence URIs and never mutate requests', async () => { + const f = await fixture(); try { + const cases = [ + [oaInput(), (m, s, x) => oa.captureOpenAIAttachments(m, s, x, { operation: 'responses', direction: 'input' }), (x, c) => oa.projectOpenAIMedia(x, c, 'responses', 'input'), 3], + [anInput(), (m, s, x) => an.captureAnthropicAttachments(m, s, x), (x, c) => an.projectAnthropicMedia(x, c), 4], + ]; + for (const [body, capture, project, count] of cases) { + const before = JSON.stringify(body); + // Nothing admitted: the exact same payload object is returned, so existing telemetry is unchanged. + assert.equal(project(body, new Map()), body); + const captured = capture(f.manager, identity(), body); + assert.equal(captured.size, count); + const projected = JSON.stringify(project(body, captured)); + assert.equal(JSON.stringify(body), before); + for (const secret of [PNG, PDF]) assert.ok(!projected.includes(secret)); + for (const { occurrenceId } of captured.values()) assert.ok(projected.includes(`fi-attachment:${occurrenceId}`)); + } + await f.manager.flush(); + assert.equal(f.uploaded.length, 7); + } finally { await f.close(); } +}); + +test('Denied media keeps the existing telemetry payload untouched', async () => { + const f = await fixture({ traceConfig: { hideInputImages: true } }); try { + const body = oaInput(), captured = oa.captureOpenAIAttachments(f.manager, identity(), body, { operation: 'responses', direction: 'input' }); + // Only the PDF is admitted; both images stay exactly as the application sent them. + assert.deepEqual([...captured.values()].map(v => v.mimeType), ['application/pdf']); + const projected = oa.projectOpenAIMedia(body, captured, 'responses', 'input'); + assert.equal(projected.input[0].content[0], body.input[0].content[0]); + assert.equal(projected.input[1], body.input[1]); + assert.ok(projected.input[0].content[1].file_data.startsWith('fi-attachment:')); + } finally { await f.close(); } +}); + +test('Hostile payloads fail capture without throwing into the application', async () => { + const f = await fixture(); try { + const hostile = new Proxy({}, { get() { throw Error('synthetic-secret'); }, has() { throw Error('synthetic-secret'); } }); + for (const payload of [hostile, { input: [hostile] }, { messages: [hostile] }, { messages: [{ role: 'user', content: [hostile] }] }, null, 7, 'x']) { + assert.equal(oa.captureOpenAIAttachments(f.manager, identity(), payload, { operation: 'responses', direction: 'input' }).size, 0); + assert.equal(oa.captureOpenAIAttachments(f.manager, identity(), payload, { operation: 'chat', direction: 'input' }).size, 0); + assert.equal(an.captureAnthropicAttachments(f.manager, identity(), payload).size, 0); + assert.doesNotThrow(() => an.projectAnthropicMedia(payload, new Map([['input/0/0', { occurrenceId: 'x', mimeType: 'image/png' }]]))); + assert.doesNotThrow(() => oa.projectOpenAIMedia(payload, new Map([['responses/input/0/0', { occurrenceId: 'x', mimeType: 'image/png' }]]), 'responses', 'input')); + } + await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } +}); + +test('Capability descriptors name every unsupported source explicitly', () => { + for (const name of ['remote_url', 'provider_file', 'partial_image', 'images.generate.stream', 'images.edit', 'speech']) assert.ok(oa.openAIAttachmentCapabilities.unsupported.includes(name)); + // Streamed Responses output is captured from the consumed terminal event; partial images never are. + assert.ok(oa.openAIAttachmentCapabilities.operations.includes('responses.stream.output')); + assert.ok(!oa.openAIAttachmentCapabilities.unsupported.includes('streamed_output')); + for (const name of ['remote_url', 'provider_file', 'native_span_hookup', 'generated_image', 'streamed_output']) assert.ok(an.anthropicAttachmentCapabilities.unsupported.includes(name)); + assert.ok(Object.isFrozen(oa.openAIAttachmentCapabilities) && Object.isFrozen(an.anthropicAttachmentCapabilities)); +}); + +test('CJS/ESM observations share deduplication without creating a second capture', async () => { + const f = await fixture(); try { + const esm = await import(pathToFileURL(path.resolve(__dirname, '../dist/esm/attachments.js'))); + const s = identity(), first = oa.captureOpenAIAttachments(f.manager, s, oaInput(), { operation: 'responses', direction: 'input' }); + const second = esm.captureOpenAIAttachments(f.manager, { ...s }, oaInput(), { operation: 'responses', direction: 'input' }); + // The second observer reuses the first occurrence, so both projections point at one upload. + assert.deepEqual([...second.entries()], [...first.entries()]); + const anthropicESM = await import(pathToFileURL(path.resolve(__dirname, '../../traceai_anthropic/dist/esm/attachments.js'))); + const a = identity(), cjs = an.captureAnthropicAttachments(f.manager, a, anInput()), mjs = anthropicESM.captureAnthropicAttachments(f.manager, { ...a }, anInput()); + assert.deepEqual([...mjs.entries()], [...cjs.entries()]); + await f.manager.flush(); assert.equal(f.uploaded.length, 7); + } finally { await f.close(); } +}); + +test('Anthropic tool-result images are not labelled computer screenshots without a typed computer tool', async () => { + const f = await fixture(); try { + an.captureAnthropicAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_photo', content: [image()] }] }] }); + await f.manager.flush(); assert.equal(f.initialized[0].purpose, 'tool_result'); assert.equal(f.initialized[0].part_index, 0); + } finally { await f.close(); } +}); + +test('Anthropic error-tool media respects hideErrorDetails before source access', async () => { + const f = await fixture({ traceConfig: { hideErrorDetails: true } }); try { + let read = false; + an.captureAnthropicAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_error', is_error: true, + content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', get data() { read = true; return PNG; } } }] }] }] }); + await f.manager.flush(); assert.equal(read, false); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } +}); + +test('OpenAI image error bodies are not treated as generated media', async () => { + const f = await fixture(); try { + oa.captureOpenAIAttachments(f.manager, identity(), { error: { message: 'synthetic' }, data: [{ b64_json: PNG }] }, { operation: 'images', direction: 'output' }); + await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } +}); + +test('Local instrumentor privacy restrictions cannot be loosened by a permissive shared manager', async () => { + for (const traceConfig of [{ hideInputs: true }, { hideInputMessages: true }, { hideInputText: true }, { piiRedaction: true }]) { + const f = await fixture(); try { + assert.equal(oa.captureOpenAIAttachments(f.manager, identity(), oaInput(), { operation: 'responses', direction: 'input', traceConfig }).size, 0); + assert.equal(an.captureAnthropicAttachments(f.manager, identity(), anInput(), { traceConfig }).size, 0); + await f.manager.flush(); assert.equal(f.initialized.length, 0, JSON.stringify(traceConfig)); + } finally { await f.close(); } + } + for (const traceConfig of [{ hideOutputs: true }, { piiRedaction: true }]) { + const f = await fixture(); try { + oa.captureOpenAIAttachments(f.manager, identity(), { data: [{ b64_json: PNG }] }, { operation: 'images', direction: 'output', traceConfig }); + await f.manager.flush(); assert.equal(f.initialized.length, 0, JSON.stringify(traceConfig)); + } finally { await f.close(); } + } + // Image-only restriction: the PDF still uploads, no image (including screenshots) does. + const f = await fixture(); try { + oa.captureOpenAIAttachments(f.manager, identity(), oaInput(), { operation: 'responses', direction: 'input', traceConfig: { hideInputImages: true } }); + an.captureAnthropicAttachments(f.manager, identity(), anInput(), { traceConfig: { hideInputImages: true } }); + await f.manager.flush(); assert.deepEqual(f.initialized.map(r => r.mime_type), ['application/pdf', 'application/pdf']); + } finally { await f.close(); } +}); + +test('Local hideErrorDetails denies Anthropic error tool media even when the manager allows it', async () => { + const f = await fixture(); try { + an.captureAnthropicAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_error', is_error: true, content: [image()] }] }] }, + { traceConfig: { hideErrorDetails: true } }); + await f.manager.flush(); assert.equal(f.initialized.length, 0); + } finally { await f.close(); } +}); + +async function providerServer() { + const requests = []; + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const c of req) chunks.push(c); + const body = JSON.parse(Buffer.concat(chunks)); requests.push({ path: req.url, body }); + if (req.url.endsWith('/responses') && body.stream) { + // A streamed Responses call whose completed response carries a generated image. + const response = { id: 'resp_stream', object: 'response', status: 'completed', model: body.model, output: [{ type: 'image_generation_call', id: 'img_stream', status: 'completed', result: PNG }], usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } }; + res.setHeader('content-type', 'text/event-stream'); + res.write(`event: response.created\ndata: ${JSON.stringify({ type: 'response.created', sequence_number: 0, response: { ...response, status: 'in_progress', output: [] } })}\n\n`); + res.end(`event: response.completed\ndata: ${JSON.stringify({ type: 'response.completed', sequence_number: 1, response })}\n\n`); + return; + } + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('/chat/completions')) res.end(JSON.stringify({ id: 'chat_fixture', object: 'chat.completion', model: body.model, choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })); + else if (req.url.endsWith('/responses')) res.end(JSON.stringify({ id: 'resp_fixture', object: 'response', status: 'completed', model: body.model, output: [{ type: 'computer_call', id: 'computer_fixture', call_id: 'call_computer', status: 'completed', action: { type: 'screenshot' } }, { type: 'image_generation_call', id: 'img_fixture', status: 'completed', result: PNG }], usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } })); + else if (req.url.endsWith('/images/generations')) res.end(JSON.stringify({ created: 1, data: [{ b64_json: PNG }] })); + else res.end(JSON.stringify({ id: 'msg_fixture', type: 'message', role: 'assistant', model: body.model, content: [{ type: 'text', text: 'ok' }], stop_reason: 'end_turn', stop_sequence: null, usage: { input_tokens: 1, output_tokens: 1 } })); + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + return { requests, url: `http://127.0.0.1:${server.address().port}/v1`, async close() { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } }; +} + +test('Real SDKs preserve APIPromise/request/response behavior and upload only through the shared manager', async () => { + const sdkModule = require(process.env.SDK_OPENAI || 'openai'); + const anthropicModule = require(process.env.SDK_ANTHROPIC || '../../traceai_anthropic/node_modules/@anthropic-ai/sdk'); + const OpenAI = sdkModule.OpenAI || sdkModule.default || sdkModule; + const Anthropic = anthropicModule.Anthropic || anthropicModule.default || anthropicModule; + const native = 'openTelemetry' in Anthropic.prototype; + const f = await fixture(), p = await providerServer(); + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + const oi = new OpenAIInstrumentation({ instrumentationConfig: { enabled: false }, attachments: f.manager }); + const ai = new AnthropicInstrumentation({ instrumentationConfig: { enabled: false }, attachments: f.manager }); + oi.setTracerProvider(provider); ai.setTracerProvider(provider); + oi.manuallyInstrument(sdkModule); ai.manuallyInstrument(anthropicModule); + try { + const client = new OpenAI({ apiKey: 'synthetic-provider-key', baseURL: p.url, maxRetries: 0 }); + const body = { model: 'synthetic-model', messages: [{ role: 'user', content: [{ type: 'image_url', image_url: { url: `data:image/png;base64,${PNG}` } }] }] }; + const pending = client.chat.completions.create(body); assert.equal(typeof pending.withResponse, 'function'); + assert.equal((await pending.withResponse()).data.choices[0].message.content, 'ok'); + let expected = 1; + if (client.responses) { + const result = await client.responses.create({ model: 'synthetic-model', ...oaInput() }); + assert.equal(result.output[0].call_id, 'call_computer'); expected += 4; + } + const generated = await client.images.generate({ prompt: 'synthetic', model: 'synthetic-model', response_format: 'b64_json' }); + assert.equal(generated.data[0].b64_json, PNG); expected++; + const ac = new Anthropic({ apiKey: 'synthetic-provider-key', baseURL: p.url, maxRetries: 0 }); + const request = ac.messages.create({ model: 'synthetic-model', max_tokens: 10, ...anInput() }); + assert.equal(typeof request.withResponse, 'function'); assert.equal((await request.withResponse()).data.content[0].text, 'ok'); + if (!native) expected += 4; + await f.manager.flush(); await provider.forceFlush(); + assert.equal(f.uploaded.length, expected); + assert.ok(p.requests.some(r => JSON.stringify(r.body).includes(PNG))); + assert.ok(!JSON.stringify(exporter.getFinishedSpans().map(s => s.attributes)).includes(PNG)); + assert.ok(exporter.getFinishedSpans().some(s => s.attributes['fi.attachment.refs'])); + // Each admitted reference appears as an occurrence URI in its own span's telemetry. + for (const span of exporter.getFinishedSpans()) { + const refs = JSON.parse(span.attributes['fi.attachment.refs'] ?? '[]'), text = JSON.stringify(span.attributes); + for (const ref of refs.filter(r => r.direction === 'input')) assert.ok(text.includes(`fi-attachment:${ref.occurrence_id}`), span.name); + } + if (client.responses) { + // Streamed Responses: the stream is delivered unchanged; the generated image in the terminal event the + // application consumed uploads once and is projected out of that span's output telemetry. + const stream = await client.responses.create({ model: 'synthetic-model', input: 'synthetic', stream: true }); + const events = []; for await (const event of stream) { events.push(event.type); if (event.type === 'response.completed') assert.equal(event.response.output[0].result, PNG); } + assert.deepEqual(events, ['response.created', 'response.completed']); + expected++; + await f.manager.flush(); await provider.forceFlush(); assert.equal(f.uploaded.length, expected); + assert.equal(f.uploaded.at(-1).toString('base64'), PNG); + const streamed = exporter.getFinishedSpans().filter(s => s.name === 'OpenAI Responses').at(-1); + const refs = JSON.parse(streamed.attributes['fi.attachment.refs'] ?? '[]'); + assert.equal(refs.length, 1); assert.equal(refs[0].direction, 'output'); assert.equal(refs[0].purpose, 'model_output'); + assert.ok(String(streamed.attributes['output.value']).includes(`fi-attachment:${refs[0].occurrence_id}`)); + assert.ok(!JSON.stringify(streamed.attributes).includes(PNG)); + // A consumer that stops early never reaches the terminal event: nothing is captured. + const early = await client.responses.create({ model: 'synthetic-model', input: 'synthetic', stream: true }); + for await (const event of early) { if (event.type === 'response.created') break; } + await f.manager.flush(); assert.equal(f.uploaded.length, expected); + } + // A raw-response caller owns the body; instrumentation must not parse for media. + const raw = await client.images.generate({ prompt: 'synthetic', response_format: 'b64_json' }).asResponse(); + assert.equal(raw.bodyUsed, false); await raw.arrayBuffer(); + await f.manager.flush(); assert.equal(f.uploaded.length, expected); + // Requests still reach the provider unchanged after privacy tightening; media egress stops. + f.manager.updatePolicy({ traceConfig: { hideInputs: true, piiRedaction: true } }); + await client.chat.completions.create(body); + await ac.messages.create({ model: 'synthetic-model', max_tokens: 10, ...anInput() }); + await client.images.generate({ prompt: 'synthetic', response_format: 'b64_json' }); + await f.manager.flush(); assert.equal(f.uploaded.length, expected); + console.log(JSON.stringify({ nativeAnthropic: native, responsesAvailable: !!client.responses, uploads: expected, spans: exporter.getFinishedSpans().length })); + } finally { + oi.disable(); ai.disable(); await provider.shutdown(); await f.close(); await p.close(); + } +}); + +test('A configured but disabled manager leaves existing span telemetry byte-identical', async () => { + const sdkModule = require(process.env.SDK_OPENAI || 'openai'); + const anthropicModule = require(process.env.SDK_ANTHROPIC || '../../traceai_anthropic/node_modules/@anthropic-ai/sdk'); + const OpenAI = sdkModule.OpenAI || sdkModule.default || sdkModule; + const Anthropic = anthropicModule.Anthropic || anthropicModule.default || anthropicModule; + const native = 'openTelemetry' in Anthropic.prototype; + const p = await providerServer(), disabled = await fixture({ enabled: false }); + const run = async attachments => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + const oi = new OpenAIInstrumentation({ instrumentationConfig: { enabled: false }, attachments }); + const ai = new AnthropicInstrumentation({ instrumentationConfig: { enabled: false }, attachments }); + oi.setTracerProvider(provider); ai.setTracerProvider(provider); + oi.manuallyInstrument(sdkModule); ai.manuallyInstrument(anthropicModule); + try { + const client = new OpenAI({ apiKey: 'synthetic-provider-key', baseURL: p.url, maxRetries: 0 }); + await client.chat.completions.create({ model: 'synthetic-model', messages: [{ role: 'user', content: [{ type: 'image_url', image_url: { url: `data:image/png;base64,${PNG}` } }] }] }); + if (client.responses) await client.responses.create({ model: 'synthetic-model', ...oaInput() }); + if (!native) await new Anthropic({ apiKey: 'synthetic-provider-key', baseURL: p.url, maxRetries: 0 }).messages.create({ model: 'synthetic-model', max_tokens: 10, ...anInput() }); + await provider.forceFlush(); + const strip = ({ name, attributes }) => ({ name, attributes: Object.fromEntries(Object.entries(attributes).filter(([k]) => !/latency|duration|time/i.test(k))) }); + return exporter.getFinishedSpans().map(strip); + } finally { oi.disable(); ai.disable(); await provider.shutdown(); } + }; + try { + const baseline = await run(undefined), off = await run(disabled.manager); + assert.ok(baseline.length >= 1); + assert.deepEqual(off, baseline); + assert.equal(disabled.calls.length, 0); + } finally { await disabled.close(); await p.close(); } +}); + +test('OpenAI data URIs of a sibling image type are admitted under their exact MIME', async () => { + const f = await fixture(); try { + const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 1, 2, 3, 0xff, 0xd9]).toString('base64'); + const captured = oa.captureOpenAIAttachments(f.manager, identity(), { messages: [{ role: 'user', content: [ + { type: 'image_url', image_url: { url: `data:image/jpeg;base64,${jpeg}` } }, + { type: 'image_url', image_url: { url: `data:image/gif;base64,${jpeg}` } }, + ] }] }, { operation: 'chat', direction: 'input' }); + await f.manager.flush(); + assert.deepEqual([...captured.values()].map(v => v.mimeType), ['image/jpeg']); + assert.deepEqual(f.initialized.map(r => r.mime_type), ['image/jpeg']); + assert.equal(f.uploaded[0].toString('base64'), jpeg); + await assertNoOpenTickets(f.manager); + } finally { await f.close(); } +}); + +test('No core degradation warning was raised by any fixture', () => { + assert.deepEqual(coreWarnings, []); +});