From 0608d16fff270adebd82b48688194120a06c2e94 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 20:03:51 -0800 Subject: [PATCH 01/10] feat(frameworks): capture typed framework media through the shared attachment manager Adds @traceai/framework-attachments, a shared typed-part normalizer with thin bindings for LangChain LLM runs, LlamaIndex chat, Vercel AI SDK model/tool events and Genkit model/tool implementations. Decode is lazy behind manager admission, inline base64/data URIs and Uint8Array only, capped at 1 MiB per part. No URL, path or stream reads. Co-Authored-By: Claude Opus 5.5 --- .../workflows/langchain-typescript-tests.yml | 6 +- .github/workflows/mastra-typescript-tests.yml | 2 +- .../traceai_framework_attachments/README.md | 23 +++ .../package.json | 60 +++++++ .../src/index.ts | 153 ++++++++++++++++++ .../test/fixture.cjs | 52 ++++++ .../test/integration.cjs | 118 ++++++++++++++ .../test/media.test.cjs | 95 +++++++++++ .../tsconfig.esm.json | 17 ++ .../tsconfig.esnext.json | 17 ++ .../tsconfig.json | 21 +++ .../packages/traceai_genkit/package.json | 3 +- .../packages/traceai_genkit/src/index.ts | 2 + .../packages/traceai_genkit/src/media.ts | 42 +++++ .../packages/traceai_genkit/tsconfig.json | 3 +- .../packages/traceai_langchain/package.json | 3 +- .../traceai_langchain/src/instrumentation.ts | 10 +- .../src/instrumentationUtils.ts | 9 +- .../packages/traceai_langchain/src/tracer.ts | 16 +- .../packages/traceai_langchain/tsconfig.json | 2 +- .../packages/traceai_llamaindex/package.json | 3 +- .../packages/traceai_llamaindex/src/types.ts | 4 +- .../traceai_llamaindex/src/wrapper.ts | 3 + .../packages/traceai_llamaindex/tsconfig.json | 3 +- .../packages/traceai_vercel/package.json | 3 +- .../packages/traceai_vercel/src/auto.ts | 29 +++- .../packages/traceai_vercel/src/media.ts | 20 +++ .../packages/traceai_vercel/src/telemetry.ts | 16 +- .../packages/traceai_vercel/tsconfig.json | 3 +- typescript/pnpm-lock.yaml | 50 ++++++ 30 files changed, 761 insertions(+), 27 deletions(-) create mode 100644 typescript/packages/traceai_framework_attachments/README.md create mode 100644 typescript/packages/traceai_framework_attachments/package.json create mode 100644 typescript/packages/traceai_framework_attachments/src/index.ts create mode 100644 typescript/packages/traceai_framework_attachments/test/fixture.cjs create mode 100644 typescript/packages/traceai_framework_attachments/test/integration.cjs create mode 100644 typescript/packages/traceai_framework_attachments/test/media.test.cjs create mode 100644 typescript/packages/traceai_framework_attachments/tsconfig.esm.json create mode 100644 typescript/packages/traceai_framework_attachments/tsconfig.esnext.json create mode 100644 typescript/packages/traceai_framework_attachments/tsconfig.json create mode 100644 typescript/packages/traceai_genkit/src/media.ts create mode 100644 typescript/packages/traceai_vercel/src/media.ts diff --git a/.github/workflows/langchain-typescript-tests.yml b/.github/workflows/langchain-typescript-tests.yml index 460bef107..02e505d44 100644 --- a/.github/workflows/langchain-typescript-tests.yml +++ b/.github/workflows/langchain-typescript-tests.yml @@ -58,9 +58,10 @@ jobs: corepack enable corepack pnpm --filter '@traceai/langchain...' install --frozen-lockfile --ignore-scripts corepack pnpm run build:deps + corepack pnpm --filter @traceai/framework-attachments run build corepack pnpm --filter @traceai/langchain run build mkdir -p "$RUNNER_TEMP/langchain-tarballs" - for package in fi-semantic-conventions fi-core traceai_langchain; do + for package in fi-semantic-conventions fi-core traceai_framework_attachments traceai_langchain; do corepack pnpm --dir "packages/$package" pack --pack-destination "$RUNNER_TEMP/langchain-tarballs" done git diff --exit-code -- pnpm-lock.yaml @@ -209,9 +210,10 @@ jobs: corepack enable corepack pnpm --filter '@traceai/langchain...' install --frozen-lockfile --ignore-scripts corepack pnpm run build:deps + corepack pnpm --filter @traceai/framework-attachments run build corepack pnpm --filter @traceai/langchain run build mkdir -p "$RUNNER_TEMP/remote-tarballs" - for package in fi-semantic-conventions fi-core traceai_langchain; do + for package in fi-semantic-conventions fi-core traceai_framework_attachments traceai_langchain; do corepack pnpm --dir "packages/$package" pack --pack-destination "$RUNNER_TEMP/remote-tarballs" done git diff --exit-code -- pnpm-lock.yaml diff --git a/.github/workflows/mastra-typescript-tests.yml b/.github/workflows/mastra-typescript-tests.yml index 4c3ec99bf..344d5424e 100644 --- a/.github/workflows/mastra-typescript-tests.yml +++ b/.github/workflows/mastra-typescript-tests.yml @@ -68,7 +68,7 @@ jobs: - name: Pack and install in a clean consumer run: | mkdir -p "$RUNNER_TEMP/mastra-package" "$RUNNER_TEMP/mastra-consumer" - for package in fi-semantic-conventions fi-core traceai_vercel traceai_mastra; do + for package in fi-semantic-conventions fi-core traceai_framework_attachments traceai_vercel traceai_mastra; do pnpm --dir "packages/$package" pack --pack-destination "$RUNNER_TEMP/mastra-package" done npm install --prefix "$RUNNER_TEMP/mastra-consumer" --ignore-scripts --no-audit --no-fund \ diff --git a/typescript/packages/traceai_framework_attachments/README.md b/typescript/packages/traceai_framework_attachments/README.md new file mode 100644 index 000000000..9ea642a1d --- /dev/null +++ b/typescript/packages/traceai_framework_attachments/README.md @@ -0,0 +1,23 @@ +# Framework attachment adapters + +Shared typed-media normalization for traceAI's framework integrations. This package is an adapter dependency, not a separate uploader or recorder. + +## Capture boundary + +`captureMediaMessages(binding, span, messages, dialect, direction)` observes typed message content. `captureMediaParts(binding, span, parts, dialect, options)` observes a known typed part array. Supported dialects are `langchain`, `llamaindex`, `vercel`, and `genkit`. + +Pass the existing provider's `attachments` manager in `binding`. Without that manager, these functions do nothing. The manager still requires explicit attachment enablement, a successful policy bootstrap and a sampled, recording span. Normal inference never waits for attachment network operations. Use the provider's existing flush/shutdown lifecycle to finish admitted transfers. + +The adapters accept inline base64/data URIs and `Uint8Array` sources at the documented typed boundaries. They do not fetch remote URLs, open local paths, read streams, reconstruct discarded media from exported text, or create new recordings. Base64 decoding is bounded to one MiB per part. A single span admits at most 64 positions through this helper; the core manager separately bounds bytes, queue length and reference projection. + +## Privacy and identity + +- Capture is off unless enabled on the shared manager. +- Directional hide flags, required PII handling, server policy and native content restrictions deny capture before decoding or copying media. +- Supply the integration's `traceConfig` and `capture` restrictions along with the manager. Binding-level denials cannot be relaxed by an individual part. +- Tool images intended for a model are subject to both output and model-input restrictions. +- No filenames, source URLs, hashes or provider credentials are added to references. +- Repeated observation of the same admitted message/part position on the same span is idempotent. Different message positions remain distinct occurrences. This is not cross-provider deduplication or permission to relink an occurrence to an ancestor span. +- The shared manager owns all transport, write capabilities and occurrence correlation. An upload completing into validation is not proof of authorized playback. + +`mediaField` reads own data descriptors only. The traversal is limited to known message/part fields; application getters and `toJSON` methods are not invoked. diff --git a/typescript/packages/traceai_framework_attachments/package.json b/typescript/packages/traceai_framework_attachments/package.json new file mode 100644 index 000000000..73f86afd1 --- /dev/null +++ b/typescript/packages/traceai_framework_attachments/package.json @@ -0,0 +1,60 @@ +{ + "name": "@traceai/framework-attachments", + "version": "0.1.0", + "description": "Typed framework media adapters for the shared traceAI attachment manager", + "main": "./dist/src/index.js", + "module": "./dist/esm/index.js", + "esnext": "./dist/esnext/index.js", + "types": "./dist/src/index.d.ts", + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/esm/index.js", + "require": "./dist/src/index.js" + } + }, + "scripts": { + "prebuild": "rimraf dist", + "build": "tsc --build tsconfig.json tsconfig.esm.json tsconfig.esnext.json && tsc-alias -p tsconfig.esm.json && pnpm run postbuild", + "postbuild": "echo '{\"type\": \"module\"}' > ./dist/esm/package.json", + "test": "node --test test/*.test.cjs", + "prepare": "pnpm run build" + }, + "author": "Future AGI", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/future-agi/traceAI.git", + "directory": "typescript/packages/traceai_framework_attachments" + }, + "bugs": { + "url": "https://github.com/future-agi/traceAI/issues" + }, + "engines": { + "node": ">=20.0.0" + }, + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "@traceai/fi-core": "workspace:^" + }, + "devDependencies": { + "@opentelemetry/resources": "^2.0.1", + "@opentelemetry/sdk-trace-base": "^2.0.1", + "@types/jest": "^29.5.14", + "@types/node": "^22.10.10", + "jest": "^29.7.0", + "rimraf": "^6.0.1", + "ts-jest": "^29.2.5", + "tsc-alias": "^1.8.10", + "typescript": "^5.7.3" + }, + "files": [ + "dist", + "README.md", + "!**/__tests__/**", + "!**/*.tsbuildinfo" + ], + "publishConfig": { + "access": "public" + } +} diff --git a/typescript/packages/traceai_framework_attachments/src/index.ts b/typescript/packages/traceai_framework_attachments/src/index.ts new file mode 100644 index 000000000..944840ab0 --- /dev/null +++ b/typescript/packages/traceai_framework_attachments/src/index.ts @@ -0,0 +1,153 @@ +import type { Span } from "@opentelemetry/api"; +import type { TraceConfigOptions } from "@traceai/fi-core"; +import type { AttachmentCapturePolicy, AttachmentManager, AttachmentOptions } from "@traceai/fi-core/attachments"; + +export type MediaDialect = "langchain" | "llamaindex" | "vercel" | "genkit"; +export interface FrameworkAttachmentOptions { + /** The configured provider's manager; adapters never create another uploader. */ + attachments?: AttachmentManager; + traceConfig?: TraceConfigOptions; + capture?: AttachmentCapturePolicy; +} + +/** Data descriptors only. Media observation must not invoke application getters/toJSON. */ +export function mediaField(value: unknown, key: string): unknown { + if (!value || typeof value !== "object") return undefined; + const field = Object.getOwnPropertyDescriptor(value, key); + return field && "value" in field ? field.value : undefined; +} + +// A synchronous callback may decode at most one MiB per part. Larger strings are +// unsupported here, not a reason to enlarge the core's bounded capture budget. +const MAX_DECODED = 1024 * 1024; +const MAX_ENCODED = Math.ceil(MAX_DECODED / 3) * 4; +const MAX_PARTS = 64; +const seen = new WeakMap>>(); + +function dataUriMime(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + return /^data:([a-zA-Z0-9/+.-]+);base64,/.exec(value.slice(0, 96))?.[1]; +} +function decode(value: unknown): Uint8Array { + if (value instanceof Uint8Array) return value; + // No implicit ArrayBuffer conversion, stream reads, paths or remote retrieval. + if (typeof value !== "string") throw new Error("unsupported media source"); + const prefix = /^data:[a-zA-Z0-9/+.-]+;base64,/.exec(value.slice(0, 96)); + const length = value.length - (prefix?.[0].length ?? 0); + if (!length || length > MAX_ENCODED || length % 4 !== 0) throw new Error("unsupported media size"); + const encoded = prefix ? value.slice(prefix[0].length) : value; + if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) throw new Error("invalid media encoding"); + const binary = atob(encoded); + if (binary.length > MAX_DECODED) throw new Error("unsupported media size"); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); + return bytes; +} + +function normalized(part: unknown, dialect: MediaDialect): { data: unknown; mime: string } | undefined { + const type = mediaField(part, "type"); + let data: unknown; + let mime: unknown; + if (dialect === "genkit") { + const media = mediaField(part, "media"); + data = mediaField(media, "url"); + // Genkit URLs are only captured when they contain inline data. Never fetch. + const uriMime = dataUriMime(data); + if (!uriMime) return; + mime = mediaField(media, "contentType") ?? uriMime; + } else if (dialect === "vercel") { + if (type === "image") { data = mediaField(part, "image"); mime = mediaField(part, "mediaType") ?? mediaField(part, "mimeType"); } + else if (type === "file") { + data = mediaField(part, "data"); mime = mediaField(part, "mediaType") ?? mediaField(part, "mimeType"); + // AI SDK v7 end events retain DefaultGeneratedFile backing values. Read + // only data descriptors, never its decoding getters or provider metadata. + if (data === undefined) { + const file = mediaField(part, "file"); + data = mediaField(file, "uint8ArrayData") ?? mediaField(file, "base64Data"); + mime = mediaField(file, "mediaType"); + } + } + else if (type === "image-data" || type === "file-data") { data = mediaField(part, "data"); mime = mediaField(part, "mediaType"); } + else return; + } else { + if (type === "image_url") { + const image = mediaField(part, "image_url"); + data = typeof image === "string" ? image : mediaField(image, "url"); + mime = dataUriMime(data); + if (!mime) return; + } else if ((type === "image" || type === "audio" || type === "file") && mediaField(part, "source_type") === "base64") { + data = mediaField(part, "data"); mime = mediaField(part, "mime_type"); + } else return; + } + const uriMime = dataUriMime(data); + if (uriMime && mime !== undefined && mime !== uriMime) return; + mime ??= uriMime; + if (typeof mime !== "string" || mime.length > 96) return; + return { data, mime }; +} + +function locallyDenied(config: TraceConfigOptions | undefined, options: AttachmentOptions): boolean { + if (!config) return false; + const input = options.direction !== "output" || options.capture?.modelInput || options.capture?.containsInput; + const output = options.direction !== "input" || options.capture?.containsOutput; + // Required text PII handling cannot sanitize opaque media. No sanitizer claim. + return !!(config.piiRedaction || (input && (config.hideInputs || config.hideInputMessages || config.hideInputText || + (options.mimeType.startsWith("image/") && config.hideInputImages))) || + (output && (config.hideOutputs || config.hideOutputMessages || config.hideOutputText))); +} + +/** + * Observe only explicit typed content arrays at an original-value boundary. + * No recursive scraping, uploads from serialized telemetry, or caller mutation. + * The source getter runs synchronously INSIDE core admission, never before it. + */ +export function captureMediaParts(binding: FrameworkAttachmentOptions, span: Span, parts: unknown, + dialect: MediaDialect, options: Omit): void { + if (!binding.attachments || !Array.isArray(parts)) return; + try { + const manager = binding.attachments; + let spans = seen.get(manager); + if (!spans) { spans = new WeakMap(); seen.set(manager, spans); } + let positions = spans.get(span); + if (!positions) { positions = new Set(); spans.set(span, positions); } + for (let i = 0; i < Math.min(parts.length, MAX_PARTS); i++) { + if (positions.size >= MAX_PARTS) break; + const part = mediaField(parts, String(i)); + const media = normalized(part, dialect); + if (!media) continue; + const key = `${options.direction}/${options.messageIndex ?? 0}/${i}/${options.purpose ?? ""}`; + if (positions.has(key)) continue; + const capture = { ...binding.capture, ...options.capture }; + // A per-part allowance must not relax a binding-level denial. + for (const restriction of ["captureContent", "recordInputs", "recordOutputs", "requested"] as const) { + if (binding.capture?.[restriction] === false) capture[restriction] = false; + } + for (const restriction of ["muted", "temporaryChat", "containsInput", "containsOutput", "modelInput", "errorDetail"] as const) { + if (binding.capture?.[restriction]) capture[restriction] = true; + } + const attachmentOptions: AttachmentOptions = { ...options, mimeType: media.mime, partIndex: i, capture }; + if (locallyDenied(binding.traceConfig, attachmentOptions)) continue; + // Lazy decode also ensures disabled, PII, unsampled and full-queue captures + // do not allocate decoded media. Core owns the detached mutable-byte copy. + const result = manager.attach(span, { kind: "Bytes", get data() { return decode(media.data); } }, attachmentOptions); + if (result.admission === "queued") positions.add(key); + } + } catch { /* Unsupported/malformed media must never affect framework execution. */ } +} + +/** Bounded message traversal, never an arbitrary object walk. */ +export function captureMediaMessages(binding: FrameworkAttachmentOptions, span: Span, messages: unknown, + dialect: MediaDialect, direction: "input" | "output"): void { + if (!binding.attachments || !Array.isArray(messages)) return; + try { + for (let i = 0; i < Math.min(messages.length, MAX_PARTS); i++) { + const message = mediaField(messages, String(i)); + // LangChain BaseTracer may retain its documented serialized BaseMessage form. + const content = mediaField(message, "content") ?? (dialect === "langchain" ? mediaField(mediaField(message, "kwargs"), "content") : undefined); + const nativeRole = mediaField(message, "role"); + const role = ["user", "assistant", "tool", "system"].includes(nativeRole as string) ? nativeRole as "user" | "assistant" | "tool" | "system" : "unknown"; + captureMediaParts(binding, span, content, dialect, { direction, messageIndex: i, role, + purpose: direction === "input" ? "model_input" : "model_output", capture: { modelInput: direction === "input" } }); + } + } catch { /* No diagnostics containing source data. */ } +} diff --git a/typescript/packages/traceai_framework_attachments/test/fixture.cjs b/typescript/packages/traceai_framework_attachments/test/fixture.cjs new file mode 100644 index 000000000..bb383615e --- /dev/null +++ b/typescript/packages/traceai_framework_attachments/test/fixture.cjs @@ -0,0 +1,52 @@ +const http = require('node:http'); +const { randomUUID } = require('node:crypto'); +const { once } = require('node:events'); +const { AttachmentManager, createAttachmentHttpTransport, attachmentFetchClient } = require('@traceai/fi-core/attachments'); +const { BasicTracerProvider, SimpleSpanProcessor, InMemorySpanExporter } = require('@opentelemetry/sdk-trace-base'); +const PNG = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+j5nQAAAAASUVORK5CYII=', 'base64'); +const URI = `data:image/png;base64,${PNG.toString('base64')}`; +const limits = { max_upload_bytes: 52428800, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, + recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, + initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1048576, playback_bytes_per_second: 1048576, + transfer_timeout_seconds: 120, grant_ttl_seconds: 60, revocation_check_seconds: 5 }; +async function fixture(config = {}) { + const requests = [], bodies = [], uploads = [], completed = []; + const server = http.createServer(async (req, res) => { + try { + const chunks = []; for await (const chunk of req) chunks.push(chunk); + const body = Buffer.concat(chunks); + requests.push({ method: req.method, path: req.url }); + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('/attachment-sessions')) { + const request = JSON.parse(body); + res.end(JSON.stringify({ schema_version: 1, capture_session_id: request.capture_session_id, + scope: { organization_id: randomUUID(), workspace_id: randomUUID(), project_id: randomUUID() }, limits, + policy: { version: 'test_v1', enabled: true, allowed_sources: ['Bytes'], allowed_modalities: ['image', 'audio', 'video', 'document', 'file'], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, + record_audio: false, record_video: false, ...config.serverPolicy }, + expires_at: new Date(Date.now() + 60000).toISOString(), transfer_modes: ['authenticated_stream'] })); + } else if (req.url.endsWith('/uploads')) { + const request = JSON.parse(body); uploads.push(request); res.statusCode = 201; + res.end(JSON.stringify({ upload_id: randomUUID(), attachment_id: request.attachment_id, occurrence_id: request.occurrence_id, + state: 'pending_upload', reserved_bytes: request.declared_bytes, expires_at: new Date(Date.now() + 60000).toISOString(), + transfer_mode: 'authenticated_stream', write_capability: 'synthetic-capability-only' })); + } else { + const upload_id = req.url.split('/').at(-2); + if (req.url.endsWith('/content')) { bodies.push(body); res.end(JSON.stringify({ upload_id, state: 'pending_upload' })); } + else { completed.push(upload_id); res.statusCode = 202; res.end(JSON.stringify({ upload_id, state: 'validating' })); } + } + } catch { res.statusCode = 500; res.end('{}'); } + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const manager = new AttachmentManager({ enabled: true, projectName: 'synthetic-framework', projectType: 'observe', + transport: createAttachmentHttpTransport({ endpoint: 'https://attachments.invalid', headers: { 'x-api-key': 'synthetic-test' }, + client: request => attachmentFetchClient({ ...request, url: `http://127.0.0.1:${server.address().port}${new URL(request.url).pathname}` }), + }), ...config, environment: () => config.environment ?? {} }); + await manager.ready(); + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + const tracer = provider.getTracer('framework-media-test'); + return { manager, provider, tracer, exporter, requests, bodies, uploads, completed, + async close() { await manager.shutdown(); await provider.shutdown(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } }; +} +module.exports = { fixture, PNG, URI }; diff --git a/typescript/packages/traceai_framework_attachments/test/integration.cjs b/typescript/packages/traceai_framework_attachments/test/integration.cjs new file mode 100644 index 000000000..3493c57be --- /dev/null +++ b/typescript/packages/traceai_framework_attachments/test/integration.cjs @@ -0,0 +1,118 @@ +// Real framework invocation against loopback attachment HTTP. No provider keys. +// The matrix installs exact upstream versions in a separate cloud-only directory. +const assert = require('node:assert/strict'); +const { createRequire } = require('node:module'); +const { join } = require('node:path'); +const { fixture, PNG, URI } = require('./fixture.cjs'); +const { context, trace } = require('@opentelemetry/api'); +const packageRequire = name => createRequire(join(__dirname, `../../traceai_${name}/package.json`)); +const upstream = process.env.FRAMEWORK_MATRIX_DIR ? createRequire(join(process.env.FRAMEWORK_MATRIX_DIR, 'package.json')) : require; +const [framework, version] = process.argv.slice(2); +const alias = `${framework}-${version}`; +async function main() { + const { AsyncLocalStorageContextManager } = packageRequire('langchain')('@opentelemetry/context-async-hooks'); + if (framework !== "genkit") context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable()); + const sdk = framework === "langchain" ? undefined : upstream(alias); + for (const privacy of ['allowed', 'disabled', 'hide', 'pii', 'native-disabled']) { + const f = await fixture({ enabled: privacy !== 'disabled', traceConfig: { hideInputs: privacy === 'hide', hideOutputs: privacy === 'hide', piiRedaction: privacy === 'pii' } }); + const binding = { attachments: f.manager, capture: { captureContent: privacy !== 'native-disabled' } }; + try { + if (framework === 'langchain') { + const { FITracer } = require('@traceai/fi-core'); + const { LangChainTracer } = require('../../traceai_langchain/dist/src/tracer.js'); + const { FakeListChatModel } = upstream(`${alias}/utils/testing`); + const { HumanMessage } = upstream(`${alias}/messages`); + const tracer = new LangChainTracer(new FITracer({ tracer: f.tracer }), true, binding); + const model = new FakeListChatModel({ responses: ['synthetic response'] }); + const result = await model.invoke([new HumanMessage({ content: [{ type: 'text', text: 'synthetic input' }, { type: 'image_url', image_url: { url: URI } }] })], { callbacks: [tracer] }); + assert.equal(result.content, 'synthetic response'); + } else if (framework === 'llamaindex') { + // The real OpenAI chat path is version-qualified; the provider transport + // receives a synthetic response through its supported custom fetch hook. + const { chatWrapper } = require('../../traceai_llamaindex/dist/src/wrapper.js'); + const OpenAI = sdk.OpenAI ?? upstream(`llamaopenai-${version}`).OpenAI; + const llm = new OpenAI({ apiKey: 'synthetic-key', model: 'gpt-4o', additionalSessionOptions: { + fetch: async () => new Response(JSON.stringify({ id: 'synthetic', object: 'chat.completion', created: 1, model: 'gpt-4o', + choices: [{ index: 0, message: { role: 'assistant', content: 'synthetic response' }, finish_reason: 'stop' }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } }), { headers: { 'content-type': 'application/json' } }), + } }); + const wrapped = chatWrapper({ className: 'OpenAI' }, binding, { warn() {} }, () => f.tracer)(llm.chat); + const result = await wrapped.call(llm, { messages: [{ role: 'user', content: [{ type: 'image_url', image_url: { url: URI } }] }] }); + assert.equal(result.message.content, 'synthetic response'); + } else if (framework === 'ai') { + const { VercelAIInstrumentation } = require('../../traceai_vercel/dist/src/auto.js'); + const instrumentation = new VercelAIInstrumentation({ ...binding, instrumentationConfig: { enabled: true } }); + instrumentation.setTracerProvider(f.provider); + const major = version === 'floor' ? 5 : 7; + const api = instrumentation.manuallyInstrument(sdk, `${major}.0.0`); + const mocks = upstream(`${alias}/test`); + const Model = mocks[`MockLanguageModelV${major === 5 ? 2 : 4}`]; + const usage = major === 5 ? { inputTokens: 1, outputTokens: 1 } : { + inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 }, outputTokens: { total: 1, text: 1, reasoning: 0 } }; + const finishReason = major === 5 ? 'stop' : { unified: 'stop', raw: 'stop' }; + const result = await api.generateText({ messages: [{ role: 'user', content: [{ type: 'image', image: PNG, mediaType: 'image/png' }, + { type: 'file', data: Buffer.from('%PDF-1.4 synthetic'), mediaType: 'application/pdf' }] }], model: new Model({ doGenerate: { + content: [{ type: 'text', text: 'synthetic response' }, { type: 'file', data: major === 5 ? PNG : { type: 'data', data: PNG }, mediaType: 'image/png' }], warnings: [], usage, finishReason, + } }) }); + assert.equal(result.text, 'synthetic response'); + if (privacy === 'allowed') { + await f.manager.flush(); + const before = f.bodies.length; + const controller = new AbortController(); + let consumed = 0; + const streaming = api.streamText({ abortSignal: controller.signal, + messages: [{ role: 'user', content: [{ type: 'image', image: PNG, mediaType: 'image/png' }] }], + model: new Model({ doStream: async () => ({ stream: new ReadableStream({ start(stream) { + stream.enqueue({ type: 'stream-start', warnings: [] }); + stream.enqueue({ type: 'text-start', id: 'synthetic' }); + stream.enqueue({ type: 'text-delta', id: 'synthetic', delta: 'first' }); + controller.signal.addEventListener('abort', () => stream.close(), { once: true }); + } }) }) }), + }); + for await (const part of streaming.textStream) { assert.equal(part, 'first'); consumed++; controller.abort(); break; } + await new Promise(resolve => setTimeout(resolve, 20)); + await f.manager.flush(); + assert.equal(consumed, 1); assert.equal(f.bodies.length - before, 1); + assert.ok(f.exporter.getFinishedSpans().some(span => span.name.includes('streamText'))); + console.log(JSON.stringify({ framework: alias, operation: 'stream-cancel', status: 'PASS' })); + const beforeTool = f.bodies.length; + const toolResult = await api.generateText({ prompt: 'synthetic tool', maxRetries: 0, + model: new Model({ doGenerate: { content: [{ type: 'tool-call', toolCallId: 'tool_1', toolName: 'imageTool', input: '{}' }], + warnings: [], usage, finishReason: major === 5 ? 'tool-calls' : { unified: 'tool-calls', raw: 'tool-calls' } } }), + tools: { imageTool: api.tool({ inputSchema: api.jsonSchema({ type: 'object', properties: {} }), + execute: async () => ({ type: 'content', value: [{ type: 'image-data', data: PNG.toString('base64'), mediaType: 'image/png' }] }), + }) }, + }); + assert.equal(toolResult.toolResults.length, 1); await f.manager.flush(); + assert.equal(f.bodies.length - beforeTool, major === 5 ? 0 : 1); + console.log(JSON.stringify({ framework: alias, operation: 'typed-tool', rawCapture: major !== 5, status: 'PASS' })); + } + instrumentation.disable(); + } else if (framework === 'genkit') { + const { enableTelemetry, flushTracing } = upstream(`${alias}/tracing`); + const { withGenkitModelAttachments, withGenkitToolAttachments } = require('../../traceai_genkit/dist/src/media.js'); + const ai = sdk.genkit({}); + await enableTelemetry({ spanProcessors: [{ onStart() {}, onEnd() {}, async forceFlush() {}, async shutdown() {} }] }); + const model = ai.defineModel({ name: `test/media-${privacy}`, supports: { media: true } }, withGenkitModelAttachments(binding, async () => ({ + message: { role: 'model', content: [{ media: { url: URI, contentType: 'image/png' } }] }, finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + }))); + const result = await ai.generate({ model, prompt: [{ media: { url: URI, contentType: 'image/png' } }] }); + assert.equal(result.message.content[0].media.url, URI); + const tool = ai.defineTool({ name: `media_tool_${privacy}`, description: 'synthetic tool' }, withGenkitToolAttachments(binding, async () => [{ media: { url: URI, contentType: 'image/png' } }])); + const toolResult = await tool({}); assert.equal(toolResult[0].media.url, URI); + await flushTracing(); + } else throw new Error('unknown framework'); + await f.manager.flush(); + if (privacy === 'allowed') { + const expected = framework === 'genkit' ? 3 : framework === 'ai' ? (version === 'floor' ? 3 : 5) : 1; + assert.equal(f.bodies.length, expected, `${alias}: uploaded count ${f.bodies.length}; spans ${f.exporter.getFinishedSpans().map(span => span.name).join(',')}`); + assert.ok(f.bodies.some(body => body.equals(PNG))); + assert.equal(f.completed.length, expected); + } else assert.equal(f.bodies.length, 0, `${alias} ${privacy}`); + console.log(JSON.stringify({ framework: alias, privacy, uploaded: f.bodies.length, completed: f.completed.length, status: 'PASS' })); + } finally { await f.close(); } + } + const delegate = trace.getTracerProvider().getDelegate?.(); + await delegate?.shutdown?.(); + trace.disable(); context.disable(); +} +main().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/typescript/packages/traceai_framework_attachments/test/media.test.cjs b/typescript/packages/traceai_framework_attachments/test/media.test.cjs new file mode 100644 index 000000000..ffc56d38e --- /dev/null +++ b/typescript/packages/traceai_framework_attachments/test/media.test.cjs @@ -0,0 +1,95 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { captureMediaMessages, captureMediaParts } = require('../dist/src/index.js'); +const { fixture, PNG, URI } = require('./fixture.cjs'); +const parts = { + langchain: [{ type: 'image_url', image_url: { url: URI } }], + llamaindex: [{ type: 'image_url', image_url: { url: URI } }], + vercel: [{ type: 'image', image: PNG, mediaType: 'image/png' }], + genkit: [{ media: { url: URI, contentType: 'image/png' } }], +}; +for (const [dialect, content] of Object.entries(parts)) { + test(`${dialect}: original bytes, occurrence correlation and idempotent boundary`, async () => { + const f = await fixture(); + try { + const span = f.tracer.startSpan('real-otel-span'); + const source = [{ role: 'user', content }]; + captureMediaMessages({ attachments: f.manager }, span, source, dialect, 'input'); + captureMediaMessages({ attachments: f.manager }, span, source, dialect, 'input'); + span.end(); await f.manager.flush(); + assert.equal(f.bodies.length, 1); assert.deepEqual(f.bodies[0], PNG); + assert.equal(f.completed.length, 1); + assert.equal(f.uploads[0].association.parent.trace_id.replaceAll('-', ''), span.spanContext().traceId); + assert.equal(f.uploads[0].association.parent.span_id, span.spanContext().spanId); + assert.equal(f.uploads[0].message_index, 0); assert.equal(f.uploads[0].part_index, 0); + const attributes = f.exporter.getFinishedSpans()[0].attributes; + const refs = JSON.parse(attributes['fi.attachment.refs']); + assert.equal(refs.length, 1); assert.equal(refs[0].occurrence_id, f.uploads[0].occurrence_id); + assert.ok(!JSON.stringify(attributes).includes(URI)); + } finally { await f.close(); } + }); + for (const [name, config] of Object.entries({ disabled: { enabled: false }, hideInputs: { traceConfig: { hideInputs: true } }, + hideMessages: { traceConfig: { hideInputMessages: true } }, hideImages: { traceConfig: { hideInputImages: true } }, + hideText: { traceConfig: { hideInputText: true } }, pii: { traceConfig: { piiRedaction: true } }, + metadataRemoval: { serverPolicy: { require_metadata_removal: true } }, mediaRedaction: { serverPolicy: { require_media_redaction: true } }, + })) test(`${dialect}: ${name} causes no upload, decode or occurrence`, async () => { + const f = await fixture(config); let decoded = 0; const atob = globalThis.atob; + globalThis.atob = (...args) => { decoded++; return atob(...args); }; + try { + const span = f.tracer.startSpan('denied'); + captureMediaMessages({ attachments: f.manager }, span, [{ role: 'user', content }], dialect, 'input'); + span.end(); await f.manager.flush(); + assert.equal(f.uploads.length, 0); assert.equal(f.bodies.length, 0); assert.equal(decoded, 0); + assert.equal(f.exporter.getFinishedSpans()[0].attributes['fi.attachment.refs'], undefined); + if (name === 'disabled') assert.equal(f.requests.length, 0); + } finally { globalThis.atob = atob; await f.close(); } + }); +} +test('tool images intersect input/output policy; binding denial cannot be relaxed', async () => { + for (const config of [{ traceConfig: { hideInputImages: true } }, { traceConfig: { hideOutputs: true } }, {}]) { + const f = await fixture(config); + try { + const span = f.tracer.startSpan('tool'); + captureMediaParts({ attachments: f.manager, capture: { recordOutputs: false } }, span, parts.vercel, 'vercel', { + direction: 'output', purpose: 'tool_result', capture: { modelInput: true, recordOutputs: true }, + }); + span.end(); await f.manager.flush(); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } + } +}); +test('no getters, paths, remote URLs, metadata or malformed/oversize data are recovered', async () => { + const f = await fixture(); let getters = 0; + try { + const span = f.tracer.startSpan('unsupported'); + const values = [ + { type: 'image', get image() { getters++; throw new Error('private'); }, mediaType: 'image/png' }, + { type: 'image', image: 'https://provider.invalid/secret', mediaType: 'image/png' }, + { type: 'image', image: '/private/file.png', mediaType: 'image/png' }, + { type: 'image', image: 'a'.repeat(1500000), mediaType: 'image/png' }, + { type: 'image', image: 'invalid!', mediaType: 'image/png' }, + { type: 'image', image: URI, mediaType: 'application/pdf' }, + ]; + captureMediaParts({ attachments: f.manager }, span, values, 'vercel', { direction: 'input' }); + span.end(); await f.manager.flush(); assert.equal(f.uploads.length, 0); assert.equal(getters, 0); + } finally { await f.close(); } +}); +test('mutable bytes are detached synchronously and distinct message occurrences survive', async () => { + const f = await fixture(); + try { + const span = f.tracer.startSpan('lifetime'); const mutable = new Uint8Array(PNG); + const content = [{ type: 'file', data: mutable, mediaType: 'image/png', filename: 'never-export.png' }]; + captureMediaMessages({ attachments: f.manager }, span, [{ role: 'user', content }, { role: 'user', content }], 'vercel', 'input'); + mutable.fill(0); span.end(); await f.manager.flush(); + assert.equal(f.bodies.length, 2); for (const body of f.bodies) assert.deepEqual(body, PNG); + assert.notEqual(f.uploads[0].occurrence_id, f.uploads[1].occurrence_id); + assert.ok(!JSON.stringify(f.uploads).includes('never-export')); + } finally { await f.close(); } +}); +test('integration-level trace config also wins over a permissive shared manager', async () => { + const f = await fixture(); + try { + const span = f.tracer.startSpan('local-hide'); + captureMediaMessages({ attachments: f.manager, traceConfig: { hideInputText: true } }, span, [{ content: parts.vercel }], 'vercel', 'input'); + span.end(); await f.manager.flush(); assert.equal(f.uploads.length, 0); + } finally { await f.close(); } +}); diff --git a/typescript/packages/traceai_framework_attachments/tsconfig.esm.json b/typescript/packages/traceai_framework_attachments/tsconfig.esm.json new file mode 100644 index 000000000..e5d6f9b6c --- /dev/null +++ b/typescript/packages/traceai_framework_attachments/tsconfig.esm.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.base.esm.json", + "compilerOptions": { + "outDir": "dist/esm", + "rootDir": "src", + "tsBuildInfoFile": "dist/esm/tsconfig.esm.tsbuildinfo", + "baseUrl": "./src" + }, + "include": [ + "src/**/*.ts" + ], + "references": [], + "exclude": [ + "src/**/*.test.ts", + "src/**/__tests__/**" + ] +} diff --git a/typescript/packages/traceai_framework_attachments/tsconfig.esnext.json b/typescript/packages/traceai_framework_attachments/tsconfig.esnext.json new file mode 100644 index 000000000..a99f0fe06 --- /dev/null +++ b/typescript/packages/traceai_framework_attachments/tsconfig.esnext.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.base.esnext.json", + "compilerOptions": { + "outDir": "dist/esnext", + "rootDir": "src", + "tsBuildInfoFile": "dist/esnext/tsconfig.esnext.tsbuildinfo", + "baseUrl": "./src" + }, + "include": [ + "src/**/*.ts" + ], + "references": [], + "exclude": [ + "src/**/*.test.ts", + "src/**/__tests__/**" + ] +} diff --git a/typescript/packages/traceai_framework_attachments/tsconfig.json b/typescript/packages/traceai_framework_attachments/tsconfig.json new file mode 100644 index 000000000..cf1640349 --- /dev/null +++ b/typescript/packages/traceai_framework_attachments/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": ".", + "baseUrl": "./src" + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "src/**/*.test.ts", + "src/**/__tests__/**", + "test/**/*.ts" + ], + "references": [ + { + "path": "../fi-core" + } + ] +} diff --git a/typescript/packages/traceai_genkit/package.json b/typescript/packages/traceai_genkit/package.json index d85982148..fcf4028c3 100644 --- a/typescript/packages/traceai_genkit/package.json +++ b/typescript/packages/traceai_genkit/package.json @@ -48,7 +48,8 @@ "dependencies": { "@opentelemetry/api": "^1.9.0", "@traceai/fi-core": "workspace:^", - "@traceai/fi-semantic-conventions": "workspace:^" + "@traceai/fi-semantic-conventions": "workspace:^", + "@traceai/framework-attachments": "workspace:^" }, "devDependencies": { "@opentelemetry/resources": "^2.0.1", diff --git a/typescript/packages/traceai_genkit/src/index.ts b/typescript/packages/traceai_genkit/src/index.ts index 6492faccb..97c6ed9eb 100644 --- a/typescript/packages/traceai_genkit/src/index.ts +++ b/typescript/packages/traceai_genkit/src/index.ts @@ -50,3 +50,5 @@ export { type ContentPolicy, type ScrubContext, } from "./privacy"; + +export { withGenkitModelAttachments, withGenkitToolAttachments } from "./media"; diff --git a/typescript/packages/traceai_genkit/src/media.ts b/typescript/packages/traceai_genkit/src/media.ts new file mode 100644 index 000000000..358abf853 --- /dev/null +++ b/typescript/packages/traceai_genkit/src/media.ts @@ -0,0 +1,42 @@ +import { context, trace } from "@opentelemetry/api"; +import { captureMediaMessages, captureMediaParts, mediaField, type FrameworkAttachmentOptions } from "@traceai/framework-attachments"; + +/** + * Wrap the implementation passed to Genkit defineModel, not an exported span. + * Genkit owns the active action span; this adapter never creates a second span. + * Pass the provider-owned manager, and the action's content restrictions. + * No inference or upload is awaited beyond the original implementation promise. + */ +export function withGenkitModelAttachments( + binding: FrameworkAttachmentOptions, implementation: (...args: Args) => Promise, +): (...args: Args) => Promise { + return async function (this: unknown, ...args: Args): Promise { + const span = trace.getSpan(context.active()); + if (span?.isRecording()) captureMediaMessages(binding, span, mediaField(args[0], "messages"), "genkit", "input"); + const result = await implementation.apply(this, args); + if (span?.isRecording()) { + // ModelResponseData's message is original typed content. No JSON recovery. + captureMediaMessages(binding, span, [mediaField(result, "message")], "genkit", "output"); + } + return result; + }; +} + +/** + * Only tools explicitly returning Genkit Part[] are media boundaries. Ordinary + * tool JSON is not searched for URLs or keys that happen to look like media. + * Tool output is also model input, so input and output restrictions intersect. + */ +export function withGenkitToolAttachments( + binding: FrameworkAttachmentOptions, implementation: (...args: Args) => Promise, +): (...args: Args) => Promise { + return async function (this: unknown, ...args: Args): Promise { + const span = trace.getSpan(context.active()); + const result = await implementation.apply(this, args); + if (span?.isRecording()) captureMediaParts(binding, span, result, "genkit", { + direction: "output", purpose: "tool_result", messageIndex: 0, role: "tool", capture: { modelInput: true }, + }); + return result; + }; +} +export type { FrameworkAttachmentOptions } from "@traceai/framework-attachments"; diff --git a/typescript/packages/traceai_genkit/tsconfig.json b/typescript/packages/traceai_genkit/tsconfig.json index 087f63ee4..0096c2e12 100644 --- a/typescript/packages/traceai_genkit/tsconfig.json +++ b/typescript/packages/traceai_genkit/tsconfig.json @@ -9,6 +9,7 @@ "exclude": ["src/**/*.test.ts", "src/**/__tests__/**", "test/**/*.ts"], "references": [ { "path": "../fi-core" }, - { "path": "../fi-semantic-conventions" } + { "path": "../fi-semantic-conventions" }, + { "path": "../traceai_framework_attachments" } ] } diff --git a/typescript/packages/traceai_langchain/package.json b/typescript/packages/traceai_langchain/package.json index 6dbf6397d..9abb2c1ee 100644 --- a/typescript/packages/traceai_langchain/package.json +++ b/typescript/packages/traceai_langchain/package.json @@ -46,7 +46,8 @@ "@opentelemetry/core": "^2.0.1", "@opentelemetry/instrumentation": "^0.202.0", "@traceai/fi-core": "workspace:^", - "@traceai/fi-semantic-conventions": "workspace:^" + "@traceai/fi-semantic-conventions": "workspace:^", + "@traceai/framework-attachments": "workspace:^" }, "peerDependencies": { "@langchain/core": "^0.2.0 || ^0.3.0 || ^1.0.0" diff --git a/typescript/packages/traceai_langchain/src/instrumentation.ts b/typescript/packages/traceai_langchain/src/instrumentation.ts index b89ea39a8..73b348db3 100644 --- a/typescript/packages/traceai_langchain/src/instrumentation.ts +++ b/typescript/packages/traceai_langchain/src/instrumentation.ts @@ -14,6 +14,7 @@ import { VERSION } from "./version"; import { diag, Tracer, TracerProvider } from "@opentelemetry/api"; import { addTracerToHandlers } from "./instrumentationUtils"; import { FITracer, TraceConfigOptions } from "@traceai/fi-core"; +import type { FrameworkAttachmentOptions } from "@traceai/framework-attachments"; const MODULE_NAME = "@langchain/core/callbacks"; @@ -57,6 +58,7 @@ export class LangChainInstrumentation extends InstrumentationBase { private fiTracer: FITracer; private tracerProvider?: TracerProvider; private traceConfig?: TraceConfigOptions; + private readonly media: FrameworkAttachmentOptions; private manualModule?: CallbackManagerModule; private modelModules?: Set; private runnableModules?: Set; @@ -65,11 +67,15 @@ export class LangChainInstrumentation extends InstrumentationBase { instrumentationConfig, traceConfig, tracerProvider, + attachments, + capture, }: { /** * The config for the instrumentation * @see {@link InstrumentationConfig} */ + attachments?: FrameworkAttachmentOptions["attachments"]; + capture?: FrameworkAttachmentOptions["capture"]; instrumentationConfig?: InstrumentationConfig; /** * @see {@link TraceConfigOptions} @@ -90,6 +96,7 @@ export class LangChainInstrumentation extends InstrumentationBase { ); this.tracerProvider = tracerProvider; this.traceConfig = traceConfig; + this.media = { attachments, traceConfig, capture }; this.fiTracer = new FITracer({ tracer: this.tracerProvider?.getTracer(INSTRUMENTATION_NAME, VERSION) ?? @@ -265,6 +272,7 @@ export class LangChainInstrumentation extends InstrumentationBase { const newInheritableHandlers = addTracerToHandlers( instrumentation.fiTracer, inheritableHandlers, + instrumentation.media, ); args[0] = newInheritableHandlers; @@ -277,7 +285,7 @@ export class LangChainInstrumentation extends InstrumentationBase { this: typeof CallbackManagerModuleV02, ...args: Parameters ) { - args[0] = addTracerToHandlers(instrumentation.fiTracer, args[0]); + args[0] = addTracerToHandlers(instrumentation.fiTracer, args[0], instrumentation.media); return original.apply(this, args); }; }); diff --git a/typescript/packages/traceai_langchain/src/instrumentationUtils.ts b/typescript/packages/traceai_langchain/src/instrumentationUtils.ts index cd86a2bfd..3cc4c3300 100644 --- a/typescript/packages/traceai_langchain/src/instrumentationUtils.ts +++ b/typescript/packages/traceai_langchain/src/instrumentationUtils.ts @@ -1,6 +1,7 @@ import type * as CallbackManagerModuleV02 from "@langchain/core/callbacks/manager"; import { LangChainTracer } from "./tracer"; import { FITracer } from "@traceai/fi-core"; +import type { FrameworkAttachmentOptions } from "@traceai/framework-attachments"; /** * Adds the {@link LangChainTracer} to the callback handlers if it is not already present @@ -13,16 +14,18 @@ import { FITracer } from "@traceai/fi-core"; export function addTracerToHandlers( tracer: FITracer, handlers?: CallbackManagerModuleV02.Callbacks, + media?: FrameworkAttachmentOptions, ): CallbackManagerModuleV02.Callbacks; export function addTracerToHandlers( tracer: FITracer, handlers?: CallbackManagerModuleV02.Callbacks, + media?: FrameworkAttachmentOptions, ): CallbackManagerModuleV02.Callbacks { // The injected tracer has no persistence I/O. Await its callbacks so stream // return/cancellation cannot race a background span start or end. Leave user // handlers (including subclasses with async persistence) unchanged. if (handlers == null) { - return [new LangChainTracer(tracer, true)]; + return [new LangChainTracer(tracer, true, media)]; } if (Array.isArray(handlers)) { const tracerAlreadyRegistered = handlers.some( @@ -30,7 +33,7 @@ export function addTracerToHandlers( ); if (!tracerAlreadyRegistered) { // eslint-disable-next-line @typescript-eslint/no-explicit-any - handlers.push(new LangChainTracer(tracer, true) as any); + handlers.push(new LangChainTracer(tracer, true, media) as any); } return handlers; } @@ -43,6 +46,6 @@ export function addTracerToHandlers( return handlers; } // eslint-disable-next-line @typescript-eslint/no-explicit-any - handlers.addHandler(new LangChainTracer(tracer, true) as any, true); + handlers.addHandler(new LangChainTracer(tracer, true, media) as any, true); return handlers; } \ No newline at end of file diff --git a/typescript/packages/traceai_langchain/src/tracer.ts b/typescript/packages/traceai_langchain/src/tracer.ts index 03921001e..a2441e921 100644 --- a/typescript/packages/traceai_langchain/src/tracer.ts +++ b/typescript/packages/traceai_langchain/src/tracer.ts @@ -26,6 +26,7 @@ import { } from "./utils"; import { FITracer, getErrorMessage } from "@traceai/fi-core"; import { isStreamCancelled, registerStreamRun } from "./streamLifecycle"; +import { captureMediaMessages, mediaField, type FrameworkAttachmentOptions } from "@traceai/framework-attachments"; type RunWithSpan = { run: Run; @@ -128,7 +129,7 @@ export class LangChainTracer extends BaseTracer { private runs: Record = {}; /** The error objects passed to handle*Error, keyed by run id, until the run's trace ends */ private runErrors: Record = {}; - constructor(tracer: FITracer, awaitHandlers?: boolean) { + constructor(tracer: FITracer, awaitHandlers?: boolean, private readonly media: FrameworkAttachmentOptions = {}) { super(); this.tracer = tracer; if (awaitHandlers != null) this.awaitHandlers = awaitHandlers; @@ -263,6 +264,12 @@ export class LangChainTracer extends BaseTracer { activeContext, ); + // Only LLM messages are typed media. Chain/tool arbitrary objects are not scraped. + if (run.run_type === "llm") { + const batches = mediaField(run.inputs, "messages"); + captureMediaMessages(this.media, span, mediaField(batches, "0"), "langchain", "input"); + } + const runWithSpan: RunWithSpan = { run, span, unregisterStream: () => {}, startedAt: performance.now(), firstTokenRecorded: false, }; @@ -316,6 +323,13 @@ export class LangChainTracer extends BaseTracer { span.setStatus({ code: SpanStatusCode.OK }); } + if (!cancelled && run.error == null && run.run_type === "llm") { + const generations = mediaField(mediaField(run.outputs, "generations"), "0"); + if (Array.isArray(generations)) { + captureMediaMessages(this.media, span, generations.slice(0, 64).map(generation => mediaField(generation, "message")), "langchain", "output"); + } + } + const attributes = safelyFlattenAttributes({ ...safelyFormatIO({ io: run.inputs, ioType: "input" }), ...safelyFormatInputMessages(run.inputs), diff --git a/typescript/packages/traceai_langchain/tsconfig.json b/typescript/packages/traceai_langchain/tsconfig.json index 376eafc21..2f5c12d22 100644 --- a/typescript/packages/traceai_langchain/tsconfig.json +++ b/typescript/packages/traceai_langchain/tsconfig.json @@ -7,5 +7,5 @@ "files": [], "include": ["src/**/*.ts", "test/**/*.ts"], "exclude": ["src/**/*.test.ts", "src/**/__tests__/**", "test/**/*.ts"], - "references": [] + "references": [{ "path": "../traceai_framework_attachments" }] } \ No newline at end of file diff --git a/typescript/packages/traceai_llamaindex/package.json b/typescript/packages/traceai_llamaindex/package.json index bc50c3759..450af0f33 100644 --- a/typescript/packages/traceai_llamaindex/package.json +++ b/typescript/packages/traceai_llamaindex/package.json @@ -41,7 +41,8 @@ "@opentelemetry/instrumentation": "^0.202.0", "@traceai/fi-core": "workspace:^", "@traceai/fi-semantic-conventions": "workspace:^", - "lodash": "^4.17.21" + "lodash": "^4.17.21", + "@traceai/framework-attachments": "workspace:^" }, "peerDependencies": { "llamaindex": ">=0.1.0", diff --git a/typescript/packages/traceai_llamaindex/src/types.ts b/typescript/packages/traceai_llamaindex/src/types.ts index b4b01b259..9e31b0e1a 100644 --- a/typescript/packages/traceai_llamaindex/src/types.ts +++ b/typescript/packages/traceai_llamaindex/src/types.ts @@ -1,6 +1,8 @@ import { InstrumentationConfig } from "@opentelemetry/instrumentation"; -export interface LlamaIndexInstrumentationConfig extends InstrumentationConfig { +import type { FrameworkAttachmentOptions } from "@traceai/framework-attachments"; + +export interface LlamaIndexInstrumentationConfig extends InstrumentationConfig, FrameworkAttachmentOptions { /** * A custom logger to log any exceptions that happen during span creation. */ diff --git a/typescript/packages/traceai_llamaindex/src/wrapper.ts b/typescript/packages/traceai_llamaindex/src/wrapper.ts index 4b44306b2..1fcf7cd54 100644 --- a/typescript/packages/traceai_llamaindex/src/wrapper.ts +++ b/typescript/packages/traceai_llamaindex/src/wrapper.ts @@ -13,6 +13,7 @@ import { safeExecuteInTheMiddle } from "@opentelemetry/instrumentation"; import { SemanticConventions, FISpanKind } from "@traceai/fi-semantic-conventions"; import { safelyJSONStringify } from "@traceai/fi-core"; +import { captureMediaMessages } from "@traceai/framework-attachments"; import { LlamaIndexInstrumentationConfig } from "./types"; import { shouldSendPrompts, llmGeneratorWrapper, generatorWrapper } from "./utils"; @@ -42,6 +43,7 @@ function handleResponse( try { if ((result as llamaindex.ChatResponse).message) { const chatResult = result as llamaindex.ChatResponse; + captureMediaMessages(config, span, [chatResult.message], "llamaindex", "output"); const outputContent = typeof chatResult.message.content === "string" ? chatResult.message.content : chatResult.message.content[0]?.type === "text" @@ -111,6 +113,7 @@ export function chatWrapper({ className }: { className: string }, this.metadata.model, ); if (shouldSendPrompts() && messages) { + captureMediaMessages(config, span, messages, "llamaindex", "input"); const serialized = messages.map((msg: any) => { const content = msg.content; const textContent = typeof content === "string" diff --git a/typescript/packages/traceai_llamaindex/tsconfig.json b/typescript/packages/traceai_llamaindex/tsconfig.json index 087f63ee4..0096c2e12 100644 --- a/typescript/packages/traceai_llamaindex/tsconfig.json +++ b/typescript/packages/traceai_llamaindex/tsconfig.json @@ -9,6 +9,7 @@ "exclude": ["src/**/*.test.ts", "src/**/__tests__/**", "test/**/*.ts"], "references": [ { "path": "../fi-core" }, - { "path": "../fi-semantic-conventions" } + { "path": "../fi-semantic-conventions" }, + { "path": "../traceai_framework_attachments" } ] } diff --git a/typescript/packages/traceai_vercel/package.json b/typescript/packages/traceai_vercel/package.json index 3166d1362..7a7d17b9e 100644 --- a/typescript/packages/traceai_vercel/package.json +++ b/typescript/packages/traceai_vercel/package.json @@ -41,7 +41,8 @@ "@opentelemetry/sdk-trace-base": "^1.30.1", "@traceai/fi-semantic-conventions": "workspace:^", "@opentelemetry/instrumentation": "^0.202.0", - "@traceai/fi-core": "workspace:^" + "@traceai/fi-core": "workspace:^", + "@traceai/framework-attachments": "workspace:^" }, "peerDependencies": { "@opentelemetry/api": ">=1.7.0 <2.0.0" diff --git a/typescript/packages/traceai_vercel/src/auto.ts b/typescript/packages/traceai_vercel/src/auto.ts index 79cf8545c..28f0becc0 100644 --- a/typescript/packages/traceai_vercel/src/auto.ts +++ b/typescript/packages/traceai_vercel/src/auto.ts @@ -7,6 +7,8 @@ import { safelyGetFIAttributes } from "./utils"; import { VERSION } from "./version"; import { createTelemetryIntegration, TelemetryIntegration } from "./telemetry"; import { observeStreamResult } from "./streams"; +import type { FrameworkAttachmentOptions } from "@traceai/framework-attachments"; +import { captureVercelInput } from "./media"; const METHODS = ["generateText", "streamText", "generateObject", "streamObject", "embed", "embedMany"] as const; const WRAPPED = Symbol.for("@traceai/vercel.wrapped"); @@ -82,13 +84,17 @@ function nativeSpan(span: Span, initial: Attributes, pending: Set): Span { } /** Adapts SDK-native v5/v6 spans while leaving the public return object intact. */ -function nativeTracer(tracer: Tracer, pending: Set): Tracer { +function nativeTracer(tracer: Tracer, pending: Set, media: FrameworkAttachmentOptions, input: unknown): Tracer { const noop = () => trace.wrapSpanContext(INVALID_SPAN_CONTEXT); + const capture = (name: string, span: Span): Span => { + if (/^ai\.(generateText|streamText|generateObject|streamObject)$/.test(name)) captureVercelInput(media, span, input); + return span; + }; return { startSpan(name, options, ctx) { - return safely(() => nativeSpan(tracer.startSpan(name, { + return safely(() => capture(name, nativeSpan(tracer.startSpan(name, { ...options, attributes: nativeAttributes(options?.attributes ?? {}), - }, ctx), options?.attributes ?? {}, pending)) ?? noop(); + }, ctx), options?.attributes ?? {}, pending))) ?? noop(); }, startActiveSpan(name: string, ...args: any[]): any { const fn = args[args.length - 1]; @@ -97,7 +103,7 @@ function nativeTracer(tracer: Tracer, pending: Set): Tracer { let entered = false; const callback = (span: Span) => { entered = true; - return fn(nativeSpan(span, attributes, pending)); + return fn(capture(name, nativeSpan(span, attributes, pending))); }; const callArgs = args.length === 1 ? [callback] : [ { ...options, attributes: nativeAttributes(attributes) }, @@ -116,6 +122,7 @@ function nativeTracer(tracer: Tracer, pending: Set): Tracer { export class VercelAIInstrumentation extends InstrumentationBase { readonly integration: TelemetryIntegration; private readonly traceConfig?: TraceConfigOptions; + private readonly media: FrameworkAttachmentOptions; private readonly tracers = new WeakMap(); private fiTracer(tracer: Tracer = this.tracer): Tracer { @@ -127,13 +134,16 @@ export class VercelAIInstrumentation extends InstrumentationBase { return wrapped; } - constructor({ instrumentationConfig = {}, traceConfig }: { + constructor({ instrumentationConfig = {}, traceConfig, attachments, capture }: { instrumentationConfig?: { enabled?: boolean }; traceConfig?: TraceConfigOptions; + attachments?: FrameworkAttachmentOptions["attachments"]; + capture?: FrameworkAttachmentOptions["capture"]; } = {}) { super("@traceai/vercel", VERSION, { ...instrumentationConfig, enabled: false }); this.traceConfig = traceConfig; - this.integration = createTelemetryIntegration(() => this.fiTracer()); + this.media = { attachments, traceConfig, capture }; + this.integration = createTelemetryIntegration(() => this.fiTracer(), undefined, this.media); if (instrumentationConfig.enabled !== false) this.enable(); } @@ -184,7 +194,7 @@ export class VercelAIInstrumentation extends InstrumentationBase { // Preserve the SDK's local-over-global precedence and callback receivers. const configured = telemetry?.integrations ?? (globalThis as Globals).AI_SDK_TELEMETRY_INTEGRATIONS ?? []; const integrations = Array.isArray(configured) ? configured : [configured]; - const integration = createTelemetryIntegration(() => instrumentation.fiTracer(), `ai.${name}`); + const integration = createTelemetryIntegration(() => instrumentation.fiTracer(), `ai.${name}`, instrumentation.media); dispatcher = integration; return { ...options, telemetry: { ...telemetry, integrations: [...integrations.filter(item => item !== instrumentation.integration), integration], @@ -193,7 +203,10 @@ export class VercelAIInstrumentation extends InstrumentationBase { pending = new Set(); return { ...options, experimental_telemetry: { ...telemetry, isEnabled: true, - tracer: nativeTracer(instrumentation.fiTracer(telemetry?.tracer ?? instrumentation.tracer), pending), + tracer: nativeTracer(instrumentation.fiTracer(telemetry?.tracer ?? instrumentation.tracer), pending, { ...instrumentation.media, capture: { ...instrumentation.media.capture, + recordInputs: telemetry?.recordInputs !== false && instrumentation.media.capture?.recordInputs !== false, + recordOutputs: telemetry?.recordOutputs !== false && instrumentation.media.capture?.recordOutputs !== false, + } }, options), } }; }) ?? args[0]; } diff --git a/typescript/packages/traceai_vercel/src/media.ts b/typescript/packages/traceai_vercel/src/media.ts new file mode 100644 index 000000000..c99e39b7b --- /dev/null +++ b/typescript/packages/traceai_vercel/src/media.ts @@ -0,0 +1,20 @@ +import type { Span } from "@opentelemetry/api"; +import { captureMediaMessages, captureMediaParts, mediaField, type FrameworkAttachmentOptions } from "@traceai/framework-attachments"; + +export function captureVercelInput(binding: FrameworkAttachmentOptions, span: Span, event: unknown): void { + const messages = mediaField(event, "promptMessages") ?? mediaField(event, "messages") ?? mediaField(event, "prompt"); + captureMediaMessages(binding, span, messages, "vercel", "input"); +} +export function captureVercelOutput(binding: FrameworkAttachmentOptions, span: Span, event: unknown): void { + captureMediaParts(binding, span, mediaField(event, "content"), "vercel", { + direction: "output", purpose: "model_output", messageIndex: 0, role: "assistant", + }); +} +export function captureVercelToolOutput(binding: FrameworkAttachmentOptions, span: Span, output: unknown): void { + // AI SDK's explicit content output, not arbitrary tool result objects. A tool + // image is also model input, so BOTH recordOutputs and input privacy apply. + if (mediaField(output, "type") !== "content") return; + captureMediaParts(binding, span, mediaField(output, "value"), "vercel", { + direction: "output", purpose: "tool_result", messageIndex: 0, role: "tool", capture: { modelInput: true }, + }); +} diff --git a/typescript/packages/traceai_vercel/src/telemetry.ts b/typescript/packages/traceai_vercel/src/telemetry.ts index 8ff929777..3245e66f7 100644 --- a/typescript/packages/traceai_vercel/src/telemetry.ts +++ b/typescript/packages/traceai_vercel/src/telemetry.ts @@ -1,6 +1,8 @@ import { Attributes, Context, Span, SpanKind, SpanStatusCode, Tracer, context, diag, trace, INVALID_SPAN_CONTEXT } from "@opentelemetry/api"; import { isTracingSuppressed } from "@opentelemetry/core"; import { getAttributesFromContext } from "@traceai/fi-core"; +import type { FrameworkAttachmentOptions } from "@traceai/framework-attachments"; +import { captureVercelInput, captureVercelOutput, captureVercelToolOutput } from "./media"; // Structural types deliberately avoid a runtime dependency on a particular AI SDK major. type Event = Record; @@ -120,8 +122,11 @@ function run(ctx: Context | undefined, execute: () => T): T { } /** v7 dispatcher callbacks, shared by the global registry and isolated per-call adapters. */ -export function createTelemetryIntegration(getTracer: () => Tracer, operation?: string): TelemetryIntegration { +export function createTelemetryIntegration(getTracer: () => Tracer, operation?: string, media: FrameworkAttachmentOptions = {}): TelemetryIntegration { const calls = new Map(); + const binding = (call: Call): FrameworkAttachmentOptions => ({ ...media, capture: { ...media.capture, + recordInputs: call.inputs && media.capture?.recordInputs !== false, + recordOutputs: call.outputs && media.capture?.recordOutputs !== false } }); let operationParent: Context | undefined; let operationSpan: Span | undefined; const modelKey = (event: Event) => typeof event.embedCallId === "string" ? event.embedCallId : "model"; @@ -164,6 +169,7 @@ export function createTelemetryIntegration(getTracer: () => Tracer, operation?: const call = calls.get(event.callId); if (!call) return; if (rootKind(call.operation) !== "CHAIN") { + captureVercelInput(binding(call), call.span, event); call.span.setAttributes(attributes(event, call)); return; } @@ -174,11 +180,13 @@ export function createTelemetryIntegration(getTracer: () => Tracer, operation?: const span = getTracer().startSpan(name, { kind: SpanKind.CLIENT, attributes: { ...getAttributesFromContext(call.ctx), ...attributes(event, call), "gen_ai.span.kind": "LLM", "fi.span.kind": "LLM", "traceai.capture.layer": "library", } }, call.ctx); + captureVercelInput(binding(call), span, event); call.models.set(key, span); }; const endModel = (event: Event) => { const call = calls.get(event.callId); if (call && rootKind(call.operation) !== "CHAIN") { + captureVercelOutput(binding(call), call.span, event); call.span.setAttributes(attributes(event, call)); return; } @@ -187,6 +195,7 @@ export function createTelemetryIntegration(getTracer: () => Tracer, operation?: call.models.delete(modelKey(event)); // Capture the end before normalization/attribute serialization work in this adapter. const end = Date.now(); + captureVercelOutput(binding(call), span, event); span.setAttributes(attributes(event, call)); span.end(end); }; @@ -222,7 +231,10 @@ export function createTelemetryIntegration(getTracer: () => Tracer, operation?: const end = Date.now(); const output = event.toolOutput; if (output?.type === "tool-error" || event.success === false) fail(span, output?.error ?? event.error); - else if (call.outputs) span.setAttributes({ "output.value": json(output?.output ?? event.output), "output.mime_type": "application/json" }); + else if (call.outputs) { + captureVercelToolOutput(binding(call), span, output?.output ?? event.output); + span.setAttributes({ "output.value": json(output?.output ?? event.output), "output.mime_type": "application/json" }); + } span.end(end); }); }, onEnd(event) { guard(() => finish(event.callId, event)); }, diff --git a/typescript/packages/traceai_vercel/tsconfig.json b/typescript/packages/traceai_vercel/tsconfig.json index e1a0eeb1f..95ffa5947 100644 --- a/typescript/packages/traceai_vercel/tsconfig.json +++ b/typescript/packages/traceai_vercel/tsconfig.json @@ -9,6 +9,7 @@ "exclude": ["src/**/*.test.ts", "test/**/*.ts"], "references": [ { "path": "../fi-core" }, - { "path": "../fi-semantic-conventions" } + { "path": "../fi-semantic-conventions" }, + { "path": "../traceai_framework_attachments" } ] } \ No newline at end of file diff --git a/typescript/pnpm-lock.yaml b/typescript/pnpm-lock.yaml index 114266c1c..04a719405 100644 --- a/typescript/pnpm-lock.yaml +++ b/typescript/pnpm-lock.yaml @@ -1188,6 +1188,44 @@ importers: specifier: ^5.8.3 version: 5.9.3 + packages/traceai_framework_attachments: + dependencies: + '@opentelemetry/api': + specifier: ^1.9.0 + version: 1.9.1 + '@traceai/fi-core': + specifier: workspace:^ + version: link:../fi-core + devDependencies: + '@opentelemetry/resources': + specifier: ^2.0.1 + version: 2.7.1(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': + specifier: ^2.0.1 + version: 2.7.1(@opentelemetry/api@1.9.1) + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 + '@types/node': + specifier: ^22.10.10 + version: 22.15.33 + jest: + specifier: ^29.7.0 + version: 29.7.0(@types/node@22.15.33)(ts-node@10.9.2(@types/node@22.15.33)(typescript@5.8.3)) + rimraf: + specifier: ^6.0.1 + version: 6.1.2 + ts-jest: + specifier: ^29.2.5 + version: 29.4.6(@babel/core@7.27.1)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.27.1))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.15.33)(ts-node@10.9.2(@types/node@22.15.33)(typescript@5.8.3)))(typescript@5.8.3) + tsc-alias: + specifier: ^1.8.10 + version: 1.8.16 + typescript: + specifier: ^5.7.3 + version: 5.8.3 + + packages/traceai_genkit: dependencies: '@opentelemetry/api': @@ -1199,6 +1237,9 @@ importers: '@traceai/fi-semantic-conventions': specifier: workspace:^ version: link:../fi-semantic-conventions + '@traceai/framework-attachments': + specifier: workspace:^ + version: link:../traceai_framework_attachments devDependencies: '@opentelemetry/context-async-hooks': specifier: 2.7.1 @@ -1747,6 +1788,9 @@ importers: '@traceai/fi-semantic-conventions': specifier: workspace:^ version: link:../fi-semantic-conventions + '@traceai/framework-attachments': + specifier: workspace:^ + version: link:../traceai_framework_attachments devDependencies: '@langchain/openai': specifier: 0.3.16 @@ -1866,6 +1910,9 @@ importers: lodash: specifier: ^4.17.21 version: 4.17.23 + '@traceai/framework-attachments': + specifier: workspace:^ + version: link:../traceai_framework_attachments devDependencies: '@llamaindex/openai': specifier: ^0.4.0 @@ -2912,6 +2959,9 @@ importers: '@traceai/fi-semantic-conventions': specifier: workspace:^ version: link:../fi-semantic-conventions + '@traceai/framework-attachments': + specifier: workspace:^ + version: link:../traceai_framework_attachments devDependencies: '@ai-sdk/openai': specifier: ^3.0.0 From 4217872cae4137713e47d1c3b8117b9e869ef904 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 20:11:42 -0800 Subject: [PATCH 02/10] feat(frameworks): Genkit middleware, Mastra disposition and framework media CI Adds genkitAttachmentMiddleware for models the application does not implement. The Mastra exporter removes fi.attachment.* keys it did not issue and documents that it never captures media. Adds a pull-request workflow for the framework media matrix, packs the shared helper in the Vercel and Mastra consumers, and registers it with release-please. Co-Authored-By: Claude Opus 5.5 --- .../workflows/framework-attachments-tests.yml | 94 ++++++++++++++++ .../workflows/langchain-typescript-tests.yml | 2 + .github/workflows/mastra-typescript-tests.yml | 4 +- .github/workflows/vercel-ai-sdk-tests.yml | 2 + .release-please-manifest.json | 1 + release-please-config.json | 5 + .../traceai_framework_attachments/README.md | 25 +++++ .../test/integration.cjs | 11 +- typescript/packages/traceai_genkit/README.md | 4 + .../packages/traceai_genkit/src/index.ts | 2 +- .../packages/traceai_genkit/src/media.ts | 17 +++ .../packages/traceai_langchain/README.md | 4 + .../packages/traceai_llamaindex/README.md | 4 + typescript/packages/traceai_mastra/README.md | 4 + .../contract/attachments.test.cjs | 102 ++++++++++++++++++ .../src/FIMastraSpanExporter.ts | 4 + .../src/__tests__/enrichment.test.ts | 15 +++ typescript/packages/traceai_vercel/README.md | 4 + 18 files changed, 300 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/framework-attachments-tests.yml create mode 100644 typescript/packages/traceai_mastra/contract/attachments.test.cjs diff --git a/.github/workflows/framework-attachments-tests.yml b/.github/workflows/framework-attachments-tests.yml new file mode 100644 index 000000000..d3c4b6d15 --- /dev/null +++ b/.github/workflows/framework-attachments-tests.yml @@ -0,0 +1,94 @@ +name: TypeScript framework attachments +on: + pull_request: + paths: + - 'typescript/packages/traceai_framework_attachments/**' + - 'typescript/packages/traceai_langchain/**' + - 'typescript/packages/traceai_llamaindex/**' + - 'typescript/packages/traceai_vercel/**' + - 'typescript/packages/traceai_genkit/**' + - 'typescript/packages/traceai_mastra/**' + - 'typescript/packages/fi-core/src/attachments/**' + - 'typescript/pnpm-lock.yaml' + - '.github/workflows/framework-attachments-tests.yml' + workflow_dispatch: +permissions: + contents: read +jobs: + frameworks: + name: Framework media / Node ${{ matrix.node }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + node: ['22', '24'] + defaults: + run: + shell: bash + working-directory: typescript + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + - uses: pnpm/action-setup@v4 + with: + version: 10.30.1 + - name: Install and build the framework packages + run: | + pnpm install --frozen-lockfile --ignore-scripts \ + --filter '@traceai/framework-attachments...' --filter '@traceai/langchain...' --filter '@traceai/llamaindex...' \ + --filter '@traceai/vercel...' --filter '@traceai/genkit...' --filter '@traceai/mastra...' + git diff --exit-code -- pnpm-lock.yaml + for package in fi-semantic-conventions fi-core framework-attachments langchain llamaindex vercel genkit mastra; do + pnpm --filter "@traceai/$package" run build + done + - name: Shared media normalizer, privacy and lifetime tests + run: pnpm --filter @traceai/framework-attachments test + - name: Real framework invocations at floor and latest versions + run: | + mkdir -p "$RUNNER_TEMP/framework-matrix" + cd "$RUNNER_TEMP/framework-matrix" + printf '{"private":true}' > package.json + pnpm add --ignore-scripts --config.auto-install-peers=false --save-exact \ + langchain-floor@npm:@langchain/core@0.2.0 langchain-latest@npm:@langchain/core@latest \ + llamaindex-floor@npm:llamaindex@0.1.0 llamaindex-latest@npm:llamaindex@latest \ + llamaopenai-floor@npm:@llamaindex/openai@0.3.0 llamaopenai-latest@npm:@llamaindex/openai@latest \ + ai-floor@npm:ai@5.0.0 ai-latest@npm:ai@latest \ + genkit-floor@npm:genkit@1.42.0 genkit-latest@npm:genkit@latest zod@4.3.6 + node -e 'const p=require("./package.json");console.log(JSON.stringify(p.dependencies))' + cd "$GITHUB_WORKSPACE/typescript" + export FRAMEWORK_MATRIX_DIR="$RUNNER_TEMP/framework-matrix" + for framework in langchain llamaindex ai genkit; do + for version in floor latest; do + echo "::group::$framework $version" + timeout 180 node packages/traceai_framework_attachments/test/integration.cjs "$framework" "$version" + echo "::endgroup::" + done + done + - name: Existing framework suites + run: | + for package in langchain llamaindex vercel genkit; do + echo "::group::$package" + NODE_OPTIONS=--experimental-vm-modules pnpm --filter "@traceai/$package" exec jest --runInBand --passWithNoTests --testPathIgnorePatterns e2e.test.ts + echo "::endgroup::" + done + pnpm --filter @traceai/mastra exec vitest run + - name: Mastra exporter attachment disposition (packed, real Agent over loopback) + run: | + mkdir -p "$RUNNER_TEMP/mastra-package" "$RUNNER_TEMP/mastra-consumer" + for package in fi-semantic-conventions fi-core traceai_framework_attachments traceai_vercel traceai_mastra; do + pnpm --dir "packages/$package" pack --pack-destination "$RUNNER_TEMP/mastra-package" + done + npm install --prefix "$RUNNER_TEMP/mastra-consumer" --ignore-scripts --no-audit --no-fund \ + "$RUNNER_TEMP"/mastra-package/*.tgz @mastra/core@latest @mastra/observability@latest @mastra/otel-exporter@latest \ + @opentelemetry/api@1.9.0 @opentelemetry/exporter-trace-otlp-proto@0.201.1 @opentelemetry/otlp-transformer@0.201.1 \ + @ai-sdk/openai@2.0.0 zod@3.25.76 + for entrypoint in src esm; do + MASTRA_TEST_DEPS="$RUNNER_TEMP/mastra-consumer" \ + MASTRA_TEST_PACKAGE="$RUNNER_TEMP/mastra-consumer/node_modules/@traceai/mastra/dist/$entrypoint/index.js" \ + node --test packages/traceai_mastra/contract/attachments.test.cjs + done diff --git a/.github/workflows/langchain-typescript-tests.yml b/.github/workflows/langchain-typescript-tests.yml index 02e505d44..ab63dd358 100644 --- a/.github/workflows/langchain-typescript-tests.yml +++ b/.github/workflows/langchain-typescript-tests.yml @@ -5,6 +5,7 @@ on: branches: [main, dev] paths: - 'typescript/packages/traceai_langchain/**' + - 'typescript/packages/traceai_framework_attachments/**' - 'typescript/packages/fi-core/**' - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/pnpm-lock.yaml' @@ -15,6 +16,7 @@ on: branches: [main, dev] paths: - 'typescript/packages/traceai_langchain/**' + - 'typescript/packages/traceai_framework_attachments/**' - 'typescript/packages/fi-core/**' - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/pnpm-lock.yaml' diff --git a/.github/workflows/mastra-typescript-tests.yml b/.github/workflows/mastra-typescript-tests.yml index 344d5424e..5d69a37cd 100644 --- a/.github/workflows/mastra-typescript-tests.yml +++ b/.github/workflows/mastra-typescript-tests.yml @@ -8,6 +8,7 @@ on: - 'typescript/packages/fi-core/**' - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/packages/traceai_vercel/**' + - 'typescript/packages/traceai_framework_attachments/**' - 'typescript/pnpm-lock.yaml' - '.github/workflows/mastra-typescript-tests.yml' push: @@ -90,7 +91,8 @@ jobs: "$PWD/packages/traceai_mastra/contract/http.test.cjs" \ "$PWD/packages/traceai_mastra/contract/tool-errors.test.cjs" \ "$PWD/packages/traceai_mastra/contract/continued-history.test.cjs" \ - "$PWD/packages/traceai_mastra/contract/terminal-history.test.cjs" + "$PWD/packages/traceai_mastra/contract/terminal-history.test.cjs" \ + "$PWD/packages/traceai_mastra/contract/attachments.test.cjs" done sudo unshare --net bash -c 'ip link set lo up; exec "$@"' _ \ "$(command -v node)" "$PWD/packages/traceai_mastra/contract/readme.test.mjs" "$MASTRA_TEST_DEPS" diff --git a/.github/workflows/vercel-ai-sdk-tests.yml b/.github/workflows/vercel-ai-sdk-tests.yml index 5488b71de..8b9a69267 100644 --- a/.github/workflows/vercel-ai-sdk-tests.yml +++ b/.github/workflows/vercel-ai-sdk-tests.yml @@ -5,6 +5,7 @@ on: branches: [dev, main] paths: - 'typescript/packages/traceai_vercel/**' + - 'typescript/packages/traceai_framework_attachments/**' - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/packages/fi-core/**' - 'typescript/pnpm-lock.yaml' @@ -42,6 +43,7 @@ jobs: # stage the workspace dependencies as tarballs too, as strands-typescript-tests.yml does. pnpm --dir packages/fi-core pack --pack-destination "$RUNNER_TEMP/package" pnpm --dir packages/fi-semantic-conventions pack --pack-destination "$RUNNER_TEMP/package" + pnpm --dir packages/traceai_framework_attachments pack --pack-destination "$RUNNER_TEMP/package" - name: Install clean consumer with published dependencies env: SDK: ${{ matrix.sdk }} diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 96c9307d2..93ef2e894 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -84,6 +84,7 @@ "typescript/packages/traceai_cursor_sdk": "0.1.1", "typescript/packages/traceai_deepseek": "0.1.2", "typescript/packages/traceai_fireworks": "0.1.2", + "typescript/packages/traceai_framework_attachments": "0.1.0", "typescript/packages/traceai_genkit": "0.1.1", "typescript/packages/traceai_google_adk": "0.1.2", "typescript/packages/traceai_google_genai": "0.1.1", diff --git a/release-please-config.json b/release-please-config.json index fb4d37dd0..930bb8e54 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -689,6 +689,11 @@ "src/version.ts" ] }, + "typescript/packages/traceai_framework_attachments": { + "release-type": "node", + "component": "npm-traceai-framework-attachments", + "package-name": "@traceai/framework-attachments" + }, "typescript/packages/traceai_genkit": { "release-type": "node", "component": "npm-traceai-genkit", diff --git a/typescript/packages/traceai_framework_attachments/README.md b/typescript/packages/traceai_framework_attachments/README.md index 9ea642a1d..5551a2e61 100644 --- a/typescript/packages/traceai_framework_attachments/README.md +++ b/typescript/packages/traceai_framework_attachments/README.md @@ -21,3 +21,28 @@ The adapters accept inline base64/data URIs and `Uint8Array` sources at the docu - The shared manager owns all transport, write capabilities and occurrence correlation. An upload completing into validation is not proof of authorized playback. `mediaField` reads own data descriptors only. The traversal is limited to known message/part fields; application getters and `toJSON` methods are not invoked. + +## Coverage by framework operation + +E means the adapter reads the original typed media value. R means the integration only keeps references that another integration already issued. N means not supported. + +| Framework | Operation | Disposition | +| --- | --- | --- | +| LangChain (`@langchain/core` 0.2 to 1.x) | Chat model input messages (`image_url` data URIs; `image`/`audio`/`file` parts with `source_type: "base64"`) | E | +| LangChain | Chat model output generations with the same typed parts | E | +| LangChain | Chain, tool and retriever inputs/outputs | N: arbitrary objects are not scanned | +| LlamaIndex (legacy package) | `chat` input messages and non-streaming chat response (`image_url` data URIs) | E | +| LlamaIndex | Streaming chat output | N | +| Vercel AI SDK 5, 6, 7 | `generateText`/`streamText`/`generateObject`/`streamObject` prompt messages (`image`, `file` parts) | E | +| Vercel AI SDK 7 | Model response `file` content and typed tool `content` results (`image-data`, `file-data`) | E | +| Vercel AI SDK 5, 6 | Model output and tool results | N: the native telemetry path exposes only serialized text | +| Genkit 1.42+ | `defineModel` implementation wrapped with `withGenkitModelAttachments` (request messages and response message) | E | +| Genkit | Any model, including plugin models, through `use: [genkitAttachmentMiddleware(binding)]` on `generate` | E | +| Genkit | `defineTool` implementation returning `Part[]`, wrapped with `withGenkitToolAttachments` | E | +| Mastra (`@traceai/mastra`) | All exporter operations | N for raw capture. The exporter receives ended, serialized spans, has no manager and never uploads. It removes `fi.attachment.*` keys it did not issue. | + +Streams are never read for media, remote URLs and file paths are never fetched, and creating new recordings is not supported. + +## Known limitation: framework plus provider instrumentation + +When a framework integration and a provider integration (for example LangChain plus `@traceai/openai`) are both given the same manager, the same image is uploaded once for the framework span and once for the provider span. The current attachment core has no cross-span occurrence claim, so these adapters cannot deduplicate across span boundaries. Give the manager to one layer only until that core API exists. diff --git a/typescript/packages/traceai_framework_attachments/test/integration.cjs b/typescript/packages/traceai_framework_attachments/test/integration.cjs index 3493c57be..02c2793d5 100644 --- a/typescript/packages/traceai_framework_attachments/test/integration.cjs +++ b/typescript/packages/traceai_framework_attachments/test/integration.cjs @@ -89,7 +89,7 @@ async function main() { instrumentation.disable(); } else if (framework === 'genkit') { const { enableTelemetry, flushTracing } = upstream(`${alias}/tracing`); - const { withGenkitModelAttachments, withGenkitToolAttachments } = require('../../traceai_genkit/dist/src/media.js'); + const { withGenkitModelAttachments, withGenkitToolAttachments, genkitAttachmentMiddleware } = require('../../traceai_genkit/dist/src/media.js'); const ai = sdk.genkit({}); await enableTelemetry({ spanProcessors: [{ onStart() {}, onEnd() {}, async forceFlush() {}, async shutdown() {} }] }); const model = ai.defineModel({ name: `test/media-${privacy}`, supports: { media: true } }, withGenkitModelAttachments(binding, async () => ({ @@ -99,11 +99,18 @@ async function main() { assert.equal(result.message.content[0].media.url, URI); const tool = ai.defineTool({ name: `media_tool_${privacy}`, description: 'synthetic tool' }, withGenkitToolAttachments(binding, async () => [{ media: { url: URI, contentType: 'image/png' } }])); const toolResult = await tool({}); assert.equal(toolResult[0].media.url, URI); + // A model the application does not implement (as with plugin models) is + // observed only through Genkit's documented generate middleware. + const plain = ai.defineModel({ name: `test/plain-${privacy}`, supports: { media: true } }, async () => ({ + message: { role: 'model', content: [{ media: { url: URI, contentType: 'image/png' } }] }, finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + })); + const viaMiddleware = await ai.generate({ model: plain, prompt: [{ media: { url: URI, contentType: 'image/png' } }], use: [genkitAttachmentMiddleware(binding)] }); + assert.equal(viaMiddleware.message.content[0].media.url, URI); await flushTracing(); } else throw new Error('unknown framework'); await f.manager.flush(); if (privacy === 'allowed') { - const expected = framework === 'genkit' ? 3 : framework === 'ai' ? (version === 'floor' ? 3 : 5) : 1; + const expected = framework === 'genkit' ? 5 : framework === 'ai' ? (version === 'floor' ? 3 : 5) : 1; assert.equal(f.bodies.length, expected, `${alias}: uploaded count ${f.bodies.length}; spans ${f.exporter.getFinishedSpans().map(span => span.name).join(',')}`); assert.ok(f.bodies.some(body => body.equals(PNG))); assert.equal(f.completed.length, expected); diff --git a/typescript/packages/traceai_genkit/README.md b/typescript/packages/traceai_genkit/README.md index cc144656a..adcdcc299 100644 --- a/typescript/packages/traceai_genkit/README.md +++ b/typescript/packages/traceai_genkit/README.md @@ -210,6 +210,10 @@ The model span's `genkit:output` is the model response JSON: `message`, `finishR `contract/auto_s5.cjs` includes three failing release regressions: `catalog-header` uses real instrumented fetch beneath an intermediate native SDK span while the Genkit model's wire span is suppressed, with an immediately exported application child's ordinary-key attribute array, status and event; `catalog-field` and `catalog-media` compare real flow `output.value` exactly against `pattern-password-response` and `magic-png-standard-response_body` in the shared golden bundle. The header case also reads its request from `pattern-token-headers`. No local predicate, copied fixture expectation, manual registration or mocked sanitizer masks the shared defect. Both source and built Genkit mappings currently fail all three privacy assertions while application results, native exports, span IDs and timing remain intact. Fixing this gate requires edits in the shared core's `src/wire/privacy.ts` and its export-time catalog integration, outside this package; it cannot be corrected by restoring forbidden integration-local discovery. This release gate is separate from the OTHER limitations below. +## Attachments (opt-in) + +Three explicit bindings pass inline Genkit media (data URI `media` parts) to the provider's attachment manager: `genkitAttachmentMiddleware(binding)` for `generate({ use: [...] })` with any model, `withGenkitModelAttachments(binding, implementation)` for a `defineModel` implementation, and `withGenkitToolAttachments(binding, implementation)` for a tool returning `Part[]`. `binding` is `{ attachments, traceConfig, capture }`. Remote media URLs are never fetched and streaming chunks are not read. See `@traceai/framework-attachments` for coverage and limits. + ## Known limitations - **2026-10-30 — Bootstrap contract.** `enable()` requires the agent's existing FI-only export branch (versioned process-state contract `traceai.auto.agent.v1`), or the explicit `spanProcessor` option. It does not create an agent or activate itself on import. This package change alone does not enable dispatch in the separate `@traceai/auto` catalog. diff --git a/typescript/packages/traceai_genkit/src/index.ts b/typescript/packages/traceai_genkit/src/index.ts index 97c6ed9eb..5cec3a6ac 100644 --- a/typescript/packages/traceai_genkit/src/index.ts +++ b/typescript/packages/traceai_genkit/src/index.ts @@ -51,4 +51,4 @@ export { type ScrubContext, } from "./privacy"; -export { withGenkitModelAttachments, withGenkitToolAttachments } from "./media"; +export { withGenkitModelAttachments, withGenkitToolAttachments, genkitAttachmentMiddleware } from "./media"; diff --git a/typescript/packages/traceai_genkit/src/media.ts b/typescript/packages/traceai_genkit/src/media.ts index 358abf853..e49ffad8c 100644 --- a/typescript/packages/traceai_genkit/src/media.ts +++ b/typescript/packages/traceai_genkit/src/media.ts @@ -22,6 +22,23 @@ export function withGenkitModelAttachments( }; } +/** + * Genkit model middleware for any model, including plugin models the application + * cannot wrap: `ai.generate({ ..., use: [genkitAttachmentMiddleware(binding)] })`. + * It observes the original GenerateRequest/GenerateResponseData on the span that + * is active when Genkit runs middleware. Streaming chunks are not read; only the + * final response message is observed, after Genkit has assembled it. + */ +export function genkitAttachmentMiddleware(binding: FrameworkAttachmentOptions) { + return async function (request: Req, next: (request?: Req) => Promise): Promise { + const span = trace.getSpan(context.active()); + if (span?.isRecording()) captureMediaMessages(binding, span, mediaField(request, "messages"), "genkit", "input"); + const result = await next(request); + if (span?.isRecording()) captureMediaMessages(binding, span, [mediaField(result, "message")], "genkit", "output"); + return result; + }; +} + /** * Only tools explicitly returning Genkit Part[] are media boundaries. Ordinary * tool JSON is not searched for URLs or keys that happen to look like media. diff --git a/typescript/packages/traceai_langchain/README.md b/typescript/packages/traceai_langchain/README.md index 47529e398..fb23dea7c 100644 --- a/typescript/packages/traceai_langchain/README.md +++ b/typescript/packages/traceai_langchain/README.md @@ -162,6 +162,10 @@ header alone does not cause server spans to appear. Stream spans end after the stream is fully consumed or closed, not when `runs.stream()` returns. Requests and outputs are not copied onto these client spans. +## Attachments (opt-in) + +Pass the provider's attachment manager as `attachments` (and optional `capture` restrictions) to `LangChainInstrumentation`. Chat model input and output messages with inline typed media (`image_url` data URIs, or `image`/`audio`/`file` parts with `source_type: "base64"`) are then attached to the LLM span through the shared manager. Nothing is captured unless attachments are enabled on that manager. Remote URLs are not fetched, and chain, tool and retriever objects are not scanned. Hide flags and PII redaction deny capture before any decoding. If a provider integration uses the same manager, the same media is uploaded once per span; see `@traceai/framework-attachments` for coverage and this limitation. + ## Peer Dependencies This package requires the following peer dependencies: diff --git a/typescript/packages/traceai_llamaindex/README.md b/typescript/packages/traceai_llamaindex/README.md index b88104eca..54252922a 100644 --- a/typescript/packages/traceai_llamaindex/README.md +++ b/typescript/packages/traceai_llamaindex/README.md @@ -120,6 +120,10 @@ This instrumentation automatically traces: - **LLM Operations**: Chat completions and text generation +## Attachments (opt-in) + +`LlamaIndexInstrumentation` accepts `attachments`, `traceConfig` and `capture`. With an enabled manager, inline `image_url` data URIs in `chat` input messages and non-streaming chat responses are attached to the LLM span. Streaming output, remote URLs and files are not captured. See `@traceai/framework-attachments` for coverage and limits. + ## Peer Dependencies This package requires the following peer dependencies: diff --git a/typescript/packages/traceai_mastra/README.md b/typescript/packages/traceai_mastra/README.md index 0e8418d3a..89e1577c8 100644 --- a/typescript/packages/traceai_mastra/README.md +++ b/typescript/packages/traceai_mastra/README.md @@ -95,6 +95,10 @@ For a long-running application, call this from your application shutdown lifecyc If you already manage a Mastra `Observability` instance, add `createFIMastraExporter(options)` to its `exporters` array instead of creating a second observability instance. Pass `projectName` explicitly to this lower-level factory (or set `FI_PROJECT_NAME`); it does not default the project name to the service name. +## Attachments + +The Mastra exporter does not capture media. It receives ended, serialized Mastra spans after the run, has no attachment manager and never uploads. It also removes any `fi.attachment.*` attribute it did not issue, so application metadata cannot forge an attachment reference. + ## Mastra v0 The deprecated `FITraceExporter` and `isFISpan` exports remain available from `@traceai/mastra/legacy` for existing Mastra v0 applications. They are not root exports and do not work with Mastra v1's removed `telemetry` configuration. For new applications, use the v1 setup above. diff --git a/typescript/packages/traceai_mastra/contract/attachments.test.cjs b/typescript/packages/traceai_mastra/contract/attachments.test.cjs new file mode 100644 index 000000000..6fe414a07 --- /dev/null +++ b/typescript/packages/traceai_mastra/contract/attachments.test.cjs @@ -0,0 +1,102 @@ +// Mastra attachment disposition (F06): the Future AGI Mastra exporter receives +// ended, serialized Mastra spans. It has no attachment manager and never uploads. +// A real Agent sends an image through the OpenAI SDK to loopback HTTP; the decoded +// OTLP output must carry no attachment session traffic and no unissued references. +const assert = require('node:assert/strict'); +const http = require('node:http'); +const path = require('node:path'); +const { createRequire } = require('node:module'); +const { pathToFileURL } = require('node:url'); +const { test, before, after, afterEach } = require('node:test'); +const packagePath = process.env.MASTRA_TEST_PACKAGE || path.resolve(__dirname, '../dist/src/index.js'); +const req = createRequire(process.env.MASTRA_TEST_DEPS + ? path.join(process.env.MASTRA_TEST_DEPS, 'package.json') : packagePath); +const { Mastra } = req('@mastra/core'); +const { Agent } = req('@mastra/core/agent'); +const { createOpenAI } = req('@ai-sdk/openai'); +const transformer = createRequire(req.resolve('@opentelemetry/exporter-trace-otlp-proto')); +const wire = transformer('@opentelemetry/otlp-transformer/build/src/generated/root.js') + .opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +const attrs = list => Object.fromEntries((list || []).map(a => [a.key, Object.values(a.value)[0]])); +// A synthetic payload larger than common serializer string limits, so the test +// records whether the exported span still holds the original bytes. +const IMAGE = Buffer.alloc(6000, 7).toString('base64'); +const URI = `data:image/png;base64,${IMAGE}`; +let createFIObservability, obs, collectorPaths, providerBodies, spans, ended; +const collector = http.createServer((request, response) => { + const chunks = []; + collectorPaths.push(request.url); + request.on('data', chunk => chunks.push(chunk)); + request.on('end', () => { + const data = wire.toObject(wire.decode(Buffer.concat(chunks)), { longs: String, bytes: String }); + for (const resource of data.resourceSpans || []) for (const scope of resource.scopeSpans || []) { + for (const span of scope.spans || []) spans.push({ ...span, attrs: attrs(span.attributes) }); + } + response.writeHead(200, { 'content-type': 'application/x-protobuf' }); + response.end(); + }); +}); +const provider = http.createServer((request, response) => { + let body = ''; + request.on('data', chunk => { body += chunk; }); + request.on('end', () => { + providerBodies.push(body); + response.writeHead(200, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ id: 'chatcmpl-synthetic', created: 1700000000, model: 'gpt-4o-mini', object: 'chat.completion', + usage: { prompt_tokens: 11, completion_tokens: 5, total_tokens: 16 }, + choices: [{ index: 0, message: { role: 'assistant', content: 'SYNTHETIC_RESPONSE' }, finish_reason: 'stop' }] })); + }); +}); +const listen = server => new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); +before(async () => { + ({ createFIObservability } = await import(pathToFileURL(packagePath).href)); + await listen(collector); + await listen(provider); +}); +after(async () => { + for (const server of [provider, collector]) { + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + } +}); +afterEach(async () => { await obs?.shutdown(); obs = undefined; delete process.env.FI_HIDE_INPUT_IMAGES; }); +function setup() { + collectorPaths = []; providerBodies = []; spans = []; ended = []; + obs = createFIObservability({ apiKey: 'synthetic-fi-key', secretKey: 'synthetic-fi-secret', + serviceName: 'attachment-contract', endpoint: `http://127.0.0.1:${collector.address().port}/tracer/v1/traces` }); + const exporter = obs.getDefaultInstance().getExporters()[0]; + const exportEvent = exporter.exportTracingEvent.bind(exporter); + exporter.exportTracingEvent = event => { + if (event.type === 'span_ended') ended.push(event.exportedSpan); + return exportEvent(event); + }; + const openai = createOpenAI({ apiKey: 'synthetic-provider-key', baseURL: `http://127.0.0.1:${provider.address().port}/v1` }); + const instance = new Agent({ id: 'media-agent', name: 'Media agent', instructions: 'Synthetic instruction', model: openai.chat('gpt-4o-mini') }); + return new Mastra({ agents: { agent: instance }, observability: obs }).getAgent('agent'); +} +const forged = { 'fi.attachment.schema_version': 1, 'fi.attachment.refs': JSON.stringify([{ attachment_id: 'forged', occurrence_id: 'forged', + direction: 'input', modality: 'image', mime_type: 'image/png' }]), 'fi.attachment.capture_session_id': 'forged', 'fi.attachment.correlation_nonce': 'forged' }; +const message = [{ role: 'user', content: [{ type: 'text', text: 'SYNTHETIC_PROMPT' }, { type: 'image', image: URI, mimeType: 'image/png' }] }]; + +for (const flag of [undefined, 'FI_HIDE_INPUT_IMAGES']) { + test(`real image input exports no attachment traffic or unissued references (${flag ?? 'default'})`, async () => { + if (flag) process.env[flag] = 'true'; + const agent = setup(); + const result = await agent.generate(message, { tracingOptions: { metadata: forged } }); + assert.equal(result.text, 'SYNTHETIC_RESPONSE'); + await obs.getDefaultInstance().flush(); + // The real provider request carried the image, so media did exist at runtime. + assert.equal(providerBodies.length, 1); + assert.match(providerBodies[0], new RegExp(IMAGE.slice(0, 64))); + assert.ok(spans.length > 0); + // The exporter only ships OTLP; it never opens an attachment session or upload. + assert.deepEqual([...new Set(collectorPaths)], ['/tracer/v1/traces']); + for (const span of spans) { + assert.deepEqual(Object.keys(span.attrs).filter(key => key.startsWith('fi.attachment.')), [], span.name); + } + // Disposition evidence: does the ended Mastra span still hold the whole original? + const retained = JSON.stringify(ended).includes(IMAGE); + console.log(JSON.stringify({ test: 'mastra-attachment-disposition', flag: flag ?? 'default', exportedSpanRetainsOriginal: retained, + wireRetainsOriginal: JSON.stringify(spans).includes(IMAGE), raw_capture: 'unsupported' })); + }); +} diff --git a/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts b/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts index 24d54c358..7705b5f87 100644 --- a/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts +++ b/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts @@ -133,6 +133,10 @@ export function enrichSpan( inferenceEnabled = runtimeEmitsInference(), ): void { const attrs = otelSpan.attributes as Record; + // Attachment references are issued only by a live attachment manager on the + // span that owns the media. This exporter sees ended, serialized Mastra spans + // and has no manager, so any such key came from application metadata/attributes. + for (const key of Object.keys(attrs)) if (key.startsWith("fi.attachment.")) delete attrs[key]; const kind = isModelCall(span.type, inferenceEnabled) ? FISpanKind.LLM : span.type === SpanType.MODEL_GENERATION ? FISpanKind.CHAIN : SPAN_KIND_BY_TYPE[span.type]; diff --git a/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts b/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts index 9fe539733..ac4acb70c 100644 --- a/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts +++ b/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts @@ -240,3 +240,18 @@ describe("native provider cost ownership", () => { expect(inference.attributes["llm.cost.total"]).toBeUndefined(); }); }); + +describe("enrichSpan — attachment references", () => { + it("never forwards fi.attachment.* keys it did not issue", () => { + const otelSpan = fakeOtelSpan({ + "fi.attachment.schema_version": 1, + "fi.attachment.refs": JSON.stringify([{ attachment_id: "forged", occurrence_id: "forged" }]), + "fi.attachment.capture_session_id": "forged", + "fi.attachment.correlation_nonce": "forged", + "mastra.metadata.note": "kept", + }); + enrichSpan(otelSpan, fakeMastraSpan(SpanType.AGENT_RUN), true); + expect(Object.keys(otelSpan.attributes).filter(key => key.startsWith("fi.attachment."))).toEqual([]); + expect(otelSpan.attributes["mastra.metadata.note"]).toBe("kept"); + }); +}); diff --git a/typescript/packages/traceai_vercel/README.md b/typescript/packages/traceai_vercel/README.md index e81107c53..b5c105bc6 100644 --- a/typescript/packages/traceai_vercel/README.md +++ b/typescript/packages/traceai_vercel/README.md @@ -199,6 +199,10 @@ The Jest regressions compare raw transport failures and unread streams across bo The auto-instrumentation regression configuration leaves content hiding off. The manual quick-start above deliberately disables input/output recording; enabling auto-instrumentation does not remove explicit recording opt-outs or FI hiding settings. +## Attachments (opt-in) + +`VercelAIInstrumentation` accepts `attachments` (the provider's manager) and `capture` restrictions. With an enabled manager, inline `image` and `file` parts in prompt messages are attached on AI SDK 5, 6 and 7. On AI SDK 7 the model response's `file` content and typed tool `content` results (`image-data`, `file-data`) are attached too. AI SDK 5 and 6 only expose serialized output through native telemetry, so their outputs are not captured. A call's `recordInputs: false` / `recordOutputs: false` also denies the matching media. Streams are never read for media. See `@traceai/framework-attachments` for coverage and limits. + ## Known limitations As of **2026-10-30**: From ce75951828da588e30a338678dc6f66bb60cd63c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 20:18:33 -0800 Subject: [PATCH 03/10] ci(frameworks): dispatchable Vercel and Mastra gates; LangChain runs its own consumer suites Co-Authored-By: Claude Opus 5.5 --- .github/workflows/framework-attachments-tests.yml | 4 +++- .github/workflows/mastra-typescript-tests.yml | 1 + .github/workflows/vercel-ai-sdk-tests.yml | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/framework-attachments-tests.yml b/.github/workflows/framework-attachments-tests.yml index d3c4b6d15..c819b4fb4 100644 --- a/.github/workflows/framework-attachments-tests.yml +++ b/.github/workflows/framework-attachments-tests.yml @@ -71,7 +71,9 @@ jobs: done - name: Existing framework suites run: | - for package in langchain llamaindex vercel genkit; do + # LangChain's own workflow runs its runtime suites against a packed + # consumer per core version; it is dispatched for this branch instead. + for package in llamaindex vercel genkit; do echo "::group::$package" NODE_OPTIONS=--experimental-vm-modules pnpm --filter "@traceai/$package" exec jest --runInBand --passWithNoTests --testPathIgnorePatterns e2e.test.ts echo "::endgroup::" diff --git a/.github/workflows/mastra-typescript-tests.yml b/.github/workflows/mastra-typescript-tests.yml index 5d69a37cd..aed442356 100644 --- a/.github/workflows/mastra-typescript-tests.yml +++ b/.github/workflows/mastra-typescript-tests.yml @@ -19,6 +19,7 @@ on: - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/pnpm-lock.yaml' - '.github/workflows/mastra-typescript-tests.yml' + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/vercel-ai-sdk-tests.yml b/.github/workflows/vercel-ai-sdk-tests.yml index 8b9a69267..6efc602d4 100644 --- a/.github/workflows/vercel-ai-sdk-tests.yml +++ b/.github/workflows/vercel-ai-sdk-tests.yml @@ -10,6 +10,7 @@ on: - 'typescript/packages/fi-core/**' - 'typescript/pnpm-lock.yaml' - '.github/workflows/vercel-ai-sdk-tests.yml' + workflow_dispatch: permissions: contents: read From aeb947e03d7fc1f7e5cfdcf90fecff7f11198798 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 20:24:57 -0800 Subject: [PATCH 04/10] test(frameworks): real PNG in the Mastra disposition test; README installer uses candidate transitive tarballs Co-Authored-By: Claude Opus 5.5 --- .../packages/traceai_langchain/contract/readme.mjs | 14 ++++++++++++++ .../traceai_mastra/contract/attachments.test.cjs | 8 +++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/typescript/packages/traceai_langchain/contract/readme.mjs b/typescript/packages/traceai_langchain/contract/readme.mjs index f69c6a94f..a813319fd 100644 --- a/typescript/packages/traceai_langchain/contract/readme.mjs +++ b/typescript/packages/traceai_langchain/contract/readme.mjs @@ -43,6 +43,16 @@ if (phase === "install") { assert.equal((await readdir(consumer)).length, 0, "Install must start in an empty consumer"); await mkdir(env.HOME); const tarballs = await readdir(tarballPath); + // Candidate @traceai packages that the documented ones depend on, but that the + // README does not name (for example an unpublished shared helper), come from + // the same candidate build. They are added only next to a documented @traceai + // dependency, never as an extra application dependency of their own. + const documented = new Set(dependencies.filter((dep) => dep.startsWith("@traceai/")) + .map((dep) => packageName(dep).slice(1).replace("/", "-") + "-")); + const transitive = tarballs.filter((name) => name.startsWith("traceai-") && name.endsWith(".tgz") + && ![...documented].some((prefix) => name.startsWith(prefix))) + .map((name) => resolve(tarballPath, name)); + let transitiveAdded = false; for (const command of commands) { // Replace only documented Future AGI dependencies with the candidate release. // No preinstalled adapters, runner, peer bypasses or extra dependencies. @@ -53,6 +63,10 @@ if (phase === "install") { assert.equal(matches.length, 1, `Expected exactly one tarball for ${dep}`); return resolve(tarballPath, matches[0]); }); + if (!transitiveAdded && command.some((dep) => dep.startsWith("@traceai/"))) { + args.push(...transitive); + transitiveAdded = true; + } const result = spawnSync("npm", ["install", "--ignore-scripts", "--no-audit", "--no-fund", ...args], { cwd: consumer, env, stdio: "inherit" }); assert.equal(result.status, 0, `README install failed: ${result.error ?? result.signal ?? "npm"}`); diff --git a/typescript/packages/traceai_mastra/contract/attachments.test.cjs b/typescript/packages/traceai_mastra/contract/attachments.test.cjs index 6fe414a07..cf6b69ff7 100644 --- a/typescript/packages/traceai_mastra/contract/attachments.test.cjs +++ b/typescript/packages/traceai_mastra/contract/attachments.test.cjs @@ -20,7 +20,9 @@ const wire = transformer('@opentelemetry/otlp-transformer/build/src/generated/ro const attrs = list => Object.fromEntries((list || []).map(a => [a.key, Object.values(a.value)[0]])); // A synthetic payload larger than common serializer string limits, so the test // records whether the exported span still holds the original bytes. -const IMAGE = Buffer.alloc(6000, 7).toString('base64'); +// A valid 1x1 PNG followed by padding: runtimes that sniff the signature accept it. +const PNG = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+j5nQAAAAASUVORK5CYII=', 'base64'); +const IMAGE = Buffer.concat([PNG, Buffer.alloc(6000, 7)]).toString('base64'); const URI = `data:image/png;base64,${IMAGE}`; let createFIObservability, obs, collectorPaths, providerBodies, spans, ended; const collector = http.createServer((request, response) => { @@ -76,7 +78,7 @@ function setup() { } const forged = { 'fi.attachment.schema_version': 1, 'fi.attachment.refs': JSON.stringify([{ attachment_id: 'forged', occurrence_id: 'forged', direction: 'input', modality: 'image', mime_type: 'image/png' }]), 'fi.attachment.capture_session_id': 'forged', 'fi.attachment.correlation_nonce': 'forged' }; -const message = [{ role: 'user', content: [{ type: 'text', text: 'SYNTHETIC_PROMPT' }, { type: 'image', image: URI, mimeType: 'image/png' }] }]; +const message = [{ role: 'user', content: [{ type: 'text', text: 'SYNTHETIC_PROMPT' }, { type: 'image', image: URI, mimeType: 'image/png', mediaType: 'image/png' }] }]; for (const flag of [undefined, 'FI_HIDE_INPUT_IMAGES']) { test(`real image input exports no attachment traffic or unissued references (${flag ?? 'default'})`, async () => { @@ -87,7 +89,7 @@ for (const flag of [undefined, 'FI_HIDE_INPUT_IMAGES']) { await obs.getDefaultInstance().flush(); // The real provider request carried the image, so media did exist at runtime. assert.equal(providerBodies.length, 1); - assert.match(providerBodies[0], new RegExp(IMAGE.slice(0, 64))); + assert.ok(providerBodies[0].includes(IMAGE.slice(-64)), 'the provider request must carry the original image'); assert.ok(spans.length > 0); // The exporter only ships OTLP; it never opens an attachment session or upload. assert.deepEqual([...new Set(collectorPaths)], ['/tracer/v1/traces']); From c889341bc603f41364a40c6f25ba7dde3f83381f Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 20:37:45 -0800 Subject: [PATCH 05/10] ci(langchain): resolve the shared attachment helper from the candidate tarball in the strict pnpm load Co-Authored-By: Claude Opus 5.5 --- .github/workflows/langchain-typescript-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/langchain-typescript-tests.yml b/.github/workflows/langchain-typescript-tests.yml index ab63dd358..a695f2ced 100644 --- a/.github/workflows/langchain-typescript-tests.yml +++ b/.github/workflows/langchain-typescript-tests.yml @@ -178,6 +178,7 @@ jobs: const local = { "@traceai/fi-core": tarball("traceai-fi-core-"), "@traceai/fi-semantic-conventions": tarball("traceai-fi-semantic-conventions-"), + "@traceai/framework-attachments": tarball("traceai-framework-attachments-"), }; fs.writeFileSync(consumer + "/package.json", JSON.stringify({ private: true, From 6aa72cb2574a75efb27ba2c3612a2eebd75aef8d Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 20:39:34 -0800 Subject: [PATCH 06/10] docs(mastra): record that newer runtimes retain inline images in exported input Co-Authored-By: Claude Opus 5.5 --- typescript/packages/traceai_framework_attachments/README.md | 2 +- typescript/packages/traceai_mastra/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/typescript/packages/traceai_framework_attachments/README.md b/typescript/packages/traceai_framework_attachments/README.md index 5551a2e61..a44ecc862 100644 --- a/typescript/packages/traceai_framework_attachments/README.md +++ b/typescript/packages/traceai_framework_attachments/README.md @@ -39,7 +39,7 @@ E means the adapter reads the original typed media value. R means the integratio | Genkit 1.42+ | `defineModel` implementation wrapped with `withGenkitModelAttachments` (request messages and response message) | E | | Genkit | Any model, including plugin models, through `use: [genkitAttachmentMiddleware(binding)]` on `generate` | E | | Genkit | `defineTool` implementation returning `Part[]`, wrapped with `withGenkitToolAttachments` | E | -| Mastra (`@traceai/mastra`) | All exporter operations | N for raw capture. The exporter receives ended, serialized spans, has no manager and never uploads. It removes `fi.attachment.*` keys it did not issue. | +| Mastra (`@traceai/mastra`) | All exporter operations | N for raw capture. The exporter sees spans only after they end, has no manager and never uploads. Newer runtimes keep inline images in span input, but the core has no API to add references to an ended span. The exporter removes `fi.attachment.*` keys it did not issue. | Streams are never read for media, remote URLs and file paths are never fetched, and creating new recordings is not supported. diff --git a/typescript/packages/traceai_mastra/README.md b/typescript/packages/traceai_mastra/README.md index 89e1577c8..29cc8dcd7 100644 --- a/typescript/packages/traceai_mastra/README.md +++ b/typescript/packages/traceai_mastra/README.md @@ -97,7 +97,7 @@ If you already manage a Mastra `Observability` instance, add `createFIMastraExpo ## Attachments -The Mastra exporter does not capture media. It receives ended, serialized Mastra spans after the run, has no attachment manager and never uploads. It also removes any `fi.attachment.*` attribute it did not issue, so application metadata cannot forge an attachment reference. +The Mastra exporter does not capture media as attachments. It receives Mastra spans after they end, has no attachment manager and never uploads. It also removes any `fi.attachment.*` attribute it did not issue, so application metadata cannot forge an attachment reference. Newer Mastra runtimes keep inline image data in the span input, and the exporter sends that input as text in `input.value`. Set `FI_HIDE_INPUT_IMAGES=true` to keep image data out of exported spans. ## Mastra v0 From 2ada9bc1b0ff706650a2adc3e6a82bb50826b737 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 22:28:09 -0800 Subject: [PATCH 07/10] fix(frameworks): move the shared media helper out of the traceai_ instrumentation prefix The @traceai/auto registry test requires every packages/traceai_* directory to be a registered instrumentation. @traceai/framework-attachments is a shared helper with no instrumentor, so its directory moves to packages/framework_attachments (npm name unchanged). Release-please keys, repository.directory, consumer tsconfig references, lockfile importer and workflow path filters and pack loops follow the move. Adds test/layout.test.cjs, which fails if the helper re-enters the traceai_ prefix or if any release key, tsconfig reference or lockfile link still points at the old directory. Co-Authored-By: Claude Opus 5.5 --- .../workflows/framework-attachments-tests.yml | 6 +- .../workflows/langchain-typescript-tests.yml | 8 +- .github/workflows/mastra-typescript-tests.yml | 4 +- .github/workflows/vercel-ai-sdk-tests.yml | 4 +- .release-please-manifest.json | 2 +- release-please-config.json | 2 +- .../README.md | 0 .../package.json | 2 +- .../src/index.ts | 0 .../test/fixture.cjs | 0 .../test/integration.cjs | 0 .../test/layout.test.cjs | 48 +++++++++++ .../test/media.test.cjs | 0 .../tsconfig.esm.json | 0 .../tsconfig.esnext.json | 0 .../tsconfig.json | 0 .../packages/traceai_genkit/tsconfig.json | 2 +- .../packages/traceai_langchain/tsconfig.json | 2 +- .../packages/traceai_llamaindex/tsconfig.json | 2 +- .../packages/traceai_vercel/tsconfig.json | 2 +- typescript/pnpm-lock.yaml | 83 +++++++++---------- 21 files changed, 107 insertions(+), 60 deletions(-) rename typescript/packages/{traceai_framework_attachments => framework_attachments}/README.md (100%) rename typescript/packages/{traceai_framework_attachments => framework_attachments}/package.json (95%) rename typescript/packages/{traceai_framework_attachments => framework_attachments}/src/index.ts (100%) rename typescript/packages/{traceai_framework_attachments => framework_attachments}/test/fixture.cjs (100%) rename typescript/packages/{traceai_framework_attachments => framework_attachments}/test/integration.cjs (100%) create mode 100644 typescript/packages/framework_attachments/test/layout.test.cjs rename typescript/packages/{traceai_framework_attachments => framework_attachments}/test/media.test.cjs (100%) rename typescript/packages/{traceai_framework_attachments => framework_attachments}/tsconfig.esm.json (100%) rename typescript/packages/{traceai_framework_attachments => framework_attachments}/tsconfig.esnext.json (100%) rename typescript/packages/{traceai_framework_attachments => framework_attachments}/tsconfig.json (100%) diff --git a/.github/workflows/framework-attachments-tests.yml b/.github/workflows/framework-attachments-tests.yml index c819b4fb4..85668003e 100644 --- a/.github/workflows/framework-attachments-tests.yml +++ b/.github/workflows/framework-attachments-tests.yml @@ -2,7 +2,7 @@ name: TypeScript framework attachments on: pull_request: paths: - - 'typescript/packages/traceai_framework_attachments/**' + - 'typescript/packages/framework_attachments/**' - 'typescript/packages/traceai_langchain/**' - 'typescript/packages/traceai_llamaindex/**' - 'typescript/packages/traceai_vercel/**' @@ -65,7 +65,7 @@ jobs: for framework in langchain llamaindex ai genkit; do for version in floor latest; do echo "::group::$framework $version" - timeout 180 node packages/traceai_framework_attachments/test/integration.cjs "$framework" "$version" + timeout 180 node packages/framework_attachments/test/integration.cjs "$framework" "$version" echo "::endgroup::" done done @@ -82,7 +82,7 @@ jobs: - name: Mastra exporter attachment disposition (packed, real Agent over loopback) run: | mkdir -p "$RUNNER_TEMP/mastra-package" "$RUNNER_TEMP/mastra-consumer" - for package in fi-semantic-conventions fi-core traceai_framework_attachments traceai_vercel traceai_mastra; do + for package in fi-semantic-conventions fi-core framework_attachments traceai_vercel traceai_mastra; do pnpm --dir "packages/$package" pack --pack-destination "$RUNNER_TEMP/mastra-package" done npm install --prefix "$RUNNER_TEMP/mastra-consumer" --ignore-scripts --no-audit --no-fund \ diff --git a/.github/workflows/langchain-typescript-tests.yml b/.github/workflows/langchain-typescript-tests.yml index a695f2ced..ad7549b2c 100644 --- a/.github/workflows/langchain-typescript-tests.yml +++ b/.github/workflows/langchain-typescript-tests.yml @@ -5,7 +5,7 @@ on: branches: [main, dev] paths: - 'typescript/packages/traceai_langchain/**' - - 'typescript/packages/traceai_framework_attachments/**' + - 'typescript/packages/framework_attachments/**' - 'typescript/packages/fi-core/**' - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/pnpm-lock.yaml' @@ -16,7 +16,7 @@ on: branches: [main, dev] paths: - 'typescript/packages/traceai_langchain/**' - - 'typescript/packages/traceai_framework_attachments/**' + - 'typescript/packages/framework_attachments/**' - 'typescript/packages/fi-core/**' - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/pnpm-lock.yaml' @@ -63,7 +63,7 @@ jobs: corepack pnpm --filter @traceai/framework-attachments run build corepack pnpm --filter @traceai/langchain run build mkdir -p "$RUNNER_TEMP/langchain-tarballs" - for package in fi-semantic-conventions fi-core traceai_framework_attachments traceai_langchain; do + for package in fi-semantic-conventions fi-core framework_attachments traceai_langchain; do corepack pnpm --dir "packages/$package" pack --pack-destination "$RUNNER_TEMP/langchain-tarballs" done git diff --exit-code -- pnpm-lock.yaml @@ -216,7 +216,7 @@ jobs: corepack pnpm --filter @traceai/framework-attachments run build corepack pnpm --filter @traceai/langchain run build mkdir -p "$RUNNER_TEMP/remote-tarballs" - for package in fi-semantic-conventions fi-core traceai_framework_attachments traceai_langchain; do + for package in fi-semantic-conventions fi-core framework_attachments traceai_langchain; do corepack pnpm --dir "packages/$package" pack --pack-destination "$RUNNER_TEMP/remote-tarballs" done git diff --exit-code -- pnpm-lock.yaml diff --git a/.github/workflows/mastra-typescript-tests.yml b/.github/workflows/mastra-typescript-tests.yml index aed442356..cf52a6761 100644 --- a/.github/workflows/mastra-typescript-tests.yml +++ b/.github/workflows/mastra-typescript-tests.yml @@ -8,7 +8,7 @@ on: - 'typescript/packages/fi-core/**' - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/packages/traceai_vercel/**' - - 'typescript/packages/traceai_framework_attachments/**' + - 'typescript/packages/framework_attachments/**' - 'typescript/pnpm-lock.yaml' - '.github/workflows/mastra-typescript-tests.yml' push: @@ -70,7 +70,7 @@ jobs: - name: Pack and install in a clean consumer run: | mkdir -p "$RUNNER_TEMP/mastra-package" "$RUNNER_TEMP/mastra-consumer" - for package in fi-semantic-conventions fi-core traceai_framework_attachments traceai_vercel traceai_mastra; do + for package in fi-semantic-conventions fi-core framework_attachments traceai_vercel traceai_mastra; do pnpm --dir "packages/$package" pack --pack-destination "$RUNNER_TEMP/mastra-package" done npm install --prefix "$RUNNER_TEMP/mastra-consumer" --ignore-scripts --no-audit --no-fund \ diff --git a/.github/workflows/vercel-ai-sdk-tests.yml b/.github/workflows/vercel-ai-sdk-tests.yml index 6efc602d4..7d183f867 100644 --- a/.github/workflows/vercel-ai-sdk-tests.yml +++ b/.github/workflows/vercel-ai-sdk-tests.yml @@ -5,7 +5,7 @@ on: branches: [dev, main] paths: - 'typescript/packages/traceai_vercel/**' - - 'typescript/packages/traceai_framework_attachments/**' + - 'typescript/packages/framework_attachments/**' - 'typescript/packages/fi-semantic-conventions/**' - 'typescript/packages/fi-core/**' - 'typescript/pnpm-lock.yaml' @@ -44,7 +44,7 @@ jobs: # stage the workspace dependencies as tarballs too, as strands-typescript-tests.yml does. pnpm --dir packages/fi-core pack --pack-destination "$RUNNER_TEMP/package" pnpm --dir packages/fi-semantic-conventions pack --pack-destination "$RUNNER_TEMP/package" - pnpm --dir packages/traceai_framework_attachments pack --pack-destination "$RUNNER_TEMP/package" + pnpm --dir packages/framework_attachments pack --pack-destination "$RUNNER_TEMP/package" - name: Install clean consumer with published dependencies env: SDK: ${{ matrix.sdk }} diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 93ef2e894..666bfa7cc 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -84,7 +84,7 @@ "typescript/packages/traceai_cursor_sdk": "0.1.1", "typescript/packages/traceai_deepseek": "0.1.2", "typescript/packages/traceai_fireworks": "0.1.2", - "typescript/packages/traceai_framework_attachments": "0.1.0", + "typescript/packages/framework_attachments": "0.1.0", "typescript/packages/traceai_genkit": "0.1.1", "typescript/packages/traceai_google_adk": "0.1.2", "typescript/packages/traceai_google_genai": "0.1.1", diff --git a/release-please-config.json b/release-please-config.json index 930bb8e54..61c5176ef 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -689,7 +689,7 @@ "src/version.ts" ] }, - "typescript/packages/traceai_framework_attachments": { + "typescript/packages/framework_attachments": { "release-type": "node", "component": "npm-traceai-framework-attachments", "package-name": "@traceai/framework-attachments" diff --git a/typescript/packages/traceai_framework_attachments/README.md b/typescript/packages/framework_attachments/README.md similarity index 100% rename from typescript/packages/traceai_framework_attachments/README.md rename to typescript/packages/framework_attachments/README.md diff --git a/typescript/packages/traceai_framework_attachments/package.json b/typescript/packages/framework_attachments/package.json similarity index 95% rename from typescript/packages/traceai_framework_attachments/package.json rename to typescript/packages/framework_attachments/package.json index 73f86afd1..de89ef6a4 100644 --- a/typescript/packages/traceai_framework_attachments/package.json +++ b/typescript/packages/framework_attachments/package.json @@ -25,7 +25,7 @@ "repository": { "type": "git", "url": "git+https://github.com/future-agi/traceAI.git", - "directory": "typescript/packages/traceai_framework_attachments" + "directory": "typescript/packages/framework_attachments" }, "bugs": { "url": "https://github.com/future-agi/traceAI/issues" diff --git a/typescript/packages/traceai_framework_attachments/src/index.ts b/typescript/packages/framework_attachments/src/index.ts similarity index 100% rename from typescript/packages/traceai_framework_attachments/src/index.ts rename to typescript/packages/framework_attachments/src/index.ts diff --git a/typescript/packages/traceai_framework_attachments/test/fixture.cjs b/typescript/packages/framework_attachments/test/fixture.cjs similarity index 100% rename from typescript/packages/traceai_framework_attachments/test/fixture.cjs rename to typescript/packages/framework_attachments/test/fixture.cjs diff --git a/typescript/packages/traceai_framework_attachments/test/integration.cjs b/typescript/packages/framework_attachments/test/integration.cjs similarity index 100% rename from typescript/packages/traceai_framework_attachments/test/integration.cjs rename to typescript/packages/framework_attachments/test/integration.cjs diff --git a/typescript/packages/framework_attachments/test/layout.test.cjs b/typescript/packages/framework_attachments/test/layout.test.cjs new file mode 100644 index 000000000..cd15ac8e3 --- /dev/null +++ b/typescript/packages/framework_attachments/test/layout.test.cjs @@ -0,0 +1,48 @@ +// Workspace layout guards for this shared helper. Filesystem only: no build output needed. +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { readFileSync, readdirSync, existsSync } = require('node:fs'); +const { basename, join, relative, resolve } = require('node:path'); + +const here = resolve(__dirname, '..'); +const packages = resolve(here, '..'); +const repo = resolve(packages, '../..'); +const manifest = JSON.parse(readFileSync(join(here, 'package.json'), 'utf8')); +const dir = relative(repo, here).split('\\').join('/'); +const consumers = ['traceai_langchain', 'traceai_llamaindex', 'traceai_vercel', 'traceai_genkit']; + +test('helper stays outside the instrumentation directory prefix used by the @traceai/auto registry', () => { + // traceai_auto/test/config.test.ts requires every packages/traceai_* directory (other than + // traceai_auto and coding-agent hooks) to be an auto-registry instrumentation. This package is + // a shared helper with no instrumentor, so it must not use that prefix. + assert.doesNotMatch(basename(here), /^traceai_/); + const enumerated = readdirSync(packages).filter(p => p.startsWith('traceai_') && p !== 'traceai_auto' + && existsSync(join(packages, p, 'package.json'))) + .map(p => JSON.parse(readFileSync(join(packages, p, 'package.json'), 'utf8')).name); + assert.ok(!enumerated.includes(manifest.name), `${manifest.name} is enumerated as an instrumentation package`); +}); + +test('release, manifest and repository metadata point at the real directory', () => { + const config = JSON.parse(readFileSync(join(repo, 'release-please-config.json'), 'utf8')).packages; + const versions = JSON.parse(readFileSync(join(repo, '.release-please-manifest.json'), 'utf8')); + assert.equal(config[dir]?.['package-name'], manifest.name); + assert.equal(versions[dir], manifest.version); + assert.equal(manifest.repository.directory, dir); + for (const key of [...Object.keys(config), ...Object.keys(versions)]) { + if (key !== dir) assert.notEqual(basename(key).replace(/^traceai_/, ''), basename(dir), `stale release key ${key}`); + } +}); + +test('every consuming package references this directory in its tsconfig and lockfile link', () => { + const lock = readFileSync(join(packages, '..', 'pnpm-lock.yaml'), 'utf8'); + assert.ok(lock.includes(`\n packages/${basename(here)}:\n`), 'lockfile importer key'); + for (const consumer of consumers) { + const root = join(packages, consumer); + const deps = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8')).dependencies ?? {}; + assert.ok(manifest.name in deps, `${consumer} depends on ${manifest.name}`); + const refs = (JSON.parse(readFileSync(join(root, 'tsconfig.json'), 'utf8')).references ?? []).map(r => resolve(root, r.path)); + assert.ok(refs.includes(here), `${consumer}/tsconfig.json references ${basename(here)}`); + const importer = lock.split(`\n packages/${consumer}:\n`)[1]?.split('\n\n')[0] ?? ''; + assert.match(importer, new RegExp(`version: link:\\.\\./${basename(here)}\\n`), `${consumer} lockfile link`); + } +}); diff --git a/typescript/packages/traceai_framework_attachments/test/media.test.cjs b/typescript/packages/framework_attachments/test/media.test.cjs similarity index 100% rename from typescript/packages/traceai_framework_attachments/test/media.test.cjs rename to typescript/packages/framework_attachments/test/media.test.cjs diff --git a/typescript/packages/traceai_framework_attachments/tsconfig.esm.json b/typescript/packages/framework_attachments/tsconfig.esm.json similarity index 100% rename from typescript/packages/traceai_framework_attachments/tsconfig.esm.json rename to typescript/packages/framework_attachments/tsconfig.esm.json diff --git a/typescript/packages/traceai_framework_attachments/tsconfig.esnext.json b/typescript/packages/framework_attachments/tsconfig.esnext.json similarity index 100% rename from typescript/packages/traceai_framework_attachments/tsconfig.esnext.json rename to typescript/packages/framework_attachments/tsconfig.esnext.json diff --git a/typescript/packages/traceai_framework_attachments/tsconfig.json b/typescript/packages/framework_attachments/tsconfig.json similarity index 100% rename from typescript/packages/traceai_framework_attachments/tsconfig.json rename to typescript/packages/framework_attachments/tsconfig.json diff --git a/typescript/packages/traceai_genkit/tsconfig.json b/typescript/packages/traceai_genkit/tsconfig.json index 0096c2e12..adf9ed807 100644 --- a/typescript/packages/traceai_genkit/tsconfig.json +++ b/typescript/packages/traceai_genkit/tsconfig.json @@ -10,6 +10,6 @@ "references": [ { "path": "../fi-core" }, { "path": "../fi-semantic-conventions" }, - { "path": "../traceai_framework_attachments" } + { "path": "../framework_attachments" } ] } diff --git a/typescript/packages/traceai_langchain/tsconfig.json b/typescript/packages/traceai_langchain/tsconfig.json index 2f5c12d22..7248bb8cd 100644 --- a/typescript/packages/traceai_langchain/tsconfig.json +++ b/typescript/packages/traceai_langchain/tsconfig.json @@ -7,5 +7,5 @@ "files": [], "include": ["src/**/*.ts", "test/**/*.ts"], "exclude": ["src/**/*.test.ts", "src/**/__tests__/**", "test/**/*.ts"], - "references": [{ "path": "../traceai_framework_attachments" }] + "references": [{ "path": "../framework_attachments" }] } \ No newline at end of file diff --git a/typescript/packages/traceai_llamaindex/tsconfig.json b/typescript/packages/traceai_llamaindex/tsconfig.json index 0096c2e12..adf9ed807 100644 --- a/typescript/packages/traceai_llamaindex/tsconfig.json +++ b/typescript/packages/traceai_llamaindex/tsconfig.json @@ -10,6 +10,6 @@ "references": [ { "path": "../fi-core" }, { "path": "../fi-semantic-conventions" }, - { "path": "../traceai_framework_attachments" } + { "path": "../framework_attachments" } ] } diff --git a/typescript/packages/traceai_vercel/tsconfig.json b/typescript/packages/traceai_vercel/tsconfig.json index 95ffa5947..55705e842 100644 --- a/typescript/packages/traceai_vercel/tsconfig.json +++ b/typescript/packages/traceai_vercel/tsconfig.json @@ -10,6 +10,6 @@ "references": [ { "path": "../fi-core" }, { "path": "../fi-semantic-conventions" }, - { "path": "../traceai_framework_attachments" } + { "path": "../framework_attachments" } ] } \ No newline at end of file diff --git a/typescript/pnpm-lock.yaml b/typescript/pnpm-lock.yaml index 04a719405..ebd0d5c23 100644 --- a/typescript/pnpm-lock.yaml +++ b/typescript/pnpm-lock.yaml @@ -268,6 +268,43 @@ importers: specifier: ^5.8.3 version: 5.8.3 + packages/framework_attachments: + dependencies: + '@opentelemetry/api': + specifier: ^1.9.0 + version: 1.9.1 + '@traceai/fi-core': + specifier: workspace:^ + version: link:../fi-core + devDependencies: + '@opentelemetry/resources': + specifier: ^2.0.1 + version: 2.7.1(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': + specifier: ^2.0.1 + version: 2.7.1(@opentelemetry/api@1.9.1) + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 + '@types/node': + specifier: ^22.10.10 + version: 22.15.33 + jest: + specifier: ^29.7.0 + version: 29.7.0(@types/node@22.15.33)(ts-node@10.9.2(@types/node@22.15.33)(typescript@5.8.3)) + rimraf: + specifier: ^6.0.1 + version: 6.1.2 + ts-jest: + specifier: ^29.2.5 + version: 29.4.6(@babel/core@7.27.1)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.27.1))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.15.33)(ts-node@10.9.2(@types/node@22.15.33)(typescript@5.8.3)))(typescript@5.8.3) + tsc-alias: + specifier: ^1.8.10 + version: 1.8.16 + typescript: + specifier: ^5.7.3 + version: 5.8.3 + packages/traceai_agent_core: dependencies: '@opentelemetry/api': @@ -1188,44 +1225,6 @@ importers: specifier: ^5.8.3 version: 5.9.3 - packages/traceai_framework_attachments: - dependencies: - '@opentelemetry/api': - specifier: ^1.9.0 - version: 1.9.1 - '@traceai/fi-core': - specifier: workspace:^ - version: link:../fi-core - devDependencies: - '@opentelemetry/resources': - specifier: ^2.0.1 - version: 2.7.1(@opentelemetry/api@1.9.1) - '@opentelemetry/sdk-trace-base': - specifier: ^2.0.1 - version: 2.7.1(@opentelemetry/api@1.9.1) - '@types/jest': - specifier: ^29.5.14 - version: 29.5.14 - '@types/node': - specifier: ^22.10.10 - version: 22.15.33 - jest: - specifier: ^29.7.0 - version: 29.7.0(@types/node@22.15.33)(ts-node@10.9.2(@types/node@22.15.33)(typescript@5.8.3)) - rimraf: - specifier: ^6.0.1 - version: 6.1.2 - ts-jest: - specifier: ^29.2.5 - version: 29.4.6(@babel/core@7.27.1)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.27.1))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.15.33)(ts-node@10.9.2(@types/node@22.15.33)(typescript@5.8.3)))(typescript@5.8.3) - tsc-alias: - specifier: ^1.8.10 - version: 1.8.16 - typescript: - specifier: ^5.7.3 - version: 5.8.3 - - packages/traceai_genkit: dependencies: '@opentelemetry/api': @@ -1239,7 +1238,7 @@ importers: version: link:../fi-semantic-conventions '@traceai/framework-attachments': specifier: workspace:^ - version: link:../traceai_framework_attachments + version: link:../framework_attachments devDependencies: '@opentelemetry/context-async-hooks': specifier: 2.7.1 @@ -1790,7 +1789,7 @@ importers: version: link:../fi-semantic-conventions '@traceai/framework-attachments': specifier: workspace:^ - version: link:../traceai_framework_attachments + version: link:../framework_attachments devDependencies: '@langchain/openai': specifier: 0.3.16 @@ -1912,7 +1911,7 @@ importers: version: 4.17.23 '@traceai/framework-attachments': specifier: workspace:^ - version: link:../traceai_framework_attachments + version: link:../framework_attachments devDependencies: '@llamaindex/openai': specifier: ^0.4.0 @@ -2961,7 +2960,7 @@ importers: version: link:../fi-semantic-conventions '@traceai/framework-attachments': specifier: workspace:^ - version: link:../traceai_framework_attachments + version: link:../framework_attachments devDependencies: '@ai-sdk/openai': specifier: ^3.0.0 From 4e24a9ceaa506ce817d45e120dae61c168ff8874 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 22:36:44 -0800 Subject: [PATCH 08/10] test(frameworks): fixture status bodies carry the F01 reason member The TypeScript core base (feat/attachments-ts-core 031178f6) now validates content and complete status bodies strictly ({id, state, reason}). The helper's loopback fixture returned {upload_id, state} only, so the PR merge ref failed five upload tests. Status bodies now include `reason`, and session and upload expiry sit an hour out so no renewal runs inside a test. Both the previous and the current core accept this fixture. Co-Authored-By: Claude Opus 5.5 --- .../packages/framework_attachments/test/fixture.cjs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/typescript/packages/framework_attachments/test/fixture.cjs b/typescript/packages/framework_attachments/test/fixture.cjs index bb383615e..7e1e816bb 100644 --- a/typescript/packages/framework_attachments/test/fixture.cjs +++ b/typescript/packages/framework_attachments/test/fixture.cjs @@ -5,6 +5,9 @@ const { AttachmentManager, createAttachmentHttpTransport, attachmentFetchClient const { BasicTracerProvider, SimpleSpanProcessor, InMemorySpanExporter } = require('@opentelemetry/sdk-trace-base'); const PNG = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+j5nQAAAAASUVORK5CYII=', 'base64'); const URI = `data:image/png;base64,${PNG.toString('base64')}`; +// Status bodies carry the F01 `reason` member; the strict v1 transport rejects a status without it. +// Session and upload expiry sit an hour out so no renewal runs inside a test. +const expiry = () => new Date(Date.now() + 3600000).toISOString(); const limits = { max_upload_bytes: 52428800, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1048576, playback_bytes_per_second: 1048576, @@ -24,16 +27,16 @@ async function fixture(config = {}) { policy: { version: 'test_v1', enabled: true, allowed_sources: ['Bytes'], allowed_modalities: ['image', 'audio', 'video', 'document', 'file'], require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false, ...config.serverPolicy }, - expires_at: new Date(Date.now() + 60000).toISOString(), transfer_modes: ['authenticated_stream'] })); + expires_at: expiry(), transfer_modes: ['authenticated_stream'] })); } else if (req.url.endsWith('/uploads')) { const request = JSON.parse(body); uploads.push(request); res.statusCode = 201; res.end(JSON.stringify({ upload_id: randomUUID(), attachment_id: request.attachment_id, occurrence_id: request.occurrence_id, - state: 'pending_upload', reserved_bytes: request.declared_bytes, expires_at: new Date(Date.now() + 60000).toISOString(), + state: 'pending_upload', reserved_bytes: request.declared_bytes, expires_at: expiry(), transfer_mode: 'authenticated_stream', write_capability: 'synthetic-capability-only' })); } else { const upload_id = req.url.split('/').at(-2); - if (req.url.endsWith('/content')) { bodies.push(body); res.end(JSON.stringify({ upload_id, state: 'pending_upload' })); } - else { completed.push(upload_id); res.statusCode = 202; res.end(JSON.stringify({ upload_id, state: 'validating' })); } + if (req.url.endsWith('/content')) { bodies.push(body); res.end(JSON.stringify({ upload_id, state: 'pending_upload', reason: 'queued' })); } + else { completed.push(upload_id); res.statusCode = 202; res.end(JSON.stringify({ upload_id, state: 'validating', reason: 'binding_pending' })); } } } catch { res.statusCode = 500; res.end('{}'); } }); From 47d6b724d6b57acc3d7087280d010cbf74375032 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 03:48:36 -0800 Subject: [PATCH 09/10] feat(frameworks): preflight every media part before decode Adopt the ts-core preflight API (SDK design section 1) in the shared framework media helper used by the LangChain, LlamaIndex, Vercel AI SDK and Genkit bindings. - Size each part from its shape only (exact decoded length, no decode), then manager.preflight() with the merged capture policy. Only an admitted part is decoded and passed to ticket.attach() with its message/part fields. The lazy `get data()` getter through manager.attach() is gone. - A part over the owned-byte budget is now denied queue_full before any decode (the old path decoded inside the core, then denied). - An admitted part that fails strict decode releases its ticket at once; decode no longer throws, so later parts in the same message still run. - Tests: queue_full denial with zero atob calls for all four dialects, preflight+ticket.attach used and manager.attach never called, ticket release on a decode failure, and no FI_ATTACHMENTS_* warning from the v1 fixture transport. No consumed byte stream exists at these boundaries (every part arrives as one complete inline value), so startAttachmentStream is not used. Co-Authored-By: Claude Opus 5.5 --- .../packages/framework_attachments/README.md | 3 +- .../framework_attachments/src/index.ts | 66 ++++++++++++++----- .../framework_attachments/test/media.test.cjs | 50 ++++++++++++++ 3 files changed, 100 insertions(+), 19 deletions(-) diff --git a/typescript/packages/framework_attachments/README.md b/typescript/packages/framework_attachments/README.md index a44ecc862..e3584d797 100644 --- a/typescript/packages/framework_attachments/README.md +++ b/typescript/packages/framework_attachments/README.md @@ -14,6 +14,7 @@ The adapters accept inline base64/data URIs and `Uint8Array` sources at the docu - Capture is off unless enabled on the shared manager. - Directional hide flags, required PII handling, server policy and native content restrictions deny capture before decoding or copying media. +- Each part is sized from its shape (exact decoded length, no decode) and then preflighted with the shared manager (`manager.preflight`), which applies the deny-wins policy and reserves that many bytes. Only an admitted part is decoded and handed to `ticket.attach`. A part that fails strict decoding releases its ticket at once. A part that does not fit the manager's owned-byte budget is denied (`queue_full`) without being decoded. - Supply the integration's `traceConfig` and `capture` restrictions along with the manager. Binding-level denials cannot be relaxed by an individual part. - Tool images intended for a model are subject to both output and model-input restrictions. - No filenames, source URLs, hashes or provider credentials are added to references. @@ -41,7 +42,7 @@ E means the adapter reads the original typed media value. R means the integratio | Genkit | `defineTool` implementation returning `Part[]`, wrapped with `withGenkitToolAttachments` | E | | Mastra (`@traceai/mastra`) | All exporter operations | N for raw capture. The exporter sees spans only after they end, has no manager and never uploads. Newer runtimes keep inline images in span input, but the core has no API to add references to an ended span. The exporter removes `fi.attachment.*` keys it did not issue. | -Streams are never read for media, remote URLs and file paths are never fetched, and creating new recordings is not supported. +Streams are never read for media, remote URLs and file paths are never fetched, and creating new recordings is not supported. Every supported framework boundary delivers each media part as one complete inline value (streamed AI SDK 7 files arrive whole in the end event), so no consumed byte stream exists here and the core's `startAttachmentStream` is not used. ## Known limitation: framework plus provider instrumentation diff --git a/typescript/packages/framework_attachments/src/index.ts b/typescript/packages/framework_attachments/src/index.ts index 944840ab0..518f1aae3 100644 --- a/typescript/packages/framework_attachments/src/index.ts +++ b/typescript/packages/framework_attachments/src/index.ts @@ -1,6 +1,6 @@ import type { Span } from "@opentelemetry/api"; import type { TraceConfigOptions } from "@traceai/fi-core"; -import type { AttachmentCapturePolicy, AttachmentManager, AttachmentOptions } from "@traceai/fi-core/attachments"; +import type { AttachmentCapturePolicy, AttachmentFields, AttachmentManager, AttachmentOptions } from "@traceai/fi-core/attachments"; export type MediaDialect = "langchain" | "llamaindex" | "vercel" | "genkit"; export interface FrameworkAttachmentOptions { @@ -28,20 +28,39 @@ function dataUriMime(value: unknown): string | undefined { if (typeof value !== "string") return undefined; return /^data:([a-zA-Z0-9/+.-]+);base64,/.exec(value.slice(0, 96))?.[1]; } -function decode(value: unknown): Uint8Array { - if (value instanceof Uint8Array) return value; +const BASE64_PREFIX = /^data:[a-zA-Z0-9/+.-]+;base64,/; +function encodedPayload(value: string): string { + const prefix = BASE64_PREFIX.exec(value.slice(0, 96)); + return prefix ? value.slice(prefix[0].length) : value; +} +/** + * Exact decoded length from the source's shape only: no decode, copy or allocation. + * 0 means unsupported (wrong type, empty, malformed length or over the per-part cap), + * and the part is skipped before preflight. + */ +function exactLength(value: unknown): number { // No implicit ArrayBuffer conversion, stream reads, paths or remote retrieval. - if (typeof value !== "string") throw new Error("unsupported media source"); - const prefix = /^data:[a-zA-Z0-9/+.-]+;base64,/.exec(value.slice(0, 96)); - const length = value.length - (prefix?.[0].length ?? 0); - if (!length || length > MAX_ENCODED || length % 4 !== 0) throw new Error("unsupported media size"); - const encoded = prefix ? value.slice(prefix[0].length) : value; - if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) throw new Error("invalid media encoding"); - const binary = atob(encoded); - if (binary.length > MAX_DECODED) throw new Error("unsupported media size"); - const bytes = new Uint8Array(binary.length); - for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); - return bytes; + if (value instanceof Uint8Array) return value.byteLength <= MAX_DECODED ? value.byteLength : 0; + if (typeof value !== "string") return 0; + const encoded = encodedPayload(value); + if (!encoded.length || encoded.length > MAX_ENCODED || encoded.length % 4 !== 0) return 0; + const length = (encoded.length / 4) * 3 - (encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0); + return length > 0 && length <= MAX_DECODED ? length : 0; +} +/** Strict decode, called only AFTER preflight admitted the part. Never throws; undefined = unusable. */ +function decode(value: unknown, maxBytes: number): Uint8Array | undefined { + // The core detaches mutable bytes synchronously inside ticket.attach. + if (value instanceof Uint8Array) return value.byteLength && value.byteLength <= maxBytes ? value : undefined; + if (typeof value !== "string") return undefined; + const encoded = encodedPayload(value); + if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded)) return undefined; + try { + const binary = atob(encoded); + if (!binary.length || binary.length > maxBytes) return undefined; + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); + return bytes; + } catch { return undefined; } } function normalized(part: unknown, dialect: MediaDialect): { data: unknown; mime: string } | undefined { @@ -99,7 +118,7 @@ function locallyDenied(config: TraceConfigOptions | undefined, options: Attachme /** * Observe only explicit typed content arrays at an original-value boundary. * No recursive scraping, uploads from serialized telemetry, or caller mutation. - * The source getter runs synchronously INSIDE core admission, never before it. + * Every part is preflighted against the shared manager before its source is decoded or copied. */ export function captureMediaParts(binding: FrameworkAttachmentOptions, span: Span, parts: unknown, dialect: MediaDialect, options: Omit): void { @@ -127,9 +146,20 @@ export function captureMediaParts(binding: FrameworkAttachmentOptions, span: Spa } const attachmentOptions: AttachmentOptions = { ...options, mimeType: media.mime, partIndex: i, capture }; if (locallyDenied(binding.traceConfig, attachmentOptions)) continue; - // Lazy decode also ensures disabled, PII, unsampled and full-queue captures - // do not allocate decoded media. Core owns the detached mutable-byte copy. - const result = manager.attach(span, { kind: "Bytes", get data() { return decode(media.data); } }, attachmentOptions); + // Size from shape only, then preflight (deny-wins, budget reserved for the exact length) + // BEFORE any decode or copy (SDK design section 1). Denied, disabled, PII, unsampled and + // full-budget parts therefore never allocate decoded media. + const maxBytes = exactLength(media.data); + if (!maxBytes) continue; + const ticket = manager.preflight(span, { mimeType: media.mime, direction: options.direction, purpose: options.purpose, + capture, maxBytes }); + if (!ticket.admitted) continue; + const bytes = decode(media.data, ticket.maxBytes); + // An admitted ticket that is not used returns its reservation at once. + if (!bytes) { ticket.release(); continue; } + const fields: AttachmentFields = { messageIndex: options.messageIndex, partIndex: i, role: options.role, + channel: options.channel, conversationId: options.conversationId, turnId: options.turnId, toolCallId: options.toolCallId }; + const result = ticket.attach({ kind: "Bytes", data: bytes }, fields); if (result.admission === "queued") positions.add(key); } } catch { /* Unsupported/malformed media must never affect framework execution. */ } diff --git a/typescript/packages/framework_attachments/test/media.test.cjs b/typescript/packages/framework_attachments/test/media.test.cjs index ffc56d38e..1114fb3d7 100644 --- a/typescript/packages/framework_attachments/test/media.test.cjs +++ b/typescript/packages/framework_attachments/test/media.test.cjs @@ -93,3 +93,53 @@ test('integration-level trace config also wins over a permissive shared manager' span.end(); await f.manager.flush(); assert.equal(f.uploads.length, 0); } finally { await f.close(); } }); +// SDK design section 1 (T-PF-09 analog): every part is preflighted before its source is decoded or copied. +const warnings = []; +process.on('warning', warning => { if (String(warning.code ?? '').startsWith('FI_ATTACHMENTS_')) warnings.push(warning.code); }); +test('preflight denies a part over the owned budget before any decode (queue_full, zero atob)', async () => { + for (const [dialect, content] of Object.entries(parts)) { + const f = await fixture({ maxOwnedBytes: 16 }); let decoded = 0; const atob = globalThis.atob; + globalThis.atob = (...args) => { decoded++; return atob(...args); }; + try { + const span = f.tracer.startSpan('budget'); + captureMediaMessages({ attachments: f.manager }, span, [{ role: 'user', content }], dialect, 'input'); + span.end(); await f.manager.flush(); + assert.equal(decoded, 0, dialect); assert.equal(f.uploads.length, 0, dialect); assert.equal(f.bodies.length, 0, dialect); + const status = f.manager.status(); + assert.equal(status.openTickets, 0, dialect); assert.equal(status.ownedBytes, 0, dialect); + } finally { globalThis.atob = atob; await f.close(); } + } +}); +test('every part goes through preflight + ticket.attach; manager.attach is never called', async () => { + for (const [dialect, content] of Object.entries(parts)) { + const f = await fixture(); const calls = { preflight: 0, attach: 0 }; + const preflight = f.manager.preflight.bind(f.manager); + f.manager.preflight = (target, options) => { calls.preflight++; assert.equal(options.maxBytes, PNG.length, dialect); return preflight(target, options); }; + f.manager.attach = () => { calls.attach++; throw new Error('legacy attach path'); }; + try { + const span = f.tracer.startSpan('ticket'); + captureMediaMessages({ attachments: f.manager }, span, [{ role: 'user', content }], dialect, 'input'); + span.end(); await f.manager.flush(); + assert.deepEqual(calls, { preflight: 1, attach: 0 }, dialect); + assert.equal(f.bodies.length, 1, dialect); assert.deepEqual(f.bodies[0], PNG); + } finally { await f.close(); } + } +}); +test('an admitted part that fails strict decode releases its ticket; later parts still upload', async () => { + const f = await fixture(); + try { + const span = f.tracer.startSpan('release'); + // Shape-valid length (8 chars) with an invalid alphabet: admitted by size, rejected by the strict decoder. + captureMediaParts({ attachments: f.manager }, span, [{ type: 'image', image: 'invalid!', mediaType: 'image/png' }, + { type: 'image', image: PNG.toString('base64'), mediaType: 'image/png' }], 'vercel', { direction: 'input' }); + const status = f.manager.status(); + assert.equal(status.openTickets, 0); + span.end(); await f.manager.flush(); + assert.equal(f.uploads.length, 1); assert.equal(f.uploads[0].part_index, 1); assert.deepEqual(f.bodies[0], PNG); + assert.equal(f.manager.status().ownedBytes, 0); + } finally { await f.close(); } +}); +test('the fixture speaks the v1 wire: no legacy-transport or wire-rejected warning fired', async () => { + await new Promise(resolve => setImmediate(resolve)); + assert.deepEqual(warnings, []); +}); From 2b38d3f0b6647bea04c48d2778853b2f027de5c5 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 04:15:45 -0800 Subject: [PATCH 10/10] fix(frameworks): keep Uint8Array parts bounded by the manager, not the base64 cap The preflight rewrite applied the 1 MiB base64 decode cap to Uint8Array parts too, so a typed-bytes image over 1 MiB (for example an AI SDK `image` part holding an uploaded photo) was skipped where the previous helper captured it. Bytes are never decoded here: size them by byteLength and let preflight apply the manager's upload cap and owned-byte budget. Base64 strings keep the 1 MiB decode cap. Test: a 1.5 MiB Uint8Array part uploads byte-identical, and the same media as base64 in the same array is skipped. Co-Authored-By: Claude Opus 5.5 --- typescript/packages/framework_attachments/README.md | 2 +- .../packages/framework_attachments/src/index.ts | 7 ++++--- .../framework_attachments/test/media.test.cjs | 13 +++++++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/typescript/packages/framework_attachments/README.md b/typescript/packages/framework_attachments/README.md index e3584d797..32c31a8be 100644 --- a/typescript/packages/framework_attachments/README.md +++ b/typescript/packages/framework_attachments/README.md @@ -8,7 +8,7 @@ Shared typed-media normalization for traceAI's framework integrations. This pack Pass the existing provider's `attachments` manager in `binding`. Without that manager, these functions do nothing. The manager still requires explicit attachment enablement, a successful policy bootstrap and a sampled, recording span. Normal inference never waits for attachment network operations. Use the provider's existing flush/shutdown lifecycle to finish admitted transfers. -The adapters accept inline base64/data URIs and `Uint8Array` sources at the documented typed boundaries. They do not fetch remote URLs, open local paths, read streams, reconstruct discarded media from exported text, or create new recordings. Base64 decoding is bounded to one MiB per part. A single span admits at most 64 positions through this helper; the core manager separately bounds bytes, queue length and reference projection. +The adapters accept inline base64/data URIs and `Uint8Array` sources at the documented typed boundaries. They do not fetch remote URLs, open local paths, read streams, reconstruct discarded media from exported text, or create new recordings. Base64 decoding is bounded to one MiB per part. `Uint8Array` parts are not decoded, so only the shared manager bounds them (its upload cap and owned-byte budget, checked at preflight). A single span admits at most 64 positions through this helper; the core manager separately bounds bytes, queue length and reference projection. ## Privacy and identity diff --git a/typescript/packages/framework_attachments/src/index.ts b/typescript/packages/framework_attachments/src/index.ts index 518f1aae3..eafc5fa97 100644 --- a/typescript/packages/framework_attachments/src/index.ts +++ b/typescript/packages/framework_attachments/src/index.ts @@ -35,12 +35,13 @@ function encodedPayload(value: string): string { } /** * Exact decoded length from the source's shape only: no decode, copy or allocation. - * 0 means unsupported (wrong type, empty, malformed length or over the per-part cap), - * and the part is skipped before preflight. + * 0 means unsupported (wrong type, empty, malformed length, or base64 over the per-part + * decode cap), and the part is skipped before preflight. */ function exactLength(value: unknown): number { // No implicit ArrayBuffer conversion, stream reads, paths or remote retrieval. - if (value instanceof Uint8Array) return value.byteLength <= MAX_DECODED ? value.byteLength : 0; + // Bytes are not decoded here, so only the manager bounds them (preflight: too_large / queue_full). + if (value instanceof Uint8Array) return value.byteLength; if (typeof value !== "string") return 0; const encoded = encodedPayload(value); if (!encoded.length || encoded.length > MAX_ENCODED || encoded.length % 4 !== 0) return 0; diff --git a/typescript/packages/framework_attachments/test/media.test.cjs b/typescript/packages/framework_attachments/test/media.test.cjs index 1114fb3d7..e547a8b9f 100644 --- a/typescript/packages/framework_attachments/test/media.test.cjs +++ b/typescript/packages/framework_attachments/test/media.test.cjs @@ -143,3 +143,16 @@ test('the fixture speaks the v1 wire: no legacy-transport or wire-rejected warni await new Promise(resolve => setImmediate(resolve)); assert.deepEqual(warnings, []); }); +test('Uint8Array parts are bounded by the manager, not the 1 MiB base64 decode cap', async () => { + const f = await fixture(); + try { + const span = f.tracer.startSpan('large-bytes'); + const large = new Uint8Array(1536 * 1024); large.set(PNG); large[large.length - 1] = 7; + captureMediaParts({ attachments: f.manager }, span, [{ type: 'image', image: large, mediaType: 'image/png' }, + { type: 'image', image: Buffer.from(large).toString('base64'), mediaType: 'image/png' }], 'vercel', { direction: 'input' }); + span.end(); await f.manager.flush(10000); + // The bytes part uploads byte-identical; the same media as base64 is over the decode cap and is skipped. + assert.equal(f.uploads.length, 1); assert.equal(f.uploads[0].part_index, 0); + assert.equal(f.bodies.length, 1); assert.ok(Buffer.from(large).equals(f.bodies[0])); + } finally { await f.close(); } +});