From cc3d18d800d9f25cc97cfe3fa11aca7cf10ee186 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 20:00:51 -0800 Subject: [PATCH 01/12] feat(agent-core): opt-in typed image attachments for Pi-family coding agents Shares the provider-owned fi-core AttachmentManager through createTurnTracer, adds a bounded typed image-block helper, attachment status in inspectStatus, and Pi/oh-my-pi bindings for user prompt and tool-result images. Co-Authored-By: Claude Opus 5.5 --- .../src/attachment-runtime.ts | 53 +++++ .../src/attachment-status.ts | 49 +++++ .../traceai_agent_core/src/attachments.ts | 96 +++++++++ .../packages/traceai_agent_core/src/index.ts | 2 + .../traceai_agent_core/src/lifecycle.ts | 4 + .../packages/traceai_agent_core/src/tracer.ts | 19 ++ .../test/attachments.test.cjs | 183 ++++++++++++++++++ .../traceai_oh_my_pi_plugin/src/index.ts | 5 + .../packages/traceai_pi_plugin/src/index.ts | 6 + .../test/attachments-native.cjs | 116 +++++++++++ 10 files changed, 533 insertions(+) create mode 100644 typescript/packages/traceai_agent_core/src/attachment-runtime.ts create mode 100644 typescript/packages/traceai_agent_core/src/attachment-status.ts create mode 100644 typescript/packages/traceai_agent_core/src/attachments.ts create mode 100644 typescript/packages/traceai_agent_core/test/attachments.test.cjs create mode 100644 typescript/packages/traceai_pi_plugin/test/attachments-native.cjs diff --git a/typescript/packages/traceai_agent_core/src/attachment-runtime.ts b/typescript/packages/traceai_agent_core/src/attachment-runtime.ts new file mode 100644 index 000000000..fd55bde4d --- /dev/null +++ b/typescript/packages/traceai_agent_core/src/attachment-runtime.ts @@ -0,0 +1,53 @@ +import { + AttachmentTransportError, + attachmentNodeHttpClient, + createAttachmentHttpTransport, + type NodeAttachmentManagerOptions, +} from "@traceai/fi-core/attachments/node"; +import type { AttachmentTransport } from "@traceai/fi-core/attachments"; +import type { ResolvedConfig } from "./config.js"; +import { collectionEnabled } from "./collection.js"; + +/** One provider-owned runtime with the same credential and revocation boundary as OTLP. */ +export function agentAttachmentRuntime( + config: ResolvedConfig, + options: NodeAttachmentManagerOptions = {}, +): NodeAttachmentManagerOptions { + // Neither the collector URL nor provider/tool content is a control-plane origin. + const endpoint = options.endpoint ?? process.env.FI_ATTACHMENTS_ENDPOINT; + const headers: Readonly> = config.credentials ? { + "x-api-key": config.credentials.apiKey, + "x-secret-key": config.credentials.secretKey, + } : {}; + const allowed = () => collectionEnabled(config) && config.captureContent && + config.destinationApproved && !!config.credentials; + let transport = options.transport; + if (!transport && endpoint && allowed()) { + try { transport = createAttachmentHttpTransport({ endpoint, headers, client: attachmentNodeHttpClient }); } + catch { /* Unsupported/untrusted control origin remains unconfigured. */ } + } + const delegate = transport; + const check = async () => { + if (!allowed() || (config.revalidate && !(await config.revalidate()))) + throw new AttachmentTransportError("policy_denied"); + }; + const guarded: AttachmentTransport | undefined = delegate && { + async bootstrap(request, signal) { await check(); return delegate.bootstrap(request, signal); }, + async initialize(session, request, signal) { await check(); return delegate.initialize(session, request, signal); }, + async upload(session, upload, data, signal) { await check(); return delegate.upload(session, upload, data, signal); }, + async complete(session, upload, request, signal) { await check(); return delegate.complete(session, upload, request, signal); }, + }; + return { + ...options, endpoint, headers, transport: guarded, + enabled: allowed() ? options.enabled : false, + projectName: config.projectName, + projectType: "observe", + environment: () => allowed() ? process.env : { FI_ATTACHMENTS_ENABLED: "false" }, + traceConfig: { + ...options.traceConfig, + hideInputs: config.hideInputs || options.traceConfig?.hideInputs, + hideOutputs: config.hideOutputs || options.traceConfig?.hideOutputs, + hideErrorDetails: config.hideErrorDetails || options.traceConfig?.hideErrorDetails, + }, + }; +} diff --git a/typescript/packages/traceai_agent_core/src/attachment-status.ts b/typescript/packages/traceai_agent_core/src/attachment-status.ts new file mode 100644 index 000000000..6c598c2b7 --- /dev/null +++ b/typescript/packages/traceai_agent_core/src/attachment-status.ts @@ -0,0 +1,49 @@ +import type { ResolvedConfig } from "./config.js"; + +/** Source readiness is separate from span support, server authorization and product playback. */ +export const AGENT_ATTACHMENT_SOURCES = Object.freeze({ + pi: "typed-image-blocks", + "oh-my-pi": "typed-image-blocks", + "pi-coding-agent": "source_unavailable", + "claude-code": "source_unavailable", + codex: "source_unavailable", + "codex-sdk": "source_unavailable", + cursor: "source_unavailable", + "cursor-cli": "source_unavailable", + "cursor-cloud": "source_unavailable", + "cursor-sdk": "source_unavailable", + opencode: "source_unavailable", + "opencode-v1": "source_unavailable", + openclaw: "source_unavailable", + hermes: "source_unavailable", + goose: "source_unavailable", + gemini: "source_unavailable", + copilot: "source_unavailable", + kiro: "source_unavailable", + antigravity: "source_unavailable", + augment: "source_unavailable", + "augment-ide": "source_unavailable", + "devin-cascade": "source_unavailable", + grok: "source_unavailable", + muse: "source_unavailable", + "copilot-vscode": "source_unavailable", +} as const); + +/** Local inspection does not mint a capability or confuse OTLP acceptance with attachment scope. */ +export function agentAttachmentStatus(config: ResolvedConfig, env: NodeJS.ProcessEnv = process.env) { + const source = AGENT_ATTACHMENT_SOURCES[config.agent as keyof typeof AGENT_ATTACHMENT_SOURCES] ?? "source_unavailable"; + const requested = /^(true|1|yes)$/i.test(env.FI_ATTACHMENTS_ENABLED?.trim() ?? ""); + return { + state: !requested ? "disabled" : + !config.enabled || !config.captureContent ? "policy_denied" : + source === "source_unavailable" ? "source_unavailable" : + !env.FI_ATTACHMENTS_ENDPOINT ? "endpoint_unconfigured" : "scope_unverified", + source, + credentialScope: "unverified-runtime-bootstrap-required", + productPlayback: "unverified", + recording: "not-created", + maxImageBytes: 256 * 1024, + localFile: "source_unavailable", + remoteUrl: "source_unavailable", + } as const; +} diff --git a/typescript/packages/traceai_agent_core/src/attachments.ts b/typescript/packages/traceai_agent_core/src/attachments.ts new file mode 100644 index 000000000..a5b81fc8c --- /dev/null +++ b/typescript/packages/traceai_agent_core/src/attachments.ts @@ -0,0 +1,96 @@ +import type { TracerProvider } from "@opentelemetry/api"; +import type { + AttachmentCapturePolicy, + AttachmentHandle, + AttachmentManager, + AttachmentOptions, + AttachmentSpan, +} from "@traceai/fi-core/attachments"; + +/** A bounded callback budget, not a claim that every provider image is capturable. */ +export const MAX_AGENT_IMAGE_BYTES = 256 * 1024; +const MAX_BASE64_LENGTH = 4 * Math.ceil(MAX_AGENT_IMAGE_BYTES / 3); +const IMAGE_MIMES = new Set(["image/png", "image/jpeg", "image/webp"]); +// Identity-only replay guard. Distinct parts/positions/spans keep distinct occurrences. +const admitted = new WeakMap>>>(); +export type AgentImageOptions = Omit; + +/** Reuse the configured provider's manager. Never create another uploader or inspect credentials. */ +export function providerAttachments(provider: TracerProvider): AttachmentManager | undefined { + const candidate = provider as TracerProvider & { + attachments?: AttachmentManager; + getDelegate?: () => TracerProvider; + }; + return candidate.attachments ?? (candidate.getDelegate?.() as typeof candidate | undefined)?.attachments; +} + +function decodeImage(value: unknown): Uint8Array { + if (typeof value !== "string" || !value.length || value.length > MAX_BASE64_LENGTH || + value.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(value)) + throw new Error("UNAVAILABLE_IMAGE"); + const bytes = Buffer.from(value, "base64"); + if (!bytes.length || bytes.length > MAX_AGENT_IMAGE_BYTES) throw new Error("UNAVAILABLE_IMAGE"); + return bytes; +} + +/** + * Pi/OMP's typed image content boundary. Not a recursive object walker. + * Data access and base64 decoding happen inside core admission, after policy, + * bootstrap, sampling, correlation and descriptor checks. No paths/URLs fetched. + * The caller must select the original occurrence, not replayed provider history. + */ +export function captureAgentImages( + manager: AttachmentManager | undefined, + span: AttachmentSpan, + blocks: unknown, + options: AgentImageOptions, +): readonly AttachmentHandle[] { + if (!manager || !Array.isArray(blocks) || manager.status().state !== "queued") return []; + const handles: AttachmentHandle[] = []; + let remaining = MAX_AGENT_IMAGE_BYTES; + let spans = admitted.get(manager); + if (!spans) admitted.set(manager, spans = new WeakMap()); + let seen = spans.get(span); + if (!seen) spans.set(span, seen = new WeakMap()); + for (let partIndex = 0; partIndex < Math.min(blocks.length, 64); partIndex++) { + try { + const part = blocks[partIndex]; + if (!part || part.type !== "image" || !IMAGE_MIMES.has(part.mimeType) || seen.get(part)?.has(partIndex)) continue; + const result = manager.attach(span, { + kind: "Bytes", + get data() { + const value = part.data; + if (typeof value !== "string" || value.length > 4 * Math.ceil(remaining / 3)) + throw new Error("UNAVAILABLE_IMAGE"); + const bytes = decodeImage(value); + if (bytes.length > remaining) throw new Error("UNAVAILABLE_IMAGE"); + remaining -= bytes.length; + return bytes; + }, + }, { ...options, mimeType: part.mimeType, partIndex }); + handles.push(result); + if (result.admission === "queued") { + let positions = seen.get(part); + if (!positions) seen.set(part, positions = new Set()); + positions.add(partIndex); + } + } catch { + // Host data, including throwing accessors, must not affect inference or diagnostics. + } + } + return handles; +} + +/** Restrictions only: adapter defaults cannot undo the provider's hide/PII policy. */ +export function agentImagePolicy(policy: { + hideInputs: boolean; + hideOutputs: boolean; + hideErrorDetails: boolean; +}, error = false): AttachmentCapturePolicy { + return { + recordInputs: !policy.hideInputs, + recordOutputs: !policy.hideOutputs, + errorDetail: error, + requested: !(error && policy.hideErrorDetails), + }; +} diff --git a/typescript/packages/traceai_agent_core/src/index.ts b/typescript/packages/traceai_agent_core/src/index.ts index 1157c79fe..205a908b5 100644 --- a/typescript/packages/traceai_agent_core/src/index.ts +++ b/typescript/packages/traceai_agent_core/src/index.ts @@ -9,4 +9,6 @@ export * from "./lifecycle.js"; export * from "./delivery.js"; export * from "./tracer.js"; export * from "./capture.js"; +export * from "./attachments.js"; +export * from "./attachment-status.js"; export { VERSION } from "./version.js"; diff --git a/typescript/packages/traceai_agent_core/src/lifecycle.ts b/typescript/packages/traceai_agent_core/src/lifecycle.ts index 28643a301..ed36bb5d3 100644 --- a/typescript/packages/traceai_agent_core/src/lifecycle.ts +++ b/typescript/packages/traceai_agent_core/src/lifecycle.ts @@ -584,7 +584,10 @@ export interface DeliveryState { activatedAt?: string; binding?: string; } +import { agentAttachmentStatus } from "./attachment-status.js"; + export interface AgentStatus { + attachments: ReturnType; agent: string; pendingTurns?: number; registrationDetail?: string; @@ -682,6 +685,7 @@ export async function inspectStatus( destination: config.endpoint, projectName: config.projectName, privacy: config.captureContent ? "content-on-by-default" : "structure-only", + attachments: agentAttachmentStatus(config, options.env), provenance: config.provenance, credentials: config.credentials ? "present" : "missing", credentialCheck: check, diff --git a/typescript/packages/traceai_agent_core/src/tracer.ts b/typescript/packages/traceai_agent_core/src/tracer.ts index 5efcd12e4..dfa06de8a 100644 --- a/typescript/packages/traceai_agent_core/src/tracer.ts +++ b/typescript/packages/traceai_agent_core/src/tracer.ts @@ -28,6 +28,10 @@ import { redact } from "./privacy.js"; import { contentHash } from "./files.js"; import { VERSION } from "./version.js"; import type { DeliveryResult } from "./validation.js"; +import type { NodeAttachmentManagerOptions } from "@traceai/fi-core/attachments/node"; +import type { AttachmentManager } from "@traceai/fi-core/attachments"; +import { captureAgentImages, type AgentImageOptions } from "./attachments.js"; +import { agentAttachmentRuntime } from "./attachment-runtime.js"; export type TurnOutcome = "success" | "error" | "interrupted" | "incomplete"; export type EventTiming = @@ -94,6 +98,8 @@ export interface ChildInput { fidelity?: Fidelity; } export interface ChildHandle { + /** Original typed Pi/OMP images only, before end(); no history replay or file retrieval. */ + images?(blocks: unknown, options: AgentImageOptions): void; end(value?: EndInput): void; context?: SpanContext; } @@ -108,6 +114,7 @@ export interface TurnHandle extends ChildHandle { child(value: ChildInput): ChildHandle; } export interface TurnTracer { + readonly attachments?: Pick; startTurn(value: TurnInput): TurnHandle; flush(): Promise; shutdown(): Promise; @@ -144,6 +151,7 @@ function content( /** A private FI provider: no global tracer/context registration, host/process detectors or model gateway. */ export function createTurnTracer(options: { config: ResolvedConfig; + attachments?: NodeAttachmentManagerOptions; identityKey?: Uint8Array; onDelivery?: (result: DeliveryResult) => void; /** Revalidate a durable snapshot's privacy/routing policy immediately before each HTTP batch. */ @@ -191,6 +199,7 @@ export function createTurnTracer(options: { project_type: ProjectType.OBSERVE, }), spanProcessors: [processor], + attachments: agentAttachmentRuntime(config, options.attachments), verbose: false, idGenerator: { generateTraceId: () => nextTrace, @@ -412,6 +421,10 @@ export function createTurnTracer(options: { eventCount = 0; const turn: TurnHandle = { context: root.spanContext(), + images(blocks, options) { + if (!ended && canCollect() && config.captureContent) + captureAgentImages(provider.attachments, root, blocks, options); + }, gap(reason): void { if (!ended && canCollect() && /^[A-Z_]{1,80}$/.test(reason)) root.setAttribute(SC.FI_AGENT_GAP, reason); @@ -551,6 +564,10 @@ export function createTurnTracer(options: { children.set(child.id, state); return { context: span.spanContext(), + images(blocks, options) { + if (!state.done && !state.skipped && !ended && canCollect() && config.captureContent) + captureAgentImages(provider.attachments, span, blocks, options); + }, end(end = {}) { if (state.done) return; state.done = true; @@ -685,6 +702,7 @@ export function createTurnTracer(options: { } return { startTurn, + attachments: provider.attachments, state: processor.state, async flush() { try { @@ -695,6 +713,7 @@ export function createTurnTracer(options: { }, async discard() { closed = true; + provider.attachments.invalidate("policy_denied"); active.clear(); processor.discard(); try { diff --git a/typescript/packages/traceai_agent_core/test/attachments.test.cjs b/typescript/packages/traceai_agent_core/test/attachments.test.cjs new file mode 100644 index 000000000..96777b3a8 --- /dev/null +++ b/typescript/packages/traceai_agent_core/test/attachments.test.cjs @@ -0,0 +1,183 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const http = require('node:http'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const { AttachmentManager, AttachmentTransportError } = require('@traceai/fi-core/attachments'); +const { captureAgentImages, MAX_AGENT_IMAGE_BYTES, createTurnTracer, agentAttachmentStatus } = require('../dist/src/index.js'); +const { identity, fake } = require('../../fi-core/test/attachments-fixtures.cjs'); +const image = () => ({ type: 'image', mimeType: 'image/png', data: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVQIHWP4z8DwHwAFgAI/ScLbtAAAAABJRU5ErkJggg==' }); +const input = { direction: 'input', purpose: 'model_input', role: 'user' }; +const tool = { direction: 'output', purpose: 'tool_result', role: 'tool', toolCallId: 'call_1', capture: { modelInput: true } }; +async function fixture(t, overrides = {}, transportOverrides = {}) { + const f = fake(transportOverrides); + const manager = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', transport: f.transport, ...overrides }); + t.after(() => manager.shutdown()); + await manager.ready(); + return { ...f, manager }; +} + +test('default-OFF reads no typed blocks, decodes nothing and makes no request', async t => { + const f = await fixture(t, { enabled: false }); + const parts = [{ get type() { throw Error('must not inspect denied blocks'); } }]; + assert.deepEqual(captureAgentImages(f.manager, identity, parts, input), []); + assert.deepEqual(f.calls, []); +}); +for (const [key, options] of [ + ['hideInputs', input], ['hideInputMessages', input], ['hideInputText', input], + ['hideOutputs', tool], ['hideOutputMessages', tool], ['hideOutputText', tool], + ['hideInputImages', tool], ['hideInputs', tool], ['piiRedaction', input], + ['hideErrorDetails', { ...tool, capture: { modelInput: true, errorDetail: true } }], +]) test(`${key} denies before base64 access and upload`, async t => { + const f = await fixture(t, { traceConfig: { [key]: true } }); + let reads = 0; + const part = { type: 'image', mimeType: 'image/png', get data() { reads++; throw Error('synthetic-secret'); } }; + const result = captureAgentImages(f.manager, identity, [part], options); + assert.equal(result.length, 1); + assert.equal(result[0].admission, key === 'piiRedaction' ? 'redaction_unavailable' : 'policy_denied'); + await f.manager.flush(); + assert.equal(reads, 0); + assert.deepEqual(f.calls, ['bootstrap']); +}); +for (const capture of [{ muted: true }, { temporaryChat: true }, { captureContent: false }, { recordInputs: false }]) + test(`integration restriction ${Object.keys(capture)[0]} cannot be overridden`, async t => { + const f = await fixture(t); + const part = { ...image(), get data() { throw Error('denied source read'); } }; + assert.equal(captureAgentImages(f.manager, identity, [part], { ...input, capture })[0].admission, 'policy_denied'); + await f.manager.flush(); assert.deepEqual(f.calls, ['bootstrap']); + }); + +test('only explicit typed image bytes; arbitrary paths, URLs and nested objects are not fetched', async t => { + const f = await fixture(t); + const parts = [ + { type: 'image', mimeType: 'image/svg+xml', data: '