diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 8fbbd06c9..115bdf0bf 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -70,7 +70,7 @@ "python/frameworks/weaviate": "0.1.2", "python/frameworks/xai": "0.1.1", "python/traceai": "0.1.0", - "typescript/packages/fi-core": "1.2.2", + "typescript/packages/fi-core": "1.2.3", "typescript/packages/fi-semantic-conventions": "1.1.0", "typescript/packages/traceai_anthropic": "0.1.5", "typescript/packages/traceai_auto": "0.1.1", diff --git a/typescript/packages/fi-core/CHANGELOG.md b/typescript/packages/fi-core/CHANGELOG.md index 8256e3551..d2c127112 100644 --- a/typescript/packages/fi-core/CHANGELOG.md +++ b/typescript/packages/fi-core/CHANGELOG.md @@ -1,3 +1,7 @@ +## [1.2.3] - pending release +### Fix +- With `hideEmbeddingVectors`, recognize `object: "embedding"` as embedding context before oversized agent content is shortened. A late discriminator must not leave its vector data visible. + ## [1.2.2] - pending release ### Fix - Preserve numeric `gen_ai.usage.cache_read_tokens.audio` counts through admission, projection and export. The 1.2.0 shared privacy runtime mistook this modality count for an audio payload. Only nonnegative safe integers receive the metric role; strings, arrays, media ancestry and registered secrets retain their existing privacy protections. diff --git a/typescript/packages/fi-core/package.json b/typescript/packages/fi-core/package.json index e8fce8874..a1dcb3102 100644 --- a/typescript/packages/fi-core/package.json +++ b/typescript/packages/fi-core/package.json @@ -1,6 +1,6 @@ { "name": "@traceai/fi-core", - "version": "1.2.2", + "version": "1.2.3", "description": "OpenTelemetry tracing for AI applications: tracer setup, span helpers, and exporters for Future AGI.", "main": "./dist/src/index.js", "module": "./dist/esm/index.js", diff --git a/typescript/packages/fi-core/src/__tests__/contentPrivacyOrder.test.ts b/typescript/packages/fi-core/src/__tests__/contentPrivacyOrder.test.ts new file mode 100644 index 000000000..15840c1c6 --- /dev/null +++ b/typescript/packages/fi-core/src/__tests__/contentPrivacyOrder.test.ts @@ -0,0 +1,10 @@ +import { redactEmbeddingValues } from "../trace/trace-config/structuredRedaction"; + +test("late embedding discriminator protects data without erasing public metadata", () => { + const input = JSON.stringify({ visible: "PUBLIC", data: [0.314159265, 0.271828182], padding: "x".repeat(20000), object: "embedding" }); + const clean = redactEmbeddingValues(input); + expect(clean).not.toContain("0.314159265"); + expect(clean).toContain("PUBLIC"); + expect(JSON.parse(clean).object).toBe("embedding"); + expect(redactEmbeddingValues(JSON.stringify({ object: "table", data: [1, 2] }))).toBe('{"object":"table","data":[1,2]}'); +}); diff --git a/typescript/packages/fi-core/src/trace/trace-config/structuredRedaction.ts b/typescript/packages/fi-core/src/trace/trace-config/structuredRedaction.ts index 6c258258a..0062dca4c 100644 --- a/typescript/packages/fi-core/src/trace/trace-config/structuredRedaction.ts +++ b/typescript/packages/fi-core/src/trace/trace-config/structuredRedaction.ts @@ -25,6 +25,7 @@ export function redactEmbeddingValues(value: string): string { return Object.fromEntries(Object.entries(item).map(([key, child]) => [ key, visit(child, depth + 1, /^(?:embedding|embeddings|vector|vectors)$/i.test(key) || + key === "data" && (item as Record).object === "embedding" || vector && !/^(?:text|input|input_text|model|index|dimensions?|count|type|mime_type)$/i.test(key)), ])); } diff --git a/typescript/packages/traceai_agent_core/src/content.ts b/typescript/packages/traceai_agent_core/src/content.ts new file mode 100644 index 000000000..687b40620 --- /dev/null +++ b/typescript/packages/traceai_agent_core/src/content.ts @@ -0,0 +1,148 @@ +import { indexPrivacyHints, normalizePrivacyName } from "@traceai/fi-core/privacy"; +import { generateTraceConfig, privacyFilter, seedSecrets, getRegisteredSecrets, type TraceConfigOptions } from "@traceai/fi-core"; +import { redact } from "./privacy.js"; +import { LIMITS } from "./schema.js"; + +/** Private, bounded originals for spans that have not finalized yet. No original + * is attached to an OTel span: finalization runs complete-value privacy with the + * current registry, then truncates, then goes through FITracer admission/export. + * A tracer-wide bound prevents many concurrently open turns retaining unbounded + * source content. Consumed/discarded entries release their original immediately. */ +export function createContentBuffer(credentials: readonly string[] = [], maxBytes = 8 * LIMITS.ingress): { + capture(value: string | undefined): number | undefined; + take(id: number | undefined): string | undefined; + clear(): void; +} { + const pending = new Map(); + let next = 0, size = 0; + return { + capture(value: string | undefined): number | undefined { + if (typeof value !== "string") return undefined; + const length = value.length > LIMITS.ingress ? Infinity : Buffer.byteLength(value); + if (length > LIMITS.ingress || size + length > maxBytes) return -1; + // Keep capture-time discovery too: an environment credential may be removed + // before finalization, but must remain protected by the shared registry. + try { seedSecrets(process.env, credentials); } catch { return -2; } + const id = next++; + pending.set(id, value); + size += length; + return id; + }, + take(id: number | undefined): string | undefined { + if (id === -1) return "[REDACTED_SIZE_LIMIT]"; + if (id === -2) return "[REDACTED_PRIVACY_FAILURE]"; + if (id === undefined) return undefined; + const value = pending.get(id); + if (value !== undefined) { + size -= Buffer.byteLength(value); + pending.delete(id); + } + return value; + }, + clear(): void { pending.clear(); size = 0; }, + }; +} + +const TRUNCATED = "[TRUNCATED]"; +const bytes = (text: string): number => Buffer.byteLength(text); +// Keep fi-core recognition at the tracer boundary, not in lightweight hook code. +const contextKey = (key: string): boolean => ["type", "object"].includes(normalizePrivacyName(key)) || + indexPrivacyHints([[key, undefined]]).size > 0; + +/** Resolve policy once, just like FITracer. Never shorten a value before the + * complete credential, media, PII and embedding inspection has succeeded. + * This is preparation, not an alternative to FITracer admission/export. */ +export function createContentSanitizer( + credentials: readonly string[] = [], + options: TraceConfigOptions = {}, +): (text: string, limit?: number) => string { + const keys = [...credentials], policy = generateTraceConfig(options); + const privacy = privacyFilter(policy, keys); + return (text: string, limit = 20_000): string => { + if (text.length > LIMITS.ingress || bytes(text) > LIMITS.ingress) + return "[REDACTED_SIZE_LIMIT]"; + try { + seedSecrets(process.env, keys); + const key = "fi.agent.content"; + const secrets = [...new Set([...keys, ...getRegisteredSecrets()])].sort((a, b) => b.length - a.length); + text = redact(text, secrets, contextKey); + // Exact-value matching just consumed the complete registry snapshot above. + // Apply the remaining FI policies without repeatedly charging every absent + // credential against every quoted prose token. The actual FITracer still + // performs its normal admission/discovery/export with the full registry. + const clean = privacy.attributes({ [key]: text }, [])[key]; + if (typeof clean !== "string") return "[REDACTED]"; + return truncateContent(clean, limit); + } catch { + // Budget/discovery errors may never fall back to raw or partially read data. + return "[REDACTED_PRIVACY_FAILURE]"; + } + }; +} + +/** Internal: the input MUST already have passed complete-value privacy. + * Retain valid JSON at every encoded layer for downstream FI privacy checks. */ +export function truncateContent(text: string, limit = 20_000): string { + if (!Number.isSafeInteger(limit) || limit < 64) + throw new RangeError("Content limit must be at least 64 bytes"); + if (bytes(text) <= limit) return text; + let parsed: unknown; + try { parsed = JSON.parse(text); } catch { + const buffer = Buffer.from(text); + let end = limit; + while ((buffer[end] & 0xc0) === 0x80) end--; + return buffer.subarray(0, end).toString("utf8"); + } + return shorten(parsed, limit) ?? JSON.stringify(TRUNCATED); +} + +// Cost after escaping inside each enclosing JSON string, excluding its quotes. +function cost(text: string, layers: number): number { + for (let i = 0; i < layers; i++) text = JSON.stringify(text).slice(1, -1); + return bytes(text); +} +function shorten(value: unknown, budget: number, layers = 0, depth = 0): string | undefined { + const full = JSON.stringify(value); + if (cost(full, layers) <= budget) return full; + const marker = JSON.stringify(TRUNCATED); + if (depth > 16) return cost(marker, layers) <= budget ? marker : undefined; + if (typeof value === "string") { + let parsed: unknown; + try { parsed = JSON.parse(value); } catch { + let remaining = budget - cost(marker, layers); + if (remaining < 0) return undefined; + let prefix = ""; + for (const char of value) { + const size = cost(JSON.stringify(char).slice(1, -1), layers); + if (size > remaining) break; + prefix += char; + remaining -= size; + } + return JSON.stringify(prefix + TRUNCATED); + } + const inner = shorten(parsed, budget - cost('""', layers), layers + 1, depth + 1); + return inner === undefined ? undefined : JSON.stringify(inner); + } + if (value === null || typeof value !== "object") return undefined; + const array = Array.isArray(value); + let markerKey = "_truncated"; + while (Object.hasOwn(value, markerKey)) markerKey += "_"; + const tail = array ? '{"_truncated":true}' : `${JSON.stringify(markerKey)}:true`; + let remaining = budget - cost((array ? "[]" : "{}") + tail, layers); + if (remaining < 0) return undefined; + const parts: string[] = []; + for (const [key, child] of Object.entries(value)) { + const prefix = array ? "" : `${JSON.stringify(key)}:`; + const overhead = cost(prefix + ",", layers); + if (overhead > remaining) continue; + const encoded = shorten(child, remaining - overhead, layers, depth + 1); + if (encoded === undefined) { + if (array) break; // Never shift array indices by skipping an item. + continue; + } + parts.push(prefix + encoded); + remaining -= overhead + cost(encoded, layers); + } + parts.push(tail); + return (array ? "[" : "{") + parts.join(",") + (array ? "]" : "}"); +} diff --git a/typescript/packages/traceai_agent_core/src/index.ts b/typescript/packages/traceai_agent_core/src/index.ts index 1157c79fe..2beb6caab 100644 --- a/typescript/packages/traceai_agent_core/src/index.ts +++ b/typescript/packages/traceai_agent_core/src/index.ts @@ -10,3 +10,4 @@ export * from "./delivery.js"; export * from "./tracer.js"; export * from "./capture.js"; export { VERSION } from "./version.js"; +export { createContentSanitizer } from "./content.js"; diff --git a/typescript/packages/traceai_agent_core/src/privacy.ts b/typescript/packages/traceai_agent_core/src/privacy.ts index c8de0fb8e..ee0c42aaf 100644 --- a/typescript/packages/traceai_agent_core/src/privacy.ts +++ b/typescript/packages/traceai_agent_core/src/privacy.ts @@ -28,23 +28,73 @@ export const isSensitiveKey = isSensitiveField; export const redactValue = (value: unknown, depth = 0): unknown => redactStructured(value, depth); /** Defense in depth for content-on-by-default. Structure-only omits content entirely. */ -export function redact(text: string): string { - return redactStructured(text) as string; +export function redact(text: string, secrets: readonly string[] = [], contextKey: (key: string) => boolean = () => false): string { + return redactStructured(text, 0, secrets, contextKey) as string; } +/** Callers supply longest-first credentials. Keep inserted markers out of later + * matches. This runs on complete text and again on decoded JSON leaves, before + * assignments/PII can change a credential that overlaps their syntax. */ +function exactSecrets(text: string, secrets: readonly string[]): string { + for (const secret of secrets) { + if (!secret || !text.includes(secret)) continue; + text = text.split("[REDACTED]").map(part => part.split(secret).join("[REDACTED]")).join("[REDACTED]"); + } + return text; +} +/** A match inside a JSON string is not framing, even if it contains brackets. + * Inspect a successfully parsed container at its original offsets, before any + * embedded rewrite can split a credential. Matches overlapping delimiters (or + * entering/leaving an embedded container) own that whole text instead. */ +function crossesJsonFraming(text: string, start: number, end: number, secrets: readonly string[]): boolean { + if (!secrets.some(secret => secret && text.includes(secret))) return false; + const strings: [number, number][] = []; + let open = -1, escaped = false; + for (let i = start; i < end; i++) { + if (open < 0) { if (text[i] === '"') open = i + 1; } + else if (escaped) escaped = false; + else if (text[i] === "\\") escaped = true; + else if (text[i] === '"') { strings.push([open, i]); open = -1; } + } + for (const secret of secrets) { + if (!secret) continue; + let at = text.indexOf(secret, Math.max(0, start - secret.length + 1)); + while (at >= 0 && at < end) { + // Binary search the string token starting at or before this occurrence. + let lo = 0, hi = strings.length; + while (lo < hi) { + const mid = (lo + hi) >>> 1; + if (strings[mid][0] <= at) lo = mid + 1; else hi = mid; + } + const token = strings[lo - 1]; + if (!token || at + secret.length > token[1]) return true; + at = text.indexOf(secret, at + 1); + } + } + return false; +} + /** Check normalized keys before serialization, including nested JSON-encoded strings. */ -export function redactStructured(value: unknown, depth = 0): unknown { +export function redactStructured(value: unknown, depth = 0, secrets: readonly string[] = [], contextKey: (key: string) => boolean = () => false): unknown { if (depth > 16) return "[REDACTED_DEPTH_LIMIT]"; if (Array.isArray(value)) - return value.map((item) => redactStructured(item, depth + 1)); - if (value !== null && typeof value === "object") + return value.map((item) => redactStructured(item, depth + 1, secrets, contextKey)); + if (value !== null && typeof value === "object") { + // A credential may itself be a key or media/embedding discriminator. Removing + // it must not erase the context that protects its siblings in later FI passes. + if (secrets.length && Object.entries(value).some(([key, item]) => + exactSecrets(key, secrets) !== key || + contextKey(key) && + (Array.isArray(item) ? item : [item]).some(v => typeof v === "string" && exactSecrets(v, secrets) !== v), + )) return "[REDACTED]"; return Object.fromEntries( Object.entries(value).map(([key, item]) => [ - key, + exactSecrets(key, secrets), isSensitiveField(key) ? "[REDACTED]" - : redactStructured(item, depth + 1), + : redactStructured(item, depth + 1, secrets, contextKey), ]), ); + } if (typeof value === "string") { // Inspect complete strings before any display truncation. Refuse oversized // content rather than treating a truncated JSON prefix as ordinary text. @@ -54,13 +104,12 @@ export function redactStructured(value: unknown, depth = 0): unknown { ) return "[REDACTED_SIZE_LIMIT]"; if (/^\s*[\[{"]/.test(value)) { - try { - return JSON.stringify(redactStructured(JSON.parse(value), depth + 1)); - } catch { - /* Opaque text still receives the free-text policy. */ - } + let parsed: unknown; + try { parsed = JSON.parse(value); } catch { return redactText(value, depth, secrets, contextKey); } + if (crossesJsonFraming(value, 0, value.length, secrets)) return "[REDACTED]"; + return JSON.stringify(redactStructured(parsed, depth + 1, secrets, contextKey)); } - return redactText(value, depth); + return redactText(value, depth, secrets, contextKey); } return value; } @@ -163,7 +212,7 @@ function redactAssignments(text: string): string { } return result + text.slice(offset); } -function redactEmbeddedJson(text: string, depth: number): string { +function redactEmbeddedJson(text: string, depth: number, secrets: readonly string[], contextKey: (key: string) => boolean): string { if (text.length > LIMITS.ingress || Buffer.byteLength(text) > LIMITS.ingress) return "[REDACTED_SIZE_LIMIT]"; let start = -1, @@ -200,7 +249,17 @@ function redactEmbeddedJson(text: string, depth: number): string { if (!stack.length) { try { const parsed = JSON.parse(text.slice(start, i + 1)); - const clean = redactStructured(parsed, depth + 1); + // A complete JSON-looking island wholly inside a prose credential is + // part of that credential, not surrounding framing. Leave it intact + // until exactSecrets removes the complete match below. + const covered = secrets.some(secret => { + if (!secret) return false; + const at = text.lastIndexOf(secret, start); + return at >= 0 && at + secret.length >= i + 1; + }); + if (covered) { start = -1; continue; } + if (crossesJsonFraming(text, start, i + 1, secrets)) return "[REDACTED]"; + const clean = redactStructured(parsed, depth + 1, secrets, contextKey); const sanitized = JSON.stringify(clean); // Do not rewrite harmless JSON-looking code or change prompt correlation. result += text.slice(offset, start) + (sanitized === JSON.stringify(parsed) @@ -215,8 +274,8 @@ function redactEmbeddedJson(text: string, depth: number): string { } return result + text.slice(offset); } -function redactText(text: string, depth = 0): string { - return redactAssignments(redactEmbeddedJson(text, depth)) +function redactText(text: string, depth: number, secrets: readonly string[], contextKey: (key: string) => boolean): string { + return redactAssignments(exactSecrets(redactEmbeddedJson(text, depth, secrets, contextKey), secrets)) // URI userinfo is credential-bearing for every scheme, including local databases. .replace(/\b[a-z][a-z0-9+.-]*:\/\/[^\s/"'<>]*@/gi, (uri) => uri.slice(0, uri.indexOf("://") + 3) + "[REDACTED]@") .replace(/\b(?:bearer|basic)\s+[^\s,;"']+/gi, "[REDACTED]") diff --git a/typescript/packages/traceai_agent_core/src/tracer.ts b/typescript/packages/traceai_agent_core/src/tracer.ts index 5efcd12e4..8713d5832 100644 --- a/typescript/packages/traceai_agent_core/src/tracer.ts +++ b/typescript/packages/traceai_agent_core/src/tracer.ts @@ -7,7 +7,7 @@ import { type SpanContext, type TimeInput, } from "@opentelemetry/api"; -import { FITracer, FITracerProvider, ProjectType } from "@traceai/fi-core"; +import { FITracer, FITracerProvider, ProjectType, generateTraceConfig } from "@traceai/fi-core"; import { resourceFromAttributes } from "@opentelemetry/resources"; import { SemanticConventions as SC } from "@traceai/fi-semantic-conventions"; import { @@ -24,7 +24,7 @@ import { opaqueId, } from "./identity.js"; import { normalizeUsage, type Counters, type UsagePlan } from "./usage.js"; -import { redact } from "./privacy.js"; +import { createContentBuffer, createContentSanitizer } from "./content.js"; import { contentHash } from "./files.js"; import { VERSION } from "./version.js"; import type { DeliveryResult } from "./validation.js"; @@ -128,19 +128,6 @@ const label = (value: string | undefined): string | undefined => value && /^[a-zA-Z0-9_.:/-]{1,128}$/.test(value) && !value.includes("://") ? value : undefined; -function content( - value: string | undefined, - credentials?: ResolvedConfig["credentials"], -): string | undefined { - if (typeof value !== "string") return undefined; - // redact() inspects complete structured content within its processing bound. - // Display truncation must never remove JSON structure before that inspection. - let clean = redact(value); - for (const key of [credentials?.apiKey, credentials?.secretKey]) - if (key) clean = clean.split(key).join("[REDACTED]"); - return clean.slice(0, 20000); -} - /** A private FI provider: no global tracer/context registration, host/process detectors or model gateway. */ export function createTurnTracer(options: { config: ResolvedConfig; @@ -160,6 +147,35 @@ export function createTurnTracer(options: { : undefined, identityKey: options.config.identityKey, }; + const traceConfig = generateTraceConfig({ + hideInputs: config.hideInputs, + hideOutputs: config.hideOutputs, + hideErrorDetails: config.hideErrorDetails, + }); + const sanitize = createContentSanitizer( + config.credentials ? [config.credentials.apiKey, config.credentials.secretKey] : [], + traceConfig, + ); + const content = (value: string | undefined): string | undefined => + typeof value === "string" ? sanitize(value) : undefined; + const contentBuffer = createContentBuffer( + config.credentials ? [config.credentials.apiKey, config.credentials.secretKey] : [], + ); + const pendingContent = new WeakMap(); + function drainContent(span: Span, emit = false): void { + const pending = pendingContent.get(span); + pendingContent.delete(span); + const input = contentBuffer.take(pending?.input); + if (emit && input !== undefined) span.setAttribute(SC.INPUT_VALUE, sanitize(input)); + for (const event of pending?.events ?? []) { + const text = contentBuffer.take(event.input); + if (emit) span.addEvent(event.name, + text === undefined ? {} : { [SC.INPUT_VALUE]: sanitize(text) }, event.time); + } + } const processor = new BoundedSpanProcessor( config, options.onDelivery, @@ -199,11 +215,7 @@ export function createTurnTracer(options: { }); const tracer = new FITracer({ tracer: provider.getTracer("@traceai/agent-core", VERSION), - traceConfig: { - hideInputs: config.hideInputs, - hideOutputs: config.hideOutputs, - hideErrorDetails: config.hideErrorDetails, - }, + traceConfig, }); const active = new Map(); const completed = new Set(); @@ -296,10 +308,11 @@ export function createTurnTracer(options: { additive: boolean, measuredTiming = false, ): void { + drainContent(span, canCollect()); const outcome = endOutcome(value); const out = !config.hideOutputs && !(config.hideErrorDetails && outcome === "error") - ? content(value.output, config.credentials) + ? content(value.output) : undefined; if (out !== undefined && canCollect()) span.setAttribute(SC.OUTPUT_VALUE, out); @@ -380,10 +393,8 @@ export function createTurnTracer(options: { attributes[SC.FI_AGENT_TAGS] = JSON.stringify(config.tags); if (config.environment) attributes["deployment.environment.name"] = config.environment; - const input = !config.hideInputs - ? content(value.input, config.credentials) - : undefined; - if (input !== undefined) attributes[SC.INPUT_VALUE] = input; + const input = !config.hideInputs ? contentBuffer.capture(value.input) : undefined; + value.input = undefined; const root = tracer.startSpan( value.unit === "run" ? `${config.agent}.run` : config.agent, { @@ -395,6 +406,8 @@ export function createTurnTracer(options: { }, ROOT_CONTEXT, ); + const events: { name: string; input?: number; time: TimeInput }[] = []; + pendingContent.set(root, { input, events }); const checkpoint = new Set(value.checkpoint ?? []); const children = new Map< string, @@ -406,6 +419,7 @@ export function createTurnTracer(options: { skipped: boolean; outcome?: TurnOutcome; pending?: EndInput; + output?: number; } >(); let ended = false, @@ -428,14 +442,12 @@ export function createTurnTracer(options: { ) return; try { - const input = !config.hideInputs - ? content(event.input, config.credentials) - : undefined; - root.addEvent( - event.name, - input === undefined ? {} : { [SC.INPUT_VALUE]: input }, - event.time, - ); + events.push({ + name: event.name, + input: !config.hideInputs ? contentBuffer.capture(event.input) : undefined, + time: Array.isArray(event.time) ? [...event.time] as [number, number] + : event.time instanceof Date ? new Date(event.time) : event.time ?? Date.now(), + }); eventCount++; } catch { /* fail open */ @@ -513,15 +525,15 @@ export function createTurnTracer(options: { if (label(child.toolCallId)) attrs[SC.TOOL_CALL_ID] = child.toolCallId!; } - const text = !config.hideInputs - ? content(child.input, config.credentials) - : undefined; - if (text !== undefined) attrs[SC.INPUT_VALUE] = text; + const skipped = checkpoint.has(contentHash(child.id)); + const input = !config.hideInputs && !skipped ? contentBuffer.capture(child.input) : undefined; + child.input = undefined; const span = tracer.startSpan( label(child.name) ?? child.kind, { attributes: attrs, startTime: child.startTime }, trace.setSpan(ROOT_CONTEXT, parent), ); + pendingContent.set(span, { input }); if (isModel) span.setAttribute( SC.FI_USAGE_SCOPE, @@ -532,6 +544,7 @@ export function createTurnTracer(options: { done: boolean; isModel: boolean; pending?: EndInput; + output?: number; measuredTiming: boolean; skipped: boolean; outcome?: TurnOutcome; @@ -539,7 +552,7 @@ export function createTurnTracer(options: { span, done: false, isModel, - skipped: checkpoint.has(contentHash(child.id)), + skipped, measuredTiming: isModel && child.fidelity === "native-request" && @@ -564,14 +577,13 @@ export function createTurnTracer(options: { span.setAttribute(SC.LLM_PROVIDER, end.provider!); } if (isModel && value.usagePlan === "turn") { + state.output = canCollect() && !config.hideOutputs && + !(config.hideErrorDetails && state.outcome === "error") + ? contentBuffer.capture(end.output) : undefined; state.pending = { ...end, outcome: state.outcome, - output: - config.hideOutputs || - (config.hideErrorDetails && state.outcome === "error") - ? undefined - : content(end.output, config.credentials), + output: undefined, usage: end.usage ? { ...end.usage } : undefined, endTime: end.endTime ?? Date.now(), }; @@ -638,7 +650,7 @@ export function createTurnTracer(options: { item.span.setAttribute(SC.FI_AGENT_USAGE_ADDITIVE, ownsUsage); finish( item.span, - { ...modelEnd, ...(ownsUsage ? { usage: end.usage } : {}) }, + { ...modelEnd, output: contentBuffer.take(item.output), ...(ownsUsage ? { usage: end.usage } : {}) }, ownsUsage, item.measuredTiming, ); @@ -671,6 +683,13 @@ export function createTurnTracer(options: { } catch { /* fail open */ } + // Deferred/checkpointed roots and children must also release originals. + drainContent(root); + for (const item of children.values()) { + drainContent(item.span); + contentBuffer.take(item.output); + } + events.length = 0; active.delete(turnWork); completed.add(turnWork); if (completed.size > 4096) @@ -696,6 +715,7 @@ export function createTurnTracer(options: { async discard() { closed = true; active.clear(); + contentBuffer.clear(); processor.discard(); try { await provider.shutdown(); @@ -705,6 +725,7 @@ export function createTurnTracer(options: { }, async shutdown() { for (const turn of active.values()) turn.end({ outcome: "incomplete" }); + contentBuffer.clear(); closed = true; try { await provider.shutdown(); diff --git a/typescript/packages/traceai_agent_core/test/conformance/host-fixture.cjs b/typescript/packages/traceai_agent_core/test/conformance/host-fixture.cjs index c1fcba349..a4c327c16 100644 --- a/typescript/packages/traceai_agent_core/test/conformance/host-fixture.cjs +++ b/typescript/packages/traceai_agent_core/test/conformance/host-fixture.cjs @@ -37,6 +37,7 @@ async function prepareHost(f) { userDir: path.dirname(f.config.paths.user), codexEvents: require(packages + "/traceai_codex_plugin/dist/events.js") .EVENTS, + codexVersion: require(packages + "/traceai_codex_plugin/package.json").version, hermesSource: path.resolve( packages, "../../python/frameworks/hermes-agent/hermes-plugin/futureagi", diff --git a/typescript/packages/traceai_agent_core/test/conformance/native-host.cjs b/typescript/packages/traceai_agent_core/test/conformance/native-host.cjs index 678e80e65..4fda50ff9 100644 --- a/typescript/packages/traceai_agent_core/test/conformance/native-host.cjs +++ b/typescript/packages/traceai_agent_core/test/conformance/native-host.cjs @@ -135,7 +135,7 @@ else if (agent === "copilot" || agent === "claude" || agent === "goose") { ? [ { pluginId: "futureagi-tracing@futureagi-local", - version: "0.1.0", + version: spec.codexVersion, enabled: true, }, ] diff --git a/typescript/packages/traceai_agent_core/test/content-finalization.test.cjs b/typescript/packages/traceai_agent_core/test/content-finalization.test.cjs new file mode 100644 index 000000000..e50401e23 --- /dev/null +++ b/typescript/packages/traceai_agent_core/test/content-finalization.test.cjs @@ -0,0 +1,126 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const http = require('node:http'); +const core = require('../dist/src/index.js'); +const { registerSecret } = require('@traceai/fi-core'); + +async function collector(t) { + const home = await fs.mkdtemp(path.join(os.tmpdir(), 'fi-content-final-')); + const bodies = []; + const server = http.createServer((req, res) => { + const chunks = []; + req.on('data', chunk => chunks.push(chunk)); + req.on('end', () => { + bodies.push(JSON.parse(Buffer.concat(chunks).toString('utf8'))); + res.setHeader('content-type', 'application/json'); res.end('{}'); + }); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const config = await core.resolveConfig({ agent: 'pi', home, env: { + FI_AGENT_ENABLED: 'true', FI_API_KEY: 'synthetic-final-api', FI_SECRET_KEY: 'synthetic-final-secret', + FI_OTLP_ENDPOINT: `http://127.0.0.1:${server.address().port}/v1/traces`, + } }); + const tracer = core.createTurnTracer({ config }); + t.after(async () => { + await tracer.shutdown(); server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + await fs.rm(home, { recursive: true, force: true }); + }); + return { tracer, async read() { + await tracer.flush(); + const spans = bodies.splice(0).flatMap(b => b.resourceSpans.flatMap(r => r.scopeSpans.flatMap(s => s.spans))); + const content = spans.flatMap(s => [...s.attributes, ...(s.events ?? []).flatMap(e => e.attributes)]) + .filter(a => a.key === 'input.value' || a.key === 'output.value').map(a => a.value.stringValue); + assert.equal(spans.length, 2); + assert.equal(content.length, 5); + return content; + } }; +} + +for (const shape of ['json', 'encoded', 'prose', 'short']) { + test(`late registration cannot expose a credential prefix on any OTLP surface: ${shape}`, async t => { + const { tracer, read } = await collector(t); + const prefix = `LATE_BOUNDARY_PRIVATE_${shape}_`; + const secret = prefix + 'q'.repeat(140); + let input = 'PUBLIC_RESULT ' + (shape === 'short' ? '' : 'p '.repeat(9940)) + secret + ' tail'; + if (shape === 'json' || shape === 'short') input = JSON.stringify({ stdout: input }); + if (shape === 'encoded') input = JSON.stringify({ encoded: JSON.stringify({ stdout: input }) }); + const turn = tracer.startTurn({ sessionId: 'late', turnId: shape, input }); + turn.event({ name: 'input', input }); + const tool = turn.child({ id: 'tool', kind: 'tool', name: 'read', input }); + registerSecret(secret); + tool.end({ output: input }); turn.end({ output: input }); + for (const value of await read()) { + assert.ok(!value.includes(prefix), 'registered credential prefix escaped'); + assert.match(value, /PUBLIC_RESULT/); + assert.ok(Buffer.byteLength(value) <= 20000); + } + }); +} + +test('deferred model output is re-scrubbed before turn-owned span finalization', async t => { + const { tracer, read } = await collector(t); + const prefix = 'LATE_DEFERRED_PRIVATE_'; + const secret = prefix + 'q'.repeat(140); + const input = JSON.stringify({ stdout: 'PUBLIC_RESULT ' + 'p '.repeat(9940) + secret + ' tail' }); + const turn = tracer.startTurn({ sessionId: 'late', turnId: 'deferred', input, usagePlan: 'turn' }); + turn.event({ name: 'input', input }); + turn.child({ id: 'model', kind: 'llm', fidelity: 'native-request', input }).end({ output: input }); + registerSecret(secret); + turn.end({ output: input }); + for (const value of await read()) { + assert.ok(!value.includes(prefix), 'deferred credential prefix escaped'); + assert.match(value, /PUBLIC_RESULT/); + } +}); + +test('capture-time environment credentials stay protected after removal before end', async t => { + const { tracer, read } = await collector(t); + const prefix = 'CAPTURE_ENV_PRIVATE_'; + const secret = prefix + 'q'.repeat(140); + const old = process.env.TEST_PRIVATE_KEY; + t.after(() => { if (old === undefined) delete process.env.TEST_PRIVATE_KEY; else process.env.TEST_PRIVATE_KEY = old; }); + process.env.TEST_PRIVATE_KEY = secret; + const input = JSON.stringify({ stdout: 'PUBLIC_RESULT ' + 'p '.repeat(9940) + secret + ' tail' }); + const turn = tracer.startTurn({ sessionId: 'late', turnId: 'environment', input }); + turn.event({ name: 'input', input }); + const tool = turn.child({ id: 'tool', kind: 'tool', input }); + delete process.env.TEST_PRIVATE_KEY; + tool.end({ output: input }); turn.end({ output: input }); + for (const value of await read()) { + assert.ok(!value.includes(prefix)); + assert.match(value, /PUBLIC_RESULT/); + } +}); + +for (const punctuation of ['[PASSWORD]', '{PASSWORD}', '"PASSWORD"', '[1,2]', '{"x":"y"}']) { + for (const shape of ['sensitive-field', 'ordinary-field', 'encoded', 'prose']) { + for (const large of [false, true]) { + test(`punctuation credential retains unrelated public OTLP content: ${punctuation} ${shape} large=${large}`, async t => { + const secret = `SYNTHETIC_NORMAL_${punctuation}_END`; + const old = process.env.DATABASE_PASSWORD; + process.env.DATABASE_PASSWORD = secret; + t.after(() => { if (old === undefined) delete process.env.DATABASE_PASSWORD; else process.env.DATABASE_PASSWORD = old; }); + const { tracer, read } = await collector(t); + const pub = 'PUBLIC_EXPECTED_TOOL_RESULT'; + const stdout = large ? (pub + ' file contents\n').repeat(800) : pub; + let input = JSON.stringify({ stdout, [shape === 'sensitive-field' ? 'password' : 'echo']: secret, exitCode: 0 }); + if (shape === 'encoded') input = JSON.stringify({ result: input }); + if (shape === 'prose') input = stdout + ' with credential ' + secret + ' then public suffix'; + const turn = tracer.startTurn({ sessionId: 'punctuation', turnId: 'one', input }); + turn.event({ name: 'input', input }); + turn.child({ id: 'tool', kind: 'tool', name: 'read', input }).end({ output: input }); + turn.end({ output: input }); + for (const value of await read()) { + assert.match(value, /PUBLIC_EXPECTED_TOOL_RESULT/); + assert.doesNotMatch(value, /SYNTHETIC_NORMAL_|PASSWORD/); + assert.ok(Buffer.byteLength(value) <= 20000); + if (shape !== 'prose') assert.doesNotThrow(() => JSON.parse(value)); + } + }); + } + } +} diff --git a/typescript/packages/traceai_agent_core/test/content-wire.test.cjs b/typescript/packages/traceai_agent_core/test/content-wire.test.cjs new file mode 100644 index 000000000..2d04bd026 --- /dev/null +++ b/typescript/packages/traceai_agent_core/test/content-wire.test.cjs @@ -0,0 +1,52 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const http = require('node:http'); +const core = require('../dist/src/index.js'); + +test('all five OTLP content surfaces retain readable JSON within 20,000 UTF-8 bytes', async t => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), 'fi-content-wire-')); + const bodies = []; + const server = http.createServer((req, res) => { + // Buffer chunks, not string concatenation: a TCP chunk may bisect a UTF-8 code point. + const chunks = []; + req.on('data', chunk => chunks.push(chunk)); + req.on('end', () => { + bodies.push(JSON.parse(Buffer.concat(chunks).toString('utf8'))); + res.setHeader('content-type', 'application/json'); res.end('{}'); + }); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const config = await core.resolveConfig({ agent: 'pi', home, env: { + FI_AGENT_ENABLED: 'true', FI_API_KEY: 'synthetic-content-api', FI_SECRET_KEY: 'synthetic-content-secret', + FI_OTLP_ENDPOINT: `http://127.0.0.1:${server.address().port}/v1/traces`, + } }); + const tracer = core.createTurnTracer({ config }); + t.after(async () => { + await tracer.shutdown(); server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + await fs.rm(home, { recursive: true, force: true }); + }); + for (const line of ['PUBLIC_RESULT ordinary stdout\n', 'PUBLIC_RESULT 漢字測試 😀\n']) { + const text = JSON.stringify({ stdout: line.repeat(1100), exitCode: 0 }); + const at = bodies.length; + const turn = tracer.startTurn({ sessionId: 'readable', turnId: line, input: text }); + turn.event({ name: 'input', input: text }); + turn.child({ id: 'tool', kind: 'tool', name: 'read', input: text }).end({ output: text }); + turn.end({ output: text }); + await tracer.flush(); + const spans = bodies.slice(at).flatMap(b => b.resourceSpans.flatMap(r => r.scopeSpans.flatMap(s => s.spans))); + const content = spans.flatMap(s => [...s.attributes, ...(s.events ?? []).flatMap(e => e.attributes)]) + .filter(a => a.key === 'input.value' || a.key === 'output.value').map(a => a.value.stringValue); + assert.equal(spans.length, 2); assert.equal(content.length, 5); + for (const value of content) { + assert.match(value, /PUBLIC_RESULT/); + assert.doesNotMatch(value, /�|/); + assert.doesNotThrow(() => JSON.parse(value)); + assert.ok(Buffer.byteLength(value) <= 20000); + assert.ok(Buffer.byteLength(value) > 19000); + } + } +}); diff --git a/typescript/packages/traceai_agent_core/test/content.test.cjs b/typescript/packages/traceai_agent_core/test/content.test.cjs new file mode 100644 index 000000000..3e8648e89 --- /dev/null +++ b/typescript/packages/traceai_agent_core/test/content.test.cjs @@ -0,0 +1,127 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { createContentBuffer, createContentSanitizer } = require('../dist/src/content.js'); +const { registerSecret } = require('@traceai/fi-core'); + +for (const shape of ['text', 'json', 'encoded', 'double']) { + for (const source of ['configured', 'registered', 'environment']) { + test(`complete ${source} secret before PII and UTF-8 cut: ${shape}`, () => { + const secret = `SYNTHETIC_OVERLAP-${source}-${shape}-123-45-6789-PRIVATE_SUFFIX`; + const old = process.env.TEST_PRIVATE_KEY; + if (source === 'environment') process.env.TEST_PRIVATE_KEY = secret; + if (source === 'registered') registerSecret(secret); + try { + const sanitize = createContentSanitizer(source === 'configured' ? [secret] : [], { piiRedaction: true }); + let value = 'PUBLIC ' + '😀'.repeat(4980) + secret + ' trailing public'; + if (shape === 'json') value = JSON.stringify({ stdout: value }); + if (shape === 'encoded') value = JSON.stringify({ encoded: JSON.stringify({ stdout: value }) }); + if (shape === 'double') value = JSON.stringify(JSON.stringify({ stdout: value })); + const clean = sanitize(value); + assert.ok(Buffer.byteLength(clean) <= 20000); + assert.match(clean, /PUBLIC/); + assert.doesNotMatch(clean, /SYNTHETIC_OVERLAP|PRIVATE_SUFFIX|123-45-6|�/); + if (shape !== 'text') assert.doesNotThrow(() => JSON.parse(clean)); + } finally { + if (old === undefined) delete process.env.TEST_PRIVATE_KEY; + else process.env.TEST_PRIVATE_KEY = old; + } + }); + } +} + +for (const text of ['PUBLIC_TOOL_RESULT file contents\n', 'PUBLIC_TOOL_RESULT 漢字測試 😀\n', 'PUBLIC_TOOL_RESULT "quote" \\slash\n']) { + for (let layers = 0; layers <= 3; layers++) { + test(`readable byte-bounded JSON at ${layers} encoded layers: ${text.slice(-10)}`, () => { + let value = JSON.stringify({ stdout: text.repeat(1100), exitCode: 0 }); + for (let i = 0; i < layers; i++) value = JSON.stringify(value); + const clean = createContentSanitizer()(value); + assert.ok(Buffer.byteLength(clean) <= 20000); + assert.ok(Buffer.byteLength(clean) > 19000); + assert.match(clean, /PUBLIC_TOOL_RESULT/); + let decoded = clean; + for (let i = 0; i <= layers; i++) decoded = JSON.parse(decoded); + assert.match(decoded.stdout, /PUBLIC_TOOL_RESULT/); + }); + } +} + +test('65 and 100 item public arrays are not replaced with a size marker', () => { + for (const length of [65, 100]) { + const value = JSON.stringify({ purpose: 'PUBLIC', files: Array.from({ length }, (_, i) => `src/public-${i}.ts`) }); + assert.equal(createContentSanitizer()(value), value); + } +}); + +test('embedding context is inspected before its late discriminator is cut', () => { + const value = JSON.stringify({ visible: 'PUBLIC', data: [0.314159265, 0.271828182], padding: 'x'.repeat(22000), object: 'embedding' }); + const clean = createContentSanitizer([], { hideEmbeddingVectors: true })(value); + assert.match(clean, /PUBLIC/); + assert.doesNotMatch(clean, /0\.314159265|0\.271828182/); +}); + +test('JSON-encoded configured credential is matched before any structural rewrite', () => { + const secret = JSON.stringify({ first: 'SYNTHETIC_FIRST', second: 'SYNTHETIC_SECOND' }); + const value = JSON.stringify({ visible: 'PUBLIC', padding: 'x'.repeat(19800), echo: secret, tail: 'y'.repeat(500) }); + const clean = createContentSanitizer([secret], { piiRedaction: true })(value); + assert.match(clean, /PUBLIC/); + assert.doesNotMatch(clean, /SYNTHETIC_FIRST|SYNTHETIC_SECOND/); +}); + +test('short configured keys cannot split longer registered credentials', () => { + registerSecret('LONG_PRIVATE_abcde_SUFFIX'); + const clean = createContentSanitizer(['abcde'])('PUBLIC LONG_PRIVATE_abcde_SUFFIX and abcde'); + assert.match(clean, /PUBLIC/); + assert.doesNotMatch(clean, /LONG_PRIVATE|SUFFIX|abcde/); +}); + +test('processing limits fail closed and short configured keys retain their protection', () => { + assert.equal(createContentSanitizer()('x'.repeat(1048577)), '[REDACTED_SIZE_LIMIT]'); + const key = 'a"b\\c'; + const clean = createContentSanitizer([key])(JSON.stringify({ public: 'PUBLIC', echo: key })); + assert.equal(JSON.parse(clean).echo, '[REDACTED]'); +}); + +test('bounded originals are consumed once and cleared on discard', () => { + const buffer = createContentBuffer([], 16); + const first = buffer.capture('😀'.repeat(4)); + assert.equal(buffer.take(buffer.capture('overflow')), '[REDACTED_SIZE_LIMIT]'); + assert.equal(buffer.take(first), '😀'.repeat(4)); + assert.equal(buffer.take(first), undefined); + const next = buffer.capture('PUBLIC'); + buffer.clear(); + assert.equal(buffer.take(next), undefined); + assert.equal(buffer.take(buffer.capture('PUBLIC')), 'PUBLIC'); + assert.equal(buffer.take(buffer.capture('x'.repeat(1048577))), '[REDACTED_SIZE_LIMIT]'); +}); + +test('JSON-looking punctuation inside prose credentials does not erase public text', () => { + for (const secret of ['SYNTHETIC_[1,2]_END', 'SYNTHETIC_{"x":"y"}_END', 'SYNTHETIC_[PASSWORD]_END']) { + const clean = createContentSanitizer([secret])('PUBLIC ' + secret + ' public suffix'); + assert.equal(clean, 'PUBLIC [REDACTED] public suffix'); + } +}); + +test('cross-string matches cannot silently remove privacy framing', () => { + const secret = 'end","mime_type":"audio/x-wav","after":"start'; + const json = '{"before":"end","mime_type":"audio/x-wav","after":"start","data":[987654,876543]}'; + for (const value of [json, 'tool says ' + json, JSON.stringify({ encoded: json })]) { + const clean = createContentSanitizer([secret])(value); + assert.doesNotMatch(clean, /987654|876543/); + assert.match(clean, /REDACTED/); + } +}); + +test('credential matches cannot erase media context or damage JSON framing', () => { + for (const [secret, value] of [ + ['audio/x-wav', JSON.stringify({ mime_type: 'audio/x-wav', data: [987654, 876543] })], + ['audio/x-wav', JSON.stringify({ mime_type: ['audio/x-wav'], data: [987654, 876543] })], + ['"mime_type":"audio/x-wav"', '{"mime_type":"audio/x-wav","data":[987654,876543]}'], + ['opaque-audio-marker', JSON.stringify({ 'opaque-audio-marker': [987654, 876543] })], + ]) { + for (const form of [value, 'tool says ' + value, JSON.stringify({ encoded: value })]) { + const clean = createContentSanitizer([secret])(form); + assert.doesNotMatch(clean, /987654|876543/); + assert.match(clean, /REDACTED/); + } + } +}); diff --git a/typescript/packages/traceai_agent_core/test/forks/cursor/tracer.test.cjs b/typescript/packages/traceai_agent_core/test/forks/cursor/tracer.test.cjs index 48f030067..beb9bd7e9 100644 --- a/typescript/packages/traceai_agent_core/test/forks/cursor/tracer.test.cjs +++ b/typescript/packages/traceai_agent_core/test/forks/cursor/tracer.test.cjs @@ -195,15 +195,12 @@ for (const usagePlan of ["request", "turn"]) for (const field of ["input.value", "output.value"]) { const text = attrs(span)[field]; if (size > 1048576) assert.equal(text, "[REDACTED_SIZE_LIMIT]"); - else if (size > 20000 && text === "") { - // fi-core >=1.2.0 omits a truncated JSON prefix it cannot classify as - // non-media. Content is lost but nothing leaks (asserted above); restoring - // readable truncation of oversized JSON is tracked as a follow-up. - } else { + else { assert.match(text, /PUBLIC_CONTENT/); assert.match(text, /\[REDACTED\]|/); - assert.ok(text.length <= 20000); - if (size > 20000) assert.equal(text.length, 20000); + assert.ok(Buffer.byteLength(text) <= 20000); + assert.doesNotThrow(() => JSON.parse(text)); + if (size > 20000) assert.ok(Buffer.byteLength(text) >= 19000); } } }); diff --git a/typescript/packages/traceai_agent_core/test/tracer.test.cjs b/typescript/packages/traceai_agent_core/test/tracer.test.cjs index 3ef96e103..8e73e5ee1 100644 --- a/typescript/packages/traceai_agent_core/test/tracer.test.cjs +++ b/typescript/packages/traceai_agent_core/test/tracer.test.cjs @@ -202,15 +202,12 @@ for (const usagePlan of ["request", "turn"]) for (const field of ["input.value", "output.value"]) { const text = attrs(span)[field]; if (size > 1048576) assert.equal(text, "[REDACTED_SIZE_LIMIT]"); - else if (size > 20000 && text === "") { - // fi-core >=1.2.0 omits a truncated JSON prefix it cannot classify as - // non-media. Content is lost but nothing leaks (asserted above); restoring - // readable truncation of oversized JSON is tracked as a follow-up. - } else { + else { assert.match(text, /PUBLIC_CONTENT/); assert.match(text, /\[REDACTED\]|/); - assert.ok(text.length <= 20000); - if (size > 20000) assert.equal(text.length, 20000); + assert.ok(Buffer.byteLength(text) <= 20000); + assert.doesNotThrow(() => JSON.parse(text)); + if (size > 20000) assert.ok(Buffer.byteLength(text) >= 19000); } } }); diff --git a/typescript/packages/traceai_opencode_v1_plugin/package.json b/typescript/packages/traceai_opencode_v1_plugin/package.json index cfefb8943..cd0e4da54 100644 --- a/typescript/packages/traceai_opencode_v1_plugin/package.json +++ b/typescript/packages/traceai_opencode_v1_plugin/package.json @@ -9,7 +9,7 @@ "files": ["dist", "README.md", "LICENSE"], "scripts": { "build": "tsc -p tsconfig.json", "test": "node --test test/*.test.mjs", "prepare": "pnpm run build" }, "engines": { "node": ">=22.16.0", "opencode": ">=1.19.0 <2.0.0" }, - "dependencies": { "@traceai/agent-core": "workspace:^" }, + "dependencies": { "@traceai/agent-core": "workspace:^", "@traceai/fi-core": "workspace:^" }, "devDependencies": { "@opencode-ai/plugin": "1.19.0", "@types/node": "^22.15.29", "typescript": "^5.8.3" }, "repository": { "type": "git", "url": "git+https://github.com/future-agi/traceAI.git", "directory": "typescript/packages/traceai_opencode_v1_plugin" }, "publishConfig": { "access": "public" }, diff --git a/typescript/packages/traceai_opencode_v1_plugin/src/observer.ts b/typescript/packages/traceai_opencode_v1_plugin/src/observer.ts index 08719e29c..be1bebfab 100644 --- a/typescript/packages/traceai_opencode_v1_plugin/src/observer.ts +++ b/typescript/packages/traceai_opencode_v1_plugin/src/observer.ts @@ -1,6 +1,7 @@ import type { Hooks } from "@opencode-ai/plugin"; +import { seedSecrets } from "@traceai/fi-core"; import { - collectionEnabled, redact, isSensitiveField, + collectionEnabled, createContentSanitizer, LIMITS, type ResolvedConfig, type TurnTracer, type TurnHandle, type TurnOutcome, type UsageInput, } from "@traceai/agent-core"; @@ -64,29 +65,46 @@ export function observe(config: ResolvedConfig, tracer: TurnTracer): Hooks { let retained = 0; let disposed = false; const MAX_CONTENT = 4 * 1024 * 1024; + const credentials = config.credentials ? [config.credentials.apiKey, config.credentials.secretKey] : []; + const sanitize = createContentSanitizer(credentials, { + hideInputs: config.hideInputs, hideOutputs: config.hideOutputs, hideErrorDetails: config.hideErrorDetails, + }); + // Sanitize only during synchronous final emission. Preserve complete values + // for core admission; the observer never performs the 20,000-byte display cut. + const finalContent = (value: string | undefined): string | undefined => + value === undefined ? undefined : sanitize(value, LIMITS.ingress); + // Originals stay memory-only until final emission: even a complete-value + // PII rewrite here could split a credential registered between hook events. + const bound = (text: string): string => text.length > LIMITS.ingress || Buffer.byteLength(text) > LIMITS.ingress + ? "[REDACTED_SIZE_LIMIT]" : text; function capture(value: unknown, direction: "input" | "output"): string | undefined { if ((direction === "input" ? config.hideInputs : config.hideOutputs) || value === undefined) return; - // Bound graph traversal, but sanitize complete string leaves and keys BEFORE - // truncation can remove JSON delimiters or split a saved credential. - let budget = 20000, nodes = 2048; - function clean(text: string): string { - text = redact(text); - for (const secret of [config.credentials?.apiKey, config.credentials?.secretKey]) if (secret) text = text.split(secret).join("[REDACTED]"); - return text; - } - function bounded(v: unknown, depth = 0): unknown { - if (budget <= 0 || --nodes < 0 || depth > 8) return "[TRUNCATED]"; - if (typeof v === "string") { const s = clean(v).slice(0, budget); budget -= s.length; return s; } + // Inspect complete data, not a 64-item projection that could drop privacy + // context. Refuse an over-budget graph whole; never truncate a raw leaf/key. + let budget = LIMITS.ingress, nodes = 20000; + function materialize(v: unknown, depth = 0): unknown { + if (--nodes < 0 || depth > 16) throw new Error("Content graph budget"); + if (typeof v === "string") { + if (v.length > budget || (budget -= Buffer.byteLength(v)) < 0) throw new Error("Content byte budget"); + return v; + } if (v === null || typeof v === "number" || typeof v === "boolean") return v; - if (Array.isArray(v)) return v.slice(0, 64).map(x => bounded(x, depth + 1)); - if (v && typeof v === "object") return Object.fromEntries(Object.entries(v).slice(0, 64).map(([k, x]) => [clean(k).slice(0, 128), isSensitiveField(k) ? "[REDACTED]" : bounded(x, depth + 1)])); + if (Array.isArray(v)) { + if (v.length > nodes) throw new Error("Content array budget"); + return v.map(x => materialize(x, depth + 1)); + } + if (v && typeof v === "object") return Object.fromEntries(Object.entries(v).map(([k, x]) => [materialize(k, depth + 1), materialize(x, depth + 1)])); return undefined; } - const text = typeof value === "string" ? clean(value).slice(0, 20000) : JSON.stringify(bounded(value))?.slice(0, 20000); - if (text === undefined) return; - if (retained + text.length > MAX_CONTENT) return; - // Conservative budget: repeated updates count too; reset only after all turns settle. - retained += text.length; + let text: string | undefined; + try { + const complete = typeof value === "string" ? value : JSON.stringify(materialize(value)); + text = complete === undefined ? undefined : bound(complete); + } catch { text = "[REDACTED_SIZE_LIMIT]"; } + try { seedSecrets(process.env, credentials); } catch { text = "[REDACTED_PRIVACY_FAILURE]"; } + if (text === undefined || retained + Buffer.byteLength(text) > MAX_CONTENT) return; + // Repeated updates count too; reset only after all turns settle. + retained += Buffer.byteLength(text); return text; } function active(session: string): Turn | undefined { return turns.get(current.get(session) ?? ""); } @@ -132,7 +150,7 @@ export function observe(config: ResolvedConfig, tracer: TurnTracer): Hooks { return; } if (turns.size >= 64) return; - const text = config.hideInputs ? undefined : parts.slice(0, 64).filter(p => p.type === "text" && !p.synthetic).map(p => typeof p.text === "string" ? capture(p.text, "input") ?? "" : "").join("\n").slice(0, 20000); + const text = config.hideInputs ? undefined : bound(parts.slice(0, 64).filter(p => p.type === "text" && !p.synthetic).map(p => typeof p.text === "string" ? capture(p.text, "input") ?? "" : "").join("\n")); const parent = invocations.get(message.sessionID); const t: Turn = { id: message.id, session: message.sessionID, start: time(message.time?.created), input: text, steps: new Map(), idle: false, observed: Date.now(), parent: parent && turns.has(parent.turn) ? { ...parent } : undefined }; @@ -210,9 +228,9 @@ export function observe(config: ResolvedConfig, tracer: TurnTracer): Hooks { function emit(t: Turn, root?: TurnHandle, parentId?: string, depth = 0): void { if (depth > 16 || !turns.has(t.id)) return; const own = !root; - const handle = root ?? tracer.startTurn({ sessionId: t.session, turnId: t.id, input: t.input, startTime: t.start, usagePlan: "request" }); + const handle = root ?? tracer.startTurn({ sessionId: t.session, turnId: t.id, input: finalContent(t.input), startTime: t.start, usagePlan: "request" }); const childId = `session-${t.id}`; - const branch = own ? undefined : handle.child({ id: childId, kind: "agent", parentId, name: "subagent", input: t.input, startTime: t.start }); + const branch = own ? undefined : handle.child({ id: childId, kind: "agent", parentId, name: "subagent", input: finalContent(t.input), startTime: t.start }); const parent = own ? undefined : childId; let answer: string | undefined; if (t.compactionStart) { @@ -222,17 +240,17 @@ export function observe(config: ResolvedConfig, tracer: TurnTracer): Hooks { for (const s of t.steps.values()) { // Subtask placeholder assistant messages can still contain genuine tools. const generation = s.native || s.outcome ? handle.child({ id: s.id, kind: s.native ? "llm" : "generation", parentId: parent, - name: s.compaction ? "compaction-generation" : "generation", startTime: s.start, input: s.input, + name: s.compaction ? "compaction-generation" : "generation", startTime: s.start, input: finalContent(s.input), model: s.model, provider: s.provider, fidelity: s.native ? "native-request" : "reconstructed-message" }) : undefined; - const text = [...s.text.values()].join("\n").slice(0, 20000) || undefined; + const text = finalContent([...s.text.values()].join("\n")) || undefined; if (text) answer = text; for (const tool of s.tools.values()) { const key = `${s.id}-${tool.id}`; - const h = handle.child({ id: key, kind: "tool", toolCallId: tool.id, parentId: generation ? s.id : parent, name: tool.name, input: tool.input, startTime: tool.start }); + const h = handle.child({ id: key, kind: "tool", toolCallId: tool.id, parentId: generation ? s.id : parent, name: tool.name, input: finalContent(tool.input), startTime: tool.start }); for (const child of [...turns.values()]) if (child.parent?.turn === t.id && child.parent.tool === key) emit(child, handle, key, depth + 1); - h.end({ output: tool.output, outcome: tool.outcome ?? "incomplete", endTime: tool.end ?? t.end }); + h.end({ output: finalContent(tool.output), outcome: tool.outcome ?? "incomplete", endTime: tool.end ?? t.end }); } - const output = text ?? (s.tools.size ? capture({ tool_calls: [...s.tools.values()].map(tool => ({ name: tool.name, arguments: tool.input })) }, "output") : undefined); + const output = text ?? (s.tools.size ? finalContent(capture({ tool_calls: [...s.tools.values()].map(tool => ({ name: tool.name, arguments: tool.input })) }, "output")) : undefined); generation?.end({ output, outcome: s.outcome ?? (s.end ? "success" : "incomplete"), endTime: s.end ?? t.end, usage: s.usage }); } // Context overflow may be recovered by compaction. Only a later completed, diff --git a/typescript/packages/traceai_opencode_v1_plugin/test/unified-core.test.mjs b/typescript/packages/traceai_opencode_v1_plugin/test/unified-core.test.mjs index 52f39e4e7..d24a0cfe9 100644 --- a/typescript/packages/traceai_opencode_v1_plugin/test/unified-core.test.mjs +++ b/typescript/packages/traceai_opencode_v1_plugin/test/unified-core.test.mjs @@ -6,6 +6,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { createTurnTracer, resolveConfig } from '@traceai/agent-core'; import { observe } from '../dist/observer.js'; +import { registerSecret } from '@traceai/fi-core'; const attrs = span => Object.fromEntries(span.attributes.map(a => [a.key, a.value.stringValue ?? a.value.intValue ?? a.value.doubleValue ?? a.value.boolValue])); async function fixture(t, env = {}) { @@ -80,16 +81,17 @@ test('replayed pre-error success cannot recover a later session failure', async test('full structured fields are redacted before the adapter truncates prompt and tool data', async t => { const f = await fixture(t); const marker = 'PRIVATE_STRUCTURED_VALUE'; - const prompt = JSON.stringify({ 'api key': { nested: marker }, padding: 'x'.repeat(24000) }); + const prompt = JSON.stringify({ visible: 'PUBLIC_PROMPT', 'api key': { nested: marker }, padding: 'x'.repeat(24000) }); await f.user(prompt); await f.message('model'); await f.part('model', { type: 'step-start' }); await f.part('model', { type: 'tool', tool: 'read', callID: 'call-source', state: { - status: 'completed', input: { 'access token': [marker], nested: JSON.stringify({ 'private key': marker }), boundary: 'x'.repeat(19990) + 'synthetic-secret-boundary' }, - output: JSON.stringify({ 'authorization': { password: marker }, padding: 'x'.repeat(24000) }), time: { start: Date.now() - 5, end: Date.now() }, + status: 'completed', input: { visible: 'PUBLIC_INPUT', 'access token': [marker], nested: JSON.stringify({ 'private key': marker }), boundary: 'x'.repeat(19990) + 'synthetic-secret-boundary' }, + output: JSON.stringify({ visible: 'PUBLIC_OUTPUT', 'authorization': { password: marker }, padding: 'x'.repeat(24000) }), time: { start: Date.now() - 5, end: Date.now() }, } }); const spans = await f.finish(), serialized = JSON.stringify(spans); assert.equal(serialized.includes(marker), false); assert.equal(serialized.includes('synthetic-secret'), false); - // Oversized JSON may be redacted in place or omitted whole by fi-core (>=1.2.0); never leaked. - assert.ok(/\[REDACTED\]||/.test(serialized)); + assert.ok(/\[REDACTED\]|/.test(serialized)); + for (const publicText of ['PUBLIC_PROMPT', 'PUBLIC_INPUT', 'PUBLIC_OUTPUT']) assert.ok(serialized.includes(publicText)); + assert.ok(!serialized.includes('')); }); for (const hideErrorDetails of ['false', 'true']) test(`successful JSON fixture read stays successful (hideErrorDetails=${hideErrorDetails})`, async t => { @@ -118,3 +120,39 @@ test('native tool failures stay errors and hidden details cannot leak through la assert.equal(tool.status.code, 2); assert.equal(attrs(tool)['output.value'], undefined); assert.equal(JSON.stringify(spans).includes('PRIVATE_FAILURE_DETAIL'), false); }); + +for (const overlap of [false, true]) test(`late observer credentials are scrubbed before any cut or PII rewrite: overlap=${overlap}`, async t => { + const previous = process.env.FI_PII_REDACTION; + process.env.FI_PII_REDACTION = 'true'; + t.after(() => { if (previous === undefined) delete process.env.FI_PII_REDACTION; else process.env.FI_PII_REDACTION = previous; }); + const f = await fixture(t); + const prefix = `LATE_OBSERVER_PRIVATE_${overlap}_`; + const secret = prefix + (overlap ? '123-45-6789_' : '') + 'q'.repeat(140); + const text = JSON.stringify({ stdout: 'PUBLIC_RESULT ' + 'p '.repeat(9940) + secret + ' tail' }); + await f.user(text); await f.message('model'); await f.part('model', { type: 'step-start' }); + await f.part('model', { type: 'text', id: 'answer', text }); + await f.part('model', { type: 'tool', tool: 'read', callID: 'call-source', state: { + status: 'completed', input: text, output: text, time: { start: Date.now() - 5, end: Date.now() }, + } }); + registerSecret(secret); + const spans = await f.finish(); + assert.equal(spans.length, 3); + for (const span of spans) for (const key of ['input.value', 'output.value']) { + const value = attrs(span)[key]; + assert.equal(typeof value, 'string'); + assert.match(value, /PUBLIC_RESULT/); + assert.ok(!value.includes(prefix)); + assert.ok(Buffer.byteLength(value) <= 20000); + } +}); + +for (const length of [65, 100]) test(`tool input keeps a public ${length}-item file list`, async t => { + const f = await fixture(t); await f.user('Read public files'); + await f.message('model'); await f.part('model', { type: 'step-start' }); + const input = { purpose: 'PUBLIC_FILES', files: Array.from({ length }, (_, i) => `src/public-file-${i}.ts`) }; + await f.part('model', { type: 'tool', tool: 'read_files', callID: 'call-source', state: { + status: 'completed', input, output: 'PUBLIC_RESULT', time: { start: Date.now() - 5, end: Date.now() }, + } }); + const spans = await f.finish(), tool = spans.find(s => attrs(s)['tool.name'] === 'read_files'); + assert.deepEqual(JSON.parse(attrs(tool)['input.value']), input); +}); diff --git a/typescript/pnpm-lock.yaml b/typescript/pnpm-lock.yaml index 2dd325b78..92f083415 100644 --- a/typescript/pnpm-lock.yaml +++ b/typescript/pnpm-lock.yaml @@ -2377,6 +2377,9 @@ importers: '@traceai/agent-core': specifier: workspace:^ version: link:../traceai_agent_core + '@traceai/fi-core': + specifier: workspace:^ + version: link:../fi-core devDependencies: '@opencode-ai/plugin': specifier: 1.19.0