Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .release-please-manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@
"python/frameworks/weaviate": "0.1.2",
"python/frameworks/xai": "0.1.1",
"python/traceai": "0.1.0",
"typescript/packages/fi-core": "1.2.2",
"typescript/packages/fi-core": "1.2.3",
"typescript/packages/fi-semantic-conventions": "1.1.0",
"typescript/packages/traceai_anthropic": "0.1.5",
"typescript/packages/traceai_auto": "0.1.1",
Expand Down
4 changes: 4 additions & 0 deletions typescript/packages/fi-core/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
## [1.2.3] - pending release
### Fix
- With `hideEmbeddingVectors`, recognize `object: "embedding"` as embedding context before oversized agent content is shortened. A late discriminator must not leave its vector data visible.

## [1.2.2] - pending release
### Fix
- Preserve numeric `gen_ai.usage.cache_read_tokens.audio` counts through admission, projection and export. The 1.2.0 shared privacy runtime mistook this modality count for an audio payload. Only nonnegative safe integers receive the metric role; strings, arrays, media ancestry and registered secrets retain their existing privacy protections.
Expand Down
2 changes: 1 addition & 1 deletion typescript/packages/fi-core/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@traceai/fi-core",
"version": "1.2.2",
"version": "1.2.3",
"description": "OpenTelemetry tracing for AI applications: tracer setup, span helpers, and exporters for Future AGI.",
"main": "./dist/src/index.js",
"module": "./dist/esm/index.js",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
import { redactEmbeddingValues } from "../trace/trace-config/structuredRedaction";

test("late embedding discriminator protects data without erasing public metadata", () => {
const input = JSON.stringify({ visible: "PUBLIC", data: [0.314159265, 0.271828182], padding: "x".repeat(20000), object: "embedding" });
const clean = redactEmbeddingValues(input);
expect(clean).not.toContain("0.314159265");
expect(clean).toContain("PUBLIC");
expect(JSON.parse(clean).object).toBe("embedding");
expect(redactEmbeddingValues(JSON.stringify({ object: "table", data: [1, 2] }))).toBe('{"object":"table","data":[1,2]}');
});
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ export function redactEmbeddingValues(value: string): string {
return Object.fromEntries(Object.entries(item).map(([key, child]) => [
key,
visit(child, depth + 1, /^(?:embedding|embeddings|vector|vectors)$/i.test(key) ||
key === "data" && (item as Record<string, unknown>).object === "embedding" ||
vector && !/^(?:text|input|input_text|model|index|dimensions?|count|type|mime_type)$/i.test(key)),
]));
}
Expand Down
148 changes: 148 additions & 0 deletions typescript/packages/traceai_agent_core/src/content.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
import { indexPrivacyHints, normalizePrivacyName } from "@traceai/fi-core/privacy";
import { generateTraceConfig, privacyFilter, seedSecrets, getRegisteredSecrets, type TraceConfigOptions } from "@traceai/fi-core";
import { redact } from "./privacy.js";
import { LIMITS } from "./schema.js";

/** Private, bounded originals for spans that have not finalized yet. No original
* is attached to an OTel span: finalization runs complete-value privacy with the
* current registry, then truncates, then goes through FITracer admission/export.
* A tracer-wide bound prevents many concurrently open turns retaining unbounded
* source content. Consumed/discarded entries release their original immediately. */
export function createContentBuffer(credentials: readonly string[] = [], maxBytes = 8 * LIMITS.ingress): {
capture(value: string | undefined): number | undefined;
take(id: number | undefined): string | undefined;
clear(): void;
} {
const pending = new Map<number, string>();
let next = 0, size = 0;
return {
capture(value: string | undefined): number | undefined {
if (typeof value !== "string") return undefined;
const length = value.length > LIMITS.ingress ? Infinity : Buffer.byteLength(value);
if (length > LIMITS.ingress || size + length > maxBytes) return -1;
// Keep capture-time discovery too: an environment credential may be removed
// before finalization, but must remain protected by the shared registry.
try { seedSecrets(process.env, credentials); } catch { return -2; }
const id = next++;
pending.set(id, value);
size += length;
return id;
},
take(id: number | undefined): string | undefined {
if (id === -1) return "[REDACTED_SIZE_LIMIT]";
if (id === -2) return "[REDACTED_PRIVACY_FAILURE]";
if (id === undefined) return undefined;
const value = pending.get(id);
if (value !== undefined) {
size -= Buffer.byteLength(value);
pending.delete(id);
}
return value;
},
clear(): void { pending.clear(); size = 0; },
};
}

const TRUNCATED = "[TRUNCATED]";
const bytes = (text: string): number => Buffer.byteLength(text);
// Keep fi-core recognition at the tracer boundary, not in lightweight hook code.
const contextKey = (key: string): boolean => ["type", "object"].includes(normalizePrivacyName(key)) ||
indexPrivacyHints([[key, undefined]]).size > 0;

/** Resolve policy once, just like FITracer. Never shorten a value before the
* complete credential, media, PII and embedding inspection has succeeded.
* This is preparation, not an alternative to FITracer admission/export. */
export function createContentSanitizer(
credentials: readonly string[] = [],
options: TraceConfigOptions = {},
): (text: string, limit?: number) => string {
const keys = [...credentials], policy = generateTraceConfig(options);
const privacy = privacyFilter(policy, keys);
return (text: string, limit = 20_000): string => {
if (text.length > LIMITS.ingress || bytes(text) > LIMITS.ingress)
return "[REDACTED_SIZE_LIMIT]";
try {
seedSecrets(process.env, keys);
const key = "fi.agent.content";
const secrets = [...new Set([...keys, ...getRegisteredSecrets()])].sort((a, b) => b.length - a.length);
text = redact(text, secrets, contextKey);
// Exact-value matching just consumed the complete registry snapshot above.
// Apply the remaining FI policies without repeatedly charging every absent
// credential against every quoted prose token. The actual FITracer still
// performs its normal admission/discovery/export with the full registry.
const clean = privacy.attributes({ [key]: text }, [])[key];
if (typeof clean !== "string") return "[REDACTED]";
return truncateContent(clean, limit);
} catch {
// Budget/discovery errors may never fall back to raw or partially read data.
return "[REDACTED_PRIVACY_FAILURE]";
}
};
}

/** Internal: the input MUST already have passed complete-value privacy.
* Retain valid JSON at every encoded layer for downstream FI privacy checks. */
export function truncateContent(text: string, limit = 20_000): string {
if (!Number.isSafeInteger(limit) || limit < 64)
throw new RangeError("Content limit must be at least 64 bytes");
if (bytes(text) <= limit) return text;
let parsed: unknown;
try { parsed = JSON.parse(text); } catch {
const buffer = Buffer.from(text);
let end = limit;
while ((buffer[end] & 0xc0) === 0x80) end--;
return buffer.subarray(0, end).toString("utf8");
}
return shorten(parsed, limit) ?? JSON.stringify(TRUNCATED);
}

// Cost after escaping inside each enclosing JSON string, excluding its quotes.
function cost(text: string, layers: number): number {
for (let i = 0; i < layers; i++) text = JSON.stringify(text).slice(1, -1);
return bytes(text);
}
function shorten(value: unknown, budget: number, layers = 0, depth = 0): string | undefined {
const full = JSON.stringify(value);
if (cost(full, layers) <= budget) return full;
const marker = JSON.stringify(TRUNCATED);
if (depth > 16) return cost(marker, layers) <= budget ? marker : undefined;
if (typeof value === "string") {
let parsed: unknown;
try { parsed = JSON.parse(value); } catch {
let remaining = budget - cost(marker, layers);
if (remaining < 0) return undefined;
let prefix = "";
for (const char of value) {
const size = cost(JSON.stringify(char).slice(1, -1), layers);
if (size > remaining) break;
prefix += char;
remaining -= size;
}
return JSON.stringify(prefix + TRUNCATED);
}
const inner = shorten(parsed, budget - cost('""', layers), layers + 1, depth + 1);
return inner === undefined ? undefined : JSON.stringify(inner);
}
if (value === null || typeof value !== "object") return undefined;
const array = Array.isArray(value);
let markerKey = "_truncated";
while (Object.hasOwn(value, markerKey)) markerKey += "_";
const tail = array ? '{"_truncated":true}' : `${JSON.stringify(markerKey)}:true`;
let remaining = budget - cost((array ? "[]" : "{}") + tail, layers);
if (remaining < 0) return undefined;
const parts: string[] = [];
for (const [key, child] of Object.entries(value)) {
const prefix = array ? "" : `${JSON.stringify(key)}:`;
const overhead = cost(prefix + ",", layers);
if (overhead > remaining) continue;
const encoded = shorten(child, remaining - overhead, layers, depth + 1);
if (encoded === undefined) {
if (array) break; // Never shift array indices by skipping an item.
continue;
}
parts.push(prefix + encoded);
remaining -= overhead + cost(encoded, layers);
}
parts.push(tail);
return (array ? "[" : "{") + parts.join(",") + (array ? "]" : "}");
}
1 change: 1 addition & 0 deletions typescript/packages/traceai_agent_core/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,4 @@ export * from "./delivery.js";
export * from "./tracer.js";
export * from "./capture.js";
export { VERSION } from "./version.js";
export { createContentSanitizer } from "./content.js";
93 changes: 76 additions & 17 deletions typescript/packages/traceai_agent_core/src/privacy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,23 +28,73 @@ export const isSensitiveKey = isSensitiveField;
export const redactValue = (value: unknown, depth = 0): unknown =>
redactStructured(value, depth);
/** Defense in depth for content-on-by-default. Structure-only omits content entirely. */
export function redact(text: string): string {
return redactStructured(text) as string;
export function redact(text: string, secrets: readonly string[] = [], contextKey: (key: string) => boolean = () => false): string {
return redactStructured(text, 0, secrets, contextKey) as string;
}
/** Callers supply longest-first credentials. Keep inserted markers out of later
* matches. This runs on complete text and again on decoded JSON leaves, before
* assignments/PII can change a credential that overlaps their syntax. */
function exactSecrets(text: string, secrets: readonly string[]): string {
for (const secret of secrets) {
if (!secret || !text.includes(secret)) continue;
text = text.split("[REDACTED]").map(part => part.split(secret).join("[REDACTED]")).join("[REDACTED]");
}
return text;
}
/** A match inside a JSON string is not framing, even if it contains brackets.
* Inspect a successfully parsed container at its original offsets, before any
* embedded rewrite can split a credential. Matches overlapping delimiters (or
* entering/leaving an embedded container) own that whole text instead. */
function crossesJsonFraming(text: string, start: number, end: number, secrets: readonly string[]): boolean {
if (!secrets.some(secret => secret && text.includes(secret))) return false;
const strings: [number, number][] = [];
let open = -1, escaped = false;
for (let i = start; i < end; i++) {
if (open < 0) { if (text[i] === '"') open = i + 1; }
else if (escaped) escaped = false;
else if (text[i] === "\\") escaped = true;
else if (text[i] === '"') { strings.push([open, i]); open = -1; }
}
for (const secret of secrets) {
if (!secret) continue;
let at = text.indexOf(secret, Math.max(0, start - secret.length + 1));
while (at >= 0 && at < end) {
// Binary search the string token starting at or before this occurrence.
let lo = 0, hi = strings.length;
while (lo < hi) {
const mid = (lo + hi) >>> 1;
if (strings[mid][0] <= at) lo = mid + 1; else hi = mid;
}
const token = strings[lo - 1];
if (!token || at + secret.length > token[1]) return true;
at = text.indexOf(secret, at + 1);
}
}
return false;
}

/** Check normalized keys before serialization, including nested JSON-encoded strings. */
export function redactStructured(value: unknown, depth = 0): unknown {
export function redactStructured(value: unknown, depth = 0, secrets: readonly string[] = [], contextKey: (key: string) => boolean = () => false): unknown {
if (depth > 16) return "[REDACTED_DEPTH_LIMIT]";
if (Array.isArray(value))
return value.map((item) => redactStructured(item, depth + 1));
if (value !== null && typeof value === "object")
return value.map((item) => redactStructured(item, depth + 1, secrets, contextKey));
if (value !== null && typeof value === "object") {
// A credential may itself be a key or media/embedding discriminator. Removing
// it must not erase the context that protects its siblings in later FI passes.
if (secrets.length && Object.entries(value).some(([key, item]) =>
exactSecrets(key, secrets) !== key ||
contextKey(key) &&
(Array.isArray(item) ? item : [item]).some(v => typeof v === "string" && exactSecrets(v, secrets) !== v),
)) return "[REDACTED]";
return Object.fromEntries(
Object.entries(value).map(([key, item]) => [
key,
exactSecrets(key, secrets),
isSensitiveField(key)
? "[REDACTED]"
: redactStructured(item, depth + 1),
: redactStructured(item, depth + 1, secrets, contextKey),
]),
);
}
if (typeof value === "string") {
// Inspect complete strings before any display truncation. Refuse oversized
// content rather than treating a truncated JSON prefix as ordinary text.
Expand All @@ -54,13 +104,12 @@ export function redactStructured(value: unknown, depth = 0): unknown {
)
return "[REDACTED_SIZE_LIMIT]";
if (/^\s*[\[{"]/.test(value)) {
try {
return JSON.stringify(redactStructured(JSON.parse(value), depth + 1));
} catch {
/* Opaque text still receives the free-text policy. */
}
let parsed: unknown;
try { parsed = JSON.parse(value); } catch { return redactText(value, depth, secrets, contextKey); }
if (crossesJsonFraming(value, 0, value.length, secrets)) return "[REDACTED]";
return JSON.stringify(redactStructured(parsed, depth + 1, secrets, contextKey));
}
return redactText(value, depth);
return redactText(value, depth, secrets, contextKey);
}
return value;
}
Expand Down Expand Up @@ -163,7 +212,7 @@ function redactAssignments(text: string): string {
}
return result + text.slice(offset);
}
function redactEmbeddedJson(text: string, depth: number): string {
function redactEmbeddedJson(text: string, depth: number, secrets: readonly string[], contextKey: (key: string) => boolean): string {
if (text.length > LIMITS.ingress || Buffer.byteLength(text) > LIMITS.ingress)
return "[REDACTED_SIZE_LIMIT]";
let start = -1,
Expand Down Expand Up @@ -200,7 +249,17 @@ function redactEmbeddedJson(text: string, depth: number): string {
if (!stack.length) {
try {
const parsed = JSON.parse(text.slice(start, i + 1));
const clean = redactStructured(parsed, depth + 1);
// A complete JSON-looking island wholly inside a prose credential is
// part of that credential, not surrounding framing. Leave it intact
// until exactSecrets removes the complete match below.
const covered = secrets.some(secret => {
if (!secret) return false;
const at = text.lastIndexOf(secret, start);
return at >= 0 && at + secret.length >= i + 1;
});
if (covered) { start = -1; continue; }
if (crossesJsonFraming(text, start, i + 1, secrets)) return "[REDACTED]";
const clean = redactStructured(parsed, depth + 1, secrets, contextKey);
const sanitized = JSON.stringify(clean);
// Do not rewrite harmless JSON-looking code or change prompt correlation.
result += text.slice(offset, start) + (sanitized === JSON.stringify(parsed)
Expand All @@ -215,8 +274,8 @@ function redactEmbeddedJson(text: string, depth: number): string {
}
return result + text.slice(offset);
}
function redactText(text: string, depth = 0): string {
return redactAssignments(redactEmbeddedJson(text, depth))
function redactText(text: string, depth: number, secrets: readonly string[], contextKey: (key: string) => boolean): string {
return redactAssignments(exactSecrets(redactEmbeddedJson(text, depth, secrets, contextKey), secrets))
// URI userinfo is credential-bearing for every scheme, including local databases.
.replace(/\b[a-z][a-z0-9+.-]*:\/\/[^\s/"'<>]*@/gi, (uri) => uri.slice(0, uri.indexOf("://") + 3) + "[REDACTED]@")
.replace(/\b(?:bearer|basic)\s+[^\s,;"']+/gi, "[REDACTED]")
Expand Down
Loading
Loading