Skip to content
Draft
70 changes: 70 additions & 0 deletions .github/workflows/openai-anthropic-attachments-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
name: OpenAI and Anthropic TypeScript attachments
on:
pull_request:
paths:
- 'typescript/packages/traceai_openai/**'
- 'typescript/packages/traceai_anthropic/**'
- 'typescript/packages/fi-core/**'
- 'typescript/packages/fi-semantic-conventions/**'
- 'typescript/pnpm-lock.yaml'
- '.github/workflows/openai-anthropic-attachments-tests.yml'
workflow_dispatch:
permissions:
contents: read
jobs:
attachments:
name: OpenAI ${{ matrix.openai }} / Anthropic ${{ matrix.anthropic }}
runs-on: ubuntu-24.04
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
include:
- { openai: '4.0.0', anthropic: '0.20.0' }
- { openai: '5.23.2', anthropic: '0.27.3' }
- { openai: '6.0.0', anthropic: '0.133.0' }
- { openai: '7.47.0', anthropic: '0.134.0' }
- { openai: 'latest', anthropic: 'latest' }
defaults:
run:
shell: bash
working-directory: typescript
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
persist-credentials: false
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: '22.23.4'
- name: Install pinned workspace dependencies and real SDK fixtures
run: |
mkdir -p "$RUNNER_TEMP/corepack-bin"
corepack enable --install-directory "$RUNNER_TEMP/corepack-bin" pnpm
echo "$RUNNER_TEMP/corepack-bin" >> "$GITHUB_PATH"
corepack pnpm --filter @traceai/openai... --filter @traceai/anthropic... install --frozen-lockfile --ignore-scripts
npm install --prefix "$RUNNER_TEMP/sdks" --ignore-scripts --no-audit --no-fund \
"openai-under-test@npm:openai@${{ matrix.openai }}" \
"anthropic-under-test@npm:@anthropic-ai/sdk@${{ matrix.anthropic }}"
- name: Build CJS and ESM
run: |
pnpm run build:deps
pnpm --filter @traceai/openai run build
pnpm --filter @traceai/anthropic run build
- name: Network-isolated attachment adapters against real SDKs and loopback mocks
run: |
sudo unshare --net -- bash -c 'ip link set lo up; exec "$@"' bash \
env PATH="$PATH" \
SDK_OPENAI="$RUNNER_TEMP/sdks/node_modules/openai-under-test" \
SDK_ANTHROPIC="$RUNNER_TEMP/sdks/node_modules/anthropic-under-test" \
node --test packages/traceai_openai/test/attachments-providers.test.cjs
- name: Network-isolated package unit suites
if: matrix.openai == 'latest'
run: |
sudo unshare --net -- bash -c 'ip link set lo up; exec "$@"' bash \
env PATH="$PATH" node packages/traceai_openai/node_modules/jest/bin/jest.js \
--config packages/traceai_openai/jest.config.js --runInBand
sudo unshare --net -- bash -c 'ip link set lo up; exec "$@"' bash \
env PATH="$PATH" node node_modules/jest/bin/jest.js \
--config packages/traceai_anthropic/jest.config.js --runInBand
git diff --exit-code -- pnpm-lock.yaml
168 changes: 168 additions & 0 deletions typescript/packages/traceai_anthropic/src/attachments.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
import type { TraceConfigOptions } from "@traceai/fi-core";
import type { AttachmentManager, AttachmentSpan, AttachmentCorrelation, AttachmentCapturePolicy, AttachmentOptions } from "@traceai/fi-core/attachments";

/** Native SDK span ownership is deliberately not overridden by this adapter. */
export const anthropicAttachmentCapabilities = Object.freeze({
schemaVersion: 1,
operations: ["messages.create.input", "tool_result.input"] as const,
sources: ["inline_base64"] as const,
/** Fail closed: no capture is attempted for these. */
unsupported: ["remote_url", "provider_file", "native_span_hookup", "generated_image", "beta_messages_hookup", "streamed_output"] as const,
maxInlineBytes: 256 * 1024,
});

/** Admitted media positions for one observation: position key -> occurrence. */
export type AnthropicCapturedMedia = ReadonlyMap<string, Readonly<{ occurrenceId: string; mimeType: string }>>;
type Captured = Map<string, Readonly<{ occurrenceId: string; mimeType: string }>>;
type RecordValue = Record<string, unknown>;
const record = (value: unknown): RecordValue => value !== null && typeof value === "object" ? value as RecordValue : {};
const MAX_PARTS = 256;
const MAX_SPANS = 1024;
const EMPTY: AnthropicCapturedMedia = new Map();
const registryKey = Symbol.for("@traceai/anthropic/attachment-observations/v1");
type Registry = WeakMap<AttachmentManager, Map<string, Captured>>;
const host = globalThis as typeof globalThis & { [registryKey]?: Registry };
const observations = (host[registryKey] ??= new WeakMap());
const TOOL_ID = /^[a-zA-Z0-9_-]{1,128}$/;

const MAX_ENCODED = 4 * Math.ceil(anthropicAttachmentCapabilities.maxInlineBytes / 3);

/** Exact decoded length from the encoded string's shape only (no decode); 0 when out of bounds. */
function bound(encoded: unknown): number {
if (typeof encoded !== "string" || !encoded.length || encoded.length % 4 !== 0 || encoded.length > MAX_ENCODED) return 0;
return (encoded.length / 4) * 3 - (encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0);
}

/** Strict decode after admission. The ticket's budget is a hard bound; padding and alphabet are strict. */
function decode(encoded: unknown, maxBytes: number): Uint8Array | undefined {
// Bound allocation BEFORE decoding.
if (!bound(encoded) || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded as string)) return undefined;
const bytes = Buffer.from(encoded as string, "base64");
return bytes.length && bytes.length <= maxBytes && bytes.length <= anthropicAttachmentCapabilities.maxInlineBytes &&
bytes.toString("base64") === encoded ? bytes : undefined;
}

/** Observe only Messages' typed image/document blocks and one level of tool_result content.
* Every part is preflighted (deny-wins, budget reserved) before its base64 is read or decoded; an admitted
* ticket that is not used is released at once. The manager owns policy, upload, lifecycle and references. A native integration
* must supply its actual emitted span/correlation, never an ambient guessed span.
* Returns the admitted positions so the caller can project them out of span telemetry.
*/
export function captureAnthropicAttachments(manager: AttachmentManager | undefined,
span: AttachmentSpan | AttachmentCorrelation, payload: unknown,
options: { traceConfig?: TraceConfigOptions; capture?: AttachmentCapturePolicy } = {}): AnthropicCapturedMedia {
if (!manager) return EMPTY;
const captured: Captured = new Map();
try {
const correlation = "snapshot" in span ? span : manager.correlation(span);
const identity = correlation.snapshot();
if (!identity?.sampled) return EMPTY;
const messages = record(payload).messages;
if (!Array.isArray(messages)) return EMPTY;
let spans = observations.get(manager);
if (!spans) { spans = new Map(); observations.set(manager, spans); }
const spanKey = `${identity.traceId}/${identity.spanId}`;
const config = options.traceConfig ?? {};
const hidden = config.hideInputs || config.hideInputMessages || config.hideInputText || config.piiRedaction;
// Screenshot classification requires a typed computer tool AND an assistant tool_use naming it.
const computerNames = new Set<string>();
const tools = record(payload).tools;
if (Array.isArray(tools)) for (const value of tools.slice(0, MAX_PARTS)) {
const tool = record(value);
if (typeof tool.type === "string" && /^computer_\d{8}$/.test(tool.type) && typeof tool.name === "string") computerNames.add(tool.name);
}
const computerCalls = new Set<string>();
for (const value of messages.slice(0, MAX_PARTS)) {
const message = record(value);
if (message.role !== "assistant" || !Array.isArray(message.content)) continue;
for (const value of message.content.slice(0, MAX_PARTS)) {
const block = record(value);
if (block.type === "tool_use" && typeof block.id === "string" && typeof block.name === "string" && computerNames.has(block.name)) computerCalls.add(block.id);
}
}
let visited = 0;
messages.slice(0, MAX_PARTS).forEach((value, messageIndex) => {
const message = record(value);
if (!Array.isArray(message.content)) return;
// Nested positions are relative to their tool result, identified by toolCallId.
const visit = (value: unknown, index: number, position: string, toolCallId?: string, errorDetail = false) => {
if (++visited > MAX_PARTS) return;
const part = record(value);
if (part.type !== "image" && part.type !== "document") return;
const source = record(part.source);
if (source.type !== "base64") return;
const mime = source.media_type;
if (typeof mime !== "string" || !(part.type === "image" ? ["image/png", "image/jpeg", "image/webp"] : ["application/pdf"]).includes(mime)) return;
const positionKey = `input/${messageIndex}/${position}`;
const previous = spans!.get(spanKey)?.get(positionKey);
if (previous) { captured.set(positionKey, previous); return; }
const role: AttachmentOptions["role"] = toolCallId ? "tool" : message.role === "assistant" ? "assistant" : "user";
const request = { mimeType: mime, direction: "input" as const,
purpose: (part.type === "document" ? "document" : toolCallId ? computerCalls.has(toolCallId) ? "computer_screenshot" : "tool_result" : "model_input") as AttachmentOptions["purpose"],
capture: { ...options.capture, modelInput: true, errorDetail: errorDetail || options.capture?.errorDetail,
...(hidden || (part.type === "image" && config.hideInputImages) || (errorDetail && config.hideErrorDetails) ? { captureContent: false } : {}) } };
// Preflight (deny-wins, budget reserved) before the base64 source is read or decoded (design section 1).
// The source is read at most once.
let data: unknown, dataRead = false;
const read = () => { if (!dataRead) { dataRead = true; data = source.data; } return data; };
let ticket = manager.preflight(correlation, { ...request, maxBytes: anthropicAttachmentCapabilities.maxInlineBytes });
// too_large (negotiated limit below the cap) and queue_full (owned budget nearly used) are evaluated after every
// privacy, correlation and sampling step, so sizing the source exactly and retrying once is safe here.
if (ticket.reason === "too_large" || ticket.reason === "queue_full") {
const exact = bound(read());
if (exact > 0) ticket = manager.preflight(correlation, { ...request, maxBytes: exact });
}
if (!ticket.admitted) return;
const bytes = decode(read(), ticket.maxBytes);
if (!bytes) { ticket.release(); return; }
const handle = ticket.attach({ kind: "Bytes", data: bytes }, { messageIndex, partIndex: index, role, ...(toolCallId ? { toolCallId } : {}) });
if (handle.admission === "queued") {
const occurrence = Object.freeze({ occurrenceId: handle.occurrenceId, mimeType: mime });
let positions = spans!.get(spanKey);
if (!positions) {
if (spans!.size >= MAX_SPANS) spans!.delete(spans!.keys().next().value!);
positions = new Map(); spans!.set(spanKey, positions);
}
if (positions.size < MAX_PARTS) positions.set(positionKey, occurrence);
captured.set(positionKey, occurrence);
}
};
message.content.slice(0, MAX_PARTS).forEach((value, index) => {
const block = record(value);
if (block.type === "tool_result" && Array.isArray(block.content)) {
// Invalid tool IDs fail closed instead of losing causal identity.
if (typeof block.tool_use_id !== "string" || !TOOL_ID.test(block.tool_use_id)) return;
block.content.slice(0, MAX_PARTS).forEach((child, childIndex) => visit(child, childIndex, `${index}/${childIndex}`, block.tool_use_id as string, block.is_error === true));
} else visit(value, index, String(index));
});
});
} catch { /* No source-bearing exceptions or changes to the application's result. */ }
return captured;
}

/** Telemetry-only projection. ONLY admitted positions are replaced with a `fi-attachment:` occurrence URI
* source, so uploaded bytes are not exported twice. Everything else is returned exactly as before, and the
* request passed to the SDK is never mutated.
*/
export function projectAnthropicMedia(payload: unknown, captured: AnthropicCapturedMedia): any {
if (!captured.size) return payload;
try {
const body = record(payload);
if (!Array.isArray(body.messages)) return payload;
const replace = (value: unknown, position: string): unknown => {
const occurrence = captured.get(position);
return occurrence ? { ...record(value), source: { type: "url", url: `fi-attachment:${occurrence.occurrenceId}` } } : value;
};
return { ...body, messages: body.messages.map((value, messageIndex) => {
const message = record(value);
if (!Array.isArray(message.content)) return value;
return { ...message, content: message.content.map((value, index) => {
const block = record(value);
if (block.type === "tool_result" && Array.isArray(block.content)) {
return { ...block, content: block.content.map((child, childIndex) => replace(child, `input/${messageIndex}/${index}/${childIndex}`)) };
}
return replace(value, `input/${messageIndex}/${index}`);
}) };
}) };
} catch { return payload; }
}
3 changes: 2 additions & 1 deletion typescript/packages/traceai_anthropic/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
export * from './version';
export * from './instrumentation';
export * from './instrumentation';
export * from './attachments';
21 changes: 17 additions & 4 deletions typescript/packages/traceai_anthropic/src/instrumentation.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import Anthropic from "@anthropic-ai/sdk";
import type { AttachmentManager } from "@traceai/fi-core/attachments";
import { captureAnthropicAttachments, projectAnthropicMedia } from "./attachments";
import {
InstrumentationBase,
InstrumentationConfig,
Expand Down Expand Up @@ -334,16 +336,22 @@ function finishInstrumentationStream(
export class AnthropicInstrumentation extends InstrumentationBase{
private fiTracer!: FITracer;
private _traceConfig?: TraceConfigOptions;
private attachments?: AttachmentManager;

constructor({
instrumentationConfig,
traceConfig,
attachments,
}: {
instrumentationConfig?: InstrumentationConfig;
traceConfig?: TraceConfigOptions;
/** The provider-owned shared attachment manager. */
attachments?: AttachmentManager;
} = {}) {
super("@traceai/instrumentation-anthropic", VERSION,Object.assign({}, instrumentationConfig));
this._traceConfig = traceConfig;
this.attachments = attachments;
this.fiTracer = new FITracer({ tracer: this.tracer, traceConfig: this._traceConfig });
}

public override enable(): void {
Expand Down Expand Up @@ -404,21 +412,26 @@ export class AnthropicInstrumentation extends InstrumentationBase{
// Not async: the SDK returns an APIPromise whose withResponse(), asResponse()
// and lazy body parsing callers (including messages.stream()) depend on.
return function patchedMessagesCreate(this: Anthropic.Messages, ...args: Parameters<Anthropic.Messages['create']>) {
const params = args[0] as Anthropic.MessageCreateParams;
const { messages: _messages, system: _system, tools: _tools, ...invocationParameters } = params;
const originalParams = args[0] as Anthropic.MessageCreateParams;
const { messages: _messages, system: _system, tools: _tools, ...invocationParameters } = originalParams;

// Content attributes are set after capture so admitted media can be projected out first.
const span = instrumentation.fiTracer.startSpan("Anthropic Messages Create", {
kind: SpanKind.INTERNAL,
attributes: {
[SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM,
[SemanticConventions.LLM_MODEL_NAME]: params.model,
[SemanticConventions.LLM_MODEL_NAME]: originalParams.model,
[SemanticConventions.LLM_PROVIDER]: LLMProvider.ANTHROPIC,
[SemanticConventions.GEN_AI_OPERATION_NAME]: "chat",
},
});
const captured = captureAnthropicAttachments(instrumentation.attachments, span, originalParams, { traceConfig: instrumentation._traceConfig });
const params = projectAnthropicMedia(originalParams, captured) as Anthropic.MessageCreateParams;
span.setAttributes({
[SemanticConventions.LLM_INVOCATION_PARAMETERS]: safelyJSONStringify(invocationParameters) ?? "",
...getAnthropicInputMessagesAttributes(params),
...getAnthropicToolsAttributes(params),
[SemanticConventions.RAW_INPUT]: safelyJSONStringify(params) ?? "",
},
});

const execContext = getExecContext(span);
Expand Down
Loading
Loading