Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions .github/workflows/coding-agent-attachments-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
name: Coding-agent attachments
on:
pull_request:
paths:
- 'typescript/packages/fi-core/src/attachments/**'
- 'typescript/packages/traceai_agent_core/**'
- 'typescript/packages/traceai_pi_plugin/**'
- 'typescript/packages/traceai_oh_my_pi_plugin/**'
- 'typescript/packages/traceai_claude_code/**'
- 'typescript/packages/traceai_cursor_sdk/src/**'
- '.github/workflows/coding-agent-attachments-tests.yml'
workflow_dispatch:
permissions:
contents: read
jobs:
attachments:
name: Coding-agent attachments (Node ${{ matrix.node }})
runs-on: ubuntu-24.04
timeout-minutes: 25
strategy:
fail-fast: false
matrix:
# 22.19.0 is the Pi plugin engines floor.
node: ['22.19.0', '24']
defaults:
run:
working-directory: typescript
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node }}
- uses: pnpm/action-setup@v4
with:
version: 10.30.1
- name: Frozen filtered install
run: >-
pnpm install --frozen-lockfile --ignore-scripts
--filter '@traceai/agent-core...' --filter '@traceai/pi-plugin...'
--filter '@traceai/oh-my-pi-plugin...' --filter '@traceai/claude-code...'
--filter '@traceai/cursor-sdk...'
- name: Build
run: |
for p in fi-semantic-conventions fi-core agent-core pi-coding-agent pi-plugin oh-my-pi-plugin claude-code cursor-sdk; do
pnpm --filter "@traceai/$p" run build
done
- name: Package suites
run: |
pnpm --filter @traceai/agent-core run test
pnpm --filter @traceai/pi-plugin run test
pnpm --filter @traceai/oh-my-pi-plugin run test
pnpm --filter @traceai/claude-code run test
pnpm --filter @traceai/cursor-sdk exec jest --runInBand
- name: Real HTTPS attachment wire with real Pi host (floor) and OMP plugin API
run: |
mkdir -p "$RUNNER_TEMP/att-tls"
openssl req -x509 -newkey rsa:2048 -nodes -keyout "$RUNNER_TEMP/att-tls/key.pem" -out "$RUNNER_TEMP/att-tls/cert.pem" \
-days 2 -subj '/CN=127.0.0.1' -addext 'subjectAltName=IP:127.0.0.1,DNS:localhost' > /dev/null 2>&1
export NODE_EXTRA_CA_CERTS="$RUNNER_TEMP/att-tls/cert.pem" ATTACHMENT_TEST_KEY="$RUNNER_TEMP/att-tls/key.pem"
node packages/traceai_pi_plugin/test/attachments-native.cjs
node packages/traceai_oh_my_pi_plugin/test/attachments-wire.cjs
- name: Real Pi host (latest)
run: |
npm install --prefix "$RUNNER_TEMP/pi-latest" --ignore-scripts --no-audit --no-fund @earendil-works/pi-coding-agent@latest
export NODE_EXTRA_CA_CERTS="$RUNNER_TEMP/att-tls/cert.pem" ATTACHMENT_TEST_KEY="$RUNNER_TEMP/att-tls/key.pem"
PI_TEST_BINARY="$RUNNER_TEMP/pi-latest/node_modules/.bin/pi" node packages/traceai_pi_plugin/test/attachments-native.cjs
102 changes: 102 additions & 0 deletions typescript/packages/traceai_agent_core/src/attachment-runtime.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
import {
ATTACHMENT_TRANSPORT_V1,
AttachmentTransportError,
attachmentNodeHttpClient,
createAttachmentHttpTransportV1,
type NodeAttachmentManagerOptions,
} from "@traceai/fi-core/attachments/node";
import type { AttachmentTransport, AttachmentTransportV1 } from "@traceai/fi-core/attachments";
import type { ResolvedConfig } from "./config.js";
import { collectionEnabled } from "./collection.js";
import { approvedAttachmentEndpoint } from "./attachment-status.js";

/** The versioned surface of a caller transport: V1 itself, or the V1 member a legacy-shaped HTTP transport carries. */
function versionedTransport(transport: AttachmentTransport | AttachmentTransportV1): AttachmentTransportV1 | undefined {
const carried = (transport as { [ATTACHMENT_TRANSPORT_V1]?: AttachmentTransportV1 })[ATTACHMENT_TRANSPORT_V1];
if (carried && typeof carried.putContent === "function") return carried;
return typeof (transport as AttachmentTransportV1).putContent === "function" ? transport as AttachmentTransportV1 : undefined;
}

/** Every wire call re-checks the provider's collection policy and route revalidation first. */
function guardV1(delegate: AttachmentTransportV1, check: () => Promise<void>): AttachmentTransportV1 {
return {
async bootstrap(body, key, signal) { await check(); return delegate.bootstrap(body, key, signal); },
async renewSession(current, sequence, signal) { await check(); return delegate.renewSession(current, sequence, signal); },
async initialize(session, body, key, signal) { await check(); return delegate.initialize(session, body, key, signal); },
async renewUpload(session, upload, body, signal) { await check(); return delegate.renewUpload(session, upload, body, signal); },
async putContent(session, upload, source, signal, pacer) { await check(); return delegate.putContent(session, upload, source, signal, pacer); },
async complete(session, upload, key, signal) { await check(); return delegate.complete(session, upload, key, signal); },
async abort(session, upload, key, signal) { await check(); return delegate.abort(session, upload, key, signal); },
async uploadStatus(session, upload, signal) { await check(); return delegate.uploadStatus(session, upload, signal); },
async putManifest(session, capability, recordingId, body, key, signal) {
await check(); return delegate.putManifest(session, capability, recordingId, body, key, signal);
},
async submitImport(session, body, key, signal) { await check(); return delegate.submitImport(session, body, key, signal); },
async importStatus(session, jobId, signal) { await check(); return delegate.importStatus(session, jobId, signal); },
};
}

/**
* A caller-supplied four-method transport stays four-method, so fi-core still marks it legacy and emits
* FI_ATTACHMENTS_LEGACY_TRANSPORT once (no renewal, recordings or imports). Pass an AttachmentTransportV1 instead.
*/
function guardLegacy(delegate: AttachmentTransport, check: () => Promise<void>): AttachmentTransport {
return {
async bootstrap(request, signal) { await check(); return delegate.bootstrap(request, signal); },
async initialize(session, request, signal) { await check(); return delegate.initialize(session, request, signal); },
async upload(session, upload, data, signal) { await check(); return delegate.upload(session, upload, data, signal); },
async complete(session, upload, request, signal) { await check(); return delegate.complete(session, upload, request, signal); },
};
}

/** One provider-owned runtime with the same credential and revocation boundary as OTLP. */
export function agentAttachmentRuntime(
config: ResolvedConfig,
options: NodeAttachmentManagerOptions = {},
): NodeAttachmentManagerOptions {
// Neither the collector URL nor provider/tool content is a control-plane origin, and the
// credential goes only to the approved destination's origin (destination approval binds a
// saved credential to that origin). Any other endpoint stays unconfigured, with no headers.
const endpoint = approvedAttachmentEndpoint(config, options.endpoint ?? process.env.FI_ATTACHMENTS_ENDPOINT);
const headers: Readonly<Record<string, string>> = config.credentials && endpoint ? {
"x-api-key": config.credentials.apiKey,
"x-secret-key": config.credentials.secretKey,
} : {};
const allowed = () => collectionEnabled(config) && config.captureContent &&
config.destinationApproved && !!config.credentials;
const check = async () => {
if (!allowed() || (config.revalidate && !(await config.revalidate())))
throw new AttachmentTransportError("policy_denied");
};
let delegate: AttachmentTransportV1 | undefined;
let legacy: AttachmentTransport | undefined;
if (options.transport) {
delegate = versionedTransport(options.transport);
if (!delegate) legacy = options.transport as AttachmentTransport;
} else if (endpoint && allowed()) {
try { delegate = createAttachmentHttpTransportV1({ endpoint, headers, client: attachmentNodeHttpClient }); }
catch { /* Unsupported/untrusted control origin remains unconfigured. */ }
}
// Always hand the manager a transport, so fi-core never builds its own from
// FI_ATTACHMENTS_ENDPOINT. Without an approved endpoint every call is endpoint_unconfigured.
// The stub is V1-shaped so an unapproved endpoint is not reported as a legacy transport.
const unconfigured = async (): Promise<never> => { throw new AttachmentTransportError("endpoint_unconfigured"); };
const transport = legacy ? guardLegacy(legacy, check) : guardV1(delegate ?? {
bootstrap: unconfigured, renewSession: unconfigured, initialize: unconfigured, renewUpload: unconfigured,
putContent: unconfigured, complete: unconfigured, abort: unconfigured, uploadStatus: unconfigured,
putManifest: unconfigured, submitImport: unconfigured, importStatus: unconfigured,
}, check);
return {
...options, endpoint, headers, transport,
enabled: allowed() ? options.enabled : false,
projectName: config.projectName,
projectType: "observe",
environment: () => allowed() ? process.env : { FI_ATTACHMENTS_ENABLED: "false" },
traceConfig: {
...options.traceConfig,
hideInputs: config.hideInputs || options.traceConfig?.hideInputs,
hideOutputs: config.hideOutputs || options.traceConfig?.hideOutputs,
hideErrorDetails: config.hideErrorDetails || options.traceConfig?.hideErrorDetails,
},
};
}
74 changes: 74 additions & 0 deletions typescript/packages/traceai_agent_core/src/attachment-status.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import type { ResolvedConfig } from "./config.js";

export type AgentAttachmentSource = "typed-image-blocks" | "source_unavailable";

/**
* Per-host media source readiness, keyed by adapter id. Separate from span support,
* server authorization and product playback. A plugin, hook or native OTel feed is
* not screenshot access: only hosts that hand the integration inline typed image
* bytes are capturable with the current Bytes/Blob attachment core.
*/
export const AGENT_ATTACHMENT_SOURCES: Readonly<Record<string, AgentAttachmentSource>> = Object.freeze({
pi: "typed-image-blocks",
"oh-my-pi": "typed-image-blocks",
claude: "typed-image-blocks",
codex: "source_unavailable",
cursor: "source_unavailable",
"cursor-cli": "source_unavailable",
"cursor-cloud": "source_unavailable",
opencode: "source_unavailable",
"opencode-v1": "source_unavailable",
openclaw: "source_unavailable",
hermes: "source_unavailable",
goose: "source_unavailable",
gemini: "source_unavailable",
copilot: "source_unavailable",
"copilot-vscode": "source_unavailable",
"kiro-cli": "source_unavailable",
"kiro-ide": "source_unavailable",
antigravity: "source_unavailable",
"antigravity-ide": "source_unavailable",
augment: "source_unavailable",
"augment-ide": "source_unavailable",
"devin-cascade": "source_unavailable",
grok: "source_unavailable",
muse: "source_unavailable",
});

/**
* The attachment control-plane endpoint, only when its origin is the approved OTLP
* destination's origin. Destination approval binds a saved credential to that origin, so an
* endpoint from elsewhere (env, flags or options) never receives the credential.
*/
export function approvedAttachmentEndpoint(
config: Pick<ResolvedConfig, "endpoint">,
candidate: string | undefined,
): string | undefined {
if (!candidate || !config.endpoint) return undefined;
try {
return new URL(candidate).origin === new URL(config.endpoint).origin ? candidate : undefined;
} catch {
return undefined;
}
}

/** Local inspection does not mint a capability or confuse OTLP acceptance with attachment scope. */
export function agentAttachmentStatus(config: ResolvedConfig, env: NodeJS.ProcessEnv = process.env) {
const source: AgentAttachmentSource = Object.prototype.hasOwnProperty.call(AGENT_ATTACHMENT_SOURCES, config.agent)
? AGENT_ATTACHMENT_SOURCES[config.agent]
: "source_unavailable";
const requested = /^(true|1|yes)$/i.test(env.FI_ATTACHMENTS_ENABLED?.trim() ?? "");
return {
state: !requested ? "disabled" :
!config.enabled || !config.captureContent || (config.hideInputs && config.hideOutputs) ? "policy_denied" :
source === "source_unavailable" ? "source_unavailable" :
!approvedAttachmentEndpoint(config, env.FI_ATTACHMENTS_ENDPOINT) ? "endpoint_unconfigured" : "scope_unverified",
source,
credentialScope: "unverified-runtime-bootstrap-required",
productPlayback: "unverified",
recording: "not-created",
maxImageBytes: 256 * 1024,
localFile: "source_unavailable",
remoteUrl: "source_unavailable",
} as const;
}
128 changes: 128 additions & 0 deletions typescript/packages/traceai_agent_core/src/attachments.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
import { randomUUID } from "node:crypto";
import type { TracerProvider } from "@opentelemetry/api";
import type {
AttachmentAdmission,
AttachmentAdmissionTicket,
AttachmentHandle,
AttachmentManager,
AttachmentOptions,
AttachmentSpan,
} from "@traceai/fi-core/attachments";

/** A bounded callback budget, not a claim that every provider image is capturable. */
export const MAX_AGENT_IMAGE_BYTES = 256 * 1024;
const IMAGE_MIMES = new Set(["image/png", "image/jpeg", "image/webp"]);
// Identity-only replay guard. Distinct parts/positions/spans keep distinct occurrences.
const admitted = new WeakMap<AttachmentManager, WeakMap<object, WeakMap<object, Set<number>>>>();
export type AgentImageOptions = Omit<AttachmentOptions, "mimeType" | "partIndex">;

/** Reuse the configured provider's manager. Never create another uploader or inspect credentials. */
export function providerAttachments(provider: TracerProvider): AttachmentManager | undefined {
const candidate = provider as TracerProvider & {
attachments?: AttachmentManager;
getDelegate?: () => TracerProvider;
};
return candidate.attachments ?? (candidate.getDelegate?.() as typeof candidate | undefined)?.attachments;
}

/**
* Exact decoded length of strict, padded base64, computed without decoding. Returns "too_large" for a
* string longer than `limit` bytes can encode (never scanned), and undefined for anything malformed.
*/
function strictBase64Length(value: unknown, limit: number): number | "too_large" | undefined {
if (typeof value !== "string" || !value.length || value.length % 4 !== 0) return undefined;
if (value.length > 4 * Math.ceil(Math.max(limit, 0) / 3)) return "too_large";
if (!/^[A-Za-z0-9+/]*={0,2}$/.test(value)) return undefined;
const length = (value.length / 4) * 3 - (value.endsWith("==") ? 2 : value.endsWith("=") ? 1 : 0);
if (length <= 0) return undefined;
return length > limit ? "too_large" : length;
}

/** A terminal handle for a part the adapter refused after admission; the ticket is already released. */
function refused(admission: AttachmentAdmission): AttachmentHandle {
return Object.freeze({ attachmentId: randomUUID(), occurrenceId: randomUUID(), admission,
completion: Promise.resolve(Object.freeze({ status: admission })) });
}
const NO_SOURCE = Object.freeze({ kind: "Bytes" as const, data: new Uint8Array(0) });

/** Typed inline image shapes: Pi/OMP `{type, mimeType, data}` and Anthropic/Claude Code base64 `source`. */
function imageShape(part: unknown): { mimeType: string; read: () => unknown } | undefined {
if (!part || typeof part !== "object") return undefined;
const block = part as { type?: unknown; mimeType?: unknown; data?: unknown; source?: unknown };
if (block.type !== "image") return undefined;
if (block.source === undefined) {
const mimeType = block.mimeType;
return typeof mimeType === "string" && IMAGE_MIMES.has(mimeType) ? { mimeType, read: () => block.data } : undefined;
}
const source = block.source as { type?: unknown; media_type?: unknown; data?: unknown } | null;
// URL/file sources are retrieval, not inline bytes; they stay unsupported.
if (!source || typeof source !== "object" || source.type !== "base64") return undefined;
const mimeType = source.media_type;
return typeof mimeType === "string" && IMAGE_MIMES.has(mimeType) ? { mimeType, read: () => source.data } : undefined;
}

/**
* The typed inline image content boundary shared by coding-agent hosts. Not a recursive object walker.
* Each part is preflighted (deny-wins policy, sampling, correlation, budget) BEFORE its base64 is read;
* only an admitted part is read, strictly validated, decoded and handed to `ticket.attach`. A part the
* per-call budget or the manager's limits refuse is re-preflighted once at its exact decoded length
* (policy already passed, so reading the string length is allowed). No paths/URLs fetched.
* The caller must select the original occurrence, not replayed provider history.
*/
export function captureAgentImages(
manager: AttachmentManager | undefined,
span: AttachmentSpan,
blocks: unknown,
options: AgentImageOptions,
): readonly AttachmentHandle[] {
if (!manager || !Array.isArray(blocks) || manager.status().state !== "ready") return [];
const handles: AttachmentHandle[] = [];
let remaining = MAX_AGENT_IMAGE_BYTES;
let spans = admitted.get(manager);
if (!spans) admitted.set(manager, spans = new WeakMap());
let seen = spans.get(span);
if (!seen) spans.set(span, seen = new WeakMap());
const { direction, purpose, capture, ...fields } = options;
for (let partIndex = 0; partIndex < Math.min(blocks.length, 64); partIndex++) {
let ticket: AttachmentAdmissionTicket | undefined;
try {
const part = blocks[partIndex];
const shape = imageShape(part);
if (!shape || seen.get(part)?.has(partIndex)) continue;
const preflight = (maxBytes: number) => manager.preflight(span, { mimeType: shape.mimeType, direction, purpose, capture, maxBytes });
ticket = preflight(remaining);
let value: unknown;
let read = false;
if (!ticket.admitted && (ticket.reason === "too_large" || ticket.reason === "queue_full")) {
// Budget denials come only after the policy gate passed.
value = shape.read(); read = true;
const exact = strictBase64Length(value, remaining);
if (typeof exact === "number") ticket = preflight(exact);
}
// A denied ticket mints its own terminal handle without touching the source.
if (!ticket.admitted) { handles.push(ticket.attach(NO_SOURCE)); ticket = undefined; continue; }
if (!read) value = shape.read();
const length = strictBase64Length(value, Math.min(remaining, ticket.maxBytes));
if (typeof length !== "number") {
ticket.release(); ticket = undefined;
handles.push(refused(length === "too_large" ? "too_large" : "source_unavailable"));
continue;
}
const result = ticket.attach({ kind: "Bytes", data: Buffer.from(value as string, "base64") }, { ...fields, partIndex });
ticket = undefined;
handles.push(result);
if (result.admission === "queued") {
// Only bytes the core actually owns count against the per-call budget.
remaining -= length;
let positions = seen.get(part);
if (!positions) seen.set(part, positions = new Set());
positions.add(partIndex);
}
} catch {
// Host data, including throwing accessors, must not affect inference or diagnostics.
// A part that was preflighted reports why; a malformed block before preflight is skipped.
if (ticket) { ticket.release(); handles.push(refused("source_unavailable")); }
}
}
return handles;
}
2 changes: 2 additions & 0 deletions typescript/packages/traceai_agent_core/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,6 @@ export * from "./lifecycle.js";
export * from "./delivery.js";
export * from "./tracer.js";
export * from "./capture.js";
export * from "./attachments.js";
export * from "./attachment-status.js";
export { VERSION } from "./version.js";
Loading
Loading