Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 96 additions & 0 deletions .github/workflows/framework-attachments-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
name: TypeScript framework attachments
on:
pull_request:
paths:
- 'typescript/packages/framework_attachments/**'
- 'typescript/packages/traceai_langchain/**'
- 'typescript/packages/traceai_llamaindex/**'
- 'typescript/packages/traceai_vercel/**'
- 'typescript/packages/traceai_genkit/**'
- 'typescript/packages/traceai_mastra/**'
- 'typescript/packages/fi-core/src/attachments/**'
- 'typescript/pnpm-lock.yaml'
- '.github/workflows/framework-attachments-tests.yml'
workflow_dispatch:
permissions:
contents: read
jobs:
frameworks:
name: Framework media / Node ${{ matrix.node }}
runs-on: ubuntu-24.04
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
node: ['22', '24']
defaults:
run:
shell: bash
working-directory: typescript
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node }}
- uses: pnpm/action-setup@v4
with:
version: 10.30.1
- name: Install and build the framework packages
run: |
pnpm install --frozen-lockfile --ignore-scripts \
--filter '@traceai/framework-attachments...' --filter '@traceai/langchain...' --filter '@traceai/llamaindex...' \
--filter '@traceai/vercel...' --filter '@traceai/genkit...' --filter '@traceai/mastra...'
git diff --exit-code -- pnpm-lock.yaml
for package in fi-semantic-conventions fi-core framework-attachments langchain llamaindex vercel genkit mastra; do
pnpm --filter "@traceai/$package" run build
done
- name: Shared media normalizer, privacy and lifetime tests
run: pnpm --filter @traceai/framework-attachments test
- name: Real framework invocations at floor and latest versions
run: |
mkdir -p "$RUNNER_TEMP/framework-matrix"
cd "$RUNNER_TEMP/framework-matrix"
printf '{"private":true}' > package.json
pnpm add --ignore-scripts --config.auto-install-peers=false --save-exact \
langchain-floor@npm:@langchain/core@0.2.0 langchain-latest@npm:@langchain/core@latest \
llamaindex-floor@npm:llamaindex@0.1.0 llamaindex-latest@npm:llamaindex@latest \
llamaopenai-floor@npm:@llamaindex/openai@0.3.0 llamaopenai-latest@npm:@llamaindex/openai@latest \
ai-floor@npm:ai@5.0.0 ai-latest@npm:ai@latest \
genkit-floor@npm:genkit@1.42.0 genkit-latest@npm:genkit@latest zod@4.3.6
node -e 'const p=require("./package.json");console.log(JSON.stringify(p.dependencies))'
cd "$GITHUB_WORKSPACE/typescript"
export FRAMEWORK_MATRIX_DIR="$RUNNER_TEMP/framework-matrix"
for framework in langchain llamaindex ai genkit; do
for version in floor latest; do
echo "::group::$framework $version"
timeout 180 node packages/framework_attachments/test/integration.cjs "$framework" "$version"
echo "::endgroup::"
done
done
- name: Existing framework suites
run: |
# LangChain's own workflow runs its runtime suites against a packed
# consumer per core version; it is dispatched for this branch instead.
for package in llamaindex vercel genkit; do
echo "::group::$package"
NODE_OPTIONS=--experimental-vm-modules pnpm --filter "@traceai/$package" exec jest --runInBand --passWithNoTests --testPathIgnorePatterns e2e.test.ts
echo "::endgroup::"
done
pnpm --filter @traceai/mastra exec vitest run
- name: Mastra exporter attachment disposition (packed, real Agent over loopback)
run: |
mkdir -p "$RUNNER_TEMP/mastra-package" "$RUNNER_TEMP/mastra-consumer"
for package in fi-semantic-conventions fi-core framework_attachments traceai_vercel traceai_mastra; do
pnpm --dir "packages/$package" pack --pack-destination "$RUNNER_TEMP/mastra-package"
done
npm install --prefix "$RUNNER_TEMP/mastra-consumer" --ignore-scripts --no-audit --no-fund \
"$RUNNER_TEMP"/mastra-package/*.tgz @mastra/core@latest @mastra/observability@latest @mastra/otel-exporter@latest \
@opentelemetry/api@1.9.0 @opentelemetry/exporter-trace-otlp-proto@0.201.1 @opentelemetry/otlp-transformer@0.201.1 \
@ai-sdk/openai@2.0.0 zod@3.25.76
for entrypoint in src esm; do
MASTRA_TEST_DEPS="$RUNNER_TEMP/mastra-consumer" \
MASTRA_TEST_PACKAGE="$RUNNER_TEMP/mastra-consumer/node_modules/@traceai/mastra/dist/$entrypoint/index.js" \
node --test packages/traceai_mastra/contract/attachments.test.cjs
done
9 changes: 7 additions & 2 deletions .github/workflows/langchain-typescript-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ on:
branches: [main, dev]
paths:
- 'typescript/packages/traceai_langchain/**'
- 'typescript/packages/framework_attachments/**'
- 'typescript/packages/fi-core/**'
- 'typescript/packages/fi-semantic-conventions/**'
- 'typescript/pnpm-lock.yaml'
Expand All @@ -15,6 +16,7 @@ on:
branches: [main, dev]
paths:
- 'typescript/packages/traceai_langchain/**'
- 'typescript/packages/framework_attachments/**'
- 'typescript/packages/fi-core/**'
- 'typescript/packages/fi-semantic-conventions/**'
- 'typescript/pnpm-lock.yaml'
Expand Down Expand Up @@ -58,9 +60,10 @@ jobs:
corepack enable
corepack pnpm --filter '@traceai/langchain...' install --frozen-lockfile --ignore-scripts
corepack pnpm run build:deps
corepack pnpm --filter @traceai/framework-attachments run build
corepack pnpm --filter @traceai/langchain run build
mkdir -p "$RUNNER_TEMP/langchain-tarballs"
for package in fi-semantic-conventions fi-core traceai_langchain; do
for package in fi-semantic-conventions fi-core framework_attachments traceai_langchain; do
corepack pnpm --dir "packages/$package" pack --pack-destination "$RUNNER_TEMP/langchain-tarballs"
done
git diff --exit-code -- pnpm-lock.yaml
Expand Down Expand Up @@ -175,6 +178,7 @@ jobs:
const local = {
"@traceai/fi-core": tarball("traceai-fi-core-"),
"@traceai/fi-semantic-conventions": tarball("traceai-fi-semantic-conventions-"),
"@traceai/framework-attachments": tarball("traceai-framework-attachments-"),
};
fs.writeFileSync(consumer + "/package.json", JSON.stringify({
private: true,
Expand Down Expand Up @@ -209,9 +213,10 @@ jobs:
corepack enable
corepack pnpm --filter '@traceai/langchain...' install --frozen-lockfile --ignore-scripts
corepack pnpm run build:deps
corepack pnpm --filter @traceai/framework-attachments run build
corepack pnpm --filter @traceai/langchain run build
mkdir -p "$RUNNER_TEMP/remote-tarballs"
for package in fi-semantic-conventions fi-core traceai_langchain; do
for package in fi-semantic-conventions fi-core framework_attachments traceai_langchain; do
corepack pnpm --dir "packages/$package" pack --pack-destination "$RUNNER_TEMP/remote-tarballs"
done
git diff --exit-code -- pnpm-lock.yaml
Expand Down
7 changes: 5 additions & 2 deletions .github/workflows/mastra-typescript-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ on:
- 'typescript/packages/fi-core/**'
- 'typescript/packages/fi-semantic-conventions/**'
- 'typescript/packages/traceai_vercel/**'
- 'typescript/packages/framework_attachments/**'
- 'typescript/pnpm-lock.yaml'
- '.github/workflows/mastra-typescript-tests.yml'
push:
Expand All @@ -18,6 +19,7 @@ on:
- 'typescript/packages/fi-semantic-conventions/**'
- 'typescript/pnpm-lock.yaml'
- '.github/workflows/mastra-typescript-tests.yml'
workflow_dispatch:

permissions:
contents: read
Expand Down Expand Up @@ -68,7 +70,7 @@ jobs:
- name: Pack and install in a clean consumer
run: |
mkdir -p "$RUNNER_TEMP/mastra-package" "$RUNNER_TEMP/mastra-consumer"
for package in fi-semantic-conventions fi-core traceai_vercel traceai_mastra; do
for package in fi-semantic-conventions fi-core framework_attachments traceai_vercel traceai_mastra; do
pnpm --dir "packages/$package" pack --pack-destination "$RUNNER_TEMP/mastra-package"
done
npm install --prefix "$RUNNER_TEMP/mastra-consumer" --ignore-scripts --no-audit --no-fund \
Expand All @@ -90,7 +92,8 @@ jobs:
"$PWD/packages/traceai_mastra/contract/http.test.cjs" \
"$PWD/packages/traceai_mastra/contract/tool-errors.test.cjs" \
"$PWD/packages/traceai_mastra/contract/continued-history.test.cjs" \
"$PWD/packages/traceai_mastra/contract/terminal-history.test.cjs"
"$PWD/packages/traceai_mastra/contract/terminal-history.test.cjs" \
"$PWD/packages/traceai_mastra/contract/attachments.test.cjs"
done
sudo unshare --net bash -c 'ip link set lo up; exec "$@"' _ \
"$(command -v node)" "$PWD/packages/traceai_mastra/contract/readme.test.mjs" "$MASTRA_TEST_DEPS"
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/vercel-ai-sdk-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@ on:
branches: [dev, main]
paths:
- 'typescript/packages/traceai_vercel/**'
- 'typescript/packages/framework_attachments/**'
- 'typescript/packages/fi-semantic-conventions/**'
- 'typescript/packages/fi-core/**'
- 'typescript/pnpm-lock.yaml'
- '.github/workflows/vercel-ai-sdk-tests.yml'
workflow_dispatch:

permissions:
contents: read
Expand Down Expand Up @@ -42,6 +44,7 @@ jobs:
# stage the workspace dependencies as tarballs too, as strands-typescript-tests.yml does.
pnpm --dir packages/fi-core pack --pack-destination "$RUNNER_TEMP/package"
pnpm --dir packages/fi-semantic-conventions pack --pack-destination "$RUNNER_TEMP/package"
pnpm --dir packages/framework_attachments pack --pack-destination "$RUNNER_TEMP/package"
- name: Install clean consumer with published dependencies
env:
SDK: ${{ matrix.sdk }}
Expand Down
1 change: 1 addition & 0 deletions .release-please-manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@
"typescript/packages/traceai_cursor_sdk": "0.1.1",
"typescript/packages/traceai_deepseek": "0.1.2",
"typescript/packages/traceai_fireworks": "0.1.2",
"typescript/packages/framework_attachments": "0.1.0",
"typescript/packages/traceai_genkit": "0.1.1",
"typescript/packages/traceai_google_adk": "0.1.2",
"typescript/packages/traceai_google_genai": "0.1.1",
Expand Down
5 changes: 5 additions & 0 deletions release-please-config.json
Original file line number Diff line number Diff line change
Expand Up @@ -689,6 +689,11 @@
"src/version.ts"
]
},
"typescript/packages/framework_attachments": {
"release-type": "node",
"component": "npm-traceai-framework-attachments",
"package-name": "@traceai/framework-attachments"
},
"typescript/packages/traceai_genkit": {
"release-type": "node",
"component": "npm-traceai-genkit",
Expand Down
49 changes: 49 additions & 0 deletions typescript/packages/framework_attachments/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
# Framework attachment adapters

Shared typed-media normalization for traceAI's framework integrations. This package is an adapter dependency, not a separate uploader or recorder.

## Capture boundary

`captureMediaMessages(binding, span, messages, dialect, direction)` observes typed message content. `captureMediaParts(binding, span, parts, dialect, options)` observes a known typed part array. Supported dialects are `langchain`, `llamaindex`, `vercel`, and `genkit`.

Pass the existing provider's `attachments` manager in `binding`. Without that manager, these functions do nothing. The manager still requires explicit attachment enablement, a successful policy bootstrap and a sampled, recording span. Normal inference never waits for attachment network operations. Use the provider's existing flush/shutdown lifecycle to finish admitted transfers.

The adapters accept inline base64/data URIs and `Uint8Array` sources at the documented typed boundaries. They do not fetch remote URLs, open local paths, read streams, reconstruct discarded media from exported text, or create new recordings. Base64 decoding is bounded to one MiB per part. `Uint8Array` parts are not decoded, so only the shared manager bounds them (its upload cap and owned-byte budget, checked at preflight). A single span admits at most 64 positions through this helper; the core manager separately bounds bytes, queue length and reference projection.

## Privacy and identity

- Capture is off unless enabled on the shared manager.
- Directional hide flags, required PII handling, server policy and native content restrictions deny capture before decoding or copying media.
- Each part is sized from its shape (exact decoded length, no decode) and then preflighted with the shared manager (`manager.preflight`), which applies the deny-wins policy and reserves that many bytes. Only an admitted part is decoded and handed to `ticket.attach`. A part that fails strict decoding releases its ticket at once. A part that does not fit the manager's owned-byte budget is denied (`queue_full`) without being decoded.
- Supply the integration's `traceConfig` and `capture` restrictions along with the manager. Binding-level denials cannot be relaxed by an individual part.
- Tool images intended for a model are subject to both output and model-input restrictions.
- No filenames, source URLs, hashes or provider credentials are added to references.
- Repeated observation of the same admitted message/part position on the same span is idempotent. Different message positions remain distinct occurrences. This is not cross-provider deduplication or permission to relink an occurrence to an ancestor span.
- The shared manager owns all transport, write capabilities and occurrence correlation. An upload completing into validation is not proof of authorized playback.

`mediaField` reads own data descriptors only. The traversal is limited to known message/part fields; application getters and `toJSON` methods are not invoked.

## Coverage by framework operation

E means the adapter reads the original typed media value. R means the integration only keeps references that another integration already issued. N means not supported.

| Framework | Operation | Disposition |
| --- | --- | --- |
| LangChain (`@langchain/core` 0.2 to 1.x) | Chat model input messages (`image_url` data URIs; `image`/`audio`/`file` parts with `source_type: "base64"`) | E |
| LangChain | Chat model output generations with the same typed parts | E |
| LangChain | Chain, tool and retriever inputs/outputs | N: arbitrary objects are not scanned |
| LlamaIndex (legacy package) | `chat` input messages and non-streaming chat response (`image_url` data URIs) | E |
| LlamaIndex | Streaming chat output | N |
| Vercel AI SDK 5, 6, 7 | `generateText`/`streamText`/`generateObject`/`streamObject` prompt messages (`image`, `file` parts) | E |
| Vercel AI SDK 7 | Model response `file` content and typed tool `content` results (`image-data`, `file-data`) | E |
| Vercel AI SDK 5, 6 | Model output and tool results | N: the native telemetry path exposes only serialized text |
| Genkit 1.42+ | `defineModel` implementation wrapped with `withGenkitModelAttachments` (request messages and response message) | E |
| Genkit | Any model, including plugin models, through `use: [genkitAttachmentMiddleware(binding)]` on `generate` | E |
| Genkit | `defineTool` implementation returning `Part[]`, wrapped with `withGenkitToolAttachments` | E |
| Mastra (`@traceai/mastra`) | All exporter operations | N for raw capture. The exporter sees spans only after they end, has no manager and never uploads. Newer runtimes keep inline images in span input, but the core has no API to add references to an ended span. The exporter removes `fi.attachment.*` keys it did not issue. |

Streams are never read for media, remote URLs and file paths are never fetched, and creating new recordings is not supported. Every supported framework boundary delivers each media part as one complete inline value (streamed AI SDK 7 files arrive whole in the end event), so no consumed byte stream exists here and the core's `startAttachmentStream` is not used.

## Known limitation: framework plus provider instrumentation

When a framework integration and a provider integration (for example LangChain plus `@traceai/openai`) are both given the same manager, the same image is uploaded once for the framework span and once for the provider span. The current attachment core has no cross-span occurrence claim, so these adapters cannot deduplicate across span boundaries. Give the manager to one layer only until that core API exists.
60 changes: 60 additions & 0 deletions typescript/packages/framework_attachments/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
{
"name": "@traceai/framework-attachments",
"version": "0.1.0",
"description": "Typed framework media adapters for the shared traceAI attachment manager",
"main": "./dist/src/index.js",
"module": "./dist/esm/index.js",
"esnext": "./dist/esnext/index.js",
"types": "./dist/src/index.d.ts",
"exports": {
".": {
"types": "./dist/src/index.d.ts",
"import": "./dist/esm/index.js",
"require": "./dist/src/index.js"
}
},
"scripts": {
"prebuild": "rimraf dist",
"build": "tsc --build tsconfig.json tsconfig.esm.json tsconfig.esnext.json && tsc-alias -p tsconfig.esm.json && pnpm run postbuild",
"postbuild": "echo '{\"type\": \"module\"}' > ./dist/esm/package.json",
"test": "node --test test/*.test.cjs",
"prepare": "pnpm run build"
},
"author": "Future AGI",
"license": "Apache-2.0",
"repository": {
"type": "git",
"url": "git+https://github.com/future-agi/traceAI.git",
"directory": "typescript/packages/framework_attachments"
},
"bugs": {
"url": "https://github.com/future-agi/traceAI/issues"
},
"engines": {
"node": ">=20.0.0"
},
"dependencies": {
"@opentelemetry/api": "^1.9.0",
"@traceai/fi-core": "workspace:^"
},
"devDependencies": {
"@opentelemetry/resources": "^2.0.1",
"@opentelemetry/sdk-trace-base": "^2.0.1",
"@types/jest": "^29.5.14",
"@types/node": "^22.10.10",
"jest": "^29.7.0",
"rimraf": "^6.0.1",
"ts-jest": "^29.2.5",
"tsc-alias": "^1.8.10",
"typescript": "^5.7.3"
},
"files": [
"dist",
"README.md",
"!**/__tests__/**",
"!**/*.tsbuildinfo"
],
"publishConfig": {
"access": "public"
}
}
Loading
Loading