diff --git a/.github/workflows/fi-core-tanstack.yml b/.github/workflows/fi-core-tanstack.yml new file mode 100644 index 000000000..30208073e --- /dev/null +++ b/.github/workflows/fi-core-tanstack.yml @@ -0,0 +1,64 @@ +name: Core TanStack AI tracing + +on: + pull_request: + branches: [main, dev] + paths: + - 'typescript/packages/fi-core/**' + - 'typescript/packages/fi-semantic-conventions/**' + - 'typescript/pnpm-lock.yaml' + - '.github/workflows/fi-core-tanstack.yml' + push: + branches: [dev] + paths: + - 'typescript/packages/fi-core/**' + - 'typescript/packages/fi-semantic-conventions/**' + - 'typescript/pnpm-lock.yaml' + - '.github/workflows/fi-core-tanstack.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + tanstack: + # floor: @tanstack/ai 0.69.0 + OTel SDK 2.0.1; current: 1.0.1 + 2.12.0 + name: TanStack AI native tracing (${{ matrix.mode }}) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + mode: [floor, current] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22.23.4' + - name: Install, build and pack core; install the packed fixture + working-directory: typescript + run: | + mkdir -p "$RUNNER_TEMP/corepack-bin" "$RUNNER_TEMP/packages" + corepack enable --install-directory "$RUNNER_TEMP/corepack-bin" pnpm + export PATH="$RUNNER_TEMP/corepack-bin:$PATH" + echo "$RUNNER_TEMP/corepack-bin" >> "$GITHUB_PATH" + corepack pnpm --filter @traceai/fi-core... install --frozen-lockfile --ignore-scripts + corepack pnpm --filter @traceai/fi-core... run build + corepack pnpm --dir packages/fi-semantic-conventions pack --pack-destination "$RUNNER_TEMP/packages" + corepack pnpm --dir packages/fi-core pack --pack-destination "$RUNNER_TEMP/packages" + node packages/fi-core/test/install-tanstack-fixture.cjs "$RUNNER_TEMP/fixture" '${{ matrix.mode }}' \ + "$RUNNER_TEMP"/packages/traceai-fi-core-*.tgz \ + "$RUNNER_TEMP"/packages/traceai-fi-semantic-conventions-*.tgz + - name: Runtime and OTLP wire tests with loopback-only networking + working-directory: typescript + run: | + sudo unshare --net bash -c ' + set -euo pipefail + ip link set lo up + export TANSTACK_FIXTURE="$1" + node --test packages/fi-core/test/tanstack.test.mjs + node node_modules/jest/bin/jest.js --config packages/fi-core/jest.config.js --runInBand + ' bash "$RUNNER_TEMP/fixture" diff --git a/typescript/packages/fi-core/README.md b/typescript/packages/fi-core/README.md index bf808e92c..7ae06b837 100644 --- a/typescript/packages/fi-core/README.md +++ b/typescript/packages/fi-core/README.md @@ -187,6 +187,34 @@ span.setAttributes({ span.end(); ``` +## TanStack AI + +TanStack AI creates the spans with its own `otelMiddleware`. The `@traceai/fi-core/tanstack` subpath supplies the options for that middleware and an isolated Future AGI provider, so you don't add a second instrumentation: + +```ts +import { chat } from "@tanstack/ai"; +import { otelMiddleware } from "@tanstack/ai/middlewares/otel"; +import { flushTraces, registerTanStackTracing, tanStackOtelOptions } from "@traceai/fi-core/tanstack"; + +const provider = registerTanStackTracing({ projectName: "my-chatbot" }); +const tracer = provider.getTracer("tanstack-ai"); + +const stream = chat({ + adapter, + messages, + threadId, // optional: an opaque conversation id + middleware: [otelMiddleware(tanStackOtelOptions(tracer, { threadIdAsSession: Boolean(threadId) }))], +}); +``` + +- **Privacy.** Messages, tool arguments and results, error text and stack traces are not exported by default. `captureContent: true` exports content and error text through your `redact` function; a redactor that throws yields `[redaction_failed]`. Stack traces are never exported. +- **Tokens are counted once.** TanStack also writes the sum of every model call's usage onto the chat span. In the Future AGI export only, that sum moves to `tanstack.ai.root_usage.*` when the model-call spans carry usage. Other exporters on the same provider receive TanStack's spans with the Future AGI enrichment attributes, and the root keeps its usage there. +- **Sessions.** `threadIdAsSession: true` exports the `threadId` you pass to `chat()` as `session.id`. Use an opaque id, never an email address or user name. +- **Existing OpenTelemetry apps.** `registerTanStackTracing()` never replaces the global provider, and each chat starts a new trace unless you pass `joinActiveTrace: true`. To export through a provider you already own instead, wrap your exporter: `new TransformingSpanExporter(exporter, normalizeTanStackSpan)`. +- **Serverless.** `await flushTraces(provider)` waits at most 2 seconds and never throws. In a long-lived server, call `shutdownTraces(provider)` when the process stops. + +To test against packed builds (floor `@tanstack/ai` 0.69.0 with OpenTelemetry SDK 2.0.1, and current 1.0.1 with SDK 2.12.0), prepare a fixture with `node test/install-tanstack-fixture.cjs DIR floor|current CORE_TGZ SEMCONV_TGZ`, then run `TANSTACK_FIXTURE=DIR pnpm test:tanstack`. + ## Development ### Building diff --git a/typescript/packages/fi-core/package.json b/typescript/packages/fi-core/package.json index b42cbcc6f..d790a954b 100644 --- a/typescript/packages/fi-core/package.json +++ b/typescript/packages/fi-core/package.json @@ -11,6 +11,11 @@ "types": "./dist/src/index.d.ts", "import": "./dist/esm/index.js", "require": "./dist/src/index.js" + }, + "./tanstack": { + "types": "./dist/src/tanstack.d.ts", + "import": "./dist/esm/tanstack.js", + "require": "./dist/src/tanstack.js" } }, "scripts": { @@ -21,7 +26,8 @@ "test:watch": "jest --watch", "test:coverage": "jest --coverage", "lint": "eslint . --ext .ts", - "prepare": "pnpm run build" + "prepare": "pnpm run build", + "test:tanstack": "node --test test/tanstack.test.mjs" }, "files": [ "dist", diff --git a/typescript/packages/fi-core/src/index.ts b/typescript/packages/fi-core/src/index.ts index aec1553fd..79d2b4419 100644 --- a/typescript/packages/fi-core/src/index.ts +++ b/typescript/packages/fi-core/src/index.ts @@ -3,3 +3,4 @@ export * from "./utils"; export type * from "./utils/types"; export * from "./otel"; export * from "./fi_types"; +export * from "./spanTransform"; diff --git a/typescript/packages/fi-core/src/otel.ts b/typescript/packages/fi-core/src/otel.ts index 663510998..e7c47a779 100644 --- a/typescript/packages/fi-core/src/otel.ts +++ b/typescript/packages/fi-core/src/otel.ts @@ -8,6 +8,7 @@ import { SpanStatusCode, } from "@opentelemetry/api"; import { ProjectType, EvalTag, prepareEvalTags } from "./fi_types"; +import { SpanTransform, TransformingSpanExporter } from "./spanTransform"; import { ExportResult, ExportResultCode } from "@opentelemetry/core"; import { ReadableSpan, @@ -345,6 +346,7 @@ interface FITracerProviderOptions { endpoint?: string; // Custom endpoint (can be base or full, _constructFullEndpoint will handle) headers?: FIHeaders; transport?: Transport; // Transport type + spanTransform?: SpanTransform; } class FITracerProvider extends BasicTracerProvider { @@ -382,6 +384,7 @@ class FITracerProvider extends BasicTracerProvider { exporter = new _OTLPHTTPTraceExporter({ url: endpoint, headers }); } + if (config.spanTransform) exporter = new TransformingSpanExporter(exporter, config.spanTransform); const defaultProcessor = new OTelSimpleSpanProcessor(exporter); super({ resource: config.resource, idGenerator, spanProcessors: [defaultProcessor] }); this.defaultProcessorAttached = true; @@ -472,6 +475,8 @@ export interface RegisterOptions { endpoint?: string; // Can be a base URL or a full URL. _constructFullEndpoint will resolve. idGenerator?: IdGenerator; transport?: Transport; // Transport type: HTTP or GRPC + /** Transform the export view without mutating SDK spans. Failures drop the batch. */ + spanTransform?: SpanTransform; } function register(options: RegisterOptions = {}): FITracerProvider { @@ -578,6 +583,7 @@ if (!projectName) { endpoint: optEndpoint, headers: exporterHeaders, transport, + spanTransform: options.spanTransform, }); if (batch) { @@ -594,6 +600,7 @@ if (!projectName) { headers: exporterHeaders, }); } + if (options.spanTransform) batchExporter = new TransformingSpanExporter(batchExporter, options.spanTransform); const batchProcessor = new OTelBatchSpanProcessor(batchExporter); (tracerProvider as any)._registeredSpanProcessors = []; diff --git a/typescript/packages/fi-core/src/spanTransform.ts b/typescript/packages/fi-core/src/spanTransform.ts new file mode 100644 index 000000000..dbf94e437 --- /dev/null +++ b/typescript/packages/fi-core/src/spanTransform.ts @@ -0,0 +1,61 @@ +import { ExportResult, ExportResultCode } from "@opentelemetry/core"; +import { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base"; + +/** Return an export view; never mutate the SDK's ReadableSpan or its attributes. */ +export type SpanTransform = (span: ReadableSpan) => ReadableSpan; + +/** Compose a normalization/privacy boundary with any standard OTel exporter. */ +export class TransformingSpanExporter implements SpanExporter { + constructor( + private readonly exporter: SpanExporter, + private readonly transform: SpanTransform, + ) {} + + export(spans: ReadableSpan[], callback: (result: ExportResult) => void): void { + let transformed: ReadableSpan[]; + try { + transformed = spans.map(this.transform); + } catch { + // A failed privacy transform must not fall back to the original payload. + callback({ code: ExportResultCode.FAILED, error: new Error("Span transformation failed") }); + return; + } + this.exporter.export(transformed, callback); + } + + forceFlush(): Promise { + return this.exporter.forceFlush?.() ?? Promise.resolve(); + } + + shutdown(): Promise { + return this.exporter.shutdown(); + } +} + +/** Preserve SDK getters and method receivers without changing its span. */ +export function spanWithAttributes( + span: ReadableSpan, + attributes: ReadableSpan["attributes"], +): ReadableSpan { + const descriptor = Object.getOwnPropertyDescriptor(span, "attributes"); + if (descriptor && !descriptor.configurable && !descriptor.writable && !descriptor.set) { + // A frozen span cannot be proxied with different attributes; copy its + // public ReadableSpan fields instead. + return { + name: span.name, kind: span.kind, spanContext: () => span.spanContext(), + parentSpanContext: span.parentSpanContext, startTime: span.startTime, + endTime: span.endTime, status: span.status, attributes, links: span.links, + events: span.events, duration: span.duration, ended: span.ended, + resource: span.resource, instrumentationScope: span.instrumentationScope, + droppedAttributesCount: span.droppedAttributesCount, + droppedEventsCount: span.droppedEventsCount, droppedLinksCount: span.droppedLinksCount, + }; + } + return new Proxy(span, { + get(target, property) { + if (property === "attributes") return attributes; + const value = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); +} diff --git a/typescript/packages/fi-core/src/tanstack.ts b/typescript/packages/fi-core/src/tanstack.ts new file mode 100644 index 000000000..a1eae7b3e --- /dev/null +++ b/typescript/packages/fi-core/src/tanstack.ts @@ -0,0 +1,253 @@ +import { + AttributeValue, Context, Span, SpanOptions, SpanStatusCode, Tracer, diag, +} from "@opentelemetry/api"; +import { ReadableSpan } from "@opentelemetry/sdk-trace-base"; +import { ProjectType } from "./fi_types"; +import { register, RegisterOptions } from "./otel"; +import { spanWithAttributes } from "./spanTransform"; + +export const GEN_AI_SPAN_KIND = "gen_ai.span.kind"; +export const SESSION_ID = "session.id"; +export const GEN_AI_PROVIDER_NAME = "gen_ai.provider.name"; +export const ERROR_TYPE = "error.type"; +export const FLUSH_TIMEOUT_MS = 2000; +const AGGREGATE_USAGE = "traceai.tanstack.aggregate_usage"; +const REDACTION_FAILED = "[redaction_failed]"; +const providers: Readonly> = { + openai: "openai", "openai-chat": "openai", anthropic: "anthropic", + gemini: "google", "gemini-text-interactions": "google", grok: "xai", + groq: "groq", ollama: "ollama", +}; + +/** Public subset of TanStack's span hook metadata; no runtime framework dependency. */ +export interface TanStackSpanInfo { + kind: "chat" | "iteration" | "tool" | "generation"; + ctx: { provider: string; threadId?: string }; + iteration?: number; +} + +export interface TanStackTracingOptions { + captureContent?: boolean; + redact?: (text: string) => string; + /** Use only when chat() receives a caller-owned, opaque threadId. */ + threadIdAsSession?: boolean; + /** Enable only when the active application's spans also go to Future AGI. */ + joinActiveTrace?: boolean; + maxContentLength?: number; +} + +/** Options accepted by TanStack AI's public otelMiddleware(options). */ +export interface TanStackOtelOptions { + tracer: Tracer; + captureContent: boolean; + redact?: (text: string) => string; + maxContentLength?: number; + onBeforeSpanStart(info: TanStackSpanInfo, options: SpanOptions): SpanOptions; + attributeEnricher(info: TanStackSpanInfo): Record; + onSpanEnd(info: TanStackSpanInfo, span: Span): void; +} + +function promoted(key: string): boolean { + return key.startsWith("gen_ai.usage.") || key.startsWith("gen_ai.cost."); +} + +/** Export-only normalization. Original spans remain intact for other exporters. */ +export function normalizeTanStackSpan(span: ReadableSpan): ReadableSpan { + if (!Object.prototype.hasOwnProperty.call(span.attributes, AGGREGATE_USAGE)) return span; + const attributes = { ...span.attributes }; + const aggregate = attributes[AGGREGATE_USAGE] === true; + delete attributes[AGGREGATE_USAGE]; + if (aggregate) { + for (const key of Object.keys(attributes)) { + if (!promoted(key)) continue; + const suffix = key.startsWith("gen_ai.usage.") + ? key.slice("gen_ai.usage.".length) : key.slice("gen_ai.".length); + attributes[`tanstack.ai.root_usage.${suffix}`] = attributes[key]; + delete attributes[key]; + } + } + return spanWithAttributes(span, attributes); +} + +function field(value: unknown, name: string): unknown { + return value !== null && typeof value === "object" ? (value as Record)[name] : undefined; +} + +/** A provider error code, else the error class name. Never the message. */ +export function errorType(error: unknown): string { + const code = field(error, "code"); + if ((typeof code === "string" && code) || (typeof code === "number" && Number.isFinite(code))) return String(code); + const name = field(error, "name"); + return typeof name === "string" && name ? name : "Error"; +} + +function safeRedactor(redact: (text: string) => string): (text: string) => string { + return text => { + try { + const result = redact(text); + return typeof result === "string" ? result : REDACTION_FAILED; + } catch { return REDACTION_FAILED; } + }; +} + +/** + * Build the options to pass to TanStack AI's own otelMiddleware. + * + * TanStack remains the only span source. The returned tracer is a public-API + * facade that keeps error text/stack traces out by default and observes usage; + * the returned hooks add Future AGI kinds, providers and opt-in sessions. Root + * usage is marked here and moved only in the Future AGI export view by + * normalizeTanStackSpan, so no SDK internals are mutated. + */ +export function tanStackOtelOptions( + tracer: Tracer, + { + captureContent = false, redact, threadIdAsSession = false, + joinActiveTrace = false, maxContentLength, + }: TanStackTracingOptions = {}, +): TanStackOtelOptions { + const safeRedact = safeRedactor(redact ?? (text => text)); + const usageSpans = new WeakSet(); + const iterations = new WeakMap(); + const usageOnIterations = new WeakSet(); + + const wrap = (span: Span): Span => { + let lastType: string | undefined; + const noteUsage = (key: string, value: unknown): void => { + if (promoted(key) && typeof value === "number" && Number.isFinite(value)) usageSpans.add(proxy); + }; + const proxy: Span = new Proxy(span, { + get(target, property) { + if (property === "setAttribute") return (key: string, value: AttributeValue) => { + target.setAttribute(key, value); + noteUsage(key, value); + return proxy; + }; + if (property === "setAttributes") return (attributes: Record) => { + target.setAttributes(attributes); + for (const [key, value] of Object.entries(attributes ?? {})) noteUsage(key, value); + return proxy; + }; + if (property === "recordException") return (exception: unknown, time?: Parameters[1]) => { + lastType = errorType(exception); + target.setAttribute(ERROR_TYPE, lastType); + const message = typeof exception === "string" ? exception : field(exception, "message"); + target.recordException({ + name: lastType, + ...(captureContent && typeof message === "string" ? { message: safeRedact(message) } : {}), + }, time); + }; + if (property === "setStatus") return (status: Parameters[0]) => { + // TanStack's fixed "cancelled" status carries no user/provider text. + target.setStatus(status.code === SpanStatusCode.ERROR && typeof status.message === "string" && status.message !== "cancelled" + ? { code: status.code, message: captureContent ? safeRedact(status.message) : (lastType ?? "Error") } + : status); + return proxy; + }; + const value = Reflect.get(target, property, target); + return typeof value === "function" ? (...args: unknown[]) => { + const result = value.apply(target, args); + return result === target ? proxy : result; + } : value; + }, + }); + return proxy; + }; + + function startActiveSpan unknown>( + name: string, arg2: SpanOptions | F, arg3?: Context | F, arg4?: F, + ): ReturnType { + if (typeof arg2 === "function") { + return tracer.startActiveSpan<(span: Span) => unknown>(name, span => arg2(wrap(span))) as ReturnType; + } + if (typeof arg3 === "function") { + return tracer.startActiveSpan<(span: Span) => unknown>(name, arg2, span => arg3(wrap(span))) as ReturnType; + } + return tracer.startActiveSpan<(span: Span) => unknown>(name, arg2, arg3 as Context, span => arg4!(wrap(span))) as ReturnType; + } + + return { + tracer: { startSpan: (name, options, context) => wrap(tracer.startSpan(name, options, context)), startActiveSpan }, + captureContent, + ...(redact ? { redact: safeRedact } : {}), + ...(maxContentLength !== undefined ? { maxContentLength } : {}), + onBeforeSpanStart(info, options) { + if (info.kind !== "chat") return options; + return { + ...options, + ...(!joinActiveTrace ? { root: true } : {}), + // Reserve the marker and kind at creation so end-time updates never + // need a new attribute slot under the SDK's attribute-count limit. + attributes: { [AGGREGATE_USAGE]: false, [GEN_AI_SPAN_KIND]: "CHAIN", ...options.attributes }, + }; + }, + attributeEnricher(info) { + const attributes: Record = {}; + if (info.kind === "iteration") { + iterations.set(info.ctx, (info.iteration ?? 0) + 1); + attributes[GEN_AI_SPAN_KIND] = "LLM"; + } else if (info.kind === "tool") { + attributes[GEN_AI_SPAN_KIND] = "TOOL"; + } + if (info.kind === "chat" || info.kind === "iteration") { + const provider = Object.prototype.hasOwnProperty.call(providers, info.ctx.provider) + ? providers[info.ctx.provider] : undefined; + if (provider) attributes[GEN_AI_PROVIDER_NAME] = provider; + } + if (threadIdAsSession && info.ctx.threadId) attributes[SESSION_ID] = info.ctx.threadId; + return attributes; + }, + onSpanEnd(info, span) { + if (info.kind === "iteration" && usageSpans.has(span)) usageOnIterations.add(info.ctx); + if (info.kind !== "chat") return; + span.setAttribute(GEN_AI_SPAN_KIND, (iterations.get(info.ctx) ?? 0) > 1 ? "AGENT" : "CHAIN"); + // The root copy is TanStack's sum of the model-call spans. Keep it only + // when it is the sole copy (no model-call span reported usage). + span.setAttribute(AGGREGATE_USAGE, usageOnIterations.has(info.ctx)); + iterations.delete(info.ctx); + usageOnIterations.delete(info.ctx); + }, + }; +} + +export type TanStackRegisterOptions = Omit< + RegisterOptions, "spanTransform" | "setGlobalTracerProvider" | "projectType" | "batch" +>; + +/** + * Register an isolated Future AGI provider for TanStack AI. It never replaces + * the application's global provider, and normalizes only its own export view. + */ +export function registerTanStackTracing(options: TanStackRegisterOptions = {}) { + return register({ + ...options, + projectType: ProjectType.OBSERVE, + setGlobalTracerProvider: false, + spanTransform: normalizeTanStackSpan, + }); +} + +/** Bounded best-effort flush for serverless handlers; diagnostics carry no payloads. */ +export async function flushTraces( + provider: { forceFlush(): Promise }, { timeoutMs = FLUSH_TIMEOUT_MS } = {}, +): Promise { + let timer: ReturnType | undefined; + try { + const timeout = new Promise(resolve => { + timer = setTimeout(() => resolve(true), timeoutMs); + (timer as { unref?: () => void }).unref?.(); + }); + const flushed = provider.forceFlush().then(() => false); + if (await Promise.race([flushed, timeout])) { + flushed.catch(() => diag.warn("Future AGI trace flush failed")); + diag.warn("Future AGI trace flush timed out"); + } + } catch { diag.warn("Future AGI trace flush failed"); } + finally { clearTimeout(timer); } +} + +/** Flush and stop the provider at process exit. Never throws. */ +export async function shutdownTraces(provider: { shutdown(): Promise }): Promise { + try { await provider.shutdown(); } + catch { diag.warn("Future AGI trace shutdown failed"); } +} diff --git a/typescript/packages/fi-core/test/install-tanstack-fixture.cjs b/typescript/packages/fi-core/test/install-tanstack-fixture.cjs new file mode 100644 index 000000000..4b4a96623 --- /dev/null +++ b/typescript/packages/fi-core/test/install-tanstack-fixture.cjs @@ -0,0 +1,42 @@ +// Install before entering the loopback-only test sandbox. No model credentials. +const { mkdirSync, writeFileSync } = require('node:fs'); +const { resolve } = require('node:path'); +const { execFileSync } = require('node:child_process'); +const [directory, mode, coreTarball, semconvTarball] = process.argv.slice(2); +if (!directory || !['floor', 'current'].includes(mode) || !coreTarball || !semconvTarball) { + throw new Error('usage: node install-tanstack-fixture.cjs DIRECTORY floor|current CORE_TGZ SEMCONV_TGZ'); +} +const floor = mode === 'floor'; +const sdk = floor ? '2.0.1' : '2.12.0'; +const exporter = floor ? '0.202.0' : '0.223.0'; +mkdirSync(directory, { recursive: true }); +writeFileSync(resolve(directory, 'package.json'), JSON.stringify({ + private: true, + type: 'module', + dependencies: { + '@traceai/fi-core': `file:${resolve(coreTarball)}`, + '@traceai/fi-semantic-conventions': `file:${resolve(semconvTarball)}`, + '@tanstack/ai': floor ? '0.69.0' : '1.0.1', + '@tanstack/ai-openai': floor ? '0.30.0' : '1.0.1', + '@opentelemetry/api': '1.9.1', + '@opentelemetry/sdk-trace-base': sdk, + '@opentelemetry/resources': sdk, + '@opentelemetry/exporter-trace-otlp-http': exporter, + '@opentelemetry/otlp-transformer': exporter, + 'typescript': '5.8.3', + '@types/node': '22.15.29', + }, + overrides: { + '@opentelemetry/sdk-trace-base': sdk, + '@opentelemetry/sdk-trace-node': sdk, + '@opentelemetry/context-async-hooks': sdk, + '@opentelemetry/core': sdk, + '@opentelemetry/resources': sdk, + '@opentelemetry/exporter-trace-otlp-http': exporter, + '@opentelemetry/exporter-trace-otlp-grpc': exporter, + ...(floor ? { '@tanstack/openai-base': '0.14.0' } : {}), + }, +}, null, 2)); +// Resolve the subpath by package name, exactly as an ESM application does. +writeFileSync(resolve(directory, 'esm-import.mjs'), "export * from '@traceai/fi-core/tanstack';\n"); +execFileSync('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund'], { cwd: directory, stdio: 'inherit' }); diff --git a/typescript/packages/fi-core/test/tanstack.test.mjs b/typescript/packages/fi-core/test/tanstack.test.mjs new file mode 100644 index 000000000..0993f737c --- /dev/null +++ b/typescript/packages/fi-core/test/tanstack.test.mjs @@ -0,0 +1,338 @@ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import { createRequire } from 'node:module'; +import { resolve, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { test } from 'node:test'; + +// Each fixture is a clean install of the exact packed core and registry SDKs: +// node test/install-tanstack-fixture.cjs DIR floor|current CORE_TGZ SEMCONV_TGZ +if (!process.env.TANSTACK_FIXTURE) { + console.error('Set TANSTACK_FIXTURE to a directory prepared by test/install-tanstack-fixture.cjs'); + process.exit(2); +} +const fixture = resolve(process.env.TANSTACK_FIXTURE); +const require = createRequire(join(fixture, 'package.json')); +const native = async (pkg, path = 'index.js') => import(pathToFileURL(join(fixture, 'node_modules', pkg, 'dist/esm', path))); +const { otelMiddleware } = await native('@tanstack/ai', 'middlewares/otel.js'); +const { chat, toolDefinition } = await native('@tanstack/ai'); +const { createOpenaiChatCompletions } = await native('@tanstack/ai-openai'); +const { BasicTracerProvider, SimpleSpanProcessor, BatchSpanProcessor, InMemorySpanExporter } = require('@opentelemetry/sdk-trace-base'); +const { SpanStatusCode, context, trace } = require('@opentelemetry/api'); +const { resourceFromAttributes } = require('@opentelemetry/resources'); +const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http'); +const { TransformingSpanExporter } = require('@traceai/fi-core'); +const { + tanStackOtelOptions, normalizeTanStackSpan, registerTanStackTracing, + flushTraces, shutdownTraces, +} = require('@traceai/fi-core/tanstack'); +// Customers on ESM-only TanStack load the ESM build. +const esm = await import(pathToFileURL(join(fixture, 'node_modules/@traceai/fi-core/dist/esm/tanstack.js'))); +const esmByName = await import(pathToFileURL(join(fixture, 'esm-import.mjs'))); +const secret = 'private-payload-7e925'; +const sdkVersion = require('@opentelemetry/sdk-trace-base/package.json').version; +const tanstackVersion = require(join(fixture, 'node_modules/@tanstack/ai/package.json')).version; +console.log(`fixture: @tanstack/ai ${tanstackVersion}, @opentelemetry/sdk-trace-base ${sdkVersion}`); +const spanJSON = spans => JSON.stringify(spans.map(({ name, attributes, status, events, links }) => ({ name, attributes, status, events, links }))); + +function opaque(tracer) { + return { startSpan(...args) { + const span = tracer.startSpan(...args); + return Object.fromEntries(['spanContext', 'setAttribute', 'setAttributes', 'addEvent', 'addLink', 'addLinks', 'setStatus', 'updateName', 'end', 'isRecording', 'recordException'].map(k => [k, span[k].bind(span)])); + }, startActiveSpan: tracer.startActiveSpan.bind(tracer) }; +} +function memory({ options, publicOnly = true, batch = false, raw = false } = {}) { + const sink = new InMemorySpanExporter(); + const rawSink = new InMemorySpanExporter(); + const Processor = batch ? BatchSpanProcessor : SimpleSpanProcessor; + const provider = new BasicTracerProvider({ spanProcessors: [ + new Processor(new TransformingSpanExporter(sink, normalizeTanStackSpan)), + ...(raw ? [new SimpleSpanProcessor(rawSink)] : []), + ] }); + const tracer = provider.getTracer('tanstack-contract'); + return { provider, sink, rawSink, tracer, + middleware: otelMiddleware(tanStackOtelOptions(publicOnly ? opaque(tracer) : tracer, options)), + }; +} +function ctx(extra = {}) { + return { activity: 'chat', provider: 'openai-chat', model: 'gpt-4o-mini', threadId: 'opaque-session', + phase: 'beforeModel', iteration: 0, messages: [{ role: 'user', content: secret }], ...extra }; +} +const usage = { promptTokens: 11, completionTokens: 7, totalTokens: 18, cost: 0.25, + promptTokensDetails: { cachedTokens: 3, cacheWriteTokens: 2 }, completionTokensDetails: { reasoningTokens: 1 } }; +async function hooks(middleware, { turns = 1, run = ctx(), withUsage = true, fallback } = {}) { + await middleware.onStart(run); + for (let i = 0; i < turns; i++) { + run.iteration = i; + await middleware.onConfig(run, { messages: run.messages, systemPrompts: [] }); + if (withUsage) await middleware.onUsage(run, usage); + } + await middleware.onFinish(run, { duration: 1, ...(fallback ? { usage: fallback } : {}) }); +} +const rootOf = spans => spans.find(s => s.attributes['gen_ai.span.kind'] === 'AGENT' || s.attributes['gen_ai.span.kind'] === 'CHAIN'); +const sum = (spans, key) => spans.reduce((n, s) => n + (s.attributes[key] ?? 0), 0); + +for (const batch of [false, true]) { + for (const turns of [1, 2]) test(`public-only tracer counts tokens once: turns=${turns}, batch=${batch}`, async () => { + const h = memory({ batch, raw: true, options: { threadIdAsSession: true } }); + await hooks(h.middleware, { turns }); + await h.provider.forceFlush(); + const spans = h.sink.getFinishedSpans(), root = rootOf(spans); + assert.equal(spans.length, turns + 1); + assert.equal(root.attributes['gen_ai.span.kind'], turns === 1 ? 'CHAIN' : 'AGENT'); + for (const [key, value] of Object.entries({ input_tokens: 11, output_tokens: 7, total_tokens: 18, cost: 0.25, + 'cache_read.input_tokens': 3, 'cache_creation.input_tokens': 2, 'reasoning.output_tokens': 1 })) { + assert.equal(sum(spans, `gen_ai.usage.${key}`), turns * value, key); + assert.equal(root.attributes[`tanstack.ai.root_usage.${key}`], turns * value, key); + } + assert.equal(root.attributes['traceai.tanstack.aggregate_usage'], undefined); + for (const span of spans) { + assert.equal(span.attributes['session.id'], 'opaque-session'); + assert.equal(span.attributes['gen_ai.provider.name'], 'openai'); + assert.equal(span.spanContext().traceId, root.spanContext().traceId); + if (span !== root) assert.equal(span.parentSpanContext?.spanId ?? span.parentSpanId, root.spanContext().spanId); + } + const rawRoot = rootOf(h.rawSink.getFinishedSpans()); + assert.equal(rawRoot.attributes['gen_ai.usage.input_tokens'], turns * 11, 'other exporters see original usage'); + assert.equal(rawRoot.attributes['traceai.tanstack.aggregate_usage'], true); + assert.equal(spanJSON(spans).includes(secret), false); + await h.provider.shutdown(); + }); +} + +test('negative control: bare native middleware double counts root usage', async () => { + const sink = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(sink)] }); + await hooks(otelMiddleware({ tracer: opaque(provider.getTracer('bare')) }), { turns: 2 }); + assert.equal(sum(sink.getFinishedSpans(), 'gen_ai.usage.input_tokens'), 44); + await provider.shutdown(); +}); + +test('ESM entry point exposes the same maintained API', () => { + for (const name of ['tanStackOtelOptions', 'normalizeTanStackSpan', 'registerTanStackTracing', 'flushTraces', 'shutdownTraces']) { + assert.equal(typeof esm[name], 'function', name); + assert.equal(typeof esmByName[name], 'function', name); + } + assert.equal(esm.futureAgiSpanKinds, undefined, 'hooks are bound to the tracer facade, not exported alone'); +}); + +test('normalizer handles frozen ReadableSpan snapshots', () => { + const frozen = Object.freeze({ name: 'chat', attributes: Object.freeze({ 'traceai.tanstack.aggregate_usage': true, 'gen_ai.usage.input_tokens': 4 }), + spanContext: () => ({ spanId: 'frozen' }), status: { code: 0 }, links: [], events: [] }); + const output = normalizeTanStackSpan(frozen); + assert.deepEqual(output.attributes, { 'tanstack.ai.root_usage.input_tokens': 4 }); + assert.equal(output.spanContext().spanId, 'frozen'); +}); + +test('no child usage: retain root-only fallback, including zero usage', async () => { + for (const fallback of [usage, { promptTokens: 0, completionTokens: 0, totalTokens: 0 }]) { + const h = memory(); + await hooks(h.middleware, { withUsage: false, fallback }); + await h.provider.forceFlush(); + assert.equal(rootOf(h.sink.getFinishedSpans()).attributes['gen_ai.usage.input_tokens'], fallback.promptTokens); + assert.equal(rootOf(h.sink.getFinishedSpans()).attributes['session.id'], undefined); + await h.provider.shutdown(); + } +}); + +test('shared middleware isolates concurrent runs and session opt-in', async () => { + const h = memory({ options: { threadIdAsSession: true } }); + await Promise.all(Array.from({ length: 30 }, (_, i) => hooks(h.middleware, { run: ctx({ threadId: `session-${i}` }), turns: i % 2 + 1 }))); + await h.provider.forceFlush(); + const spans = h.sink.getFinishedSpans(); + assert.equal(new Set(spans.map(s => s.spanContext().traceId)).size, 30); + assert.equal(sum(spans, 'gen_ai.usage.input_tokens'), 45 * 11); + await h.provider.shutdown(); +}); + +for (const joinActiveTrace of [false, true]) test(`active app parent is ${joinActiveTrace ? 'joined' : 'isolated'}`, async () => { + const h = memory({ options: { joinActiveTrace } }); + const parent = h.tracer.startSpan('application'); + // A context-manager-free tracer still gets the explicit current parent. + const withParent = { startSpan: (name, options, explicit) => h.tracer.startSpan(name, options, explicit ?? trace.setSpan(context.active(), parent)), + startActiveSpan: h.tracer.startActiveSpan.bind(h.tracer) }; + await hooks(otelMiddleware(tanStackOtelOptions(withParent, { joinActiveTrace }))); + await h.provider.forceFlush(); + const root = rootOf(h.sink.getFinishedSpans()); + assert.equal(root.spanContext().traceId === parent.spanContext().traceId, joinActiveTrace); + parent.end(); + await h.provider.shutdown(); +}); + +for (const captureContent of [false, true]) test(`model and tool errors obey captureContent=${captureContent}`, async () => { + const h = memory({ options: { captureContent, redact: text => text.replaceAll(secret, '[hidden]') } }); + const run = ctx(); + await h.middleware.onStart(run); + await h.middleware.onConfig(run, { messages: run.messages, systemPrompts: [] }); + await h.middleware.onBeforeToolCall(run, { toolName: 'get_weather', toolCallId: 'call-1', args: { city: secret } }); + const error = Object.assign(new Error(secret), { code: 'SYNTHETIC' }); + await h.middleware.onAfterToolCall(run, { toolName: 'get_weather', toolCallId: 'call-1', ok: false, error, result: secret }); + await h.middleware.onError(run, { error, duration: 1 }); + await h.provider.forceFlush(); + const spans = h.sink.getFinishedSpans(); + assert.equal(spans.length, 3); + assert.equal(spans.filter(s => s.status.code === SpanStatusCode.ERROR).length, 3); + const encoded = spanJSON(spans); + assert.ok(!encoded.includes(secret)); + assert.ok(!encoded.includes('exception.stacktrace')); + assert.equal(encoded.includes('[hidden]'), captureContent); + assert.ok(encoded.includes('SYNTHETIC')); + await h.provider.shutdown(); +}); + +test('redactor failures fail closed for content and exceptions', async () => { + const h = memory({ options: { captureContent: true, redact: () => { throw new Error(secret); } } }); + const run = ctx(); + await h.middleware.onStart(run); + await h.middleware.onConfig(run, { messages: run.messages, systemPrompts: [] }); + await h.middleware.onError(run, { error: new Error(secret), duration: 1 }); + await h.provider.forceFlush(); + const encoded = spanJSON(h.sink.getFinishedSpans()); + assert.ok(encoded.includes('[redaction_failed]')); + assert.ok(!encoded.includes(secret)); + await h.provider.shutdown(); +}); + +test('all active-span overloads apply the same privacy facade', async () => { + const h = memory(); + const { tracer } = tanStackOtelOptions(h.tracer); + const callback = span => { span.recordException(new Error(secret)); span.setStatus({ code: SpanStatusCode.ERROR, message: secret }); span.end(); return 42; }; + assert.equal(tracer.startActiveSpan('a', callback), 42); + assert.equal(tracer.startActiveSpan('b', {}, callback), 42); + assert.equal(tracer.startActiveSpan('c', {}, context.active(), callback), 42); + await h.provider.forceFlush(); + assert.equal(h.sink.getFinishedSpans().length, 3); + assert.ok(!spanJSON(h.sink.getFinishedSpans()).includes(secret)); + await h.provider.shutdown(); +}); + +test('normalization is immutable, idempotent and scoped; preserves future cost keys', () => { + const attributes = Object.freeze({ 'traceai.tanstack.aggregate_usage': true, + 'gen_ai.cost.total': 1.25, 'gen_ai.usage.future_tokens': 6, unrelated: 'keep' }); + const span = { attributes, spanContext() { assert.equal(this, span); return { spanId: 'id' }; } }; + const output = normalizeTanStackSpan(span); + assert.deepEqual(output.attributes, { 'tanstack.ai.root_usage.cost.total': 1.25, 'tanstack.ai.root_usage.future_tokens': 6, unrelated: 'keep' }); + assert.deepEqual(output.spanContext(), { spanId: 'id' }); + assert.equal(normalizeTanStackSpan(output), output); + assert.equal(span.attributes, attributes); + const unrelated = { attributes: { 'gen_ai.usage.input_tokens': 12 } }; + assert.equal(normalizeTanStackSpan(unrelated), unrelated); +}); + +test('export-transform failure drops the batch without forwarding private text', async () => { + let exports = 0, flushes = 0, shutdowns = 0; + const exporter = new TransformingSpanExporter({ export() { exports++; }, async forceFlush() { flushes++; }, async shutdown() { shutdowns++; } }, () => { throw new Error(secret); }); + let result; + exporter.export([{}], r => { result = r; }); + assert.equal(exports, 0); + assert.equal(result.code, 1); + assert.ok(!result.error.message.includes(secret)); + await exporter.forceFlush(); await exporter.shutdown(); + assert.equal(flushes, 1); assert.equal(shutdowns, 1); +}); + +test('bounded flush and shutdown tolerate collector failures', async () => { + // flushTraces unrefs its timer so it never holds a process open; keep this + // test's event loop alive while waiting on a collector that never answers. + const keepAlive = setInterval(() => {}, 1000); + try { + await flushTraces({ forceFlush: async () => { throw new Error(secret); } }); + const started = Date.now(); + await flushTraces({ forceFlush: () => new Promise(() => {}) }, { timeoutMs: 5 }); + assert.ok(Date.now() - started < 1000); + await shutdownTraces({ shutdown: async () => { throw new Error(secret); } }); + } finally { clearInterval(keepAlive); } +}); + +async function server(handler) { + const server = createServer(handler); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + return { url: `http://127.0.0.1:${server.address().port}`, close: () => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); }) }; +} +async function body(req) { const parts = []; for await (const part of req) parts.push(part); return JSON.parse(Buffer.concat(parts)); } +const attrs = pairs => Object.fromEntries((pairs ?? []).map(({ key, value }) => [key, value.stringValue ?? value.boolValue ?? Number(value.intValue ?? value.doubleValue)])); + +for (const mode of ['tools', 'tool-error', 'model-error', 'abort', 'content']) test(`real streamed chat and OTLP wire: ${mode}`, async () => { + const requests = [], bodies = [], wire = []; + const collector = await server(async (req, res) => { + requests.push({ path: req.url, headers: req.headers }); + const payload = await body(req); + bodies.push(payload); + for (const resource of payload.resourceSpans) { + assert.equal(attrs(resource.resource.attributes).project_name, 'tanstack-contract'); + for (const scope of resource.scopeSpans) wire.push(...scope.spans); + } + res.writeHead(200, { 'content-type': 'application/json' }); res.end('{}'); + }); + const model = await server(async (req, res) => { + const request = await body(req); + if (mode === 'model-error') { + res.writeHead(400, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ error: { message: secret, type: 'invalid_request_error', code: 'SYNTHETIC' } })); return; + } + const usedTool = request.messages.some(m => m.role === 'tool'); + const wantsTool = ['tools', 'tool-error'].includes(mode) && !usedTool; + const event = (delta, finish_reason = null, usage) => `data: ${JSON.stringify({ id: 'synthetic', object: 'chat.completion.chunk', created: 1700000000, model: 'gpt-4o-mini', + choices: delta === null ? [] : [{ index: 0, delta, finish_reason }], ...(usage ? { usage } : {}) })}\n\n`; + res.writeHead(200, { 'content-type': 'text/event-stream' }); + if (wantsTool) { + res.write(event({ role: 'assistant', tool_calls: [{ index: 0, id: 'call-1', type: 'function', function: { name: 'get_weather', arguments: JSON.stringify({ city: secret }) } }] })); + } else res.write(event({ role: 'assistant', content: secret })); + if (mode === 'abort') return; // client's abort closes this connection + res.end(event({}, wantsTool ? 'tool_calls' : 'stop') + event(null, null, { prompt_tokens: 11, completion_tokens: 7, total_tokens: 18 }) + 'data: [DONE]\n\n'); + }); + const provider = registerTanStackTracing({ projectName: 'tanstack-contract', endpoint: collector.url, + headers: { 'x-api-key': 'synthetic-key', 'x-secret-key': 'synthetic-secret' } }); + try { + const abortController = new AbortController(); + const middleware = otelMiddleware(tanStackOtelOptions(provider.getTracer('tanstack'), { + threadIdAsSession: true, captureContent: mode === 'content', redact: text => text.replaceAll(secret, '[hidden]'), + })); + const weather = toolDefinition({ name: 'get_weather', description: 'Get weather', inputSchema: { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] } }) + .server(async ({ city }) => { if (mode === 'tool-error') throw new Error(secret); return { city, forecast: 'sunny' }; }); + const stream = chat({ adapter: createOpenaiChatCompletions('gpt-4o-mini', 'synthetic-model-key', { baseURL: `${model.url}/v1`, maxRetries: 0 }), + messages: [{ role: 'user', content: secret }], tools: ['tools', 'tool-error'].includes(mode) ? [weather] : [], + middleware: [middleware], threadId: 'session-wire', abortController }); + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + if (mode === 'abort' && chunk.type === 'TEXT_MESSAGE_CONTENT') abortController.abort(); + } + await provider.forceFlush(); + assert.equal(wire.length, ['tools', 'tool-error'].includes(mode) ? 4 : 2); + assert.equal(new Set(wire.map(s => s.spanId)).size, wire.length); + const spans = wire.map(s => ({ ...s, attributes: attrs(s.attributes) })); + const root = rootOf(spans); + assert.ok(root); + assert.equal(root.parentSpanId ?? '', ''); + for (const span of spans) { + assert.equal(span.traceId, root.traceId); + assert.equal(span.attributes['session.id'], 'session-wire'); + if (span !== root) assert.ok(spans.some(p => p.spanId === span.parentSpanId)); + } + for (const req of requests) { + assert.equal(req.path, '/tracer/v1/traces'); + assert.equal(req.headers['x-api-key'], 'synthetic-key'); + assert.equal(req.headers['x-secret-key'], 'synthetic-secret'); + } + const encoded = JSON.stringify(bodies); + assert.ok(!encoded.includes(secret)); + assert.ok(!encoded.includes('exception.stacktrace')); + assert.ok(!encoded.includes('synthetic-key')); + if (mode === 'content') assert.ok(encoded.includes('[hidden]')); + if (['tools', 'tool-error', 'content'].includes(mode)) assert.equal(sum(spans, 'gen_ai.usage.input_tokens'), mode === 'content' ? 11 : 22); + if (mode === 'model-error') assert.ok(spans.every(s => s.status.code === 2)); + if (mode === 'tool-error') assert.equal(spans.filter(s => s.status.code === 2).length, 1); + if (mode === 'abort') assert.ok(spans.every(s => s.status.code === 2 && s.status.message === 'cancelled')); + } finally { await provider.shutdown(); await model.close(); await collector.close(); } +}); + +test('existing SDK can compose the normalizer without replacing its provider', async () => { + const h = memory(); + const other = { attributes: { unrelated: true } }; + assert.equal(normalizeTanStackSpan(other), other); + await hooks(h.middleware); + await h.provider.forceFlush(); + assert.equal(sum(h.sink.getFinishedSpans(), 'gen_ai.usage.input_tokens'), 11); + await h.provider.shutdown(); +});