From e02163cca0c2439eae2a069b297f61e210e4fb3a Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 01:25:48 -0800 Subject: [PATCH 1/5] fix(mastra): enforce export privacy and lifecycle compatibility Honor FI content and error controls across native and FI aliases, scrub credentials, preserve runtime outcomes, and drain conversions before flush or shutdown. Respect environment project routing and classify only actual model calls as LLM observations. Add offline floor/latest SDK contracts and CI. Co-Authored-By: Claude Code --- .github/workflows/mastra-typescript-tests.yml | 74 +++++ .../traceai_mastra/contract/recall.test.cjs | 265 ++++++++++++++++++ .../src/FIMastraSpanExporter.ts | 89 ++++-- .../traceai_mastra/src/FIObservability.ts | 2 +- .../src/__tests__/enrichment.test.ts | 42 ++- .../src/__tests__/privacy.test.ts | 73 +++++ .../packages/traceai_mastra/src/privacy.ts | 91 ++++++ 7 files changed, 606 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/mastra-typescript-tests.yml create mode 100644 typescript/packages/traceai_mastra/contract/recall.test.cjs create mode 100644 typescript/packages/traceai_mastra/src/__tests__/privacy.test.ts create mode 100644 typescript/packages/traceai_mastra/src/privacy.ts diff --git a/.github/workflows/mastra-typescript-tests.yml b/.github/workflows/mastra-typescript-tests.yml new file mode 100644 index 000000000..208473d0f --- /dev/null +++ b/.github/workflows/mastra-typescript-tests.yml @@ -0,0 +1,74 @@ +name: Mastra TypeScript tests + +on: + pull_request: + branches: [dev, main] + paths: + - 'typescript/packages/traceai_mastra/**' + - 'typescript/packages/fi-core/**' + - 'typescript/packages/fi-semantic-conventions/**' + - 'typescript/pnpm-lock.yaml' + - '.github/workflows/mastra-typescript-tests.yml' + push: + branches: [dev] + paths: + - 'typescript/packages/traceai_mastra/**' + - '.github/workflows/mastra-typescript-tests.yml' + +permissions: + contents: read + +jobs: + contract: + name: Mastra ${{ matrix.version }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - { version: floor, core: '1.16.0', observability: '1.0.0', exporter: '1.0.0' } + - { version: latest, core: latest, observability: latest, exporter: latest } + defaults: + run: + shell: bash + working-directory: typescript + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22.23.4' + - name: Install frozen dependencies + run: | + corepack enable + pnpm --filter @traceai/mastra... install --frozen-lockfile --ignore-scripts + - name: Build, offline unit test and pack + run: | + pnpm --filter @traceai/mastra... run build + sudo unshare --net bash -c 'ip link set lo up; exec "$@"' _ \ + "$(command -v node)" "$PWD/node_modules/jest/bin/jest.js" \ + --config "$PWD/packages/traceai_mastra/jest.config.js" --runInBand + mkdir -p "$RUNNER_TEMP/mastra-package" + pnpm --dir packages/traceai_mastra pack --pack-destination "$RUNNER_TEMP/mastra-package" + git diff --exit-code -- pnpm-lock.yaml + - name: Install packed exporter and real SDK in a clean consumer + env: + CORE_VERSION: ${{ matrix.core }} + OBSERVABILITY_VERSION: ${{ matrix.observability }} + EXPORTER_VERSION: ${{ matrix.exporter }} + run: | + mkdir -p "$RUNNER_TEMP/mastra-consumer" + npm install --prefix "$RUNNER_TEMP/mastra-consumer" --ignore-scripts --no-audit --no-fund \ + "$RUNNER_TEMP"/mastra-package/*.tgz \ + "@mastra/core@$CORE_VERSION" "@mastra/observability@$OBSERVABILITY_VERSION" \ + "@mastra/otel-exporter@$EXPORTER_VERSION" \ + @opentelemetry/otlp-transformer@0.201.1 + - name: Native lifecycle and wire contracts (loopback-only network namespace) + run: | + export MASTRA_TEST_DEPS="$RUNNER_TEMP/mastra-consumer" + export MASTRA_TEST_PACKAGE="$RUNNER_TEMP/mastra-consumer/node_modules/@traceai/mastra/dist/src/index.js" + sudo unshare --net bash -c 'ip link set lo up; exec "$@"' _ \ + env MASTRA_TEST_DEPS="$MASTRA_TEST_DEPS" MASTRA_TEST_PACKAGE="$MASTRA_TEST_PACKAGE" \ + "$(command -v node)" --test "$PWD/packages/traceai_mastra/contract/recall.test.cjs" diff --git a/typescript/packages/traceai_mastra/contract/recall.test.cjs b/typescript/packages/traceai_mastra/contract/recall.test.cjs new file mode 100644 index 000000000..4a7eef063 --- /dev/null +++ b/typescript/packages/traceai_mastra/contract/recall.test.cjs @@ -0,0 +1,265 @@ +// Offline contract: real Mastra lifecycles and decoded OTLP/protobuf, synthetic data only. +// Run against the peer floor and latest in separate dependency installations: +// MASTRA_TEST_DEPS=/path/to/install MASTRA_TEST_PACKAGE=/path/to/built/index.js node --test contract/recall.test.cjs +const assert = require('node:assert/strict'); +const http = require('node:http'); +const path = require('node:path'); +const { createRequire } = require('node:module'); +const { test, before, after, beforeEach, afterEach } = require('node:test'); +const packagePath = process.env.MASTRA_TEST_PACKAGE || path.resolve(__dirname, '../dist/src/index.js'); +const req = createRequire(process.env.MASTRA_TEST_DEPS + ? path.join(process.env.MASTRA_TEST_DEPS, 'package.json') : packagePath); +const { createFIObservability } = require(packagePath); +const { SpanType } = req('@mastra/core/observability'); +const wire = req('@opentelemetry/otlp-transformer/build/src/generated/root.js') + .opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +const flags = ['INPUTS', 'OUTPUTS', 'INPUT_MESSAGES', 'OUTPUT_MESSAGES', 'INPUT_TEXT', + 'OUTPUT_TEXT', 'INPUT_IMAGES', 'EMBEDDING_VECTORS', 'ERROR_DETAILS'].map(key => `FI_HIDE_${key}`); +const envKeys = [...flags, 'FI_PROJECT_NAME']; +let originalEnv, obs, endpoint; +const spans = []; +const attributes = list => Object.fromEntries((list || []).map(a => [a.key, Object.values(a.value)[0]])); +const server = http.createServer((request, response) => { + const chunks = []; + request.on('data', chunk => chunks.push(chunk)); + request.on('end', () => { + assert.equal(request.url, '/tracer/v1/traces'); + assert.equal(request.headers['x-api-key'], 'synthetic-api'); + assert.equal(request.headers['x-secret-key'], 'synthetic-secret'); + const data = wire.toObject(wire.decode(Buffer.concat(chunks)), { longs: String, bytes: String }); + for (const resource of data.resourceSpans || []) { + for (const scope of resource.scopeSpans || []) { + for (const span of scope.spans || []) spans.push({ ...span, resource: attributes(resource.resource.attributes), attrs: attributes(span.attributes) }); + } + } + response.writeHead(200, { 'content-type': 'application/x-protobuf' }); + response.end(); + }); +}); +before(async () => { + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + endpoint = `http://127.0.0.1:${server.address().port}/tracer/v1/traces`; +}); +after(() => new Promise(resolve => server.close(resolve))); +beforeEach(() => { + originalEnv = Object.fromEntries(envKeys.map(key => [key, process.env[key]])); + for (const key of envKeys) delete process.env[key]; + spans.length = 0; +}); +afterEach(async () => { + await obs?.shutdown(); + obs = undefined; + for (const [key, value] of Object.entries(originalEnv)) { + if (value === undefined) delete process.env[key]; else process.env[key] = value; + } +}); +function setup(options = {}) { + obs = createFIObservability({ serviceName: 'synthetic-service', apiKey: 'synthetic-api', + secretKey: 'synthetic-secret', endpoint, ...options }); + return obs.getDefaultInstance(); +} +async function flush() { + // Wait only in content tests; lifecycle regression tests intentionally do not yield. + await new Promise(resolve => setImmediate(resolve)); + await obs.getDefaultInstance().flush(); +} +function privateSpans(instance) { + const agent = instance.startSpan({ type: SpanType.AGENT_RUN, name: 'privacy-agent', + input: 'PRIVATE_INPUT', attributes: { instructions: 'PRIVATE_SYSTEM' } }); + const tool = instance.startSpan({ type: SpanType.TOOL_CALL, name: 'privacy-tool', parent: agent, + entityName: 'tool', input: { apiKey: 'PRIVATE_CREDENTIAL', prompt: 'PRIVATE_INPUT' } }); + tool.end({ output: 'PRIVATE_OUTPUT' }); + const model = instance.startSpan({ type: SpanType.MODEL_INFERENCE || SpanType.MODEL_GENERATION, + name: 'privacy-model', parent: agent, attributes: { model: 'synthetic', usage: { inputTokens: 3, outputTokens: 2 } }, + input: [{ role: 'user', content: [{ type: 'text', text: 'PRIVATE_INPUT' }, { type: 'image', image: 'PRIVATE_IMAGE' }] }] }); + model.end({ output: [{ role: 'assistant', content: 'PRIVATE_OUTPUT' }] }); + const error = instance.startSpan({ type: SpanType.TOOL_CALL, name: 'privacy-error', parent: agent, entityName: 'error' }); + error.error({ error: new Error('PRIVATE_ERROR'), endSpan: true }); + agent.end({ output: 'PRIVATE_OUTPUT' }); +} +for (const flag of ['INPUTS', 'INPUT_MESSAGES', 'INPUT_TEXT']) { + test(`FI_HIDE_${flag} protects native and FI input aliases and instructions`, async () => { + process.env[`FI_HIDE_${flag}`] = 'true'; + privateSpans(setup()); + await flush(); + assert.equal(spans.length, 4); + assert.doesNotMatch(JSON.stringify(spans), /PRIVATE_INPUT|PRIVATE_SYSTEM|PRIVATE_ERROR/); + assert.match(JSON.stringify(spans), /PRIVATE_OUTPUT/); + }); +} +for (const flag of ['OUTPUTS', 'OUTPUT_MESSAGES', 'OUTPUT_TEXT']) { + test(`FI_HIDE_${flag} protects native and FI output aliases`, async () => { + process.env[`FI_HIDE_${flag}`] = 'true'; + privateSpans(setup()); + await flush(); + assert.equal(spans.length, 4); + assert.doesNotMatch(JSON.stringify(spans), /PRIVATE_OUTPUT|PRIVATE_ERROR/); + assert.match(JSON.stringify(spans), /PRIVATE_INPUT/); + }); +} +test('all FI privacy flags keep structure and error classification, not private content', async () => { + for (const key of flags) process.env[key] = 'true'; + privateSpans(setup()); + await flush(); + assert.equal(spans.length, 4); + assert.doesNotMatch(JSON.stringify(spans), /PRIVATE_/); + const root = spans.find(span => span.attrs['gen_ai.span.kind'] === 'AGENT'); + assert.ok(root); + assert.equal(new Set(spans.map(span => span.spanId)).size, 4); + for (const span of spans.filter(span => span !== root)) { + assert.equal(span.parentSpanId, root.spanId); + assert.equal(span.traceId, root.traceId); + } + const error = spans.find(span => span.status?.code === 2); + assert.ok(error); + assert.ok(error.attrs['error.type']); + assert.ok(error.events.some(event => attributes(event.attributes)['exception.type'])); +}); +test('FI_HIDE_ERROR_DETAILS alone hides errors without hiding input or output', async () => { + process.env.FI_HIDE_ERROR_DETAILS = 'true'; + privateSpans(setup()); + await flush(); + assert.doesNotMatch(JSON.stringify(spans), /PRIVATE_ERROR/); + assert.match(JSON.stringify(spans), /PRIVATE_INPUT/); + assert.match(JSON.stringify(spans), /PRIVATE_OUTPUT/); +}); +test('credentials are removed even with content hiding off', async () => { + privateSpans(setup()); + await flush(); + assert.doesNotMatch(JSON.stringify(spans), /PRIVATE_CREDENTIAL/); + assert.match(JSON.stringify(spans), /PRIVATE_INPUT|PRIVATE_OUTPUT/); +}); +test('explicit error-detail false overrides implied hiding', async () => { + process.env.FI_HIDE_INPUTS = 'true'; + process.env.FI_HIDE_ERROR_DETAILS = 'false'; + privateSpans(setup()); + await flush(); + assert.doesNotMatch(JSON.stringify(spans), /PRIVATE_INPUT/); + assert.match(JSON.stringify(spans), /PRIVATE_ERROR/); +}); +for (const [label, explicit, env, expected] of [ + ['environment', undefined, 'env-project', 'env-project'], + ['explicit', 'option-project', 'env-project', 'option-project'], + ['fallback', undefined, undefined, 'synthetic-service'], +]) { + test(`project precedence: ${label}`, async () => { + if (env) process.env.FI_PROJECT_NAME = env; + const instance = setup({ projectName: explicit }); + instance.startSpan({ type: SpanType.GENERIC, name: 'project' }).end(); + await flush(); + assert.equal(spans.length, 1); + assert.equal(spans[0].resource.project_name, expected); + }); +} +for (const lifecycle of ['flush', 'shutdown']) { + test(`immediate ${lifecycle} drains all conversions exactly once`, async () => { + const instance = setup(); + for (let i = 0; i < 8; i++) instance.startSpan({ type: SpanType.GENERIC, name: `span-${i}` }).end(); + await instance[lifecycle](); + assert.equal(spans.length, 8); + assert.equal(new Set(spans.map(span => span.spanId)).size, 8); + if (lifecycle === 'shutdown') await instance.shutdown(); + assert.equal(spans.length, 8); + }); +} +test('generation produces one LLM observation per real model request', async () => { + const { Mastra } = req('@mastra/core'); + const { Agent } = req('@mastra/core/agent'); + setup(); + let calls = 0; + const model = { specificationVersion: 'v2', provider: 'synthetic', modelId: 'synthetic', supportedUrls: {}, + doGenerate: async () => { calls++; return { content: [{ type: 'text', text: 'answer' }], finishReason: 'stop', + usage: { inputTokens: 11, outputTokens: 5, totalTokens: 16 }, warnings: [] }; } }; + const agent = new Agent({ id: 'test-agent', name: 'Test agent', instructions: 'Answer', model }); + const mastra = new Mastra({ agents: { agent }, observability: obs }); + const result = await mastra.getAgent('agent').generate('question'); + assert.equal(result.text, 'answer'); + await flush(); + assert.equal(calls, 1); + const llm = spans.filter(span => span.attrs['gen_ai.span.kind'] === 'LLM'); + assert.equal(llm.length, calls); + assert.equal(llm[0].status?.code, 1); + assert.equal(Number(llm[0].attrs['gen_ai.usage.input_tokens']), 11); + assert.equal(Number(llm[0].attrs['gen_ai.usage.output_tokens']), 5); +}); +test('shutdown waits for delayed conversion and rejects new events', async () => { + const instance = setup(); + const exporter = instance.getExporters()[0]; + const convert = exporter.converter.convertSpan.bind(exporter.converter); + let release, entered; + const gate = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { entered = resolve; }); + exporter.converter.convertSpan = async span => { entered(); await gate; return convert(span); }; + instance.startSpan({ type: SpanType.GENERIC, name: 'delayed' }).end(); + await started; + let done = false; + const shutdown = exporter.shutdown().then(() => { done = true; }); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(done, false); + instance.startSpan({ type: SpanType.GENERIC, name: 'too-late' }).end(); + release(); + await shutdown; + assert.equal(spans.length, 1); + assert.equal(spans[0].name, 'delayed'); +}); +test('failed conversions settle and do not prevent draining healthy spans', async () => { + const instance = setup(); + const exporter = instance.getExporters()[0]; + const convert = exporter.converter.convertSpan.bind(exporter.converter); + exporter.converter.convertSpan = async span => { + if (span.name === 'invalid') throw new Error('synthetic conversion failure'); + return convert(span); + }; + instance.startSpan({ type: SpanType.GENERIC, name: 'invalid' }).end(); + instance.startSpan({ type: SpanType.GENERIC, name: 'valid' }).end(); + await instance.shutdown(); + assert.equal(spans.length, 1); + assert.equal(spans[0].name, 'valid'); +}); +test('real stream cancellation preserves the unsuccessful SDK outcomes', { skip: !SpanType.MODEL_INFERENCE }, async () => { + const { Mastra } = req('@mastra/core'); + const { Agent } = req('@mastra/core/agent'); + setup(); + const controller = new AbortController(); + const model = { specificationVersion: 'v2', provider: 'synthetic', modelId: 'synthetic', supportedUrls: {}, + doStream: async options => ({ stream: new ReadableStream({ start(stream) { + stream.enqueue({ type: 'stream-start', warnings: [] }); + stream.enqueue({ type: 'text-start', id: 'text' }); + stream.enqueue({ type: 'text-delta', id: 'text', delta: 'partial' }); + options.abortSignal.addEventListener('abort', () => stream.error(new DOMException('caller abort', 'AbortError')), { once: true }); + } }) }) }; + const agent = new Agent({ id: 'cancel-agent', name: 'Cancel agent', instructions: 'Answer', model }); + const mastra = new Mastra({ agents: { agent }, observability: obs }); + const result = await mastra.getAgent('agent').stream('question', { abortSignal: controller.signal }); + const watchdog = setTimeout(() => controller.abort(), 3000); + let sawText = false; + try { + for await (const chunk of result.fullStream) if (chunk.type === 'text-delta') { sawText = true; controller.abort(); } + } catch (error) { + assert.equal(controller.signal.aborted, true); + } finally { clearTimeout(watchdog); } + await flush(); + assert.equal(sawText, true); + const cancelled = spans.filter(span => ['abort', 'interrupted'].includes(span.attrs['mastra.span.outcome'])); + assert.ok(cancelled.length >= 3); + for (const span of cancelled) { + assert.notEqual(span.status?.code, 1); + assert.equal(span.attrs['mastra.span.stop_reason'], 'caller_cancelled'); + } +}); +test('native unsuccessful outcomes are never OK', { skip: !SpanType.MODEL_INFERENCE }, async () => { + const instance = setup(); + for (const termination of [ + { outcome: 'abort', stopReason: 'caller_cancelled' }, + { outcome: 'abort', stopReason: 'timeout' }, + { outcome: 'error', stopReason: 'runtime_error' }, + { outcome: 'interrupted', stopReason: 'signal_received' }, + ]) instance.startSpan({ type: SpanType.WORKFLOW_RUN, name: termination.stopReason }).end(termination); + await flush(); + assert.equal(spans.length, 4); + for (const span of spans) { + assert.notEqual(span.status?.code, 1); + assert.ok(span.attrs['mastra.span.outcome']); + assert.ok(span.attrs['mastra.span.stop_reason']); + } +}); diff --git a/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts b/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts index 32c8f26d0..09bc14916 100644 --- a/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts +++ b/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts @@ -6,6 +6,9 @@ import { FISpanKind } from "@traceai/fi-semantic-conventions"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto"; import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base"; import type { ReadableSpan } from "@opentelemetry/sdk-trace-base"; +import { SpanStatusCode } from "@opentelemetry/api"; +import { generateTraceConfig } from "@traceai/fi-core"; +import { sanitizeSpan } from "./privacy.js"; export interface FISpanExporterConfig { /** Full OTLP traces endpoint URL. */ @@ -29,22 +32,23 @@ export interface FISpanExporterConfig { * (llm / agent / tool / chain / ...). Mastra emits `gen_ai.operation.name` but * NOT a span kind, so without this mapping every span renders as "unknown". */ -const SPAN_KIND_BY_TYPE: Partial> = { - [SpanType.AGENT_RUN]: FISpanKind.AGENT, - [SpanType.MODEL_GENERATION]: FISpanKind.LLM, - [SpanType.MODEL_INFERENCE]: FISpanKind.LLM, - [SpanType.MODEL_STEP]: FISpanKind.CHAIN, - [SpanType.TOOL_CALL]: FISpanKind.TOOL, - [SpanType.MCP_TOOL_CALL]: FISpanKind.TOOL, - [SpanType.CLIENT_TOOL_CALL]: FISpanKind.TOOL, - [SpanType.WORKFLOW_RUN]: FISpanKind.CHAIN, - [SpanType.WORKFLOW_STEP]: FISpanKind.CHAIN, - [SpanType.WORKFLOW_CONDITIONAL]: FISpanKind.CHAIN, - [SpanType.WORKFLOW_PARALLEL]: FISpanKind.CHAIN, - [SpanType.WORKFLOW_LOOP]: FISpanKind.CHAIN, - [SpanType.GENERIC]: FISpanKind.CHAIN, - [SpanType.RAG_EMBEDDING]: FISpanKind.EMBEDDING, - [SpanType.RAG_VECTOR_OPERATION]: FISpanKind.RETRIEVER, +// String keys also work with the peer floor, whose enum lacks newer members. +const SPAN_KIND_BY_TYPE: Readonly>> = { + agent_run: FISpanKind.AGENT, + model_inference: FISpanKind.LLM, + model_step: FISpanKind.CHAIN, + tool_call: FISpanKind.TOOL, + mcp_tool_call: FISpanKind.TOOL, + client_tool_call: FISpanKind.TOOL, + provider_tool_call: FISpanKind.TOOL, + workflow_run: FISpanKind.CHAIN, + workflow_step: FISpanKind.CHAIN, + workflow_conditional: FISpanKind.CHAIN, + workflow_parallel: FISpanKind.CHAIN, + workflow_loop: FISpanKind.CHAIN, + generic: FISpanKind.CHAIN, + rag_embedding: FISpanKind.EMBEDDING, + rag_vector_operation: FISpanKind.RETRIEVER, }; /** @@ -76,7 +80,11 @@ export function enrichSpan(otelSpan: ReadableSpan, span: AnyExportedSpan): void const attrs = otelSpan.attributes as Record; // Span kind so the span isn't typed "unknown" in Future AGI. - const kind = SPAN_KIND_BY_TYPE[span.type]; + // Let the official converter decide which generation is an actual request: + // the floor uses MODEL_GENERATION/chat; newer SDKs use MODEL_INFERENCE/chat. + const kind = span.type === SpanType.MODEL_GENERATION + ? (attrs["gen_ai.operation.name"] === "chat" ? FISpanKind.LLM : FISpanKind.CHAIN) + : SPAN_KIND_BY_TYPE[span.type]; if (kind && attrs["gen_ai.span.kind"] == null) { attrs["gen_ai.span.kind"] = kind; } @@ -87,6 +95,18 @@ export function enrichSpan(otelSpan: ReadableSpan, span: AnyExportedSpan): void const s = span as { input?: unknown; output?: unknown }; if (attrs["input.value"] == null) setIoValue(attrs, "input", s.input); if (attrs["output.value"] == null) setIoValue(attrs, "output", s.output); + + // Older converters only look at errorInfo, but current runtimes communicate + // aborts/timeouts/interruptions through the authoritative termination fields. + const termination = span as { outcome?: string | null; stopReason?: string | null }; + if (termination.outcome) attrs["mastra.span.outcome"] = termination.outcome; + if (termination.stopReason) attrs["mastra.span.stop_reason"] = termination.stopReason; + if (termination.outcome && termination.outcome !== "success" && otelSpan.status.code !== SpanStatusCode.ERROR) { + otelSpan.status.code = ["error", "abort"].includes(termination.outcome) + ? SpanStatusCode.ERROR + : SpanStatusCode.UNSET; + delete otelSpan.status.message; + } } /** @@ -101,6 +121,9 @@ export class FIMastraSpanExporter extends BaseExporter { name = "future-agi"; private readonly converter: SpanConverter; private readonly processor: BatchSpanProcessor; + private readonly traceConfig = generateTraceConfig(); + private readonly pending = new Set>(); + private shutdownPromise?: Promise; constructor(config: FISpanExporterConfig) { super(); @@ -124,26 +147,42 @@ export class FIMastraSpanExporter extends BaseExporter { ); } + exportTracingEvent(event: TracingEvent): Promise { + if (this.shutdownPromise) return Promise.resolve(); + // Track from public entry: BaseExporter awaits its formatter before calling + // _exportTracingEvent, so tracking only conversion misses immediate shutdown. + const task = super.exportTracingEvent(event); + this.pending.add(task); + const cleanup = () => { this.pending.delete(task); }; + void task.then(cleanup, cleanup); + return task; + } + protected async _exportTracingEvent(event: TracingEvent): Promise { if (event.type !== "span_ended") return; const span = event.exportedSpan; try { const otelSpan = await this.converter.convertSpan(span); enrichSpan(otelSpan, span); - this.processor.onEnd(otelSpan); - } catch (error) { - this.logger.error( - `[@traceai/mastra] Failed to export span ${span.id}`, - error as Error, - ); + this.processor.onEnd(sanitizeSpan(otelSpan, this.traceConfig)); + } catch { + // Conversion failures can quote sensitive input; do not send it to logs. + this.logger.error("[@traceai/mastra] Failed to convert or enqueue span"); } } + private async drain(): Promise { + while (this.pending.size > 0) await Promise.allSettled([...this.pending]); + } + async flush(): Promise { + if (this.shutdownPromise) return this.shutdownPromise; + await this.drain(); await this.processor.forceFlush(); } - async shutdown(): Promise { - await this.processor.shutdown(); + shutdown(): Promise { + this.shutdownPromise ??= this.drain().then(() => this.processor.shutdown()); + return this.shutdownPromise; } } diff --git a/typescript/packages/traceai_mastra/src/FIObservability.ts b/typescript/packages/traceai_mastra/src/FIObservability.ts index 3c0b3a8e1..ae766c999 100644 --- a/typescript/packages/traceai_mastra/src/FIObservability.ts +++ b/typescript/packages/traceai_mastra/src/FIObservability.ts @@ -149,7 +149,7 @@ export function createFIObservability( ...exporterOptions, serviceName, // Default the FI project to the service name so a project is created. - projectName: exporterOptions.projectName ?? serviceName, + projectName: exporterOptions.projectName ?? process.env.FI_PROJECT_NAME ?? serviceName, }), ], }, diff --git a/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts b/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts index 815c2bed6..84825696b 100644 --- a/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts +++ b/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts @@ -20,12 +20,12 @@ function fakeMastraSpan( describe("enrichSpan — gen_ai.span.kind mapping", () => { it.each([ [SpanType.AGENT_RUN, "AGENT"], - [SpanType.MODEL_GENERATION, "LLM"], - [SpanType.MODEL_INFERENCE, "LLM"], + [SpanType.MODEL_GENERATION, "CHAIN"], + ["model_inference", "LLM"], [SpanType.MODEL_STEP, "CHAIN"], [SpanType.TOOL_CALL, "TOOL"], [SpanType.MCP_TOOL_CALL, "TOOL"], - [SpanType.CLIENT_TOOL_CALL, "TOOL"], + ["client_tool_call", "TOOL"], [SpanType.WORKFLOW_RUN, "CHAIN"], [SpanType.GENERIC, "CHAIN"], ])("maps %s -> %s", (type, kind) => { @@ -36,7 +36,7 @@ describe("enrichSpan — gen_ai.span.kind mapping", () => { it("leaves span kind unset for unmapped types", () => { const otelSpan = fakeOtelSpan(); - enrichSpan(otelSpan, fakeMastraSpan(SpanType.SCORER_RUN)); + enrichSpan(otelSpan, fakeMastraSpan("scorer_run" as SpanType)); expect(otelSpan.attributes["gen_ai.span.kind"]).toBeUndefined(); }); @@ -82,3 +82,37 @@ describe("enrichSpan — input.value / output.value", () => { expect(otelSpan.attributes["input.value"]).toBe("kept"); }); }); + + +describe("enrichSpan — authoritative converter and runtime fields", () => { + it("keeps floor generations classified as the actual LLM call", () => { + const converted = fakeOtelSpan({ "gen_ai.operation.name": "chat" }); + enrichSpan(converted, fakeMastraSpan(SpanType.MODEL_GENERATION)); + expect(converted.attributes["gen_ai.span.kind"]).toBe("LLM"); + }); + + it.each([ + ["abort", "caller_cancelled", 2], + ["abort", "timeout", 2], + ["error", "runtime_error", 2], + ["interrupted", "signal_received", 0], + ["suspend", "approval_required", 0], + ["limit", "max_steps", 0], + ["unknown", undefined, 0], + ["success", undefined, 1], + ])("preserves %s (%s) without inventing success", (outcome, stopReason, code) => { + const converted = { attributes: {}, status: { code: 1 } } as unknown as ReadableSpan; + const span = { ...fakeMastraSpan(SpanType.WORKFLOW_RUN), outcome, stopReason } as unknown as AnyExportedSpan; + enrichSpan(converted, span); + expect(converted.status.code).toBe(code); + expect(converted.attributes["mastra.span.outcome"]).toBe(outcome); + expect(converted.attributes["mastra.span.stop_reason"]).toBe(stopReason); + }); + + it("does not downgrade an existing error status", () => { + const converted = { attributes: {}, status: { code: 2 } } as unknown as ReadableSpan; + const span = { ...fakeMastraSpan(SpanType.WORKFLOW_RUN), outcome: "interrupted" } as unknown as AnyExportedSpan; + enrichSpan(converted, span); + expect(converted.status.code).toBe(2); + }); +}); diff --git a/typescript/packages/traceai_mastra/src/__tests__/privacy.test.ts b/typescript/packages/traceai_mastra/src/__tests__/privacy.test.ts new file mode 100644 index 000000000..631e0b4aa --- /dev/null +++ b/typescript/packages/traceai_mastra/src/__tests__/privacy.test.ts @@ -0,0 +1,73 @@ +import type { ReadableSpan } from "@opentelemetry/sdk-trace-base"; +import { generateTraceConfig, REDACTED_VALUE } from "@traceai/fi-core"; +import { SemanticConventions } from "@traceai/fi-semantic-conventions"; +import { sanitizeSpan } from "../privacy"; + +function converted(attributes: Record = {}): ReadableSpan { + return { + attributes, + status: { code: 2, message: "PRIVATE_ERROR\n at run (/tmp/test.js:1:2)" }, + events: [{ name: "exception", time: [0, 0], attributes: { + "exception.type": "Error", "exception.message": "PRIVATE_ERROR", + "exception.stacktrace": "PRIVATE_STACK", "exception.escaped": true, + "mastra.tool.error_message": "PRIVATE_ERROR", "custom.number": 1, + } }], + } as unknown as ReadableSpan; +} + +describe("Mastra export privacy", () => { + it("applies content policy to both span and event aliases without mutation", () => { + const attributes = Object.fromEntries([ + "input.value", "raw.input", "gen_ai.input.messages", "gen_ai.prompt", + SemanticConventions.PROMPT_TEMPLATE_VARIABLES, "gen_ai.system_instructions", + "gen_ai.tool.call.arguments", "mastra.agent_run.input", + "output.value", "raw.output", "gen_ai.output.messages", "gen_ai.completion", + "gen_ai.tool.call.result", "mastra.workflow_run.output", "error.message", + ].map(key => [key, "PRIVATE_CONTENT"])); + attributes["gen_ai.usage.input_tokens"] = "11"; + const span = converted(attributes); + span.events[0].attributes = { ...span.events[0].attributes, ...attributes }; + const original = JSON.stringify(span); + const result = sanitizeSpan(span, generateTraceConfig({ hideInputs: true, hideOutputs: true, hideErrorDetails: true })); + expect(JSON.stringify(result)).not.toContain("PRIVATE_"); + expect(result.attributes["gen_ai.usage.input_tokens"]).toBe("11"); + expect(result.status).toEqual({ code: 2 }); + expect(result.events[0].attributes).toMatchObject({ "exception.type": "Error", "exception.escaped": true, "custom.number": 1 }); + expect(JSON.stringify(span)).toBe(original); + }); + + it("masks credentials in nested serialized data, flattened metadata and events", () => { + const span = converted({ + "input.value": JSON.stringify({ nested: [{ apiKey: "PRIVATE_KEY", accessToken: "PRIVATE_TOKEN", q: "keep" }] }), + "mastra.metadata.api_key": "PRIVATE_KEY", + "mastra.metadata.headers": JSON.stringify({ Authorization: "Bearer PRIVATE_AUTH", "x-api-key": "PRIVATE_KEY" }), + }); + span.events[0].attributes = { "debug": "apiKey=PRIVATE_KEY", "x-secret-key": "PRIVATE_SECRET" }; + const result = sanitizeSpan(span, generateTraceConfig({ hideErrorDetails: true })); + expect(JSON.stringify(result)).not.toContain("PRIVATE_"); + expect(result.attributes["input.value"]).toContain("keep"); + expect(result.attributes["mastra.metadata.api_key"]).toBe(REDACTED_VALUE); + }); + + it("hides image-containing aliases independently of text flags", () => { + const span = converted({ "input.value": "PRIVATE_IMAGE", "mastra.tool_call.input": "PRIVATE_IMAGE", "gen_ai.input.messages": "PRIVATE_IMAGE" }); + const result = sanitizeSpan(span, generateTraceConfig({ hideInputImages: true })); + expect(JSON.stringify(result.attributes)).not.toContain("PRIVATE_IMAGE"); + }); + + it("hides vectors from both native and FI embedding output", () => { + const result = sanitizeSpan(converted({ + "mastra.span.type": "rag_embedding", "embedding.embeddings.0.vector": [0.1234567], + "output.value": "[[0.1234567]]", "mastra.rag_embedding.output": "[[0.1234567]]", + "gen_ai.usage.input_tokens": 5, + }), generateTraceConfig({ hideEmbeddingVectors: true })); + expect(JSON.stringify(result.attributes)).not.toContain("0.1234567"); + expect(result.attributes["gen_ai.usage.input_tokens"]).toBe(5); + }); + + it("keeps error messages without status stacks when explicitly enabled", () => { + const result = sanitizeSpan(converted(), generateTraceConfig({ hideInputs: true, hideErrorDetails: false })); + expect(result.status).toEqual({ code: 2, message: "PRIVATE_ERROR" }); + expect(result.events[0].attributes?.["exception.message"]).toBe("PRIVATE_ERROR"); + }); +}); diff --git a/typescript/packages/traceai_mastra/src/privacy.ts b/typescript/packages/traceai_mastra/src/privacy.ts new file mode 100644 index 000000000..e0d728166 --- /dev/null +++ b/typescript/packages/traceai_mastra/src/privacy.ts @@ -0,0 +1,91 @@ +import type { Attributes, AttributeValue } from "@opentelemetry/api"; +import type { ReadableSpan } from "@opentelemetry/sdk-trace-base"; +import { + ERROR_DETAIL_ATTRIBUTE_KEYS, + REDACTED_VALUE, + stripStackTrace, + type TraceConfig, +} from "@traceai/fi-core"; + +import { SemanticConventions as FI } from "@traceai/fi-semantic-conventions"; + +// Both native Mastra/GenAI and FI aliases can contain the same content. Filtering +// only input.value/output.value leaves the native copies visible on the wire. +const INPUT_KEYS = new Set([ + FI.INPUT_VALUE, FI.INPUT_MIME_TYPE, FI.RAW_INPUT, FI.LLM_INPUT_MESSAGES, + FI.PROMPT_TEMPLATE_VARIABLES, "gen_ai.prompt", "gen_ai.system_instructions", + "gen_ai.tool.call.arguments", "gen_ai.tool.definitions", +]); +const OUTPUT_KEYS = new Set([ + FI.OUTPUT_VALUE, FI.OUTPUT_MIME_TYPE, FI.RAW_OUTPUT, FI.LLM_OUTPUT_MESSAGES, + "gen_ai.completion", "gen_ai.tool.call.result", +]); +const ERROR_DETAIL = /\.(?:exception\.(?:message|stacktrace)|error\.message|error_message)$/; +const CREDENTIAL = /(?:^|[._-])(?:api[_-]?key|secret[_-]?key|access[_-]?token|refresh[_-]?token|authorization|password|secret|token|cookie|set[_-]?cookie)$/i; + +/** Mask structured credentials even on older SDKs without a default data filter. */ +function redactCredentials(value: unknown, depth = 0): unknown { + if (depth > 32) return REDACTED_VALUE; + if (Array.isArray(value)) return value.map(item => redactCredentials(item, depth + 1)); + if (value !== null && typeof value === "object") { + return Object.fromEntries(Object.entries(value).map(([key, item]) => [ + key, CREDENTIAL.test(key) ? REDACTED_VALUE : redactCredentials(item, depth + 1), + ])); + } + if (typeof value !== "string") return value; + // The native converter serializes object-valued inputs, outputs and metadata. + const trimmed = value.trimStart(); + if (trimmed.startsWith("{") || trimmed.startsWith("[")) { + try { + return JSON.stringify(redactCredentials(JSON.parse(value), depth + 1)); + } catch { + // Not JSON: handle credential assignments in plain text below. + } + } + return value.replace( + /\b(api[_-]?key|secret[_-]?key|access[_-]?token|refresh[_-]?token|authorization|password|secret|token)\s*[:=]\s*(?:Bearer\s+)?[^\s,;]+/gi, + `$1=${REDACTED_VALUE}`, + ); +} + +function maskAttributes(attributes: Attributes, config: TraceConfig, embedding: boolean): Attributes { + const masked: Attributes = {}; + for (const [key, value] of Object.entries(attributes)) { + if (value == null) continue; + if (config.hideErrorDetails && (ERROR_DETAIL_ATTRIBUTE_KEYS.has(key) || ERROR_DETAIL.test(key))) continue; + const input = INPUT_KEYS.has(key) || /^mastra\..*\.(?:input|instructions)$/.test(key); + const output = OUTPUT_KEYS.has(key) || /^mastra\..*\.output$/.test(key); + // Native and FI I/O blobs do not share a reliable typed message schema. + // Hide the entire alias for granular content flags rather than leave a + // second unmasked copy (including tool arguments and system instructions). + const hideInput = input && (config.hideInputs || config.hideInputMessages || config.hideInputText || config.hideInputImages); + const hideOutput = output && (config.hideOutputs || config.hideOutputMessages || config.hideOutputText); + const hideVector = config.hideEmbeddingVectors && (/(?:embedding|embeddings).*vector/.test(key) || (embedding && output)); + if (hideInput || hideOutput || hideVector) { + if (!key.endsWith("mime_type")) masked[key] = REDACTED_VALUE; + continue; + } + masked[key] = CREDENTIAL.test(key) + ? REDACTED_VALUE + : redactCredentials(value) as AttributeValue; + } + return masked; +} + +/** Return a sanitized view; never mutate the native span seen by other exporters. */ +export function sanitizeSpan(span: ReadableSpan, config: TraceConfig): ReadableSpan { + const embedding = span.attributes["mastra.span.type"] === "rag_embedding"; + const message = span.status.message; + const status = message == null || config.hideErrorDetails + ? { code: span.status.code } + : { code: span.status.code, message: redactCredentials(stripStackTrace(message)) as string }; + return { + ...span, + status, + attributes: maskAttributes(span.attributes, config, embedding), + events: span.events.map(event => ({ + ...event, + attributes: maskAttributes(event.attributes ?? {}, config, embedding), + })), + }; +} From 042176b7684e7b0c96a21e2d1c90571e4da2b95b Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 02:09:49 -0800 Subject: [PATCH 2/5] fix(mastra): complete privacy policy and mixed-peer conversion Cover native request text, structured and error-text credentials, and opt-in shared PII redaction. Classify calls from emitting-runtime feature support, reuse native conversion across mixed peer versions, and emit deterministic integration metadata. Exercise CJS/ESM wire contracts across floor, latest and both mixed SDK sets. Co-Authored-By: Claude Code --- .github/workflows/mastra-typescript-tests.yml | 13 ++- .../fi-core/src/trace/trace-config/index.ts | 2 + .../traceai_mastra/contract/recall.test.cjs | 105 +++++++++++++++++- .../src/FIMastraSpanExporter.ts | 66 +++++++++-- .../src/__tests__/enrichment.test.ts | 30 ++++- .../src/__tests__/privacy.test.ts | 47 ++++++++ .../packages/traceai_mastra/src/privacy.ts | 31 ++++-- 7 files changed, 261 insertions(+), 33 deletions(-) diff --git a/.github/workflows/mastra-typescript-tests.yml b/.github/workflows/mastra-typescript-tests.yml index 208473d0f..6d6127bd9 100644 --- a/.github/workflows/mastra-typescript-tests.yml +++ b/.github/workflows/mastra-typescript-tests.yml @@ -29,6 +29,8 @@ jobs: include: - { version: floor, core: '1.16.0', observability: '1.0.0', exporter: '1.0.0' } - { version: latest, core: latest, observability: latest, exporter: latest } + - { version: latest-runtime-floor-converter, core: latest, observability: latest, exporter: '1.0.0' } + - { version: floor-runtime-latest-converter, core: latest, observability: '1.0.0', exporter: latest } defaults: run: shell: bash @@ -51,6 +53,7 @@ jobs: "$(command -v node)" "$PWD/node_modules/jest/bin/jest.js" \ --config "$PWD/packages/traceai_mastra/jest.config.js" --runInBand mkdir -p "$RUNNER_TEMP/mastra-package" + pnpm --dir packages/fi-core pack --pack-destination "$RUNNER_TEMP/mastra-package" pnpm --dir packages/traceai_mastra pack --pack-destination "$RUNNER_TEMP/mastra-package" git diff --exit-code -- pnpm-lock.yaml - name: Install packed exporter and real SDK in a clean consumer @@ -68,7 +71,9 @@ jobs: - name: Native lifecycle and wire contracts (loopback-only network namespace) run: | export MASTRA_TEST_DEPS="$RUNNER_TEMP/mastra-consumer" - export MASTRA_TEST_PACKAGE="$RUNNER_TEMP/mastra-consumer/node_modules/@traceai/mastra/dist/src/index.js" - sudo unshare --net bash -c 'ip link set lo up; exec "$@"' _ \ - env MASTRA_TEST_DEPS="$MASTRA_TEST_DEPS" MASTRA_TEST_PACKAGE="$MASTRA_TEST_PACKAGE" \ - "$(command -v node)" --test "$PWD/packages/traceai_mastra/contract/recall.test.cjs" + for entrypoint in src esm; do + export MASTRA_TEST_PACKAGE="$RUNNER_TEMP/mastra-consumer/node_modules/@traceai/mastra/dist/$entrypoint/index.js" + sudo unshare --net bash -c 'ip link set lo up; exec "$@"' _ \ + env MASTRA_TEST_DEPS="$MASTRA_TEST_DEPS" MASTRA_TEST_PACKAGE="$MASTRA_TEST_PACKAGE" \ + "$(command -v node)" --test "$PWD/packages/traceai_mastra/contract/recall.test.cjs" + done diff --git a/typescript/packages/fi-core/src/trace/trace-config/index.ts b/typescript/packages/fi-core/src/trace/trace-config/index.ts index 6a077d4ec..147889c0b 100644 --- a/typescript/packages/fi-core/src/trace/trace-config/index.ts +++ b/typescript/packages/fi-core/src/trace/trace-config/index.ts @@ -23,6 +23,8 @@ export { REDACTED_VALUE, } from "./constants"; export { generateTraceConfig } from "./traceConfig"; +// Native exporters reuse the same opt-in policy without creating extra spans. +export { redactPiiInValue } from "./piiRedaction"; export { ERROR_TYPE, ERROR_MESSAGE, diff --git a/typescript/packages/traceai_mastra/contract/recall.test.cjs b/typescript/packages/traceai_mastra/contract/recall.test.cjs index 4a7eef063..b5916e7d2 100644 --- a/typescript/packages/traceai_mastra/contract/recall.test.cjs +++ b/typescript/packages/traceai_mastra/contract/recall.test.cjs @@ -9,13 +9,19 @@ const { test, before, after, beforeEach, afterEach } = require('node:test'); const packagePath = process.env.MASTRA_TEST_PACKAGE || path.resolve(__dirname, '../dist/src/index.js'); const req = createRequire(process.env.MASTRA_TEST_DEPS ? path.join(process.env.MASTRA_TEST_DEPS, 'package.json') : packagePath); -const { createFIObservability } = require(packagePath); +const { pathToFileURL } = require('node:url'); +let createFIObservability; const { SpanType } = req('@mastra/core/observability'); +// Core's enum alone is not evidence that the installed observability runtime +// implements outcomes. In particular, new-core/old-observability is supported. +const { observabilityFeatures } = req('@mastra/observability'); +const supportsOutcomes = observabilityFeatures?.has('model-inference-span') === true; + const wire = req('@opentelemetry/otlp-transformer/build/src/generated/root.js') .opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; const flags = ['INPUTS', 'OUTPUTS', 'INPUT_MESSAGES', 'OUTPUT_MESSAGES', 'INPUT_TEXT', 'OUTPUT_TEXT', 'INPUT_IMAGES', 'EMBEDDING_VECTORS', 'ERROR_DETAILS'].map(key => `FI_HIDE_${key}`); -const envKeys = [...flags, 'FI_PROJECT_NAME']; +const envKeys = [...flags, 'FI_PROJECT_NAME', 'FI_PII_REDACTION']; let originalEnv, obs, endpoint; const spans = []; const attributes = list => Object.fromEntries((list || []).map(a => [a.key, Object.values(a.value)[0]])); @@ -29,7 +35,7 @@ const server = http.createServer((request, response) => { const data = wire.toObject(wire.decode(Buffer.concat(chunks)), { longs: String, bytes: String }); for (const resource of data.resourceSpans || []) { for (const scope of resource.scopeSpans || []) { - for (const span of scope.spans || []) spans.push({ ...span, resource: attributes(resource.resource.attributes), attrs: attributes(span.attributes) }); + for (const span of scope.spans || []) spans.push({ ...span, scope: scope.scope, resource: attributes(resource.resource.attributes), attrs: attributes(span.attributes) }); } } response.writeHead(200, { 'content-type': 'application/x-protobuf' }); @@ -37,6 +43,7 @@ const server = http.createServer((request, response) => { }); }); before(async () => { + ({ createFIObservability } = await import(pathToFileURL(packagePath).href)); await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); endpoint = `http://127.0.0.1:${server.address().port}/tracer/v1/traces`; }); @@ -178,6 +185,9 @@ test('generation produces one LLM observation per real model request', async () assert.equal(calls, 1); const llm = spans.filter(span => span.attrs['gen_ai.span.kind'] === 'LLM'); assert.equal(llm.length, calls); + assert.equal(spans.filter(span => span.attrs['gen_ai.operation.name'] === 'chat').length, calls); + assert.equal(spans.reduce((n, span) => n + Number(span.attrs['gen_ai.usage.input_tokens'] || 0), 0), 11); + assert.equal(spans.reduce((n, span) => n + Number(span.attrs['gen_ai.usage.output_tokens'] || 0), 0), 5); assert.equal(llm[0].status?.code, 1); assert.equal(Number(llm[0].attrs['gen_ai.usage.input_tokens']), 11); assert.equal(Number(llm[0].attrs['gen_ai.usage.output_tokens']), 5); @@ -216,7 +226,7 @@ test('failed conversions settle and do not prevent draining healthy spans', asyn assert.equal(spans.length, 1); assert.equal(spans[0].name, 'valid'); }); -test('real stream cancellation preserves the unsuccessful SDK outcomes', { skip: !SpanType.MODEL_INFERENCE }, async () => { +test('real stream cancellation preserves the unsuccessful SDK outcomes', { skip: !supportsOutcomes }, async () => { const { Mastra } = req('@mastra/core'); const { Agent } = req('@mastra/core/agent'); setup(); @@ -247,7 +257,7 @@ test('real stream cancellation preserves the unsuccessful SDK outcomes', { skip: assert.equal(span.attrs['mastra.span.stop_reason'], 'caller_cancelled'); } }); -test('native unsuccessful outcomes are never OK', { skip: !SpanType.MODEL_INFERENCE }, async () => { +test('native unsuccessful outcomes are never OK', { skip: !supportsOutcomes }, async () => { const instance = setup(); for (const termination of [ { outcome: 'abort', stopReason: 'caller_cancelled' }, @@ -263,3 +273,88 @@ test('native unsuccessful outcomes are never OK', { skip: !SpanType.MODEL_INFERE assert.ok(span.attrs['mastra.span.stop_reason']); } }); + +// Round-one recall regressions: native converter aliases, not invented wire keys. +test('input hiding covers native stop sequences and tool descriptions', async () => { + process.env.FI_HIDE_INPUTS = 'true'; + const instance = setup(); + instance.startSpan({ type: SpanType.MODEL_INFERENCE || SpanType.MODEL_GENERATION, + name: 'native-model', input: 'PRIVATE_INPUT', attributes: { + model: 'synthetic', parameters: { stopSequences: ['PRIVATE_STOP_SEQUENCE'] }, + } }).end({ output: 'safe-output' }); + instance.startSpan({ type: SpanType.TOOL_CALL, name: 'native-tool', entityName: 'tool', + input: 'PRIVATE_INPUT', attributes: { toolDescription: 'PRIVATE_TOOL_DESCRIPTION' }, + }).end({ output: 'safe-output' }); + await flush(); + assert.equal(spans.length, 2); + assert.doesNotMatch(JSON.stringify(spans), /PRIVATE_/); +}); +test('input hiding covers stop sequences supplied to a real generation', async () => { + process.env.FI_HIDE_INPUTS = 'true'; + const { Mastra } = req('@mastra/core'); + const { Agent } = req('@mastra/core/agent'); + setup(); + let calls = 0; + const model = { specificationVersion: 'v2', provider: 'synthetic', modelId: 'synthetic', supportedUrls: {}, + doGenerate: async options => { + calls++; + assert.deepEqual(options.stopSequences, ['PRIVATE_STOP_SEQUENCE']); + return { content: [{ type: 'text', text: 'safe-output' }], finishReason: 'stop', + usage: { inputTokens: 11, outputTokens: 5, totalTokens: 16 }, warnings: [] }; + } }; + const agent = new Agent({ id: 'stop-agent', name: 'Stop agent', instructions: 'PRIVATE_SYSTEM', model }); + const mastra = new Mastra({ agents: { agent }, observability: obs }); + await mastra.getAgent('agent').generate('PRIVATE_INPUT', { modelSettings: { stopSequences: ['PRIVATE_STOP_SEQUENCE'] } }); + await flush(); + assert.equal(calls, 1); + assert.ok(spans.length > 0); + assert.doesNotMatch(JSON.stringify(spans), /PRIVATE_/); +}); +test('credential scrubbing covers nested camelCase keys without upstream filtering', async () => { + const instance = setup(); + instance.startSpan({ type: SpanType.TOOL_CALL, name: 'credential-tool', entityName: 'tool', + input: { clientSecret: 'PRIVATE_CLIENT_SECRET', privateKey: 'PRIVATE_PRIVATE_KEY', + nested: [{ apiKey: 'PRIVATE_API_KEY', accessToken: 'PRIVATE_ACCESS_TOKEN', keep: 'safe' }] }, + }).end({ output: { privateKey: 'PRIVATE_OUTPUT_KEY', keep: 'safe' } }); + await flush(); + assert.equal(spans.length, 1); + assert.doesNotMatch(JSON.stringify(spans), /PRIVATE_/); + assert.match(JSON.stringify(spans), /safe/); +}); +for (const errorText of [ + 'Request failed; headers: {"Authorization":"Bearer PRIVATE_JSON_AUTH"}', + 'Authorization: Basic c3ludGhldGljOmNyZWRlbnRpYWw=', + 'Request failed; headers: {"proxy-authorization":"Basic c3ludGhldGljOmNyZWRlbnRpYWw="}', + "Request failed; clientSecret='PRIVATE_CLIENT_SECRET' privateKey=PRIVATE_PRIVATE_KEY", +]) test(`scrubs credential-bearing errors: ${errorText}`, async () => { + setup().startSpan({ type: SpanType.TOOL_CALL, name: 'credential-error', entityName: 'tool' }) + .error({ error: new Error(errorText), endSpan: true }); + await flush(); + assert.equal(spans.length, 1); + assert.equal(spans[0].status.code, 2); + assert.doesNotMatch(JSON.stringify(spans), /PRIVATE_|c3ludGhldGljOmNyZWRlbnRpYWw=/); +}); +for (const enabled of [true, false]) test(`PII redaction is opt-in (${enabled}) across native/FI content and errors`, async () => { + process.env.FI_PII_REDACTION = String(enabled); + const instance = setup(); + instance.startSpan({ type: SpanType.TOOL_CALL, name: 'pii-tool', entityName: 'tool', + input: { contact: 'customer@example.test', count: 7 }, + }).end({ output: ['customer@example.test', 'safe'] }); + instance.startSpan({ type: SpanType.TOOL_CALL, name: 'pii-error', entityName: 'tool' }) + .error({ error: new Error('Contact customer@example.test'), endSpan: true }); + await flush(); + assert.equal(spans.length, 2); + const data = JSON.stringify(spans); + if (enabled) { + assert.doesNotMatch(data, /customer@example\.test/); + assert.match(data, //); + } else assert.match(data, /customer@example\.test/); +}); +test('integration version is deterministic on the OTLP resource and scope', async () => { + setup().startSpan({ type: SpanType.GENERIC, name: 'version' }).end(); + await flush(); + const { VERSION } = await import(pathToFileURL(path.join(path.dirname(packagePath), 'version.js')).href); + assert.equal(spans.length, 1); + assert.equal(spans[0].scope.version, VERSION); + assert.equal(spans[0].resource['telemetry.sdk.version'], VERSION); +}); diff --git a/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts b/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts index 09bc14916..af220980d 100644 --- a/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts +++ b/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts @@ -1,4 +1,6 @@ import { BaseExporter } from "@mastra/observability"; +import * as observability from "@mastra/observability"; +import { coreFeatures } from "@mastra/core/features"; import { SpanConverter } from "@mastra/otel-exporter"; import { SpanType } from "@mastra/core/observability"; import type { AnyExportedSpan, TracingEvent } from "@mastra/core/observability"; @@ -6,9 +8,21 @@ import { FISpanKind } from "@traceai/fi-semantic-conventions"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto"; import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base"; import type { ReadableSpan } from "@opentelemetry/sdk-trace-base"; -import { SpanStatusCode } from "@opentelemetry/api"; +import { SpanKind, SpanStatusCode } from "@opentelemetry/api"; import { generateTraceConfig } from "@traceai/fi-core"; import { sanitizeSpan } from "./privacy.js"; +import { VERSION } from "./version.js"; + +function runtimeEmitsInference(): boolean { + // Use the emitting runtime, NOT the converter's potentially nested dependency. + // A namespace import keeps strict ESM compatible with the old peer floor. + const features = (observability as { observabilityFeatures?: ReadonlySet }).observabilityFeatures; + return features?.has("model-inference-span") === true && coreFeatures.has("model-inference-span"); +} + +function isModelCall(type: string, inferenceEnabled: boolean): boolean { + return type === "model_inference" || (type === "model_generation" && !inferenceEnabled); +} export interface FISpanExporterConfig { /** Full OTLP traces endpoint URL. */ @@ -76,15 +90,24 @@ function setIoValue( * (from the span's input/output). Existing values are not overwritten. Exported * for unit testing. */ -export function enrichSpan(otelSpan: ReadableSpan, span: AnyExportedSpan): void { +export function enrichSpan( + otelSpan: ReadableSpan, + span: AnyExportedSpan, + inferenceEnabled = runtimeEmitsInference(), +): void { const attrs = otelSpan.attributes as Record; - // Span kind so the span isn't typed "unknown" in Future AGI. - // Let the official converter decide which generation is an actual request: - // the floor uses MODEL_GENERATION/chat; newer SDKs use MODEL_INFERENCE/chat. - const kind = span.type === SpanType.MODEL_GENERATION - ? (attrs["gen_ai.operation.name"] === "chat" ? FISpanKind.LLM : FISpanKind.CHAIN) - : SPAN_KIND_BY_TYPE[span.type]; + const kind = isModelCall(span.type, inferenceEnabled) ? FISpanKind.LLM + : span.type === SpanType.MODEL_GENERATION ? FISpanKind.CHAIN : SPAN_KIND_BY_TYPE[span.type]; + if (span.type === SpanType.MODEL_GENERATION && inferenceEnabled) { + // Old converters classify the new generation loop as chat and attach its + // aggregate usage. Keep that usage only on the real provider-call spans. + attrs["gen_ai.operation.name"] = "model_generation"; + for (const key of Object.keys(attrs)) { + if (/^gen_ai\.(?:usage|request|response)\./.test(key) + || key === "gen_ai.input.messages" || key === "gen_ai.output.messages") delete attrs[key]; + } + } if (kind && attrs["gen_ai.span.kind"] == null) { attrs["gen_ai.span.kind"] = kind; } @@ -132,7 +155,10 @@ export class FIMastraSpanExporter extends BaseExporter { packageName: "@traceai/mastra", serviceName: config.serviceName, // Only `resourceAttributes` is read off this config by the converter. - config: { resourceAttributes: config.resourceAttributes } as any, + config: { resourceAttributes: { + ...config.resourceAttributes, + "telemetry.sdk.version": VERSION, + } } as any, }); const exporter = new OTLPTraceExporter({ url: config.endpoint, @@ -162,9 +188,25 @@ export class FIMastraSpanExporter extends BaseExporter { if (event.type !== "span_ended") return; const span = event.exportedSpan; try { - const otelSpan = await this.converter.convertSpan(span); - enrichSpan(otelSpan, span); - this.processor.onEnd(sanitizeSpan(otelSpan, this.traceConfig)); + const inferenceEnabled = runtimeEmitsInference(); + const modelCall = isModelCall(span.type, inferenceEnabled); + let otelSpan = await this.converter.convertSpan(span); + if (modelCall && otelSpan.attributes["gen_ai.operation.name"] !== "chat") { + // Independently versioned converters can recognize the other request + // type. Reuse their native message/usage/parameter conversion with a + // shallow view; never mutate the runtime span or emit a second span. + const converterType = span.type === "model_inference" ? "model_generation" : "model_inference"; + otelSpan = await this.converter.convertSpan({ ...span, type: converterType } as AnyExportedSpan); + otelSpan.attributes["mastra.span.type"] = span.type; + } + enrichSpan(otelSpan, span, inferenceEnabled); + const generationWrapper = span.type === SpanType.MODEL_GENERATION && inferenceEnabled; + this.processor.onEnd(sanitizeSpan({ + ...otelSpan, + ...(generationWrapper ? { name: span.name, kind: SpanKind.INTERNAL } : {}), + ...(modelCall ? { kind: SpanKind.CLIENT } : {}), + instrumentationScope: { ...otelSpan.instrumentationScope, version: VERSION }, + }, this.traceConfig)); } catch { // Conversion failures can quote sensitive input; do not send it to logs. this.logger.error("[@traceai/mastra] Failed to convert or enqueue span"); diff --git a/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts b/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts index 84825696b..27316ae37 100644 --- a/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts +++ b/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts @@ -30,7 +30,7 @@ describe("enrichSpan — gen_ai.span.kind mapping", () => { [SpanType.GENERIC, "CHAIN"], ])("maps %s -> %s", (type, kind) => { const otelSpan = fakeOtelSpan(); - enrichSpan(otelSpan, fakeMastraSpan(type as SpanType)); + enrichSpan(otelSpan, fakeMastraSpan(type as SpanType), true); expect(otelSpan.attributes["gen_ai.span.kind"]).toBe(kind); }); @@ -84,10 +84,10 @@ describe("enrichSpan — input.value / output.value", () => { }); -describe("enrichSpan — authoritative converter and runtime fields", () => { +describe("enrichSpan — authoritative runtime fields", () => { it("keeps floor generations classified as the actual LLM call", () => { const converted = fakeOtelSpan({ "gen_ai.operation.name": "chat" }); - enrichSpan(converted, fakeMastraSpan(SpanType.MODEL_GENERATION)); + enrichSpan(converted, fakeMastraSpan(SpanType.MODEL_GENERATION), false); expect(converted.attributes["gen_ai.span.kind"]).toBe("LLM"); }); @@ -116,3 +116,27 @@ describe("enrichSpan — authoritative converter and runtime fields", () => { expect(converted.status.code).toBe(2); }); }); + + +describe("enrichSpan — mismatched converter versions", () => { + it("ignores an old converter's chat and aggregate usage on a new wrapper", () => { + const converted = fakeOtelSpan({ "gen_ai.operation.name": "chat", + "gen_ai.usage.input_tokens": 11, "gen_ai.usage.output_tokens": 5, + "gen_ai.request.model": "synthetic", "gen_ai.response.model": "synthetic", + "gen_ai.input.messages": "question", "gen_ai.output.messages": "answer", + }); + enrichSpan(converted, fakeMastraSpan(SpanType.MODEL_GENERATION, "question", "answer"), true); + expect(converted.attributes["gen_ai.span.kind"]).toBe("CHAIN"); + expect(converted.attributes["gen_ai.operation.name"]).toBe("model_generation"); + expect(Object.keys(converted.attributes).some(key => key.startsWith("gen_ai.usage."))).toBe(false); + expect(converted.attributes["gen_ai.request.model"]).toBeUndefined(); + expect(converted.attributes["input.value"]).toBe("question"); + expect(converted.attributes["output.value"]).toBe("answer"); + }); + + it("does not demote a floor runtime call because the new converter lacks chat", () => { + const converted = fakeOtelSpan({ "gen_ai.operation.name": "model_generation" }); + enrichSpan(converted, fakeMastraSpan(SpanType.MODEL_GENERATION), false); + expect(converted.attributes["gen_ai.span.kind"]).toBe("LLM"); + }); +}); diff --git a/typescript/packages/traceai_mastra/src/__tests__/privacy.test.ts b/typescript/packages/traceai_mastra/src/__tests__/privacy.test.ts index 631e0b4aa..907f769b9 100644 --- a/typescript/packages/traceai_mastra/src/__tests__/privacy.test.ts +++ b/typescript/packages/traceai_mastra/src/__tests__/privacy.test.ts @@ -21,6 +21,7 @@ describe("Mastra export privacy", () => { "input.value", "raw.input", "gen_ai.input.messages", "gen_ai.prompt", SemanticConventions.PROMPT_TEMPLATE_VARIABLES, "gen_ai.system_instructions", "gen_ai.tool.call.arguments", "mastra.agent_run.input", + "gen_ai.tool.description", "gen_ai.request.stop_sequences", "output.value", "raw.output", "gen_ai.output.messages", "gen_ai.completion", "gen_ai.tool.call.result", "mastra.workflow_run.output", "error.message", ].map(key => [key, "PRIVATE_CONTENT"])); @@ -71,3 +72,49 @@ describe("Mastra export privacy", () => { expect(result.events[0].attributes?.["exception.message"]).toBe("PRIVATE_ERROR"); }); }); + + +describe("Mastra complete credential and PII policy", () => { + it.each(["clientSecret", "client_secret", "client-secret", "privateKey", "private_key", "private-key", + "APIKey", "api_key", "mastra.metadata.clientSecret", "headers.Proxy-Authorization"])("scrubs %s", key => { + const span = converted({ [key]: "PRIVATE_VALUE", "input.value": JSON.stringify({ [key]: "PRIVATE_VALUE", keep: "safe" }) }); + const result = sanitizeSpan(span, generateTraceConfig()); + expect(JSON.stringify(result)).not.toContain("PRIVATE_VALUE"); + expect(result.attributes["input.value"]).toContain("safe"); + }); + + it.each([ + 'Request failed; headers: {"Authorization":"Bearer PRIVATE_VALUE"}', + 'Authorization: Basic PRIVATE_VALUE', + 'Authorization: Bearer PRIVATE_VALUE', + 'clientSecret="PRIVATE_VALUE with spaces"', + "private_key='PRIVATE_VALUE with spaces'", + 'Request failed; {"privateKey":"PRIVATE_VALUE\\nsecond line"}', + ])("scrubs credentials from status and exception text: %s", message => { + const span = converted(); + span.status.message = message; + span.events[0].attributes = { "exception.message": message, "exception.stacktrace": message }; + const original = JSON.stringify(span); + const result = sanitizeSpan(span, generateTraceConfig()); + expect(JSON.stringify(result)).not.toContain("PRIVATE_VALUE"); + expect(JSON.stringify(result)).not.toContain("second line"); + expect(result.status.code).toBe(2); + expect(JSON.stringify(span)).toBe(original); + }); + + it("uses the shared PII policy for scalar/array/serialized attributes, events and status", () => { + const span = converted({ "input.value": JSON.stringify({ email: "customer@example.test", count: 7 }), + "gen_ai.request.stop_sequences": ["customer@example.test"], "custom.count": 7, + }); + span.status.message = "customer@example.test"; + span.events[0].attributes = { "exception.message": "customer@example.test" }; + const original = JSON.stringify(span); + const result = sanitizeSpan(span, generateTraceConfig({ piiRedaction: true })); + expect(JSON.stringify(result)).not.toContain("customer@example.test"); + expect(result.attributes["input.value"]).toContain(""); + expect(result.attributes["gen_ai.request.stop_sequences"]).toEqual([""]); + expect(result.attributes["custom.count"]).toBe(7); + expect(JSON.stringify(span)).toBe(original); + expect(JSON.stringify(sanitizeSpan(span, generateTraceConfig({ piiRedaction: false })))).toContain("customer@example.test"); + }); +}); diff --git a/typescript/packages/traceai_mastra/src/privacy.ts b/typescript/packages/traceai_mastra/src/privacy.ts index e0d728166..8cb7e8ba1 100644 --- a/typescript/packages/traceai_mastra/src/privacy.ts +++ b/typescript/packages/traceai_mastra/src/privacy.ts @@ -4,6 +4,7 @@ import { ERROR_DETAIL_ATTRIBUTE_KEYS, REDACTED_VALUE, stripStackTrace, + redactPiiInValue, type TraceConfig, } from "@traceai/fi-core"; @@ -15,13 +16,23 @@ const INPUT_KEYS = new Set([ FI.INPUT_VALUE, FI.INPUT_MIME_TYPE, FI.RAW_INPUT, FI.LLM_INPUT_MESSAGES, FI.PROMPT_TEMPLATE_VARIABLES, "gen_ai.prompt", "gen_ai.system_instructions", "gen_ai.tool.call.arguments", "gen_ai.tool.definitions", + "gen_ai.request.stop_sequences", "gen_ai.tool.description", ]); const OUTPUT_KEYS = new Set([ FI.OUTPUT_VALUE, FI.OUTPUT_MIME_TYPE, FI.RAW_OUTPUT, FI.LLM_OUTPUT_MESSAGES, "gen_ai.completion", "gen_ai.tool.call.result", ]); const ERROR_DETAIL = /\.(?:exception\.(?:message|stacktrace)|error\.message|error_message)$/; -const CREDENTIAL = /(?:^|[._-])(?:api[_-]?key|secret[_-]?key|access[_-]?token|refresh[_-]?token|authorization|password|secret|token|cookie|set[_-]?cookie)$/i; +const CREDENTIAL = /(?:^|_)(?:api_?key|secret_?key|private_?key|client_?secret|access_?token|refresh_?token|id_?token|authorization|password|secret|token|cookie|set_?cookie)$/; + +function isCredentialKey(key: string): boolean { + const normalized = key.replace(/([a-z0-9])([A-Z])/g, "$1_$2").replace(/[.-]/g, "_").toLowerCase(); + return CREDENTIAL.test(normalized); +} + +// Also match quoted fields inside non-JSON error text. Consume the complete +// quoted value or Basic/Bearer scheme + payload, not just its first word. +const CREDENTIAL_ASSIGNMENT = /\b((?:proxy[_-]?)?authorization|api[_-]?key|secret[_-]?key|private[_-]?key|client[_-]?secret|access[_-]?token|refresh[_-]?token|id[_-]?token|password|secret|token|cookie|set[_-]?cookie)(["']?)\s*[:=]\s*(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|(?:Bearer|Basic)\s+[^\s,;}"']+|[^\s,;}"']+)/gi; /** Mask structured credentials even on older SDKs without a default data filter. */ function redactCredentials(value: unknown, depth = 0): unknown { @@ -29,7 +40,7 @@ function redactCredentials(value: unknown, depth = 0): unknown { if (Array.isArray(value)) return value.map(item => redactCredentials(item, depth + 1)); if (value !== null && typeof value === "object") { return Object.fromEntries(Object.entries(value).map(([key, item]) => [ - key, CREDENTIAL.test(key) ? REDACTED_VALUE : redactCredentials(item, depth + 1), + key, isCredentialKey(key) ? REDACTED_VALUE : redactCredentials(item, depth + 1), ])); } if (typeof value !== "string") return value; @@ -42,10 +53,12 @@ function redactCredentials(value: unknown, depth = 0): unknown { // Not JSON: handle credential assignments in plain text below. } } - return value.replace( - /\b(api[_-]?key|secret[_-]?key|access[_-]?token|refresh[_-]?token|authorization|password|secret|token)\s*[:=]\s*(?:Bearer\s+)?[^\s,;]+/gi, - `$1=${REDACTED_VALUE}`, - ); + return value.replace(CREDENTIAL_ASSIGNMENT, `$1$2: "${REDACTED_VALUE}"`); +} + +function sanitizeValue(value: AttributeValue, config: TraceConfig): AttributeValue { + const scrubbed = redactCredentials(value) as AttributeValue; + return config.piiRedaction ? redactPiiInValue(scrubbed) : scrubbed; } function maskAttributes(attributes: Attributes, config: TraceConfig, embedding: boolean): Attributes { @@ -65,9 +78,9 @@ function maskAttributes(attributes: Attributes, config: TraceConfig, embedding: if (!key.endsWith("mime_type")) masked[key] = REDACTED_VALUE; continue; } - masked[key] = CREDENTIAL.test(key) + masked[key] = isCredentialKey(key) ? REDACTED_VALUE - : redactCredentials(value) as AttributeValue; + : sanitizeValue(value, config); } return masked; } @@ -78,7 +91,7 @@ export function sanitizeSpan(span: ReadableSpan, config: TraceConfig): ReadableS const message = span.status.message; const status = message == null || config.hideErrorDetails ? { code: span.status.code } - : { code: span.status.code, message: redactCredentials(stripStackTrace(message)) as string }; + : { code: span.status.code, message: sanitizeValue(stripStackTrace(message), config) as string }; return { ...span, status, From 8cd88330e1d5f82bd4c1c0162548240b10f492c7 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 02:50:34 -0800 Subject: [PATCH 3/5] fix(mastra): scrub serialized credentials and preserve native context Cover AWS credential fields, full cookie headers and nested JSON error serialization. Reconcile remote parents and links with older converters, and classify native workflow control-flow spans. Add failing-first SDK/OTLP contracts and unit coverage. Co-Authored-By: Claude Code --- .../traceai_mastra/contract/recall.test.cjs | 105 ++++++++++++++++++ .../src/FIMastraSpanExporter.ts | 40 ++++++- .../src/__tests__/enrichment.test.ts | 44 +++++++- .../src/__tests__/privacy.test.ts | 39 +++++++ .../packages/traceai_mastra/src/privacy.ts | 26 ++++- 5 files changed, 248 insertions(+), 6 deletions(-) diff --git a/typescript/packages/traceai_mastra/contract/recall.test.cjs b/typescript/packages/traceai_mastra/contract/recall.test.cjs index b5916e7d2..803e1a14a 100644 --- a/typescript/packages/traceai_mastra/contract/recall.test.cjs +++ b/typescript/packages/traceai_mastra/contract/recall.test.cjs @@ -358,3 +358,108 @@ test('integration version is deterministic on the OTLP resource and scope', asyn assert.equal(spans[0].scope.version, VERSION); assert.equal(spans[0].resource['telemetry.sdk.version'], VERSION); }); + +// Round-two recall: credential serialization and older-converter context gaps. +test('AWS credentials never leave native tool I/O with content capture enabled', async () => { + setup().startSpan({ type: SpanType.TOOL_CALL, name: 'aws-tool', input: { + credentials: { accessKeyId: 'PRIVATE_AWS_ID', secretAccessKey: 'PRIVATE_AWS_SECRET', sessionToken: 'PRIVATE_AWS_SESSION' }, + keep: 'safe-input', + } }).end({ output: { secret_access_key: 'PRIVATE_OUTPUT_SECRET', keep: 'safe-output' } }); + await flush(); + assert.equal(spans.length, 1); + assert.doesNotMatch(JSON.stringify(spans), /PRIVATE_/); + assert.match(spans[0].attrs['input.value'], /safe-input/); + assert.match(spans[0].attrs['output.value'], /safe-output/); +}); +for (const message of [ + 'Request failed; Cookie: sessionid=PRIVATE_FIRST_COOKIE; sid=PRIVATE_SECOND_COOKIE', + 'Request failed; Set-Cookie: sid=PRIVATE_COOKIE; HttpOnly; Secure', + 'Request failed; encoded config: ' + JSON.stringify(JSON.stringify({ headers: { Authorization: 'Bearer PRIVATE_ESCAPED_AUTH' }, keep: 'safe' })), + 'Request failed; encoded config: ' + JSON.stringify(JSON.stringify(JSON.stringify({ credentials: { secretAccessKey: 'PRIVATE_ENCODED_SECRET' } }))), +]) test(`serialized credentials are scrubbed in status and exception events: ${message}`, async () => { + setup().startSpan({ type: SpanType.TOOL_CALL, name: 'serialized-error' }).error({ error: new Error(message), endSpan: true }); + await flush(); + assert.equal(spans.length, 1); + assert.equal(spans[0].status.code, 2); + assert.ok(spans[0].events.length > 0); + assert.doesNotMatch(JSON.stringify(spans), /PRIVATE_/); +}); +test('a real agent remains nested under its remote parent with every supported converter', async () => { + const { Mastra } = req('@mastra/core'); + const { Agent } = req('@mastra/core/agent'); + setup(); + const model = { specificationVersion: 'v2', provider: 'synthetic', modelId: 'synthetic', supportedUrls: {}, + doGenerate: async () => ({ content: [{ type: 'text', text: 'safe' }], finishReason: 'stop', + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, warnings: [] }) }; + const agent = new Agent({ id: 'remote', name: 'Remote agent', instructions: 'Answer', model }); + const mastra = new Mastra({ agents: { agent }, observability: obs }); + await mastra.getAgent('agent').generate('question', { tracingOptions: { + traceId: '11111111111111111111111111111111', parentSpanId: '2222222222222222', + } }); + await flush(); + const root = spans.find(span => span.attrs['gen_ai.span.kind'] === 'AGENT'); + assert.ok(root); + assert.equal(root.traceId, Buffer.from('11111111111111111111111111111111', 'hex').toString('base64')); + assert.equal(root.parentSpanId, Buffer.from('2222222222222222', 'hex').toString('base64')); + assert.equal(new Set(spans.map(span => span.spanId)).size, spans.length); +}); +test('native remote parentage and explicit links survive conversion without duplicates', async () => { + const instance = setup(); + const root = instance.startSpan({ type: SpanType.AGENT_RUN, name: 'linked-agent', + tracingOptions: { traceId: '11111111111111111111111111111111', parentSpanId: '2222222222222222' }, + links: [{ traceId: '33333333333333333333333333333333', spanId: '4444444444444444' }], + }); + instance.startSpan({ type: SpanType.TOOL_CALL, name: 'local-child', parent: root }).end(); + root.end(); + await flush(); + const agent = spans.find(span => span.attrs['gen_ai.span.kind'] === 'AGENT'); + const child = spans.find(span => span.attrs['gen_ai.span.kind'] === 'TOOL'); + assert.equal(spans.length, 2); + assert.equal(agent.parentSpanId, Buffer.from('2222222222222222', 'hex').toString('base64')); + assert.equal(child.parentSpanId, agent.spanId); + // The old runtime cannot produce links; old converters still must retain + // links supplied by newer runtimes (the mixed row runs this assertion). + if (supportsOutcomes) { + assert.equal(agent.links.length, 1); + assert.equal(agent.links[0].traceId, Buffer.from('33333333333333333333333333333333', 'hex').toString('base64')); + assert.equal(agent.links[0].spanId, Buffer.from('4444444444444444', 'hex').toString('base64')); + } +}); +test('all native workflow control-flow kinds are classified as chains', async () => { + const instance = setup(); + for (const type of [SpanType.WORKFLOW_SLEEP, SpanType.WORKFLOW_WAIT_EVENT, SpanType.WORKFLOW_CONDITIONAL_EVAL]) { + assert.ok(type); + instance.startSpan({ type, name: type }).end(); + } + await flush(); + assert.equal(spans.length, 3); + for (const span of spans) assert.equal(span.attrs['gen_ai.span.kind'], 'CHAIN'); +}); +test('a successful real workflow sleep is classified without duplicating its spans', async () => { + const { Mastra } = req('@mastra/core'); + const { createWorkflow, createStep } = req('@mastra/core/workflows'); + const { z } = req('zod'); + setup(); + const schema = z.object({ q: z.string() }); + const step = createStep({ id: 'safe-step', inputSchema: schema, outputSchema: schema, execute: async ({ inputData }) => inputData }); + const workflow = createWorkflow({ id: 'sleep-workflow', inputSchema: schema, outputSchema: schema }).then(step).sleep(1).commit(); + const mastra = new Mastra({ workflows: { workflow }, observability: obs }); + const run = await mastra.getWorkflow('workflow').createRun(); + const result = await run.start({ inputData: { q: 'safe' } }); + assert.equal(result.status, 'success'); + await flush(); + const sleeps = spans.filter(span => span.attrs['mastra.span.type'] === 'workflow_sleep'); + assert.equal(sleeps.length, 1); + for (const span of spans) assert.equal(span.attrs['gen_ai.span.kind'], 'CHAIN'); + assert.equal(new Set(spans.map(span => span.spanId)).size, spans.length); +}); +test('an explicit native link survives even the oldest converter', { skip: !supportsOutcomes }, async () => { + setup().startSpan({ type: SpanType.AGENT_RUN, name: 'linked-root', + links: [{ traceId: '33333333333333333333333333333333', spanId: '4444444444444444' }], + }).end(); + await flush(); + assert.equal(spans.length, 1); + assert.equal(spans[0].links?.length, 1); + assert.equal(spans[0].links[0].traceId, Buffer.from('33333333333333333333333333333333', 'hex').toString('base64')); + assert.equal(spans[0].links[0].spanId, Buffer.from('4444444444444444', 'hex').toString('base64')); +}); diff --git a/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts b/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts index af220980d..997bd422e 100644 --- a/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts +++ b/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts @@ -8,7 +8,7 @@ import { FISpanKind } from "@traceai/fi-semantic-conventions"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto"; import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base"; import type { ReadableSpan } from "@opentelemetry/sdk-trace-base"; -import { SpanKind, SpanStatusCode } from "@opentelemetry/api"; +import { SpanKind, SpanStatusCode, TraceFlags } from "@opentelemetry/api"; import { generateTraceConfig } from "@traceai/fi-core"; import { sanitizeSpan } from "./privacy.js"; import { VERSION } from "./version.js"; @@ -60,6 +60,9 @@ const SPAN_KIND_BY_TYPE: Readonly>> = { workflow_conditional: FISpanKind.CHAIN, workflow_parallel: FISpanKind.CHAIN, workflow_loop: FISpanKind.CHAIN, + workflow_sleep: FISpanKind.CHAIN, + workflow_wait_event: FISpanKind.CHAIN, + workflow_conditional_eval: FISpanKind.CHAIN, generic: FISpanKind.CHAIN, rag_embedding: FISpanKind.EMBEDDING, rag_vector_operation: FISpanKind.RETRIEVER, @@ -132,6 +135,40 @@ export function enrichSpan( } } +/** Preserve context fields that older supported converters do not understand. */ +export function reconcileSpanContext(otelSpan: ReadableSpan, span: AnyExportedSpan): ReadableSpan { + // Structural view also compiles against the floor's pre-link span interface. + const native = span as AnyExportedSpan & { + externalParentSpanId?: string; + links?: ReadonlyArray<{ traceId: string; spanId: string }>; + }; + const parentId = native.parentSpanId ?? native.externalParentSpanId; + const links = [...otelSpan.links]; + const linked = new Set(links.map(link => `${link.context.traceId}:${link.context.spanId}`)); + for (const link of native.links ?? []) { + const key = `${link.traceId}:${link.spanId}`; + if (linked.has(key)) continue; + linked.add(key); + links.push({ context: { + traceId: link.traceId, + spanId: link.spanId, + traceFlags: TraceFlags.SAMPLED, + isRemote: true, + } }); + } + return { + ...otelSpan, + // Keep a converter's richer context and link attributes when present. + parentSpanContext: otelSpan.parentSpanContext ?? (parentId ? { + traceId: span.traceId, + spanId: parentId, + traceFlags: otelSpan.spanContext().traceFlags, + isRemote: native.parentSpanId == null, + } : undefined), + links, + }; +} + /** * Mastra v1 observability exporter for Future AGI. * @@ -199,6 +236,7 @@ export class FIMastraSpanExporter extends BaseExporter { otelSpan = await this.converter.convertSpan({ ...span, type: converterType } as AnyExportedSpan); otelSpan.attributes["mastra.span.type"] = span.type; } + otelSpan = reconcileSpanContext(otelSpan, span); enrichSpan(otelSpan, span, inferenceEnabled); const generationWrapper = span.type === SpanType.MODEL_GENERATION && inferenceEnabled; this.processor.onEnd(sanitizeSpan({ diff --git a/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts b/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts index 27316ae37..90093cabb 100644 --- a/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts +++ b/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts @@ -1,7 +1,7 @@ import { SpanType } from "@mastra/core/observability"; import type { AnyExportedSpan } from "@mastra/core/observability"; import type { ReadableSpan } from "@opentelemetry/sdk-trace-base"; -import { enrichSpan } from "../FIMastraSpanExporter"; +import { enrichSpan, reconcileSpanContext } from "../FIMastraSpanExporter"; function fakeOtelSpan( attributes: Record = {}, @@ -27,6 +27,9 @@ describe("enrichSpan — gen_ai.span.kind mapping", () => { [SpanType.MCP_TOOL_CALL, "TOOL"], ["client_tool_call", "TOOL"], [SpanType.WORKFLOW_RUN, "CHAIN"], + ["workflow_sleep", "CHAIN"], + ["workflow_wait_event", "CHAIN"], + ["workflow_conditional_eval", "CHAIN"], [SpanType.GENERIC, "CHAIN"], ])("maps %s -> %s", (type, kind) => { const otelSpan = fakeOtelSpan(); @@ -140,3 +143,42 @@ describe("enrichSpan — mismatched converter versions", () => { expect(converted.attributes["gen_ai.span.kind"]).toBe("LLM"); }); }); + +describe("context reconciliation across converter versions", () => { + const traceId = "11111111111111111111111111111111"; + const parentId = "2222222222222222"; + const nativeLink = { traceId: "33333333333333333333333333333333", spanId: "4444444444444444" }; + function converted(): ReadableSpan { + return { ...fakeOtelSpan(), links: [], spanContext: () => ({ traceId, spanId: "5555555555555555", traceFlags: 1 }) } as ReadableSpan; + } + it("recovers an external parent and native links without mutating either input", () => { + const native = { ...fakeMastraSpan(SpanType.AGENT_RUN), traceId, externalParentSpanId: parentId, + links: [nativeLink, nativeLink] } as unknown as AnyExportedSpan; + const otel = converted(); + const original = JSON.stringify({ native, otel }); + const result = reconcileSpanContext(otel, native); + expect(result.parentSpanContext).toEqual({ traceId, spanId: parentId, traceFlags: 1, isRemote: true }); + expect(result.links).toEqual([{ context: { ...nativeLink, traceFlags: 1, isRemote: true } }]); + expect(JSON.stringify({ native, otel })).toBe(original); + }); + it("uses a local parent before an inherited external parent", () => { + const native = { ...fakeMastraSpan(SpanType.TOOL_CALL), traceId, parentSpanId: parentId, + externalParentSpanId: "6666666666666666" } as unknown as AnyExportedSpan; + expect(reconcileSpanContext(converted(), native).parentSpanContext).toEqual({ traceId, spanId: parentId, traceFlags: 1, isRemote: false }); + }); + it("retains current converter context and link metadata without appending duplicates", () => { + const otel = { ...converted(), parentSpanContext: { traceId, spanId: parentId, traceFlags: 0, isRemote: true }, + links: [{ context: { ...nativeLink, traceFlags: 0 }, attributes: { reason: "handoff" } }], + } as ReadableSpan; + const native = { ...fakeMastraSpan(SpanType.AGENT_RUN), links: [nativeLink] } as unknown as AnyExportedSpan; + const result = reconcileSpanContext(otel, native); + expect(result.parentSpanContext).toBe(otel.parentSpanContext); + expect(result.links).toEqual(otel.links); + expect(result.links).not.toBe(otel.links); + }); + it("does not invent parentage or links for a root", () => { + const result = reconcileSpanContext(converted(), fakeMastraSpan(SpanType.AGENT_RUN)); + expect(result.parentSpanContext).toBeUndefined(); + expect(result.links).toEqual([]); + }); +}); diff --git a/typescript/packages/traceai_mastra/src/__tests__/privacy.test.ts b/typescript/packages/traceai_mastra/src/__tests__/privacy.test.ts index 907f769b9..4e0f72799 100644 --- a/typescript/packages/traceai_mastra/src/__tests__/privacy.test.ts +++ b/typescript/packages/traceai_mastra/src/__tests__/privacy.test.ts @@ -118,3 +118,42 @@ describe("Mastra complete credential and PII policy", () => { expect(JSON.stringify(sanitizeSpan(span, generateTraceConfig({ piiRedaction: false })))).toContain("customer@example.test"); }); }); + +describe("Mastra serialized credential boundaries", () => { + it.each(["secretAccessKey", "secret_access_key", "secret-access-key", "accessKeyId", "AWS_SECRET_ACCESS_KEY", + "credentials.secretAccessKey", "aws.sessionToken"])("scrubs AWS field %s", key => { + const span = converted({ [key]: "PRIVATE_AWS", "input.value": JSON.stringify({ [key]: "PRIVATE_AWS", keep: "safe" }) }); + const original = JSON.stringify(span); + const result = sanitizeSpan(span, generateTraceConfig()); + expect(JSON.stringify(result)).not.toContain("PRIVATE_AWS"); + expect(result.attributes["input.value"]).toContain("safe"); + expect(JSON.stringify(span)).toBe(original); + }); + + it.each([ + 'Cookie: first=PRIVATE_FIRST; second=PRIVATE_SECOND', + 'Set-Cookie: sid=PRIVATE_FIRST; Path=/; Secure', + 'Request failed; Cookie: first=PRIVATE_FIRST; second=PRIVATE_SECOND\nRetry after 2 seconds', + 'secretAccessKey=PRIVATE_FIRST access_key_id=PRIVATE_SECOND', + 'Request failed: ' + JSON.stringify(JSON.stringify({ headers: { Authorization: 'Bearer PRIVATE_FIRST', Cookie: 'first=PRIVATE_SECOND; sid=PRIVATE_THIRD' }, keep: 'safe' })), + JSON.stringify(JSON.stringify(JSON.stringify({ secretAccessKey: 'PRIVATE_FIRST' }))), + ])("scrubs serialized/text credentials without mutating the source: %s", message => { + const span = converted({ "input.value": message }); + span.status.message = message; + span.events[0].attributes = { "exception.message": message }; + const original = JSON.stringify(span); + const result = sanitizeSpan(span, generateTraceConfig()); + expect(JSON.stringify(result)).not.toContain("PRIVATE_"); + expect(JSON.stringify(span)).toBe(original); + if (message.includes("Retry after")) expect(result.status.message).toContain("Retry after 2 seconds"); + }); + + it("retains ordinary quoted diagnostics and bounds nested serialization", () => { + const safe = 'Request failed: {"message":"not authorized", "count":2}'; + const result = sanitizeSpan(converted({ "input.value": safe }), generateTraceConfig()); + expect(result.attributes["input.value"]).toBe(safe); + let nested: unknown = { secretAccessKey: "PRIVATE_DEEP" }; + for (let i = 0; i < 40; i++) nested = { nested }; + expect(JSON.stringify(sanitizeSpan(converted({ "input.value": JSON.stringify(nested) }), generateTraceConfig()))).not.toContain("PRIVATE_DEEP"); + }); +}); diff --git a/typescript/packages/traceai_mastra/src/privacy.ts b/typescript/packages/traceai_mastra/src/privacy.ts index 8cb7e8ba1..73b5b9435 100644 --- a/typescript/packages/traceai_mastra/src/privacy.ts +++ b/typescript/packages/traceai_mastra/src/privacy.ts @@ -23,7 +23,7 @@ const OUTPUT_KEYS = new Set([ "gen_ai.completion", "gen_ai.tool.call.result", ]); const ERROR_DETAIL = /\.(?:exception\.(?:message|stacktrace)|error\.message|error_message)$/; -const CREDENTIAL = /(?:^|_)(?:api_?key|secret_?key|private_?key|client_?secret|access_?token|refresh_?token|id_?token|authorization|password|secret|token|cookie|set_?cookie)$/; +const CREDENTIAL = /(?:^|_)(?:api_?key|secret_?key|secret_?access_?key|access_?key_?id|private_?key|client_?secret|access_?token|refresh_?token|id_?token|authorization|password|secret|token|cookie|set_?cookie)$/; function isCredentialKey(key: string): boolean { const normalized = key.replace(/([a-z0-9])([A-Z])/g, "$1_$2").replace(/[.-]/g, "_").toLowerCase(); @@ -32,7 +32,11 @@ function isCredentialKey(key: string): boolean { // Also match quoted fields inside non-JSON error text. Consume the complete // quoted value or Basic/Bearer scheme + payload, not just its first word. -const CREDENTIAL_ASSIGNMENT = /\b((?:proxy[_-]?)?authorization|api[_-]?key|secret[_-]?key|private[_-]?key|client[_-]?secret|access[_-]?token|refresh[_-]?token|id[_-]?token|password|secret|token|cookie|set[_-]?cookie)(["']?)\s*[:=]\s*(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|(?:Bearer|Basic)\s+[^\s,;}"']+|[^\s,;}"']+)/gi; +const CREDENTIAL_ASSIGNMENT = /\b((?:proxy[_-]?)?authorization|api[_-]?key|secret[_-]?key|secret[_-]?access[_-]?key|access[_-]?key[_-]?id|private[_-]?key|client[_-]?secret|access[_-]?token|refresh[_-]?token|id[_-]?token|password|secret|token)(["']?)\s*[:=]\s*(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|(?:Bearer|Basic)\s+[^\s,;}"']+|[^\s,;}"']+)/gi; +// A Cookie header is a list, not one scalar token. In unstructured diagnostics +// conservatively consume the complete header line, including every semicolon. +const COOKIE_ASSIGNMENT = /\b((?:set[_-]?)?cookie)(["']?)\s*[:=]\s*(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|[^\r\n"']+)/gi; +const JSON_STRING = /"(?:\\.|[^"\\])*"/g; /** Mask structured credentials even on older SDKs without a default data filter. */ function redactCredentials(value: unknown, depth = 0): unknown { @@ -46,14 +50,28 @@ function redactCredentials(value: unknown, depth = 0): unknown { if (typeof value !== "string") return value; // The native converter serializes object-valued inputs, outputs and metadata. const trimmed = value.trimStart(); - if (trimmed.startsWith("{") || trimmed.startsWith("[")) { + if (trimmed.startsWith("{") || trimmed.startsWith("[") || trimmed.startsWith('"')) { try { return JSON.stringify(redactCredentials(JSON.parse(value), depth + 1)); } catch { // Not JSON: handle credential assignments in plain text below. } } - return value.replace(CREDENTIAL_ASSIGNMENT, `$1$2: "${REDACTED_VALUE}"`); + // Error prefixes can surround JSON-stringified JSON. Decode complete string + // literals before inspecting their contents, retaining the original encoding + // and bounding recursive decoding with the same depth limit as objects. + const decoded = value.replace(JSON_STRING, literal => { + try { + const text = JSON.parse(literal) as string; + const scrubbed = redactCredentials(text, depth + 1); + return scrubbed === text ? literal : JSON.stringify(scrubbed); + } catch { + return literal; + } + }); + return decoded + .replace(COOKIE_ASSIGNMENT, `$1$2: "${REDACTED_VALUE}"`) + .replace(CREDENTIAL_ASSIGNMENT, `$1$2: "${REDACTED_VALUE}"`); } function sanitizeValue(value: AttributeValue, config: TraceConfig): AttributeValue { From 5c384e310d3549679c8229559e9ea4b23b756812 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 13:43:03 -0800 Subject: [PATCH 4/5] fix(mastra): scrub embedded diagnostic credentials and classify processors Co-Authored-By: Claude Code --- .../traceai_mastra/contract/recall.test.cjs | 27 ++++++++++ .../src/FIMastraSpanExporter.ts | 1 + .../src/__tests__/enrichment.test.ts | 1 + .../src/__tests__/privacy.test.ts | 16 ++++++ .../packages/traceai_mastra/src/privacy.ts | 49 ++++++++++++++++++- 5 files changed, 92 insertions(+), 2 deletions(-) diff --git a/typescript/packages/traceai_mastra/contract/recall.test.cjs b/typescript/packages/traceai_mastra/contract/recall.test.cjs index 803e1a14a..dcd8ddc00 100644 --- a/typescript/packages/traceai_mastra/contract/recall.test.cjs +++ b/typescript/packages/traceai_mastra/contract/recall.test.cjs @@ -192,6 +192,30 @@ test('generation produces one LLM observation per real model request', async () assert.equal(Number(llm[0].attrs['gen_ai.usage.input_tokens']), 11); assert.equal(Number(llm[0].attrs['gen_ai.usage.output_tokens']), 5); }); +test('real agent tool execution classifies processor_run as CHAIN', async () => { + const { Mastra } = req('@mastra/core'); + const { Agent } = req('@mastra/core/agent'); + const { createTool } = req('@mastra/core/tools'); + const { z } = req('zod'); + setup(); + let calls = 0; + const model = { specificationVersion: 'v2', provider: 'synthetic', modelId: 'synthetic', supportedUrls: {}, + doGenerate: async () => ++calls === 1 + ? { content: [{ type: 'tool-call', toolCallId: 'tool-1', toolName: 'lookup', input: '{}' }], + finishReason: 'tool-calls', usage: { inputTokens: 2, outputTokens: 1 }, warnings: [] } + : { content: [{ type: 'text', text: 'done' }], finishReason: 'stop', + usage: { inputTokens: 2, outputTokens: 1 }, warnings: [] } }; + const lookup = createTool({ id: 'lookup', description: 'Look up synthetic data', + inputSchema: z.object({}), execute: async () => ({ ok: true }) }); + const agent = new Agent({ id: 'processor-agent', name: 'Processor agent', instructions: 'Use tool', model, tools: { lookup } }); + const mastra = new Mastra({ agents: { agent }, observability: obs }); + await mastra.getAgent('agent').generate('Run lookup', { maxSteps: 3 }); + await flush(); + const processors = spans.filter(span => span.attrs['mastra.span.type'] === 'processor_run'); + if (supportsOutcomes) assert.ok(processors.length > 0, 'current runtime must emit processor_run'); + for (const span of processors) assert.equal(span.attrs['gen_ai.span.kind'], 'CHAIN'); +}); + test('shutdown waits for delayed conversion and rejects new events', async () => { const instance = setup(); const exporter = instance.getExporters()[0]; @@ -376,6 +400,9 @@ for (const message of [ 'Request failed; Set-Cookie: sid=PRIVATE_COOKIE; HttpOnly; Secure', 'Request failed; encoded config: ' + JSON.stringify(JSON.stringify({ headers: { Authorization: 'Bearer PRIVATE_ESCAPED_AUTH' }, keep: 'safe' })), 'Request failed; encoded config: ' + JSON.stringify(JSON.stringify(JSON.stringify({ credentials: { secretAccessKey: 'PRIVATE_ENCODED_SECRET' } }))), + 'Request failed; config: ' + JSON.stringify({ sessionToken: 'PRIVATE_SESSION', auth_token: 'PRIVATE_AUTH' }), + 'Request failed; Authorization: AWS4-HMAC-SHA256 Credential=PRIVATE_ACCESS/20261102/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature=PRIVATE_SIGNATURE', + 'Request failed; Authorization: Digest username="synthetic", realm="test", nonce="PRIVATE_NONCE", response="PRIVATE_DIGEST"', ]) test(`serialized credentials are scrubbed in status and exception events: ${message}`, async () => { setup().startSpan({ type: SpanType.TOOL_CALL, name: 'serialized-error' }).error({ error: new Error(message), endSpan: true }); await flush(); diff --git a/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts b/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts index 997bd422e..61a2f9e8e 100644 --- a/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts +++ b/typescript/packages/traceai_mastra/src/FIMastraSpanExporter.ts @@ -51,6 +51,7 @@ const SPAN_KIND_BY_TYPE: Readonly>> = { agent_run: FISpanKind.AGENT, model_inference: FISpanKind.LLM, model_step: FISpanKind.CHAIN, + processor_run: FISpanKind.CHAIN, tool_call: FISpanKind.TOOL, mcp_tool_call: FISpanKind.TOOL, client_tool_call: FISpanKind.TOOL, diff --git a/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts b/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts index 90093cabb..e44e02160 100644 --- a/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts +++ b/typescript/packages/traceai_mastra/src/__tests__/enrichment.test.ts @@ -23,6 +23,7 @@ describe("enrichSpan — gen_ai.span.kind mapping", () => { [SpanType.MODEL_GENERATION, "CHAIN"], ["model_inference", "LLM"], [SpanType.MODEL_STEP, "CHAIN"], + ["processor_run", "CHAIN"], [SpanType.TOOL_CALL, "TOOL"], [SpanType.MCP_TOOL_CALL, "TOOL"], ["client_tool_call", "TOOL"], diff --git a/typescript/packages/traceai_mastra/src/__tests__/privacy.test.ts b/typescript/packages/traceai_mastra/src/__tests__/privacy.test.ts index 4e0f72799..05f34d06d 100644 --- a/typescript/packages/traceai_mastra/src/__tests__/privacy.test.ts +++ b/typescript/packages/traceai_mastra/src/__tests__/privacy.test.ts @@ -148,6 +148,22 @@ describe("Mastra serialized credential boundaries", () => { if (message.includes("Retry after")) expect(result.status.message).toContain("Retry after 2 seconds"); }); + it.each([ + 'Tool failed: {"sessionToken":"PRIVATE_SESSION","auth_token":"PRIVATE_AUTH","message":"retry"}', + 'Tool failed: {"error":{"sessionToken":"PRIVATE_SESSION","auth_token":"PRIVATE_AUTH"},"message":"retry"}', + 'Tool failed: Authorization: AWS4-HMAC-SHA256 Credential=PRIVATE_SESSION/20261102/region/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=PRIVATE_AUTH', + 'Tool failed: Authorization: Digest username="PRIVATE_SESSION", realm="service", nonce="PRIVATE_AUTH", response="PRIVATE_RESPONSE"', + ])("scrubs embedded tool diagnostics and complete authorization headers: %s", message => { + const span = converted({ "mastra.tool_call.error_message": message }); + span.status.message = message; + span.events[0].attributes = { "exception.message": message, "exception.stacktrace": message }; + const original = JSON.stringify(span); + const result = sanitizeSpan(span, generateTraceConfig()); + expect(JSON.stringify(result)).not.toMatch(/PRIVATE_(?:SESSION|AUTH|RESPONSE)/); + expect(result.status.code).toBe(2); + expect(JSON.stringify(span)).toBe(original); + }); + it("retains ordinary quoted diagnostics and bounds nested serialization", () => { const safe = 'Request failed: {"message":"not authorized", "count":2}'; const result = sanitizeSpan(converted({ "input.value": safe }), generateTraceConfig()); diff --git a/typescript/packages/traceai_mastra/src/privacy.ts b/typescript/packages/traceai_mastra/src/privacy.ts index 73b5b9435..6c8d87887 100644 --- a/typescript/packages/traceai_mastra/src/privacy.ts +++ b/typescript/packages/traceai_mastra/src/privacy.ts @@ -32,12 +32,56 @@ function isCredentialKey(key: string): boolean { // Also match quoted fields inside non-JSON error text. Consume the complete // quoted value or Basic/Bearer scheme + payload, not just its first word. -const CREDENTIAL_ASSIGNMENT = /\b((?:proxy[_-]?)?authorization|api[_-]?key|secret[_-]?key|secret[_-]?access[_-]?key|access[_-]?key[_-]?id|private[_-]?key|client[_-]?secret|access[_-]?token|refresh[_-]?token|id[_-]?token|password|secret|token)(["']?)\s*[:=]\s*(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|(?:Bearer|Basic)\s+[^\s,;}"']+|[^\s,;}"']+)/gi; +const CREDENTIAL_ASSIGNMENT = /\b((?:proxy[_-]?)?authorization|api[_-]?key|secret[_-]?key|secret[_-]?access[_-]?key|access[_-]?key[_-]?id|private[_-]?key|client[_-]?secret|access[_-]?token|refresh[_-]?token|id[_-]?token|session[_-]?token|auth[_-]?token|password|secret|token)(["']?)\s*[:=]\s*(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|(?:Bearer|Basic)\s+[^\s,;}"']+|[^\s,;}"']+)/gi; // A Cookie header is a list, not one scalar token. In unstructured diagnostics // conservatively consume the complete header line, including every semicolon. const COOKIE_ASSIGNMENT = /\b((?:set[_-]?)?cookie)(["']?)\s*[:=]\s*(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|[^\r\n"']+)/gi; +// Signed Authorization schemes carry multiple comma/space-separated fields. +// A scalar-token matcher would leave AWS Credential/Signature and Digest nonce +// on the wire. Keep a quoted JSON value bounded, or consume the entire line. +const AUTHORIZATION_ASSIGNMENT = /\b((?:proxy[_-]?)?authorization)(["']?)\s*[:=]\s*(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|[^\r\n]+)/gi; const JSON_STRING = /"(?:\\.|[^"\\])*"/g; +/** Scrub JSON objects embedded after a diagnostic prefix, including nested fields. */ +function scrubEmbeddedJson(text: string, depth: number): string { + let result = ""; + let start = 0; + for (let i = 0; i < text.length; i++) { + if (text[i] !== "{" && text[i] !== "[") continue; + const opening = i; + const stack: string[] = []; + let quoted = false; + let escaped = false; + let end = i; + for (; end < text.length; end++) { + const char = text[end]; + if (quoted) { + if (escaped) escaped = false; + else if (char === "\\") escaped = true; + else if (char === '"') quoted = false; + } else if (char === '"') quoted = true; + else if (char === "{" || char === "[") stack.push(char); + else if (char === "}" || char === "]") { + const expected = char === "}" ? "{" : "["; + if (stack.pop() !== expected) break; + if (stack.length === 0) { end++; break; } + } + } + if (stack.length !== 0 || end > text.length) continue; + const candidate = text.slice(opening, end); + try { + const parsed = JSON.parse(candidate); + const scrubbed = JSON.stringify(redactCredentials(parsed, depth + 1)); + result += text.slice(start, opening) + (scrubbed === JSON.stringify(parsed) ? candidate : scrubbed); + start = end; + i = end - 1; + } catch { + // This brace-delimited diagnostic was not JSON. + } + } + return result + text.slice(start); +} + /** Mask structured credentials even on older SDKs without a default data filter. */ function redactCredentials(value: unknown, depth = 0): unknown { if (depth > 32) return REDACTED_VALUE; @@ -69,8 +113,9 @@ function redactCredentials(value: unknown, depth = 0): unknown { return literal; } }); - return decoded + return scrubEmbeddedJson(decoded, depth) .replace(COOKIE_ASSIGNMENT, `$1$2: "${REDACTED_VALUE}"`) + .replace(AUTHORIZATION_ASSIGNMENT, `$1$2: "${REDACTED_VALUE}"`) .replace(CREDENTIAL_ASSIGNMENT, `$1$2: "${REDACTED_VALUE}"`); } From b67ec746fdc4e0f1bc878fe30e49f72da824cdad Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 14:27:39 -0800 Subject: [PATCH 5/5] fix(mastra): require released fi-core PII helper Stage fi-core 1.1.1 for dependency-first release and constrain Mastra to a core build that exports redactPiiInValue. Add a regression guard for the packed dependency floor. Co-Authored-By: Claude Code --- .release-please-manifest.json | 2 +- typescript/packages/fi-core/CHANGELOG.md | 4 +++ typescript/packages/fi-core/package.json | 2 +- .../packages/traceai_mastra/package.json | 2 +- .../src/__tests__/coreCompatibility.test.ts | 31 +++++++++++++++++++ typescript/pnpm-lock.yaml | 2 +- 6 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 typescript/packages/traceai_mastra/src/__tests__/coreCompatibility.test.ts diff --git a/.release-please-manifest.json b/.release-please-manifest.json index d394b6567..5a4030065 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -70,7 +70,7 @@ "python/frameworks/weaviate": "0.1.2", "python/frameworks/xai": "0.1.1", "python/traceai": "0.1.0", - "typescript/packages/fi-core": "1.1.0", + "typescript/packages/fi-core": "1.1.1", "typescript/packages/fi-semantic-conventions": "1.0.0", "typescript/packages/traceai_anthropic": "0.1.4", "typescript/packages/traceai_bedrock": "0.1.2", diff --git a/typescript/packages/fi-core/CHANGELOG.md b/typescript/packages/fi-core/CHANGELOG.md index 7bb212cb1..312c1eae8 100644 --- a/typescript/packages/fi-core/CHANGELOG.md +++ b/typescript/packages/fi-core/CHANGELOG.md @@ -1,3 +1,7 @@ +## [1.1.1] - pending release +### Fix +- Export `redactPiiInValue` from the package root for integrations that apply the shared PII policy at the OTLP export boundary. `@traceai/mastra` requires at least this version; publish fi-core before publishing Mastra. + ## [1.1.0] - 2026-10-18 ### Security fix: content hiding skipped the attributes passed when a span is created - **This bug predates 1.1.0: it is in every published release, 0.1.0 to 1.0.0.** `FITracer.startSpan` handed `options.attributes` straight to the OpenTelemetry span, so attributes passed at span creation skipped `FISpan` and every masking rule: `hideInputs` and the other hide flags, PII redaction, and (new in this release) error-detail hiding. About 22 instrumentations put the prompt there, `@traceai/openai` included, so with `FI_HIDE_INPUTS=true` the prompt was still exported in `input.value`, `gen_ai.input.messages` and `raw.input`. diff --git a/typescript/packages/fi-core/package.json b/typescript/packages/fi-core/package.json index c29956ca3..d9f22f4f1 100644 --- a/typescript/packages/fi-core/package.json +++ b/typescript/packages/fi-core/package.json @@ -1,6 +1,6 @@ { "name": "@traceai/fi-core", - "version": "1.1.0", + "version": "1.1.1", "description": "OpenTelemetry tracing for AI applications: tracer setup, span helpers, and exporters for Future AGI.", "main": "./dist/src/index.js", "module": "./dist/esm/index.js", diff --git a/typescript/packages/traceai_mastra/package.json b/typescript/packages/traceai_mastra/package.json index a1e86637c..da581fb33 100644 --- a/typescript/packages/traceai_mastra/package.json +++ b/typescript/packages/traceai_mastra/package.json @@ -45,7 +45,7 @@ "@opentelemetry/exporter-trace-otlp-proto": "^0.201.1", "@opentelemetry/sdk-trace-base": "^2.0.1", "@opentelemetry/semantic-conventions": "^1.34.0", - "@traceai/fi-core": "workspace:^", + "@traceai/fi-core": "workspace:^1.1.1", "@traceai/fi-semantic-conventions": "workspace:^", "@traceai/vercel": "workspace:^" }, diff --git a/typescript/packages/traceai_mastra/src/__tests__/coreCompatibility.test.ts b/typescript/packages/traceai_mastra/src/__tests__/coreCompatibility.test.ts new file mode 100644 index 000000000..058da00cc --- /dev/null +++ b/typescript/packages/traceai_mastra/src/__tests__/coreCompatibility.test.ts @@ -0,0 +1,31 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { redactPiiInValue } from "@traceai/fi-core"; + +const packageJson = (path: string): { version: string; dependencies: Record } => + JSON.parse(readFileSync(resolve(__dirname, path), "utf8")); + +const parts = (version: string): number[] => version.split(".").map(Number); +const compare = (left: string, right: string): number => { + const a = parts(left); + const b = parts(right); + for (let i = 0; i < 3; i++) { + if (a[i] !== b[i]) return a[i] - b[i]; + } + return 0; +}; + +describe("released fi-core compatibility", () => { + it("requires a fi-core release that exports the PII helper", () => { + const mastra = packageJson("../../package.json"); + const core = packageJson("../../../fi-core/package.json"); + const range = mastra.dependencies["@traceai/fi-core"]; + const match = /^workspace:\^(\d+\.\d+\.\d+)$/.exec(range); + expect(match).not.toBeNull(); + // fi-core 1.1.0 is already published without redactPiiInValue. A plain + // workspace:^ would pack as ^1.1.0 and allow a lockfile to retain it. + expect(compare(match![1], "1.1.0")).toBeGreaterThan(0); + expect(compare(core.version, match![1])).toBeGreaterThanOrEqual(0); + expect(redactPiiInValue("test@example.com")).toBe(""); + }); +}); diff --git a/typescript/pnpm-lock.yaml b/typescript/pnpm-lock.yaml index 4f2577fb8..77a246bf4 100644 --- a/typescript/pnpm-lock.yaml +++ b/typescript/pnpm-lock.yaml @@ -1386,7 +1386,7 @@ importers: specifier: ^1.34.0 version: 1.34.0 '@traceai/fi-core': - specifier: workspace:^ + specifier: workspace:^1.1.1 version: link:../fi-core '@traceai/fi-semantic-conventions': specifier: workspace:^