diff --git a/.github/workflows/google-genai-sdk-compat.yml b/.github/workflows/google-genai-sdk-compat.yml index 18370f66d..9b9ae5dd2 100644 --- a/.github/workflows/google-genai-sdk-compat.yml +++ b/.github/workflows/google-genai-sdk-compat.yml @@ -33,7 +33,18 @@ jobs: - uses: actions/setup-node@v4 with: node-version: ${{ matrix.node }} - - name: Build and pack with published shared dependencies + - name: Build and pack candidate shared dependencies + working-directory: typescript + shell: bash + run: | + set -euo pipefail + npm install --global pnpm@10.30.1 + pnpm --filter @traceai/fi-core... install --frozen-lockfile --ignore-scripts + pnpm --filter @traceai/fi-semantic-conventions run build + pnpm --filter @traceai/fi-core run build + pnpm --dir packages/fi-semantic-conventions pack --out "$RUNNER_TEMP/semconv.tgz" + pnpm --dir packages/fi-core pack --out "$RUNNER_TEMP/core.tgz" + - name: Build adapter against published shared dependencies shell: bash run: | set -euo pipefail @@ -50,8 +61,12 @@ jobs: mkdir "$RUNNER_TEMP/consumer" cd "$RUNNER_TEMP/consumer" npm init -y + # Exercise this PR's core fix, including the legacy coexistence adapter, + # rather than silently resolving the already-published broken core. + npm pkg set 'overrides.@traceai/fi-core=$@traceai/fi-core' 'overrides.@traceai/fi-semantic-conventions=$@traceai/fi-semantic-conventions' tarball=$(find "$RUNNER_TEMP" -maxdepth 1 -name 'traceai-google-genai-sdk-*.tgz' -print) - npm install --ignore-scripts "$tarball" '@google/genai@${{ matrix.sdk }}' '@opentelemetry/sdk-trace-node@2.0.1' 'typescript@5.9.3' '@types/node@22' '@traceai/google-genai@0.1.0' '@google/generative-ai@0.24.1' + npm install --ignore-scripts "$tarball" "$RUNNER_TEMP/core.tgz" "$RUNNER_TEMP/semconv.tgz" '@google/genai@${{ matrix.sdk }}' '@opentelemetry/sdk-trace-node@2.0.1' 'typescript@5.9.3' '@types/node@22' '@traceai/google-genai@0.1.0' '@google/generative-ai@0.24.1' + npm ls @traceai/fi-core @traceai/fi-semantic-conventions - name: Real SDK, OTLP, privacy and exact customer snippet without internet shell: bash env: diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 365b297e3..1174ae307 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -70,7 +70,7 @@ "python/frameworks/weaviate": "0.1.2", "python/frameworks/xai": "0.1.1", "python/traceai": "0.1.0", - "typescript/packages/fi-core": "1.2.1", + "typescript/packages/fi-core": "1.2.2", "typescript/packages/fi-semantic-conventions": "1.1.0", "typescript/packages/traceai_anthropic": "0.1.5", "typescript/packages/traceai_auto": "0.1.0", diff --git a/spec/auto-instrumentation/privacy.json b/spec/auto-instrumentation/privacy.json index 319aad9e2..eac602779 100644 --- a/spec/auto-instrumentation/privacy.json +++ b/spec/auto-instrumentation/privacy.json @@ -1,17 +1,17 @@ { "version": 3, "scalar_metrics": { - "version": 5, + "version": 6, "canonical_decimal": { "ruling": "LANE-CALL-NIKM4-5A-I1-DIGIT-STRING-COUNTS-r2 with I1-R10B A-DIGIT-CONFINED", "regex": "^(0|[1-9][0-9]{0,15})$", "max_chars": 16, "base64_min_chars": 4, "keys": ["gen_ai.audio.bytes"], - "scope": "Only typed byte counters consult this rule. gen_ai.audio.bytes keeps full canonical decimal strings with a safe-integer numeric value unless they have base64 shape at or above base64_min_chars. elevenlabs.audio.bytes never grants a string role. The four-character shape threshold is not a special case for one count. Preserve accepted string types; no signs, leading zeros, fractions, exponents, whitespace or final newline. Other G4 core/detail keys retain their original scalar roles, including arbitrary strings and literal undefined. This rule does not change Q8b NON_CONTENT types." + "scope": "Only declared canonical-decimal counters consult this rule. gen_ai.audio.bytes keeps full canonical decimal strings with a safe-integer numeric value unless they have base64 shape at or above base64_min_chars. elevenlabs.audio.bytes never grants a string role. The four-character shape threshold is not a special case for one count. Preserve accepted string types; no signs, leading zeros, fractions, exponents, whitespace or final newline. Other G4 core/detail keys retain their original scalar roles, including arbitrary strings and literal undefined. This rule does not change Q8b NON_CONTENT types." }, "ruling": "TRACEAI-I1-G3-G4-POLICY-SCOPE-RULING-r1-FISEC G4", - "scope": "Closed exact complete keys in the declared usage/metrics containers only. Case, punctuation, prefixes, suffixes and array indices are not aliases. A nested object path must equal the complete key. gen_ai.audio and elevenlabs.audio are metric containers only for their listed bytes counters, not for arbitrary audio members. Their nonnegative_safe_integer value_type grants a role to finite nonnegative safe-integer numbers; only gen_ai.audio.bytes additionally accepts strings under canonical_decimal. All other inputs on these two byte counters retain the role-free media policy. Other declared metrics keep their original number/boolean/string roles without decimal validation.", + "scope": "Closed exact complete keys in the declared usage/metrics containers only. Case, punctuation, prefixes, suffixes and array indices are not aliases. A nested object path must equal the complete key. gen_ai.audio and elevenlabs.audio are metric containers only for their listed bytes counters, not for arbitrary audio members. Their nonnegative_safe_integer value_type grants a role to finite nonnegative safe-integer numbers; only gen_ai.audio.bytes additionally accepts strings under canonical_decimal. gen_ai.usage.cache_read_tokens.audio also accepts only nonnegative safe-integer numbers, never strings. All other inputs on these typed counters retain the role-free media policy. Other declared metrics keep their original number/boolean/string roles without decimal validation.", "precedence": "Scalar role only removes structural media classification caused by the metric spelling. Explicit media/type hints on any ancestor, inherited media/audio, established media payload ancestry, credential ancestry and known numeric/string secrets take precedence. On the two typed byte counters only, base64-shaped strings at or above canonical_decimal.base64_min_chars cannot obtain a role, even when numerically safe. Magic bytes, data URIs and other independent value checks still apply. A counter leaf under established media payload ancestry cannot use the usage metadata boundary to escape omission. Private payload-role ancestry survives parsed, stringified and diagnostic JSON handoffs; resetting a JSON root never grants a metric role. Metric-looking descendants reached through this retained payload origin remain payload, including numeric/bool scalars and numeric/bool arrays; ordinary metadata text boundaries are unchanged.", "arrays": "Arrays never receive the scalar metric role. Nonempty numeric/bool media payload arrays are omitted; existing string/object array walking and empty-array behavior remain unchanged.", "containers": [ @@ -28,6 +28,11 @@ "gen_ai.usage.output_tokens.reasoning" ] }, + { + "path": "gen_ai.usage", + "keys": ["gen_ai.usage.cache_read_tokens.audio"], + "value_type": "nonnegative_safe_integer" + }, { "path": "gen_ai.audio", "keys": ["gen_ai.audio.bytes"], diff --git a/typescript/packages/fi-core/CHANGELOG.md b/typescript/packages/fi-core/CHANGELOG.md index 8ab205788..8256e3551 100644 --- a/typescript/packages/fi-core/CHANGELOG.md +++ b/typescript/packages/fi-core/CHANGELOG.md @@ -1,3 +1,7 @@ +## [1.2.2] - pending release +### Fix +- Preserve numeric `gen_ai.usage.cache_read_tokens.audio` counts through admission, projection and export. The 1.2.0 shared privacy runtime mistook this modality count for an audio payload. Only nonnegative safe integers receive the metric role; strings, arrays, media ancestry and registered secrets retain their existing privacy protections. + ## [1.2.1] - pending release ### Fix - **Spans no longer carry the process command line or the OS user.** 1.2.0 added OpenTelemetry's full process detector to every provider resource, so `process.command_args`, `process.command`, `process.owner`, `process.executable.path` and `process.pid` were exported with every span. Command-line flags can hold secrets. By default fi-core now detects only `process.runtime.name`, `process.runtime.version` and `process.runtime.description`, as it detected nothing before 1.2.0. diff --git a/typescript/packages/fi-core/package.json b/typescript/packages/fi-core/package.json index a18f3482f..e8fce8874 100644 --- a/typescript/packages/fi-core/package.json +++ b/typescript/packages/fi-core/package.json @@ -1,6 +1,6 @@ { "name": "@traceai/fi-core", - "version": "1.2.1", + "version": "1.2.2", "description": "OpenTelemetry tracing for AI applications: tracer setup, span helpers, and exporters for Future AGI.", "main": "./dist/src/index.js", "module": "./dist/esm/index.js", diff --git a/typescript/packages/fi-core/src/__tests__/audioCacheCount.test.ts b/typescript/packages/fi-core/src/__tests__/audioCacheCount.test.ts new file mode 100644 index 000000000..7bd459d3f --- /dev/null +++ b/typescript/packages/fi-core/src/__tests__/audioCacheCount.test.ts @@ -0,0 +1,55 @@ +import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; +import { privacyFilter } from "../export-privacy"; +import { projectAttributes } from "../attribute-projection"; +import { indexPrivacyHints, mediaReplacement, scrubValue } from "../privacy-runtime"; +import { FITracer } from "../trace/trace-config/FITracer"; + +const key = "gen_ai.usage.cache_read_tokens.audio"; + +test.each([false, true])("audio cache counts survive admission and export with hiding=%s", async hidden => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + const traceConfig = { hideInputs: hidden, hideOutputs: hidden }; + try { + const tracer = new FITracer({ tracer: provider.getTracer("audio-cache-count"), traceConfig }); + for (const value of [0, 2, Number.MAX_SAFE_INTEGER]) { + expect(scrubValue(value, [], key)).toBe(value); + expect(projectAttributes({ [key]: value }, 16).attributes[key]).toBe(value); + const span = tracer.startSpan("cache-count", { attributes: { + [key]: value, "input.value": "PRIVATE_INPUT", "output.value": "PRIVATE_OUTPUT", + } }); + span.end(); + const admitted = exporter.getFinishedSpans().at(-1)!; + expect(admitted.attributes[key]).toBe(value); + const first = privacyFilter(traceConfig).span(admitted); + for (const exported of [first, privacyFilter(traceConfig).span(first)]) { + expect(exported.attributes[key]).toBe(value); + if (hidden) expect(JSON.stringify(exported.attributes)).not.toMatch(/PRIVATE_INPUT|PRIVATE_OUTPUT/); + } + } + } finally { await provider.shutdown(); } +}); + +test.each([true, false, -1, 0.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1, [2]])( + "audio cache metric role rejects non-count %s", value => { + expect(mediaReplacement(value, key)).toBe(""); + expect(scrubValue(value, [], key)).toBe(""); + }, +); + +test.each(["2", "12345678", "AQIDBAUGBwgJCgsM", "private audio payload!"])( + "audio cache metric role never exempts string %s", value => { + expect(mediaReplacement(value, key)).toMatch(/^ { + expect(scrubValue(12345678, ["12345678"], key)).toBe(""); + expect(scrubValue(2, [], `secret.${key}`)).toBe(""); + expect(mediaReplacement(2, `custom.${key}`)).toBe(""); + expect(mediaReplacement(2, key, indexPrivacyHints([["type", "input_audio"]]))).toBe(""); + expect(scrubValue({ audio: { data: { [key]: 2 } } })).toEqual({ + audio: { data: { [key]: "" } }, + }); +}); diff --git a/typescript/packages/fi-core/src/__tests__/privacyRuntimePerfEquivalence.test.ts b/typescript/packages/fi-core/src/__tests__/privacyRuntimePerfEquivalence.test.ts index c7ab3097f..e9ec8dbc8 100644 --- a/typescript/packages/fi-core/src/__tests__/privacyRuntimePerfEquivalence.test.ts +++ b/typescript/packages/fi-core/src/__tests__/privacyRuntimePerfEquivalence.test.ts @@ -243,7 +243,9 @@ const ORIGINAL_BARE_MEDIA = String.raw`function bareMedia(value: string): string }`; const PARENT_RUNTIME_SHA256 = "d4601e3aff870d917336c78510ffa0544eb62204383d8e429c7ab489fc7efc17"; const BASE_RUNTIME_SHA256 = "411d30cc45c3cdc801a6a9e172ddd3cf41a6ffdc4b193861cec6962a067b95a9"; -const BASE_CATALOG_SHA256 = "2ab8d4d7618a27612251e6195c49f0a824431f2b7f34dddb0e35ef2e12946b9c"; +// fi-core 1.2.2: rebase both runtime oracles onto scalar catalog v6. The +// algorithm is unchanged; audioCacheCount.test.ts verifies the new typed key. +const BASE_CATALOG_SHA256 = "9da3fef3a84741f8cbe345a44f6a2f8c6aa7737fdf3c1a91d590f271eadb138f"; const source = readFileSync(join(__dirname, "..", "privacy-runtime.ts"), "utf8"); const catalogSource = readFileSync(join(__dirname, "..", "privacy.json"), "utf8"); const catalog = JSON.parse(catalogSource); diff --git a/typescript/packages/fi-core/src/privacy.json b/typescript/packages/fi-core/src/privacy.json index 319aad9e2..eac602779 100644 --- a/typescript/packages/fi-core/src/privacy.json +++ b/typescript/packages/fi-core/src/privacy.json @@ -1,17 +1,17 @@ { "version": 3, "scalar_metrics": { - "version": 5, + "version": 6, "canonical_decimal": { "ruling": "LANE-CALL-NIKM4-5A-I1-DIGIT-STRING-COUNTS-r2 with I1-R10B A-DIGIT-CONFINED", "regex": "^(0|[1-9][0-9]{0,15})$", "max_chars": 16, "base64_min_chars": 4, "keys": ["gen_ai.audio.bytes"], - "scope": "Only typed byte counters consult this rule. gen_ai.audio.bytes keeps full canonical decimal strings with a safe-integer numeric value unless they have base64 shape at or above base64_min_chars. elevenlabs.audio.bytes never grants a string role. The four-character shape threshold is not a special case for one count. Preserve accepted string types; no signs, leading zeros, fractions, exponents, whitespace or final newline. Other G4 core/detail keys retain their original scalar roles, including arbitrary strings and literal undefined. This rule does not change Q8b NON_CONTENT types." + "scope": "Only declared canonical-decimal counters consult this rule. gen_ai.audio.bytes keeps full canonical decimal strings with a safe-integer numeric value unless they have base64 shape at or above base64_min_chars. elevenlabs.audio.bytes never grants a string role. The four-character shape threshold is not a special case for one count. Preserve accepted string types; no signs, leading zeros, fractions, exponents, whitespace or final newline. Other G4 core/detail keys retain their original scalar roles, including arbitrary strings and literal undefined. This rule does not change Q8b NON_CONTENT types." }, "ruling": "TRACEAI-I1-G3-G4-POLICY-SCOPE-RULING-r1-FISEC G4", - "scope": "Closed exact complete keys in the declared usage/metrics containers only. Case, punctuation, prefixes, suffixes and array indices are not aliases. A nested object path must equal the complete key. gen_ai.audio and elevenlabs.audio are metric containers only for their listed bytes counters, not for arbitrary audio members. Their nonnegative_safe_integer value_type grants a role to finite nonnegative safe-integer numbers; only gen_ai.audio.bytes additionally accepts strings under canonical_decimal. All other inputs on these two byte counters retain the role-free media policy. Other declared metrics keep their original number/boolean/string roles without decimal validation.", + "scope": "Closed exact complete keys in the declared usage/metrics containers only. Case, punctuation, prefixes, suffixes and array indices are not aliases. A nested object path must equal the complete key. gen_ai.audio and elevenlabs.audio are metric containers only for their listed bytes counters, not for arbitrary audio members. Their nonnegative_safe_integer value_type grants a role to finite nonnegative safe-integer numbers; only gen_ai.audio.bytes additionally accepts strings under canonical_decimal. gen_ai.usage.cache_read_tokens.audio also accepts only nonnegative safe-integer numbers, never strings. All other inputs on these typed counters retain the role-free media policy. Other declared metrics keep their original number/boolean/string roles without decimal validation.", "precedence": "Scalar role only removes structural media classification caused by the metric spelling. Explicit media/type hints on any ancestor, inherited media/audio, established media payload ancestry, credential ancestry and known numeric/string secrets take precedence. On the two typed byte counters only, base64-shaped strings at or above canonical_decimal.base64_min_chars cannot obtain a role, even when numerically safe. Magic bytes, data URIs and other independent value checks still apply. A counter leaf under established media payload ancestry cannot use the usage metadata boundary to escape omission. Private payload-role ancestry survives parsed, stringified and diagnostic JSON handoffs; resetting a JSON root never grants a metric role. Metric-looking descendants reached through this retained payload origin remain payload, including numeric/bool scalars and numeric/bool arrays; ordinary metadata text boundaries are unchanged.", "arrays": "Arrays never receive the scalar metric role. Nonempty numeric/bool media payload arrays are omitted; existing string/object array walking and empty-array behavior remain unchanged.", "containers": [ @@ -28,6 +28,11 @@ "gen_ai.usage.output_tokens.reasoning" ] }, + { + "path": "gen_ai.usage", + "keys": ["gen_ai.usage.cache_read_tokens.audio"], + "value_type": "nonnegative_safe_integer" + }, { "path": "gen_ai.audio", "keys": ["gen_ai.audio.bytes"], diff --git a/typescript/packages/fi-core/test/genai-privacy.test.cjs b/typescript/packages/fi-core/test/genai-privacy.test.cjs index 9472479b5..f2bb67d67 100644 --- a/typescript/packages/fi-core/test/genai-privacy.test.cjs +++ b/typescript/packages/fi-core/test/genai-privacy.test.cjs @@ -8,7 +8,7 @@ const { createServer } = require('node:http'); const { once } = require('node:events'); const { test } = require('node:test'); const load = createRequire(resolve(process.env.FI_CORE_FIXTURE || __dirname + '/..', 'package.json')); -const { FITracer } = process.env.FI_CORE_FIXTURE ? load('@traceai/fi-core') : load('./dist/src/index.js'); +const { FITracer, registerSecret } = process.env.FI_CORE_FIXTURE ? load('@traceai/fi-core') : load('./dist/src/index.js'); const { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor, SamplingDecision } = load('@opentelemetry/sdk-trace-base'); const { OTLPTraceExporter } = load('@opentelemetry/exporter-trace-otlp-http'); const { ROOT_CONTEXT, trace } = load('@opentelemetry/api'); @@ -116,13 +116,17 @@ for (const part of [ } } -test('hideInputImages: preserves declared non-image media and legacy text', async () => { +test('hideInputImages: preserves non-image structure and text, never audio payload bytes', async () => { const parts = [{ type: 'blob', modality: 'audio', mime_type: 'audio/wav', content: 'audio' }, { type: 'uri', modality: 'video', uri: 'https://example.invalid/video' }, { type: 'text', content: 'keep text' }]; const value = JSON.stringify([{ role: 'user', parts, content: 'keep legacy' }]); const { attributes } = await record({ hideInputImages: true }, INPUT, value); - assert.deepEqual(JSON.parse(attributes[INPUT]), JSON.parse(value)); + // Image hiding does not remove the audio part, but the shared media policy + // independently withholds its declared payload at every hide-flag setting. + assert.deepEqual(JSON.parse(attributes[INPUT]), [{ role: 'user', parts: [ + { ...parts[0], content: '' }, ...parts.slice(1), + ], content: 'keep legacy' }]); }); test('hideInputImages: opaque tool payloads and unknown parts fail closed', async () => { @@ -163,9 +167,14 @@ test('image hiding still applies PII redaction to retained text', async () => { assert.ok(attributes[INPUT].includes('')); }); -test('disabled and opposite-direction flags preserve bytes; broad flags remove messages', async () => { +test('disabled and opposite-direction flags preserve public bytes, never discovered secrets; broad flags remove messages', async () => { + const publicText = native([{ type: 'text', content: 'public-direction-control' }]); + registerSecret(SECRET); for (const config of [{}, { hideOutputText: true }]) { - assert.equal((await record(config, INPUT, text)).attributes[INPUT], text); + assert.equal((await record(config, INPUT, publicText)).attributes[INPUT], publicText); + // Turning hiding off must not make a process-wide known credential public. + assert.deepEqual(JSON.parse((await record(config, INPUT, text)).attributes[INPUT]), + [{ role: 'user', parts: [{ type: 'text', content: '' }] }]); } for (const flag of ['hideInputs', 'hideInputMessages']) { assert.equal((await record({ [flag]: true }, INPUT, text)).attributes[INPUT], undefined); @@ -182,14 +191,23 @@ test('empty messages and repeated masking are stable', async () => { assert.equal((await record(config, INPUT, onceMasked)).attributes[INPUT], onceMasked); }); -test('structured metadata and a large mixed message batch cannot carry payloads through', async () => { - const value = JSON.stringify(Array.from({ length: 1000 }, () => ({ role: 'user', - name: { data: SECRET }, parts: [{ type: 'text', content: SECRET, id: { data: SECRET } }, - { type: 'tool_call_response', response: [SECRET] }] }))); - const { attributes } = await record({ hideInputText: true }, INPUT, value); - assert.ok(!attributes[INPUT].includes(SECRET)); - assert.equal(JSON.parse(attributes[INPUT]).length, 1000); -}); +for (const count of [100, 1000]) { + test(`structured metadata in ${count} mixed messages stays private, including budget exhaustion`, async () => { + const value = JSON.stringify(Array.from({ length: count }, () => ({ role: 'user', + name: { data: SECRET }, parts: [{ type: 'text', content: SECRET, id: { data: SECRET } }, + { type: 'tool_call_response', response: [SECRET] }] }))); + const result = await record({ hideInputText: true }, INPUT, value); + assert.ok(!JSON.stringify(result).includes(SECRET)); + if (count === 100) { + assert.equal(JSON.parse(result.attributes[INPUT]).length, count); + } else { + // The shared privacy inspection budget fails closed, before the sampler, + // instead of partially retaining an uninspected oversized message batch. + assert.equal(result.attributes[INPUT], undefined); + assert.equal(result.sampled[0][INPUT], undefined); + } + }); +} test('environment privacy flags use the same native message masking', async () => { const previous = process.env.FI_HIDE_INPUT_TEXT; diff --git a/typescript/packages/traceai_auto/src/wire/privacy.json b/typescript/packages/traceai_auto/src/wire/privacy.json index 319aad9e2..eac602779 100644 --- a/typescript/packages/traceai_auto/src/wire/privacy.json +++ b/typescript/packages/traceai_auto/src/wire/privacy.json @@ -1,17 +1,17 @@ { "version": 3, "scalar_metrics": { - "version": 5, + "version": 6, "canonical_decimal": { "ruling": "LANE-CALL-NIKM4-5A-I1-DIGIT-STRING-COUNTS-r2 with I1-R10B A-DIGIT-CONFINED", "regex": "^(0|[1-9][0-9]{0,15})$", "max_chars": 16, "base64_min_chars": 4, "keys": ["gen_ai.audio.bytes"], - "scope": "Only typed byte counters consult this rule. gen_ai.audio.bytes keeps full canonical decimal strings with a safe-integer numeric value unless they have base64 shape at or above base64_min_chars. elevenlabs.audio.bytes never grants a string role. The four-character shape threshold is not a special case for one count. Preserve accepted string types; no signs, leading zeros, fractions, exponents, whitespace or final newline. Other G4 core/detail keys retain their original scalar roles, including arbitrary strings and literal undefined. This rule does not change Q8b NON_CONTENT types." + "scope": "Only declared canonical-decimal counters consult this rule. gen_ai.audio.bytes keeps full canonical decimal strings with a safe-integer numeric value unless they have base64 shape at or above base64_min_chars. elevenlabs.audio.bytes never grants a string role. The four-character shape threshold is not a special case for one count. Preserve accepted string types; no signs, leading zeros, fractions, exponents, whitespace or final newline. Other G4 core/detail keys retain their original scalar roles, including arbitrary strings and literal undefined. This rule does not change Q8b NON_CONTENT types." }, "ruling": "TRACEAI-I1-G3-G4-POLICY-SCOPE-RULING-r1-FISEC G4", - "scope": "Closed exact complete keys in the declared usage/metrics containers only. Case, punctuation, prefixes, suffixes and array indices are not aliases. A nested object path must equal the complete key. gen_ai.audio and elevenlabs.audio are metric containers only for their listed bytes counters, not for arbitrary audio members. Their nonnegative_safe_integer value_type grants a role to finite nonnegative safe-integer numbers; only gen_ai.audio.bytes additionally accepts strings under canonical_decimal. All other inputs on these two byte counters retain the role-free media policy. Other declared metrics keep their original number/boolean/string roles without decimal validation.", + "scope": "Closed exact complete keys in the declared usage/metrics containers only. Case, punctuation, prefixes, suffixes and array indices are not aliases. A nested object path must equal the complete key. gen_ai.audio and elevenlabs.audio are metric containers only for their listed bytes counters, not for arbitrary audio members. Their nonnegative_safe_integer value_type grants a role to finite nonnegative safe-integer numbers; only gen_ai.audio.bytes additionally accepts strings under canonical_decimal. gen_ai.usage.cache_read_tokens.audio also accepts only nonnegative safe-integer numbers, never strings. All other inputs on these typed counters retain the role-free media policy. Other declared metrics keep their original number/boolean/string roles without decimal validation.", "precedence": "Scalar role only removes structural media classification caused by the metric spelling. Explicit media/type hints on any ancestor, inherited media/audio, established media payload ancestry, credential ancestry and known numeric/string secrets take precedence. On the two typed byte counters only, base64-shaped strings at or above canonical_decimal.base64_min_chars cannot obtain a role, even when numerically safe. Magic bytes, data URIs and other independent value checks still apply. A counter leaf under established media payload ancestry cannot use the usage metadata boundary to escape omission. Private payload-role ancestry survives parsed, stringified and diagnostic JSON handoffs; resetting a JSON root never grants a metric role. Metric-looking descendants reached through this retained payload origin remain payload, including numeric/bool scalars and numeric/bool arrays; ordinary metadata text boundaries are unchanged.", "arrays": "Arrays never receive the scalar metric role. Nonempty numeric/bool media payload arrays are omitted; existing string/object array walking and empty-array behavior remain unchanged.", "containers": [ @@ -28,6 +28,11 @@ "gen_ai.usage.output_tokens.reasoning" ] }, + { + "path": "gen_ai.usage", + "keys": ["gen_ai.usage.cache_read_tokens.audio"], + "value_type": "nonnegative_safe_integer" + }, { "path": "gen_ai.audio", "keys": ["gen_ai.audio.bytes"], diff --git a/typescript/packages/traceai_codex_sdk/contract/otlp_wire.mjs b/typescript/packages/traceai_codex_sdk/contract/otlp_wire.mjs index 99b9c01a0..33669d97e 100644 --- a/typescript/packages/traceai_codex_sdk/contract/otlp_wire.mjs +++ b/typescript/packages/traceai_codex_sdk/contract/otlp_wire.mjs @@ -4,7 +4,8 @@ import assert from "node:assert/strict"; import http from "node:http"; import { fileURLToPath } from "node:url"; import * as InstalledCodexSDK from "@openai/codex-sdk"; -import { ProjectType, register } from "@traceai/fi-core"; +import { ROOT_CONTEXT } from "@opentelemetry/api"; +import { FITracer, ProjectType, register, setSession } from "@traceai/fi-core"; import { CodexSDKInstrumentation, shutdown } from "../dist/esm/index.js"; const CodexSDK = process.env.CODEX_SDK_ENTRY @@ -35,6 +36,11 @@ try { .startThread({ skipGitRepoCheck: true }); const result = await thread.run("synthetic prompt WIRE-SECRET-PROMPT"); assert.equal(result.items.length, 3); + // fi-core 1.2.0 exports only a session bound through the typed context API. + // A plain instrumentation attribute (Codex's thread-derived session) stays masked. + const tracer = new FITracer({ tracer: provider.getTracer("typed-session-wire-contract") }); + tracer.startActiveSpan("typed-session-control", {}, + setSession(ROOT_CONTEXT, { sessionId: "typed-wire-session" }), (span) => span.end()); await shutdown(provider); instrumentation.unpatch(); assert.ok(received.length > 0, "collector must receive OTLP export"); @@ -54,7 +60,9 @@ try { assert.equal(command.traceId, turn.traceId); assert.equal(command.parentSpanId, turn.spanId); assert.ok(turn.attributes.some((attribute) => attribute.key === "gen_ai.usage.input_tokens" && String(attribute.value.intValue) === "100")); - assert.ok(turn.attributes.some((attribute) => attribute.key === "session.id" && attribute.value.stringValue === "thread-contract-1")); + assert.ok(turn.attributes.some((attribute) => attribute.key === "session.id" && attribute.value.stringValue === "")); + const typed = spans.find((span) => span.name === "typed-session-control"); + assert.ok(typed?.attributes.some((attribute) => attribute.key === "session.id" && attribute.value.stringValue === "typed-wire-session")); assert.equal(JSON.stringify(spans).includes("WIRE-SECRET-PROMPT"), false, "private prompt leaked to OTLP"); console.log(`PASS OTLP wire (${process.env.CODEX_SDK_ENTRY ?? "installed SDK"}): ${received.length} exports`); } finally { diff --git a/typescript/packages/traceai_google_genai/test/release-regressions.test.cjs b/typescript/packages/traceai_google_genai/test/release-regressions.test.cjs index b5c1f695c..9028fa654 100644 --- a/typescript/packages/traceai_google_genai/test/release-regressions.test.cjs +++ b/typescript/packages/traceai_google_genai/test/release-regressions.test.cjs @@ -434,7 +434,13 @@ for (const option of ['hideInputs', 'hideInputText', 'hideInputMessages', 'hideI assert.doesNotMatch(JSON.stringify(exportedPayloads), /INPUT_SENTINEL|SYSTEM_PRIVATE_SENTINEL|STOP_PRIVATE_SENTINEL/); } else { assert.doesNotMatch(JSON.stringify(exportedPayloads), /OUTPUT_SENTINEL/); - assert.deepEqual(JSON.parse(attributes['gen_ai.request.parameters']), bodies[0].generationConfig); + // fi-core 1.2.0 projects request metadata for either input or output + // hiding. Only canonical configuration keys survive; provider-specific + // topP and content-bearing stopSequences remain fail-closed. + assert.deepEqual(JSON.parse(attributes['gen_ai.request.parameters']), { + temperature: 0.73, topP: '__REDACTED__', stopSequences: '__REDACTED__', + }); + assert.doesNotMatch(attributes['gen_ai.request.parameters'], /STOP_PRIVATE_SENTINEL/); } }); } finally { @@ -457,14 +463,19 @@ function toolModel() { }); } const effectiveKeys = ['generationConfig', 'safetySettings', 'tools', 'toolConfig', 'cachedContent']; -function assertEffectiveInput(attributes, wire) { +function assertEffectiveInput(attributes, wire, hideToolMetadata = false) { for (const key of ['raw.input', 'input.value']) { const captured = JSON.parse(attributes[key]); for (const field of effectiveKeys) assert.deepEqual(captured[field], wire[field], `${key}.${field}`); } const declarations = wire.tools?.flatMap(tool => tool.functionDeclarations || []); + // The shared metadata projection keeps tool identity, not descriptions/schemas, + // whenever either direction is hidden. Raw input still follows input hiding. + const expected = hideToolMetadata ? declarations?.map(tool => ({ + name: tool.name, description: '__REDACTED__', parameters: '__REDACTED__', + })) : declarations; assert.deepEqual(attributes['gen_ai.tool.definitions'] && JSON.parse(attributes['gen_ai.tool.definitions']), - declarations?.length ? declarations : undefined); + expected?.length ? expected : undefined); } for (const [name, input] of [['string', 'INPUT_SENTINEL'], ['strings', ['INPUT_SENTINEL']], @@ -601,7 +612,7 @@ for (const option of ['hideInputs', 'hideInputText', 'hideInputMessages', 'hideI assert.equal(attributes['raw.output'], undefined); assert.equal(attributes['gen_ai.output.messages'], undefined); assert.deepEqual(JSON.parse(attributes['gen_ai.input.messages']), expectedHistory); - assertEffectiveInput(attributes, bodies[1]); + assertEffectiveInput(attributes, bodies[1], true); } }); } finally { diff --git a/typescript/packages/traceai_google_genai_sdk/test/regressions.test.cjs b/typescript/packages/traceai_google_genai_sdk/test/regressions.test.cjs index 251fb4e17..5df944b60 100644 --- a/typescript/packages/traceai_google_genai_sdk/test/regressions.test.cjs +++ b/typescript/packages/traceai_google_genai_sdk/test/regressions.test.cjs @@ -1009,6 +1009,7 @@ for (const streaming of [false, true]) { const data = await exported(); assert.equal(Number(data["gen_ai.usage.input_tokens.audio"]), 6); assert.equal(Number(data["gen_ai.usage.output_tokens.image"]), 1); + assert.equal(Number(data["gen_ai.usage.cache_read_tokens.audio"]), 2); if (hidden) assert.doesNotMatch(JSON.stringify(data), /PRIVATE_INPUT|PRIVATE_OUTPUT/); } finally { h.dispose(); await provider.forceFlush(); } } diff --git a/typescript/packages/traceai_langchain/contract/readme.mjs b/typescript/packages/traceai_langchain/contract/readme.mjs index 1dd7a77b3..f69c6a94f 100644 --- a/typescript/packages/traceai_langchain/contract/readme.mjs +++ b/typescript/packages/traceai_langchain/contract/readme.mjs @@ -80,7 +80,8 @@ if (phase === "install") { } else if (request.url === "/tracer/v1/traces") { for (const resource of body.resourceSpans) { const attrs = Object.fromEntries(resource.resource.attributes.map((a) => [a.key, a.value.stringValue])); - assert.equal(attrs["service.name"], "your-project-name"); + assert.equal(attrs["project_name"], "your-project-name"); + assert.equal(attrs["service.name"], "langchain-readme-service"); for (const scope of resource.scopeSpans) spans.push(...scope.spans); } response.end("{}"); @@ -95,7 +96,7 @@ if (phase === "install") { await writeFile(join(consumer, runner[1]), source + "\n"); const baseUrl = `http://127.0.0.1:${server.address().port}`; const child = spawn("npx", runner, { cwd: consumer, env: { - ...env, FI_API_KEY: "synthetic-fi-key", FI_SECRET_KEY: "synthetic-fi-secret", + ...env, OTEL_SERVICE_NAME: "langchain-readme-service", FI_API_KEY: "synthetic-fi-key", FI_SECRET_KEY: "synthetic-fi-secret", OPENAI_API_KEY: "synthetic-openai-key", FI_BASE_URL: baseUrl, OPENAI_BASE_URL: `${baseUrl}/v1`, }, stdio: ["ignore", "pipe", "pipe"] }); let stdout = "", stderr = ""; diff --git a/typescript/packages/traceai_openai/src/__tests__/openai.test.ts b/typescript/packages/traceai_openai/src/__tests__/openai.test.ts index 19b833df2..a436787bb 100644 --- a/typescript/packages/traceai_openai/src/__tests__/openai.test.ts +++ b/typescript/packages/traceai_openai/src/__tests__/openai.test.ts @@ -318,7 +318,9 @@ describe("OpenAIInstrumentation", () => { expect(spans.length).toBe(1); const span = spans[0]; expect(span.name).toBe("OpenAI Chat Completions"); - expect(span.attributes).toMatchInlineSnapshot(` + expect(span.attributes).toMatchInlineSnapshot( + { "gen_ai.response.time_to_first_token": expect.any(Number) }, + ` { "fi.span.kind": "LLM", "gen_ai.input.messages": "[{"role":"user","content":"Say this is a test"}]", @@ -328,6 +330,7 @@ describe("OpenAIInstrumentation", () => { "gen_ai.request.model": "gpt-3.5-turbo", "gen_ai.request.parameters": "{"model":"gpt-3.5-turbo","stream":true}", "gen_ai.response.finish_reasons": "["stop"]", + "gen_ai.response.time_to_first_token": Any, "input.mime_type": "application/json", "input.value": "{"messages":[{"role":"user","content":"Say this is a test"}],"model":"gpt-3.5-turbo","stream":true}", "output.mime_type": "text/plain", @@ -749,7 +752,9 @@ describe("OpenAIInstrumentation", () => { expect(spans.length).toBe(1); const span = spans[0]; expect(span.name).toBe("OpenAI Chat Completions"); - expect(span.attributes).toMatchInlineSnapshot(` + expect(span.attributes).toMatchInlineSnapshot( + { "gen_ai.response.time_to_first_token": expect.any(Number) }, + ` { "fi.span.kind": "LLM", "gen_ai.input.messages": "[{"role":"user","content":"What's the weather today?"}]", @@ -761,6 +766,7 @@ describe("OpenAIInstrumentation", () => { "gen_ai.response.finish_reasons": "["tool_calls"]", "gen_ai.response.id": "chatcmpl-8iA39kCtuVHIVDr9AnBdJZjgSjNWL", "gen_ai.response.model": "gpt-3.5-turbo-0613", + "gen_ai.response.time_to_first_token": Any, "gen_ai.tool.definitions": "[{"type":"function","function":{"name":"getCurrentLocation","parameters":{"type":"object","properties":{}},"description":"Get the current location of the user."}},{"type":"function","function":{"name":"getWeather","description":"Get the weather for a location.","parameters":{"type":"object","properties":{"location":{"type":"string"}}}}}]", "input.mime_type": "application/json", "input.value": "{"messages":[{"role":"user","content":"What's the weather today?"}],"model":"gpt-3.5-turbo","tools":[{"type":"function","function":{"name":"getCurrentLocation","parameters":{"type":"object","properties":{}},"description":"Get the current location of the user."}},{"type":"function","function":{"name":"getWeather","description":"Get the weather for a location.","parameters":{"type":"object","properties":{"location":{"type":"string"}}}}}],"stream":true}", @@ -865,7 +871,9 @@ describe("OpenAIInstrumentation", () => { expect(spans.length).toBe(1); const span = spans[0]; expect(span.name).toBe("OpenAI Chat Completions"); - expect(span.attributes).toMatchInlineSnapshot(` + expect(span.attributes).toMatchInlineSnapshot( + { "gen_ai.response.time_to_first_token": expect.any(Number) }, + ` { "fi.span.kind": "LLM", "gen_ai.input.messages": "[{"role":"user","content":"What's the weather today?"}]", @@ -877,6 +885,7 @@ describe("OpenAIInstrumentation", () => { "gen_ai.response.finish_reasons": "["function_call"]", "gen_ai.response.id": "chatcmpl-8iA39kCtuVHIVDr9AnBdJZjgSjNWL", "gen_ai.response.model": "gpt-3.5-turbo-0613", + "gen_ai.response.time_to_first_token": Any, "input.mime_type": "application/json", "input.value": "{"messages":[{"role":"user","content":"What's the weather today?"}],"model":"gpt-3.5-turbo","functions":[{"name":"getWeather","description":"Get the weather for a location.","parameters":{"type":"object","properties":{"location":{"type":"string"}}}},{"name":"getCurrentLocation","description":"Get the current location of the user.","parameters":{"type":"object","properties":{}}}],"stream":true}", "output.mime_type": "text/plain", @@ -983,7 +992,7 @@ describe("OpenAIInstrumentation", () => { expect(span.attributes).toMatchInlineSnapshot(` { "fi.span.kind": "LLM", - "gen_ai.input.messages": "[{"role":"user","content":[{"type":"text","text":"Say this is a test"},{"type":"image_url","image_url":{"url":"data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="}}]}]", + "gen_ai.input.messages": "[{"role":"user","content":[{"type":"text","text":"Say this is a test"},{"type":"image_url","image_url":{"url":"[image-redacted]"}}]}]", "gen_ai.operation.name": "chat", "gen_ai.output.messages": "[{"role":"assistant","content":"This is a test."}]", "gen_ai.provider.name": "openai", @@ -996,10 +1005,10 @@ describe("OpenAIInstrumentation", () => { "gen_ai.usage.output_tokens": 5, "gen_ai.usage.total_tokens": 17, "input.mime_type": "application/json", - "input.value": "{"messages":[{"role":"user","content":[{"type":"text","text":"Say this is a test"},{"type":"image_url","image_url":{"url":"data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="}}]}],"model":"gpt-3.5-turbo"}", + "input.value": "{"messages":[{"role":"user","content":[{"type":"text","text":"Say this is a test"},{"type":"image_url","image_url":{"url":"[image-redacted]"}}]}],"model":"gpt-3.5-turbo"}", "output.mime_type": "application/json", "output.value": "{"id":"chatcmpl-8adq9JloOzNZ9TyuzrKyLpGXexh6p","object":"chat.completion","created":1703743645,"model":"gpt-3.5-turbo-0613","choices":[{"index":0,"message":{"role":"assistant","content":"This is a test."},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":12,"completion_tokens":5,"total_tokens":17}}", - "raw.input": "{"messages":[{"role":"user","content":[{"type":"text","text":"Say this is a test"},{"type":"image_url","image_url":{"url":"data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="}}]}],"model":"gpt-3.5-turbo"}", + "raw.input": "{"messages":[{"role":"user","content":[{"type":"text","text":"Say this is a test"},{"type":"image_url","image_url":{"url":"[image-redacted]"}}]}],"model":"gpt-3.5-turbo"}", "raw.output": "{"id":"chatcmpl-8adq9JloOzNZ9TyuzrKyLpGXexh6p","object":"chat.completion","created":1703743645,"model":"gpt-3.5-turbo-0613","choices":[{"index":0,"message":{"role":"assistant","content":"This is a test."},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":12,"completion_tokens":5,"total_tokens":17}}", } `); @@ -1414,7 +1423,9 @@ describe("AzureOpenAIInstrumentation", () => { expect(spans.length).toBe(1); const span = spans[0]; expect(span.name).toBe("OpenAI Chat Completions"); - expect(span.attributes).toMatchInlineSnapshot(` + expect(span.attributes).toMatchInlineSnapshot( + { "gen_ai.response.time_to_first_token": expect.any(Number) }, + ` { "fi.span.kind": "LLM", "gen_ai.input.messages": "[{"role":"user","content":"Say this is a test"}]", @@ -1424,6 +1435,7 @@ describe("AzureOpenAIInstrumentation", () => { "gen_ai.request.model": "gpt-35-turbo", "gen_ai.request.parameters": "{"model":"gpt-35-turbo","stream":true}", "gen_ai.response.finish_reasons": "["stop"]", + "gen_ai.response.time_to_first_token": Any, "input.mime_type": "application/json", "input.value": "{"messages":[{"role":"user","content":"Say this is a test"}],"model":"gpt-35-turbo","stream":true}", "output.mime_type": "text/plain",