Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions .github/workflows/google-genai-sdk-compat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,18 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node }}
- name: Build and pack with published shared dependencies
- name: Build and pack candidate shared dependencies
working-directory: typescript
shell: bash
run: |
set -euo pipefail
npm install --global pnpm@10.30.1
pnpm --filter @traceai/fi-core... install --frozen-lockfile --ignore-scripts
pnpm --filter @traceai/fi-semantic-conventions run build
pnpm --filter @traceai/fi-core run build
pnpm --dir packages/fi-semantic-conventions pack --out "$RUNNER_TEMP/semconv.tgz"
pnpm --dir packages/fi-core pack --out "$RUNNER_TEMP/core.tgz"
- name: Build adapter against published shared dependencies
shell: bash
run: |
set -euo pipefail
Expand All @@ -50,8 +61,12 @@ jobs:
mkdir "$RUNNER_TEMP/consumer"
cd "$RUNNER_TEMP/consumer"
npm init -y
# Exercise this PR's core fix, including the legacy coexistence adapter,
# rather than silently resolving the already-published broken core.
npm pkg set 'overrides.@traceai/fi-core=$@traceai/fi-core' 'overrides.@traceai/fi-semantic-conventions=$@traceai/fi-semantic-conventions'
tarball=$(find "$RUNNER_TEMP" -maxdepth 1 -name 'traceai-google-genai-sdk-*.tgz' -print)
npm install --ignore-scripts "$tarball" '@google/genai@${{ matrix.sdk }}' '@opentelemetry/sdk-trace-node@2.0.1' 'typescript@5.9.3' '@types/node@22' '@traceai/google-genai@0.1.0' '@google/generative-ai@0.24.1'
npm install --ignore-scripts "$tarball" "$RUNNER_TEMP/core.tgz" "$RUNNER_TEMP/semconv.tgz" '@google/genai@${{ matrix.sdk }}' '@opentelemetry/sdk-trace-node@2.0.1' 'typescript@5.9.3' '@types/node@22' '@traceai/google-genai@0.1.0' '@google/generative-ai@0.24.1'
npm ls @traceai/fi-core @traceai/fi-semantic-conventions
- name: Real SDK, OTLP, privacy and exact customer snippet without internet
shell: bash
env:
Expand Down
9 changes: 4 additions & 5 deletions .release-please-manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,10 @@
"python/frameworks/weaviate": "0.1.2",
"python/frameworks/xai": "0.1.1",
"python/traceai": "0.1.0",
"typescript/packages/fi-core": "1.2.1",
"typescript/packages/fi-core": "1.2.2",
"typescript/packages/fi-semantic-conventions": "1.1.0",
"typescript/packages/traceai_anthropic": "0.1.5",
"typescript/packages/traceai_auto": "0.1.0",
"typescript/packages/traceai_auto": "0.1.1",
"typescript/packages/traceai_bedrock": "0.1.3",
"typescript/packages/traceai_beeai": "0.1.2",
"typescript/packages/traceai_cerebras": "0.1.2",
Expand Down Expand Up @@ -134,7 +134,7 @@
"python/frameworks/transformers": "0.1.0",
"typescript/packages/traceai_agent_core": "0.1.0",
"typescript/packages/traceai_agents": "0.1.0",
"typescript/packages/traceai_codex_plugin": "0.1.0",
"typescript/packages/traceai_codex_plugin": "0.1.1",
"typescript/packages/traceai_opencode_v1_plugin": "0.1.0",
"typescript/packages/traceai_antigravity_plugin": "0.1.0",
"typescript/packages/traceai_copilot": "0.1.0",
Expand All @@ -152,6 +152,5 @@
"typescript/packages/traceai_oh_my_pi_plugin": "0.1.0",
"typescript/packages/traceai_opencode_plugin": "0.1.0",
"typescript/packages/traceai_pi_plugin": "0.1.0",
"typescript/packages/traceai_vscode_extension": "0.1.0",
"typescript/packages/traceai_auto": "0.1.0"
"typescript/packages/traceai_vscode_extension": "0.1.0"
}
19 changes: 17 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Trace LLM calls, agents, tools, and retrieval. Keep your application code and yo

**79 integration families · 5 languages · OpenTelemetry-native · Apache 2.0**

[Get started](#quickstart) · [Integrations](#integrations) · [Documentation](https://docs.futureagi.com/docs/tracing) · [Contribute](CONTRIBUTING.md)
[Zero-code Node.js](#zero-code-tracing-for-nodejs) · [SDK quickstarts](#quickstart) · [Integrations](#integrations) · [Documentation](https://docs.futureagi.com/docs/tracing) · [Contribute](CONTRIBUTING.md)

[![PyPI](https://img.shields.io/pypi/v/fi-instrumentation-otel?label=PyPI&color=555555)](https://pypi.org/project/fi-instrumentation-otel/)
[![npm](https://img.shields.io/npm/v/%40traceai%2Ffi-core?label=npm&color=555555)](https://www.npmjs.com/package/@traceai/fi-core)
Expand All @@ -31,9 +31,24 @@ Trace LLM calls, agents, tools, and retrieval. Keep your application code and yo
- **Follow the request.** Connect model calls, retrieval, and tool execution with OpenTelemetry spans and context.
- **Choose what leaves your app.** Configure content masking and export through your own OpenTelemetry pipeline.

## Zero-code tracing for Node.js

**See your AI calls without adding tracing code.** Install [`@traceai/auto`](typescript/packages/traceai_auto/) and preload it before your app (Node.js 20.6+, ESM or CommonJS):

```bash
npm install @traceai/auto
FI_API_KEY="your-api-key" FI_SECRET_KEY="your-secret-key" \
FI_PROJECT_NAME="my-ai-app" \
node --import @traceai/auto/register app.mjs
```

Keep your provider credentials configured as usual. Make an AI request, consume the response, then open **Observe → my-ai-app** in [Future AGI](https://app.futureagi.com). OpenAI instrumentation is included; recognized AI calls through global `fetch` are captured too. Optional framework integrations retain their own coverage limits.

**Prompts and responses are captured by default.** Set `FI_HIDE_INPUTS=true FI_HIDE_OUTPUTS=true FI_HIDE_ERROR_DETAILS=true` before startup to hide content. [Setup, supported libraries, privacy and framework recipes →](typescript/packages/traceai_auto/)

## Quickstart

Pick your language. These examples send traces to [Future AGI](https://app.futureagi.com); you can also supply your own OpenTelemetry tracer provider and exporter.
Prefer explicit instrumentation or another language? Pick your language. These examples send traces to [Future AGI](https://app.futureagi.com); you can also supply your own OpenTelemetry tracer provider and exporter.

| Language | Instrumentation | Install / coordinates |
| :--- | :--- | :--- |
Expand Down
3 changes: 2 additions & 1 deletion release-please-config.json
Original file line number Diff line number Diff line change
Expand Up @@ -604,7 +604,8 @@
"typescript/packages/traceai_auto": {
"release-type": "node",
"component": "npm-traceai-auto",
"package-name": "@traceai/auto"
"package-name": "@traceai/auto",
"extra-files": ["src/index.ts"]
},
"typescript/packages/traceai_bedrock": {
"release-type": "node",
Expand Down
2 changes: 1 addition & 1 deletion spec/auto-instrumentation/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ It runs on the HTTP clients each runtime uses for AI calls:
| SDK | Transports patched |
| --- | --- |
| Python | `httpx` and `httpx2` (sync and async), `requests`, `aiohttp` |
| TypeScript | `globalThis.fetch` (covers undici and every fetch-based SDK) |
| TypeScript | `globalThis.fetch` as observed after preload; not direct `undici.fetch`/`undici.request`, `node-fetch`, or a fetch reference cached before preload |
| Go | `http.DefaultTransport` (and any `http.Client` built from it) |
| C# | `HttpClient` through a `DiagnosticListener` handler |
| Java | `java.net.http.HttpClient` and OkHttp when present |
Expand Down
11 changes: 8 additions & 3 deletions spec/auto-instrumentation/privacy.json
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
{
"version": 3,
"scalar_metrics": {
"version": 5,
"version": 6,
"canonical_decimal": {
"ruling": "LANE-CALL-NIKM4-5A-I1-DIGIT-STRING-COUNTS-r2 with I1-R10B A-DIGIT-CONFINED",
"regex": "^(0|[1-9][0-9]{0,15})$",
"max_chars": 16,
"base64_min_chars": 4,
"keys": ["gen_ai.audio.bytes"],
"scope": "Only typed byte counters consult this rule. gen_ai.audio.bytes keeps full canonical decimal strings with a safe-integer numeric value unless they have base64 shape at or above base64_min_chars. elevenlabs.audio.bytes never grants a string role. The four-character shape threshold is not a special case for one count. Preserve accepted string types; no signs, leading zeros, fractions, exponents, whitespace or final newline. Other G4 core/detail keys retain their original scalar roles, including arbitrary strings and literal undefined. This rule does not change Q8b NON_CONTENT types."
"scope": "Only declared canonical-decimal counters consult this rule. gen_ai.audio.bytes keeps full canonical decimal strings with a safe-integer numeric value unless they have base64 shape at or above base64_min_chars. elevenlabs.audio.bytes never grants a string role. The four-character shape threshold is not a special case for one count. Preserve accepted string types; no signs, leading zeros, fractions, exponents, whitespace or final newline. Other G4 core/detail keys retain their original scalar roles, including arbitrary strings and literal undefined. This rule does not change Q8b NON_CONTENT types."
},
"ruling": "TRACEAI-I1-G3-G4-POLICY-SCOPE-RULING-r1-FISEC G4",
"scope": "Closed exact complete keys in the declared usage/metrics containers only. Case, punctuation, prefixes, suffixes and array indices are not aliases. A nested object path must equal the complete key. gen_ai.audio and elevenlabs.audio are metric containers only for their listed bytes counters, not for arbitrary audio members. Their nonnegative_safe_integer value_type grants a role to finite nonnegative safe-integer numbers; only gen_ai.audio.bytes additionally accepts strings under canonical_decimal. All other inputs on these two byte counters retain the role-free media policy. Other declared metrics keep their original number/boolean/string roles without decimal validation.",
"scope": "Closed exact complete keys in the declared usage/metrics containers only. Case, punctuation, prefixes, suffixes and array indices are not aliases. A nested object path must equal the complete key. gen_ai.audio and elevenlabs.audio are metric containers only for their listed bytes counters, not for arbitrary audio members. Their nonnegative_safe_integer value_type grants a role to finite nonnegative safe-integer numbers; only gen_ai.audio.bytes additionally accepts strings under canonical_decimal. gen_ai.usage.cache_read_tokens.audio also accepts only nonnegative safe-integer numbers, never strings. All other inputs on these typed counters retain the role-free media policy. Other declared metrics keep their original number/boolean/string roles without decimal validation.",
"precedence": "Scalar role only removes structural media classification caused by the metric spelling. Explicit media/type hints on any ancestor, inherited media/audio, established media payload ancestry, credential ancestry and known numeric/string secrets take precedence. On the two typed byte counters only, base64-shaped strings at or above canonical_decimal.base64_min_chars cannot obtain a role, even when numerically safe. Magic bytes, data URIs and other independent value checks still apply. A counter leaf under established media payload ancestry cannot use the usage metadata boundary to escape omission. Private payload-role ancestry survives parsed, stringified and diagnostic JSON handoffs; resetting a JSON root never grants a metric role. Metric-looking descendants reached through this retained payload origin remain payload, including numeric/bool scalars and numeric/bool arrays; ordinary metadata text boundaries are unchanged.",
"arrays": "Arrays never receive the scalar metric role. Nonempty numeric/bool media payload arrays are omitted; existing string/object array walking and empty-array behavior remain unchanged.",
"containers": [
Expand All @@ -28,6 +28,11 @@
"gen_ai.usage.output_tokens.reasoning"
]
},
{
"path": "gen_ai.usage",
"keys": ["gen_ai.usage.cache_read_tokens.audio"],
"value_type": "nonnegative_safe_integer"
},
{
"path": "gen_ai.audio",
"keys": ["gen_ai.audio.bytes"],
Expand Down
4 changes: 4 additions & 0 deletions typescript/packages/fi-core/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
## [1.2.2] - pending release
### Fix
- Preserve numeric `gen_ai.usage.cache_read_tokens.audio` counts through admission, projection and export. The 1.2.0 shared privacy runtime mistook this modality count for an audio payload. Only nonnegative safe integers receive the metric role; strings, arrays, media ancestry and registered secrets retain their existing privacy protections.

## [1.2.1] - pending release
### Fix
- **Spans no longer carry the process command line or the OS user.** 1.2.0 added OpenTelemetry's full process detector to every provider resource, so `process.command_args`, `process.command`, `process.owner`, `process.executable.path` and `process.pid` were exported with every span. Command-line flags can hold secrets. By default fi-core now detects only `process.runtime.name`, `process.runtime.version` and `process.runtime.description`, as it detected nothing before 1.2.0.
Expand Down
6 changes: 6 additions & 0 deletions typescript/packages/fi-core/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,12 @@

Core OpenTelemetry instrumentation library for TypeScript applications with advanced evaluation capabilities for AI systems.

## Choose your setup

For **zero-code Node.js tracing**, start with [`@traceai/auto`](https://github.com/future-agi/traceAI/blob/dev/typescript/packages/traceai_auto/README.md): install one package and preload it before your app. It includes OpenAI instrumentation, recognizes supported AI traffic through global `fetch`, and enables eligible installed integrations. Its own export route is batched; prompts and responses are captured by default, with explicit hide controls.

Use **`@traceai/fi-core`** when you want to configure the provider, create custom spans or register instrumentors yourself. Do not add a second provider or duplicate instrumentation to an app already using the auto preload. Application-owned OpenTelemetry exporters keep their own privacy policies.

## Overview

`@traceai/fi-core` provides a comprehensive tracing solution built on OpenTelemetry that's specifically designed for AI applications. It offers custom span exporters, evaluation tags, and seamless integration with the TraceAI platform for observability and performance monitoring.
Expand Down
2 changes: 1 addition & 1 deletion typescript/packages/fi-core/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@traceai/fi-core",
"version": "1.2.1",
"version": "1.2.2",
"description": "OpenTelemetry tracing for AI applications: tracer setup, span helpers, and exporters for Future AGI.",
"main": "./dist/src/index.js",
"module": "./dist/esm/index.js",
Expand Down
55 changes: 55 additions & 0 deletions typescript/packages/fi-core/src/__tests__/audioCacheCount.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base";
import { privacyFilter } from "../export-privacy";
import { projectAttributes } from "../attribute-projection";
import { indexPrivacyHints, mediaReplacement, scrubValue } from "../privacy-runtime";
import { FITracer } from "../trace/trace-config/FITracer";

const key = "gen_ai.usage.cache_read_tokens.audio";

test.each([false, true])("audio cache counts survive admission and export with hiding=%s", async hidden => {
const exporter = new InMemorySpanExporter();
const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] });
const traceConfig = { hideInputs: hidden, hideOutputs: hidden };
try {
const tracer = new FITracer({ tracer: provider.getTracer("audio-cache-count"), traceConfig });
for (const value of [0, 2, Number.MAX_SAFE_INTEGER]) {
expect(scrubValue(value, [], key)).toBe(value);
expect(projectAttributes({ [key]: value }, 16).attributes[key]).toBe(value);
const span = tracer.startSpan("cache-count", { attributes: {
[key]: value, "input.value": "PRIVATE_INPUT", "output.value": "PRIVATE_OUTPUT",
} });
span.end();
const admitted = exporter.getFinishedSpans().at(-1)!;
expect(admitted.attributes[key]).toBe(value);
const first = privacyFilter(traceConfig).span(admitted);
for (const exported of [first, privacyFilter(traceConfig).span(first)]) {
expect(exported.attributes[key]).toBe(value);
if (hidden) expect(JSON.stringify(exported.attributes)).not.toMatch(/PRIVATE_INPUT|PRIVATE_OUTPUT/);
}
}
} finally { await provider.shutdown(); }
});

test.each([true, false, -1, 0.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1, [2]])(
"audio cache metric role rejects non-count %s", value => {
expect(mediaReplacement(value, key)).toBe("<omitted: media>");
expect(scrubValue(value, [], key)).toBe("<omitted: media>");
},
);

test.each(["2", "12345678", "AQIDBAUGBwgJCgsM", "private audio payload!"])(
"audio cache metric role never exempts string %s", value => {
expect(mediaReplacement(value, key)).toMatch(/^<omitted: /);
expect(scrubValue(value, [], key)).toMatch(/^<omitted: /);
},
);

test("audio cache metric role retains secret, exact-key and payload precedence", () => {
expect(scrubValue(12345678, ["12345678"], key)).toBe("<redacted>");
expect(scrubValue(2, [], `secret.${key}`)).toBe("<redacted>");
expect(mediaReplacement(2, `custom.${key}`)).toBe("<omitted: media>");
expect(mediaReplacement(2, key, indexPrivacyHints([["type", "input_audio"]]))).toBe("<omitted: media>");
expect(scrubValue({ audio: { data: { [key]: 2 } } })).toEqual({
audio: { data: { [key]: "<omitted: media>" } },
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,9 @@ const ORIGINAL_BARE_MEDIA = String.raw`function bareMedia(value: string): string
}`;
const PARENT_RUNTIME_SHA256 = "d4601e3aff870d917336c78510ffa0544eb62204383d8e429c7ab489fc7efc17";
const BASE_RUNTIME_SHA256 = "411d30cc45c3cdc801a6a9e172ddd3cf41a6ffdc4b193861cec6962a067b95a9";
const BASE_CATALOG_SHA256 = "2ab8d4d7618a27612251e6195c49f0a824431f2b7f34dddb0e35ef2e12946b9c";
// fi-core 1.2.2: rebase both runtime oracles onto scalar catalog v6. The
// algorithm is unchanged; audioCacheCount.test.ts verifies the new typed key.
const BASE_CATALOG_SHA256 = "9da3fef3a84741f8cbe345a44f6a2f8c6aa7737fdf3c1a91d590f271eadb138f";
const source = readFileSync(join(__dirname, "..", "privacy-runtime.ts"), "utf8");
const catalogSource = readFileSync(join(__dirname, "..", "privacy.json"), "utf8");
const catalog = JSON.parse(catalogSource);
Expand Down
Loading
Loading