From e8e8fa481a9478c4a60d8c1cec0ca1f74a980890 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 09:05:31 -0700 Subject: [PATCH 1/2] fix(openllmetry): correct native trace content guidance Distinguish supported GenAI message promotion from unmapped decorator payloads, keep trace-only privacy defaults, and prevent duplicate instrumentation. Add documentation regression coverage. Co-Authored-By: Claude Code --- package.json | 2 +- scripts/openllmetry-content.test.mjs | 44 +++++++++++++++++++ .../docs/integrations/otel/openllmetry.mdx | 18 +++++--- 3 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 scripts/openllmetry-content.test.mjs diff --git a/package.json b/package.json index 273d686fd..8a5710017 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "astro": "astro", "new-doc": "node scripts/new-doc.mjs", "audit-links": "node scripts/audit-links.mjs", - "test:integrations": "node --test scripts/integrations.test.mjs scripts/new-doc.test.mjs", + "test:integrations": "node --test scripts/integrations.test.mjs scripts/new-doc.test.mjs scripts/openllmetry-content.test.mjs", "test:integrations:browser": "node scripts/integrations.browser.cjs && node scripts/integrations.regression.cjs && node scripts/widget-regression.cjs", "check:integrations": "tsc --allowJs --checkJs --noEmit --strict --types node --target ES2022 --module nodenext src/lib/integrations.mjs" }, diff --git a/scripts/openllmetry-content.test.mjs b/scripts/openllmetry-content.test.mjs new file mode 100644 index 000000000..7931d4bf0 --- /dev/null +++ b/scripts/openllmetry-content.test.mjs @@ -0,0 +1,44 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; + +const page = readFileSync( + new URL("../src/pages/docs/integrations/otel/openllmetry.mdx", import.meta.url), + "utf8", +); + +// Page contract only. OTLP wire and collector conversion evidence are recorded +// separately; these checks do not claim that Cloud runs a particular collector. +test("GenAI messages and decorator payloads have distinct storage guidance", () => { + assert.match(page, /collectors with GenAI message mapping/i); + assert.match(page, /`gen_ai\.input\.messages` and `gen_ai\.output\.messages`/); + assert.match(page, /`traceloop\.entity\.input` and `traceloop\.entity\.output`/); + assert.doesNotMatch(page, /Input and Output columns[^\n]*(?:stay empty|are empty)/i); + assert.doesNotMatch(page, /collector fills them only from/); +}); + +test("setup uses native span attributes and disables unsupported signals", () => { + assert.match(page, /use_attributes=True/); + assert.match(page, /TRACELOOP_METRICS_ENABLED=false/); + assert.match(page, /TRACELOOP_LOGGING_ENABLED=false/); +}); + +test("privacy defaults remain content off without promising error redaction", () => { + assert.match(page, /TRACELOOP_TRACE_CONTENT=false/); + assert.match(page, /exception\/status text/); + assert.match(page, /base64 images/); + assert.match(page, /telemetry_enabled=False/); +}); + +test("native SDK initialization is the only instrumentation and export route", () => { + assert.equal((page.match(/Traceloop\.init\(/g) ?? []).length, 1); + assert.match(page, /Do not instrument the same client with both/); + assert.doesNotMatch(page, /pip install[^\n]*(?:traceAI-|openinference-)/i); + assert.doesNotMatch(page, /class \w+(?:Processor|Exporter)/); +}); + +test("decorator limitation is explicit rather than hidden by content fix", () => { + assert.match(page, /does not read `traceloop\.span\.kind`/); + assert.match(page, /decorator spans show as `unknown`/); + assert.match(page, /not promoted to the Input and Output fields/); +}); diff --git a/src/pages/docs/integrations/otel/openllmetry.mdx b/src/pages/docs/integrations/otel/openllmetry.mdx index ebd472a19..6052e5f87 100644 --- a/src/pages/docs/integrations/otel/openllmetry.mdx +++ b/src/pages/docs/integrations/otel/openllmetry.mdx @@ -33,7 +33,8 @@ export TRACELOOP_HEADERS="$(python -c 'import os; from urllib.parse import quote # Does not suppress exception/status text or the OpenAI base URL; see Privacy. export TRACELOOP_TRACE_CONTENT=false # the SDK default is true -export TRACELOOP_METRICS_ENABLED=false # the collector serves no /v1/metrics +export TRACELOOP_METRICS_ENABLED=false # Future AGI ingests traces, not metrics +export TRACELOOP_LOGGING_ENABLED=false # or OpenTelemetry logs unset TRACELOOP_API_KEY # that key is for Traceloop's own host ``` @@ -53,6 +54,7 @@ Traceloop.init( "project_type": "observe", }, telemetry_enabled=False, + use_attributes=True, # keep any opted-in content on spans, not log events ) ``` @@ -105,7 +107,8 @@ Open **Observe** in Future AGI and select `openllmetry_project` to inspect the t - The model call is typed `llm` from `gen_ai.operation.name=chat`. Its model and token counts come from the `gen_ai.*` keys that 0.62.4 emits. - The collector does not read `traceloop.span.kind`, so the decorator spans show as `unknown`. Every `traceloop.*` key is kept as a span attribute, so you can still filter on `traceloop.span.kind`. - A failed model call or a decorated function that raises is stored with status `ERROR` on the whole path up to the workflow span. -- With content on, the text stays in the span attributes (`gen_ai.input.messages`, `gen_ai.output.messages`, `traceloop.entity.input`, `traceloop.entity.output`). The Input and Output columns of the trace view stay empty. +- With content on, model calls emit `gen_ai.input.messages` and `gen_ai.output.messages`. Collectors with GenAI message mapping also use these attributes to populate the Input and Output fields. On older collectors, inspect the attributes directly. +- Decorated functions emit `traceloop.entity.input` and `traceloop.entity.output` with content on. These remain span attributes; they are not promoted to the Input and Output fields by the same GenAI mapping. --- @@ -126,16 +129,15 @@ The Traceloop SDK captures content unless `TRACELOOP_TRACE_CONTENT` is `false`. - **HTTP 400 from the collector:** the resource has no `project_name`. Add it to `resource_attributes`. - **HTTP 401:** check `TRACELOOP_HEADERS`. An unencoded comma in a key cuts the value short, and a value with a space, quote, semicolon or backslash is dropped with a warning that prints the whole pair. - **`ModuleNotFoundError: httpx`:** install `httpx` and `requests` next to the SDK. -- **A `/v1/metrics` request fails:** set `TRACELOOP_METRICS_ENABLED=false`. -- **No spans in a short script:** the SDK flushes on exit. Make sure the process exits normally, or call `Traceloop.init(disable_batch=True)` while debugging. +- **A `/v1/metrics` or `/v1/logs` request fails:** set `TRACELOOP_METRICS_ENABLED=false` and `TRACELOOP_LOGGING_ENABLED=false`. Keep `use_attributes=True` in the initialization; Future AGI accepts traces only. +- **No spans in a short script:** the SDK flushes on exit. Make sure the process exits normally, or add `disable_batch=True` to the initialization above while debugging. +- **Model input or output appears only in attributes:** on a self-hosted installation, update to a collector with GenAI message mapping. This does not add mappings for decorator payloads. With content capture off, there is no prompt or response body to display. - **The SDK switches to gRPC:** a base URL without `http://` or `https://` selects gRPC. Use an HTTP or HTTPS collector origin for this setup. --- ## Limitations -- Decorator spans are stored as `unknown`; `traceloop.span.kind` remains available as a filterable attribute. -- Input and Output columns are empty even with content on, because the collector fills them only from `input.value` and `output.value`. - gRPC, Traceloop's hosted dashboard, datasets and prompt registry are not part of this integration. - This example covers OpenAI Chat Completions; it does not establish support for every OpenLLMetry instrumentor. @@ -143,4 +145,6 @@ The Traceloop SDK captures content unless `TRACELOOP_TRACE_CONTENT` is `false`. ## How this differs from traceAI instrumentors -traceAI packages (for example `traceAI-openai`) wrap the client library and set Future AGI span kinds themselves. OpenLLMetry is a separate instrumentation that exports its own attributes. This recipe sends them as they are, with no mapping of its own, so span kinds follow what the collector reads. If you want Future AGI kinds on every span, use the traceAI instrumentor for your library instead. +Keep this native route if your app already uses OpenLLMetry. It exports one set of spans directly to Future AGI; you do not need a traceAI package or an application-side converter. + +traceAI instrumentors are an alternative for supported clients, not an extra layer for this setup. Do not instrument the same client with both OpenLLMetry and traceAI, or register a second exporter for the same destination: doing so can create duplicate observations. From 36908ff68083a50447dc03fbb08909d5550f8259 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 4 Nov 2026 12:09:34 -0800 Subject: [PATCH 2/2] docs(openllmetry): explain native tool classification trade-off Show how to annotate the existing Traceloop tool span with the standard execute_tool operation recognized by the released collector, without another SDK, span, or exporter. Preserve the content-capture and decorator-payload limitations. Co-Authored-By: Claude Code --- scripts/openllmetry-content.test.mjs | 21 +++++++++++++++++++ .../docs/integrations/otel/openllmetry.mdx | 19 ++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/scripts/openllmetry-content.test.mjs b/scripts/openllmetry-content.test.mjs index 7931d4bf0..84125705d 100644 --- a/scripts/openllmetry-content.test.mjs +++ b/scripts/openllmetry-content.test.mjs @@ -40,5 +40,26 @@ test("native SDK initialization is the only instrumentation and export route", ( test("decorator limitation is explicit rather than hidden by content fix", () => { assert.match(page, /does not read `traceloop\.span\.kind`/); assert.match(page, /decorator spans show as `unknown`/); + assert.match(page, /does not infer a tool type from `gen_ai\.tool\.name`/); assert.match(page, /not promoted to the Input and Output fields/); }); + +test("native export trade-off states the user-visible filtering limitation", () => { + assert.match(page, /deliberately keeps OpenLLMetry's native export/); + assert.match(page, /no automatic decorator classification/); + assert.match(page, /observation-type filter for `tool` will not include the unannotated/); + assert.match(page, /classification limitation, not a missing span/); +}); + +test("optional tool classification annotates the existing span without another SDK", () => { + const python = [...page.matchAll(/```python\n([\s\S]*?)```/g)].map((match) => match[1]); + assert.equal(python.length, 3); + assert.match(python[2], /from opentelemetry import trace/); + assert.match(python[2], /@tool\(name="lookup_policy"\)/); + assert.match(python[2], /trace\.get_current_span\(\)\.set_attribute\("gen_ai\.operation\.name", "execute_tool"\)/); + assert.doesNotMatch(python[2], /start_span|start_as_current_span|TracerProvider|Exporter|Processor|Traceloop\.init/); + assert.match(page, /Replace `lookup_policy` in step 4/); + assert.match(page, /v1\.58\.0 maps `gen_ai\.operation\.name=execute_tool` to `tool`/); + assert.match(page, /annotation changes classification only, not content capture or Input\/Output mapping/); + assert.match(page, /other decorator spans remain `unknown`/); +}); diff --git a/src/pages/docs/integrations/otel/openllmetry.mdx b/src/pages/docs/integrations/otel/openllmetry.mdx index 6052e5f87..7fed0771b 100644 --- a/src/pages/docs/integrations/otel/openllmetry.mdx +++ b/src/pages/docs/integrations/otel/openllmetry.mdx @@ -105,11 +105,28 @@ Open **Observe** in Future AGI and select `openllmetry_project` to inspect the t | OpenAI chat call | `llm` | model, prompt tokens, completion tokens | - The model call is typed `llm` from `gen_ai.operation.name=chat`. Its model and token counts come from the `gen_ai.*` keys that 0.62.4 emits. -- The collector does not read `traceloop.span.kind`, so the decorator spans show as `unknown`. Every `traceloop.*` key is kept as a span attribute, so you can still filter on `traceloop.span.kind`. +- The collector does not read `traceloop.span.kind`, so the decorator spans show as `unknown`. It does not infer a tool type from `gen_ai.tool.name` either. The spans and their attributes are retained; `unknown` is a classification limitation, not a missing span. - A failed model call or a decorated function that raises is stored with status `ERROR` on the whole path up to the workflow span. - With content on, model calls emit `gen_ai.input.messages` and `gen_ai.output.messages`. Collectors with GenAI message mapping also use these attributes to populate the Input and Output fields. On older collectors, inspect the attributes directly. - Decorated functions emit `traceloop.entity.input` and `traceloop.entity.output` with content on. These remain span attributes; they are not promoted to the Input and Output fields by the same GenAI mapping. +### When you need typed tool spans + +This guide deliberately keeps OpenLLMetry's native export rather than adding a conversion layer. The trade-off is no automatic decorator classification: you can inspect the function spans in the trace tree, but an observation-type filter for `tool` will not include the unannotated `lookup_policy.tool` span. Use this route as-is when the trace hierarchy, raw attributes and model-call telemetry are sufficient. + +If you need the tool observation type, annotate the **existing** tool span with the standard OpenTelemetry GenAI operation `execute_tool`. Replace `lookup_policy` in step 4 with the following definition, keeping the `@tool` import and the single `Traceloop.init` call unchanged: + +```python +from opentelemetry import trace + +@tool(name="lookup_policy") +def lookup_policy(question: str) -> str: + trace.get_current_span().set_attribute("gen_ai.operation.name", "execute_tool") + return "Refunds are accepted within 30 days of purchase." +``` + +The collector in Future AGI v1.58.0 maps `gen_ai.operation.name=execute_tool` to `tool`. This uses the OpenTelemetry API already installed with Traceloop; it does not create another span, provider or exporter. The other decorator spans remain `unknown`, and tool arguments/results still stay in attributes when content capture is enabled. This annotation changes classification only, not content capture or Input/Output mapping. On an older self-hosted collector, upgrade to a release with this operation mapping before relying on tool-type filters. + --- ## Privacy