diff --git a/package.json b/package.json index 937a8d1de..45b31611e 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "astro": "astro", "new-doc": "node scripts/new-doc.mjs", "audit-links": "node scripts/audit-links.mjs", - "test:integrations": "node --test scripts/integrations.test.mjs scripts/new-doc.test.mjs scripts/langfuse-import.test.mjs scripts/n8n-page.test.mjs scripts/openlit-endpoint.test.mjs scripts/chromadb-java.test.mjs", + "test:integrations": "node --test scripts/integrations.test.mjs scripts/new-doc.test.mjs scripts/langfuse-import.test.mjs scripts/n8n-page.test.mjs scripts/openlit-endpoint.test.mjs scripts/chromadb-java.test.mjs scripts/openllmetry-content.test.mjs", "test:integrations:browser": "node scripts/integrations.browser.cjs && node scripts/integrations.regression.cjs && node scripts/widget-regression.cjs", "check:integrations": "tsc --allowJs --checkJs --noEmit --strict --types node --target ES2022 --module nodenext src/lib/integrations.mjs" }, diff --git a/scripts/openllmetry-content.test.mjs b/scripts/openllmetry-content.test.mjs new file mode 100644 index 000000000..84125705d --- /dev/null +++ b/scripts/openllmetry-content.test.mjs @@ -0,0 +1,65 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; + +const page = readFileSync( + new URL("../src/pages/docs/integrations/otel/openllmetry.mdx", import.meta.url), + "utf8", +); + +// Page contract only. OTLP wire and collector conversion evidence are recorded +// separately; these checks do not claim that Cloud runs a particular collector. +test("GenAI messages and decorator payloads have distinct storage guidance", () => { + assert.match(page, /collectors with GenAI message mapping/i); + assert.match(page, /`gen_ai\.input\.messages` and `gen_ai\.output\.messages`/); + assert.match(page, /`traceloop\.entity\.input` and `traceloop\.entity\.output`/); + assert.doesNotMatch(page, /Input and Output columns[^\n]*(?:stay empty|are empty)/i); + assert.doesNotMatch(page, /collector fills them only from/); +}); + +test("setup uses native span attributes and disables unsupported signals", () => { + assert.match(page, /use_attributes=True/); + assert.match(page, /TRACELOOP_METRICS_ENABLED=false/); + assert.match(page, /TRACELOOP_LOGGING_ENABLED=false/); +}); + +test("privacy defaults remain content off without promising error redaction", () => { + assert.match(page, /TRACELOOP_TRACE_CONTENT=false/); + assert.match(page, /exception\/status text/); + assert.match(page, /base64 images/); + assert.match(page, /telemetry_enabled=False/); +}); + +test("native SDK initialization is the only instrumentation and export route", () => { + assert.equal((page.match(/Traceloop\.init\(/g) ?? []).length, 1); + assert.match(page, /Do not instrument the same client with both/); + assert.doesNotMatch(page, /pip install[^\n]*(?:traceAI-|openinference-)/i); + assert.doesNotMatch(page, /class \w+(?:Processor|Exporter)/); +}); + +test("decorator limitation is explicit rather than hidden by content fix", () => { + assert.match(page, /does not read `traceloop\.span\.kind`/); + assert.match(page, /decorator spans show as `unknown`/); + assert.match(page, /does not infer a tool type from `gen_ai\.tool\.name`/); + assert.match(page, /not promoted to the Input and Output fields/); +}); + +test("native export trade-off states the user-visible filtering limitation", () => { + assert.match(page, /deliberately keeps OpenLLMetry's native export/); + assert.match(page, /no automatic decorator classification/); + assert.match(page, /observation-type filter for `tool` will not include the unannotated/); + assert.match(page, /classification limitation, not a missing span/); +}); + +test("optional tool classification annotates the existing span without another SDK", () => { + const python = [...page.matchAll(/```python\n([\s\S]*?)```/g)].map((match) => match[1]); + assert.equal(python.length, 3); + assert.match(python[2], /from opentelemetry import trace/); + assert.match(python[2], /@tool\(name="lookup_policy"\)/); + assert.match(python[2], /trace\.get_current_span\(\)\.set_attribute\("gen_ai\.operation\.name", "execute_tool"\)/); + assert.doesNotMatch(python[2], /start_span|start_as_current_span|TracerProvider|Exporter|Processor|Traceloop\.init/); + assert.match(page, /Replace `lookup_policy` in step 4/); + assert.match(page, /v1\.58\.0 maps `gen_ai\.operation\.name=execute_tool` to `tool`/); + assert.match(page, /annotation changes classification only, not content capture or Input\/Output mapping/); + assert.match(page, /other decorator spans remain `unknown`/); +}); diff --git a/src/pages/docs/integrations/otel/openllmetry.mdx b/src/pages/docs/integrations/otel/openllmetry.mdx index ebd472a19..7fed0771b 100644 --- a/src/pages/docs/integrations/otel/openllmetry.mdx +++ b/src/pages/docs/integrations/otel/openllmetry.mdx @@ -33,7 +33,8 @@ export TRACELOOP_HEADERS="$(python -c 'import os; from urllib.parse import quote # Does not suppress exception/status text or the OpenAI base URL; see Privacy. export TRACELOOP_TRACE_CONTENT=false # the SDK default is true -export TRACELOOP_METRICS_ENABLED=false # the collector serves no /v1/metrics +export TRACELOOP_METRICS_ENABLED=false # Future AGI ingests traces, not metrics +export TRACELOOP_LOGGING_ENABLED=false # or OpenTelemetry logs unset TRACELOOP_API_KEY # that key is for Traceloop's own host ``` @@ -53,6 +54,7 @@ Traceloop.init( "project_type": "observe", }, telemetry_enabled=False, + use_attributes=True, # keep any opted-in content on spans, not log events ) ``` @@ -103,9 +105,27 @@ Open **Observe** in Future AGI and select `openllmetry_project` to inspect the t | OpenAI chat call | `llm` | model, prompt tokens, completion tokens | - The model call is typed `llm` from `gen_ai.operation.name=chat`. Its model and token counts come from the `gen_ai.*` keys that 0.62.4 emits. -- The collector does not read `traceloop.span.kind`, so the decorator spans show as `unknown`. Every `traceloop.*` key is kept as a span attribute, so you can still filter on `traceloop.span.kind`. +- The collector does not read `traceloop.span.kind`, so the decorator spans show as `unknown`. It does not infer a tool type from `gen_ai.tool.name` either. The spans and their attributes are retained; `unknown` is a classification limitation, not a missing span. - A failed model call or a decorated function that raises is stored with status `ERROR` on the whole path up to the workflow span. -- With content on, the text stays in the span attributes (`gen_ai.input.messages`, `gen_ai.output.messages`, `traceloop.entity.input`, `traceloop.entity.output`). The Input and Output columns of the trace view stay empty. +- With content on, model calls emit `gen_ai.input.messages` and `gen_ai.output.messages`. Collectors with GenAI message mapping also use these attributes to populate the Input and Output fields. On older collectors, inspect the attributes directly. +- Decorated functions emit `traceloop.entity.input` and `traceloop.entity.output` with content on. These remain span attributes; they are not promoted to the Input and Output fields by the same GenAI mapping. + +### When you need typed tool spans + +This guide deliberately keeps OpenLLMetry's native export rather than adding a conversion layer. The trade-off is no automatic decorator classification: you can inspect the function spans in the trace tree, but an observation-type filter for `tool` will not include the unannotated `lookup_policy.tool` span. Use this route as-is when the trace hierarchy, raw attributes and model-call telemetry are sufficient. + +If you need the tool observation type, annotate the **existing** tool span with the standard OpenTelemetry GenAI operation `execute_tool`. Replace `lookup_policy` in step 4 with the following definition, keeping the `@tool` import and the single `Traceloop.init` call unchanged: + +```python +from opentelemetry import trace + +@tool(name="lookup_policy") +def lookup_policy(question: str) -> str: + trace.get_current_span().set_attribute("gen_ai.operation.name", "execute_tool") + return "Refunds are accepted within 30 days of purchase." +``` + +The collector in Future AGI v1.58.0 maps `gen_ai.operation.name=execute_tool` to `tool`. This uses the OpenTelemetry API already installed with Traceloop; it does not create another span, provider or exporter. The other decorator spans remain `unknown`, and tool arguments/results still stay in attributes when content capture is enabled. This annotation changes classification only, not content capture or Input/Output mapping. On an older self-hosted collector, upgrade to a release with this operation mapping before relying on tool-type filters. --- @@ -126,16 +146,15 @@ The Traceloop SDK captures content unless `TRACELOOP_TRACE_CONTENT` is `false`. - **HTTP 400 from the collector:** the resource has no `project_name`. Add it to `resource_attributes`. - **HTTP 401:** check `TRACELOOP_HEADERS`. An unencoded comma in a key cuts the value short, and a value with a space, quote, semicolon or backslash is dropped with a warning that prints the whole pair. - **`ModuleNotFoundError: httpx`:** install `httpx` and `requests` next to the SDK. -- **A `/v1/metrics` request fails:** set `TRACELOOP_METRICS_ENABLED=false`. -- **No spans in a short script:** the SDK flushes on exit. Make sure the process exits normally, or call `Traceloop.init(disable_batch=True)` while debugging. +- **A `/v1/metrics` or `/v1/logs` request fails:** set `TRACELOOP_METRICS_ENABLED=false` and `TRACELOOP_LOGGING_ENABLED=false`. Keep `use_attributes=True` in the initialization; Future AGI accepts traces only. +- **No spans in a short script:** the SDK flushes on exit. Make sure the process exits normally, or add `disable_batch=True` to the initialization above while debugging. +- **Model input or output appears only in attributes:** on a self-hosted installation, update to a collector with GenAI message mapping. This does not add mappings for decorator payloads. With content capture off, there is no prompt or response body to display. - **The SDK switches to gRPC:** a base URL without `http://` or `https://` selects gRPC. Use an HTTP or HTTPS collector origin for this setup. --- ## Limitations -- Decorator spans are stored as `unknown`; `traceloop.span.kind` remains available as a filterable attribute. -- Input and Output columns are empty even with content on, because the collector fills them only from `input.value` and `output.value`. - gRPC, Traceloop's hosted dashboard, datasets and prompt registry are not part of this integration. - This example covers OpenAI Chat Completions; it does not establish support for every OpenLLMetry instrumentor. @@ -143,4 +162,6 @@ The Traceloop SDK captures content unless `TRACELOOP_TRACE_CONTENT` is `false`. ## How this differs from traceAI instrumentors -traceAI packages (for example `traceAI-openai`) wrap the client library and set Future AGI span kinds themselves. OpenLLMetry is a separate instrumentation that exports its own attributes. This recipe sends them as they are, with no mapping of its own, so span kinds follow what the collector reads. If you want Future AGI kinds on every span, use the traceAI instrumentor for your library instead. +Keep this native route if your app already uses OpenLLMetry. It exports one set of spans directly to Future AGI; you do not need a traceAI package or an application-side converter. + +traceAI instrumentors are an alternative for supported clients, not an extra layer for this setup. Do not instrument the same client with both OpenLLMetry and traceAI, or register a second exporter for the same destination: doing so can create duplicate observations.