From ce20e177cfaf345cf44569ef0ca05adb12f2664f Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Thu, 3 Sep 2026 15:49:56 -0400 Subject: [PATCH 1/5] Add manual distributed tracing propagation for LLM Observability Adds LLMObs.injectDistributedHeaders/activateDistributedHeaders so applications can manually propagate LLMObs context (ml_app, session_id, agent attribution) across boundaries automatic instrumentation doesn't cover, e.g. an SQS worker reading its own message attributes. Standard APM trace context rides the normal Propagators.defaultPropagator(); LLMObs tags piggyback on the existing x-datadog-tags carrier entry to stay wire-compatible with dd-trace-py/js/go. Claude session: `15543c2c-2abe-408e-b16e-05ddbe972287` Resume: `claude --resume 15543c2c-2abe-408e-b16e-05ddbe972287` --- .../trace/llmobs/DDLLMObsPropagator.java | 129 ++++++++++++++++++ .../datadog/trace/llmobs/LLMObsSystem.java | 2 + .../trace/llmobs/domain/DDLLMObsSpan.java | 37 +++++ .../trace/llmobs/DDLLMObsPropagatorTest.java | 124 +++++++++++++++++ .../java/datadog/trace/api/llmobs/LLMObs.java | 47 +++++++ .../datadog/trace/api/llmobs/LLMObsTags.java | 9 ++ .../api/llmobs/noop/NoOpLLMObsPropagator.java | 21 +++ .../trace/api/llmobs/LLMObsInternal.java | 5 + 8 files changed, 374 insertions(+) create mode 100644 dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/DDLLMObsPropagator.java create mode 100644 dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/DDLLMObsPropagatorTest.java create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsPropagator.java diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/DDLLMObsPropagator.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/DDLLMObsPropagator.java new file mode 100644 index 00000000000..473b123c3e0 --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/DDLLMObsPropagator.java @@ -0,0 +1,129 @@ +package datadog.trace.llmobs; + +import datadog.context.Context; +import datadog.context.ContextScope; +import datadog.context.propagation.Propagators; +import datadog.trace.api.llmobs.LLMObs; +import datadog.trace.api.llmobs.LLMObsContext; +import datadog.trace.api.llmobs.LLMObsSpan; +import datadog.trace.api.llmobs.LLMObsTags; +import datadog.trace.bootstrap.instrumentation.api.AgentScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.llmobs.domain.DDLLMObsSpan; +import java.io.Closeable; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Explicit, manual distributed tracing propagation for LLM Observability, for boundaries that + * automatic instrumentation doesn't cover — e.g. an SQS worker reading its own message attributes. + * + *

The standard APM trace context (trace id, parent id, sampling, {@code x-datadog-tags}, ...) is + * injected/extracted via the normal {@link Propagators}. LLMObs-specific values (ml_app, + * session_id, agent attribution) ride as additional {@code _dd.p.llmobs_*} tags appended to the + * same {@code x-datadog-tags} carrier entry — the wire container dd-trace-py/js/go already use for + * these tags — so a mixed-language pipeline can still join a trace across this hop. + */ +public class DDLLMObsPropagator implements LLMObs.LLMObsPropagator { + private static final Logger LOGGER = LoggerFactory.getLogger(DDLLMObsPropagator.class); + + // Package-private (not exposed by dd-trace-core) but a long-stable wire header name; also + // listed in datadog.trace.util.PropagationUtils#KNOWN_PROPAGATION_HEADERS. + private static final String DATADOG_TAGS_HEADER = "x-datadog-tags"; + + @Override + public Map injectDistributedHeaders( + LLMObsSpan span, Map headers) { + Objects.requireNonNull(span, "span"); + Objects.requireNonNull(headers, "headers"); + if (!(span instanceof DDLLMObsSpan)) { + LOGGER.debug( + "injectDistributedHeaders requires a span started by the LLM Observability SDK, got {}; ignoring", + span.getClass()); + return headers; + } + DDLLMObsSpan llmObsSpan = (DDLLMObsSpan) span; + AgentSpan agentSpan = llmObsSpan.getAgentSpan(); + + Propagators.defaultPropagator().inject(agentSpan, headers, Map::put); + + StringBuilder llmObsTags = new StringBuilder(); + appendTag(llmObsTags, LLMObsTags.PROPAGATED_ML_APP, llmObsSpan.getMlApp()); + appendTag(llmObsTags, LLMObsTags.PROPAGATED_SESSION_ID, llmObsSpan.getSessionId()); + appendTag(llmObsTags, LLMObsTags.PROPAGATED_PAGENT_SPAN_ID, llmObsSpan.getParentAgentSpanId()); + appendTag(llmObsTags, LLMObsTags.PROPAGATED_PAGENT_NAME, llmObsSpan.getParentAgentName()); + + if (llmObsTags.length() > 0) { + String existing = headers.get(DATADOG_TAGS_HEADER); + headers.put( + DATADOG_TAGS_HEADER, + existing == null || existing.isEmpty() + ? llmObsTags.toString() + : existing + "," + llmObsTags); + } + return headers; + } + + @Override + public Closeable activateDistributedHeaders(Map headers) { + Objects.requireNonNull(headers, "headers"); + + Context extracted = + Propagators.defaultPropagator() + .extract(Context.root(), headers, (carrier, visitor) -> carrier.forEach(visitor)); + AgentSpan extractedSpan = AgentSpan.fromContext(extracted); + if (extractedSpan == null) { + LOGGER.debug( + "no distributed trace context found in headers; activateDistributedHeaders is a no-op"); + return () -> {}; + } + + Map llmObsTags = parseLlmObsTags(headers.get(DATADOG_TAGS_HEADER)); + String sessionId = llmObsTags.get(LLMObsTags.PROPAGATED_SESSION_ID); + String pagentSpanId = llmObsTags.get(LLMObsTags.PROPAGATED_PAGENT_SPAN_ID); + String pagentName = llmObsTags.get(LLMObsTags.PROPAGATED_PAGENT_NAME); + + AgentScope apmScope = AgentTracer.get().activateSpan(extractedSpan); + AgentSpanContext extractedContext = extractedSpan.spanContext(); + ContextScope llmObsScope = + LLMObsContext.attach(extractedContext, sessionId, null, pagentSpanId, pagentName); + + return () -> { + llmObsScope.close(); + apmScope.close(); + }; + } + + private static void appendTag(StringBuilder sb, String key, String value) { + if (value == null || value.isEmpty()) { + return; + } + if (sb.length() > 0) { + sb.append(','); + } + sb.append(key).append('=').append(value); + } + + private static Map parseLlmObsTags(String xDatadogTags) { + Map tags = new LinkedHashMap<>(); + if (xDatadogTags == null || xDatadogTags.isEmpty()) { + return tags; + } + for (String pair : xDatadogTags.split(",")) { + int eq = pair.indexOf('='); + if (eq <= 0) { + continue; + } + String key = pair.substring(0, eq); + if (key.startsWith("_dd.p.llmobs_")) { + tags.put(key, pair.substring(eq + 1)); + } + } + return tags; + } +} diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index 864cf27eb2c..a11f51c1b45 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -51,6 +51,8 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) { LLMObsInternal.setEvalProcessor(new LLMObsCustomEvalProcessor(mlApp, sco, config)); LLMObsInternal.setFeedbackProcessor(new LLMObsCustomFeedbackProcessor(mlApp, sco, config)); + + LLMObsInternal.setPropagator(new DDLLMObsPropagator()); } private static class LLMObsCustomFeedbackProcessor implements LLMObs.LLMObsFeedbackProcessor { diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index 42c6790a0c2..58362cf47d6 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -68,6 +68,9 @@ public class DDLLMObsSpan implements LLMObsSpan { private final String spanKind; private final String mlApp; private final boolean hasSessionId; + private final String sessionId; + private final String parentAgentSpanId; + private final String parentAgentName; private final ContextScope scope; // Non-null only for agent-kind spans started without an ambient APM root. Activating the // agent's APM span keeps children in the same APM trace so the trace-ID gate passes and @@ -160,6 +163,7 @@ public DDLLMObsSpan( } this.hasSessionId = sessionId != null && !sessionId.isEmpty(); + this.sessionId = this.hasSessionId ? sessionId : null; if (this.hasSessionId) { span.setTag(LLMOBS_TAG_PREFIX + LLMObsTags.SESSION_ID, sessionId); } @@ -197,6 +201,8 @@ public DDLLMObsSpan( span.setTag(PAGENT_NAME_TAG_INTERNAL, resolvedParentAgentName); } } + this.parentAgentSpanId = resolvedParentAgentSpanId; + this.parentAgentName = resolvedParentAgentName; // Propagate the effective sessionId and agent attribution to descendant LLMObs spans. scope = @@ -681,4 +687,35 @@ public DDTraceId getTraceId() { public long getSpanId() { return span.getSpanId(); } + + /** Internal accessor for the underlying APM span, used by {@code DDLLMObsPropagator}. */ + public AgentSpan getAgentSpan() { + return span; + } + + /** Internal accessor for this span's effective ml_app, used by {@code DDLLMObsPropagator}. */ + public String getMlApp() { + return mlApp; + } + + /** + * Internal accessor for this span's effective session_id (including one inherited from an + * enclosing LLMObs span), used by {@code DDLLMObsPropagator}. May be {@code null}. + */ + public String getSessionId() { + return sessionId; + } + + /** + * Internal accessor for this span's effective agent attribution, used by {@code + * DDLLMObsPropagator}. May be {@code null}. + */ + public String getParentAgentSpanId() { + return parentAgentSpanId; + } + + /** See {@link #getParentAgentSpanId()}. May be {@code null}. */ + public String getParentAgentName() { + return parentAgentName; + } } diff --git a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/DDLLMObsPropagatorTest.java b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/DDLLMObsPropagatorTest.java new file mode 100644 index 00000000000..92cd7f21a8f --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/DDLLMObsPropagatorTest.java @@ -0,0 +1,124 @@ +package datadog.trace.llmobs; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.agent.tooling.TracerInstaller; +import datadog.trace.api.WellKnownTags; +import datadog.trace.api.llmobs.LLMObsContext; +import datadog.trace.bootstrap.instrumentation.api.AgentScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.core.CoreTracer; +import datadog.trace.llmobs.domain.DDLLMObsSpan; +import java.io.Closeable; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Round-trips {@link DDLLMObsPropagator} through a plain {@code Map} carrier — the + * shape a customer's own SQS message-attribute map would take. + */ +class DDLLMObsPropagatorTest { + + private static CoreTracer tracer; + private final DDLLMObsPropagator propagator = new DDLLMObsPropagator(); + + @BeforeAll + static void installTracer() { + tracer = CoreTracer.builder().build(); + TracerInstaller.forceInstallGlobalTracer(tracer); + } + + @AfterAll + static void closeTracer() { + TracerInstaller.forceInstallGlobalTracer(null); + tracer.close(); + } + + private static DDLLMObsSpan newAgentSpan(String name, String mlApp, String sessionId) { + return newSpan(Tags.LLMOBS_AGENT_SPAN_KIND, name, mlApp, sessionId); + } + + private static DDLLMObsSpan newToolSpan(String name, String mlApp, String sessionId) { + return newSpan(Tags.LLMOBS_TOOL_SPAN_KIND, name, mlApp, sessionId); + } + + private static DDLLMObsSpan newSpan(String kind, String name, String mlApp, String sessionId) { + WellKnownTags tags = + new WellKnownTags("runtime-id", "hostname", "test", "service", "version", "java"); + return new DDLLMObsSpan(kind, name, mlApp, sessionId, "service", tags); + } + + private static AgentScope startRootApmScope() { + AgentSpan root = AgentTracer.get().buildSpan("apm", "sqs.produce").start(); + return AgentTracer.activateSpan(root); + } + + @Test + void injectRequiresNonNullSpanAndHeaders() { + assertThrows( + NullPointerException.class, + () -> propagator.injectDistributedHeaders(null, new HashMap<>())); + } + + @Test + void activateRequiresNonNullHeaders() { + assertThrows(NullPointerException.class, () -> propagator.activateDistributedHeaders(null)); + } + + @Test + void activateWithoutTraceContextIsNoOp() throws Exception { + try (Closeable scope = propagator.activateDistributedHeaders(new HashMap<>())) { + assertNull(LLMObsContext.current()); + } + } + + @Test + void injectThenActivateJoinsSameTraceAndPropagatesLlmObsContext() throws Exception { + Map headers = new HashMap<>(); + + try (AgentScope apmScope = startRootApmScope()) { + DDLLMObsSpan producerAgent = newAgentSpan("producer-agent", "my-ml-app", "session-123"); + long producerTraceId = producerAgent.getTraceId().toLong(); + try { + propagator.injectDistributedHeaders(producerAgent, headers); + } finally { + producerAgent.finish(); + } + + // Simulate the consumer side: a different message handled with no ambient context. + try (Closeable consumerScope = propagator.activateDistributedHeaders(headers)) { + DDLLMObsSpan consumerTool = newToolSpan("consumer-tool", "my-ml-app", null); + try { + assertEquals(producerTraceId, consumerTool.getTraceId().toLong()); + assertEquals("session-123", consumerTool.getSessionId()); + } finally { + consumerTool.finish(); + } + } + } + } + + @Test + void injectAlwaysIncludesMlAppEvenWithoutSessionIdOrAttribution() { + Map headers = new HashMap<>(); + try (AgentScope apmScope = startRootApmScope()) { + DDLLMObsSpan standaloneTool = newToolSpan("standalone-tool", "my-ml-app", null); + try { + propagator.injectDistributedHeaders(standaloneTool, headers); + String xDatadogTags = headers.get("x-datadog-tags"); + // ml_app is always present since it's required on every LLMObs span. + assertTrue(xDatadogTags != null && xDatadogTags.contains("_dd.p.llmobs_ml_app=my-ml-app")); + } finally { + standaloneTool.finish(); + } + } + } +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java index 2f9cfbcb619..fa8fa9429b1 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -2,7 +2,9 @@ import datadog.trace.api.llmobs.noop.NoOpLLMObsEvalProcessor; import datadog.trace.api.llmobs.noop.NoOpLLMObsFeedbackProcessor; +import datadog.trace.api.llmobs.noop.NoOpLLMObsPropagator; import datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory; +import java.io.Closeable; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -23,6 +25,7 @@ protected LLMObs() {} @Nullable protected static volatile LLMObsSpanProcessor SPAN_PROCESSOR; protected static LLMObsFeedbackProcessor FEEDBACK_PROCESSOR = NoOpLLMObsFeedbackProcessor.INSTANCE; + protected static LLMObsPropagator PROPAGATOR = NoOpLLMObsPropagator.INSTANCE; public static LLMObsSpan startLLMSpan( String spanName, @@ -182,6 +185,44 @@ public static void submitFeedback(Feedback feedback) { FEEDBACK_PROCESSOR.submitFeedback(feedback); } + /** + * Injects {@code span}'s distributed tracing context into {@code headers}, for manual propagation + * across a boundary automatic instrumentation doesn't cover — e.g. a queue/worker hop such as + * SQS, where a customer reads and writes message attributes themselves. + * + *

This populates the standard APM propagation headers (trace id, parent id, sampling, etc.) as + * well as LLMObs-specific propagating tags (ml_app, session_id, agent attribution) so a + * downstream call to {@link #activateDistributedHeaders} can resume the same LLMObs trace. + * + *

{@code headers} is mutated in place and returned for convenience. + * + * @param span the span whose context to inject; must be a span started by this SDK + * @param headers the carrier to inject propagation headers into + * @throws NullPointerException if {@code span} or {@code headers} is {@code null} + */ + public static Map injectDistributedHeaders( + LLMObsSpan span, Map headers) { + return PROPAGATOR.injectDistributedHeaders(span, headers); + } + + /** + * Activates a distributed tracing context previously injected by {@link + * #injectDistributedHeaders}, so that LLMObs spans started while the returned {@link Closeable} + * is open join the originating trace (same trace id, correct parent, inherited session_id and + * agent attribution). + * + *

Callers must close the returned {@link Closeable} once done processing under this context — + * e.g. per message in an SQS consumer loop, so unrelated messages don't share a trace. + * + * @param headers the carrier to extract propagation headers from + * @return a {@link Closeable} that deactivates the context when closed; never {@code null}, a + * no-op if {@code headers} carries no recognizable distributed tracing context + * @throws NullPointerException if {@code headers} is {@code null} + */ + public static Closeable activateDistributedHeaders(Map headers) { + return PROPAGATOR.activateDistributedHeaders(headers); + } + public interface LLMObsSpanFactory { LLMObsSpan startLLMSpan( String spanName, @@ -244,6 +285,12 @@ public interface LLMObsFeedbackProcessor { void submitFeedback(Feedback feedback); } + public interface LLMObsPropagator { + Map injectDistributedHeaders(LLMObsSpan span, Map headers); + + Closeable activateDistributedHeaders(Map headers); + } + /** * End-user feedback on a span, trace, session or customer-defined join key. * diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java index 0a7931589b5..f56f5c38104 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java @@ -19,4 +19,13 @@ public class LLMObsTags { // Agent attribution public static final String PAGENT_SPAN_ID = "pagent_span_id"; public static final String PAGENT_NAME = "pagent_name"; + + // Distributed tracing propagation tags. These ride alongside the standard APM `x-datadog-tags` + // propagating tags so a mixed-language pipeline can still join an LLMObs trace across a + // process boundary that isn't covered by automatic instrumentation (e.g. an SQS worker). + // Naming matches the `_dd.p.llmobs_*` convention used by dd-trace-py/js/go. + public static final String PROPAGATED_ML_APP = "_dd.p.llmobs_ml_app"; + public static final String PROPAGATED_SESSION_ID = "_dd.p.llmobs_sid"; + public static final String PROPAGATED_PAGENT_SPAN_ID = "_dd.p.llmobs_pagent_span_id"; + public static final String PROPAGATED_PAGENT_NAME = "_dd.p.llmobs_pagent_name"; } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsPropagator.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsPropagator.java new file mode 100644 index 00000000000..4025791b4ec --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsPropagator.java @@ -0,0 +1,21 @@ +package datadog.trace.api.llmobs.noop; + +import datadog.trace.api.llmobs.LLMObs; +import datadog.trace.api.llmobs.LLMObsSpan; +import java.io.Closeable; +import java.util.Map; + +public class NoOpLLMObsPropagator implements LLMObs.LLMObsPropagator { + public static final NoOpLLMObsPropagator INSTANCE = new NoOpLLMObsPropagator(); + + @Override + public Map injectDistributedHeaders( + LLMObsSpan span, Map headers) { + return headers; + } + + @Override + public Closeable activateDistributedHeaders(Map headers) { + return () -> {}; + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/llmobs/LLMObsInternal.java b/internal-api/src/main/java/datadog/trace/api/llmobs/LLMObsInternal.java index ed32f10118a..52335b14044 100644 --- a/internal-api/src/main/java/datadog/trace/api/llmobs/LLMObsInternal.java +++ b/internal-api/src/main/java/datadog/trace/api/llmobs/LLMObsInternal.java @@ -21,6 +21,11 @@ public static void setFeedbackProcessor(LLMObsFeedbackProcessor feedbackProcesso FEEDBACK_PROCESSOR = feedbackProcessor; } + /** Sets the LLM Observability distributed tracing propagator. */ + public static void setPropagator(LLMObsPropagator propagator) { + PROPAGATOR = propagator; + } + /** Returns the registered user span processor, if any. */ @Nullable public static LLMObsSpanProcessor getSpanProcessor() { From 14a03cde9b3bde0507387e220f08a3b3a0fce27d Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Fri, 4 Sep 2026 11:29:53 -0400 Subject: [PATCH 2/5] Propagate LLM Observability agent-attribution tags via dedicated PTags fields Replaces hand-rolled x-datadog-tags string manipulation in DDLLMObsPropagator with dedicated PropagationTags fields (mirroring dm/tid/ts/opm) for ml_app, session_id, and agent id/name, wired through both the Datadog and W3C codecs via an AgentSpanContext seam so agent-llmobs stays free of a dd-trace-core dependency. --- .../trace/llmobs/DDLLMObsPropagator.java | 79 +++-------- .../datadog/trace/core/DDSpanContext.java | 40 ++++++ .../core/propagation/PropagationTags.java | 36 +++++ .../propagation/ptags/DatadogPTagsCodec.java | 19 ++- .../propagation/ptags/LLMObsTagValues.java | 23 +++ .../core/propagation/ptags/PTagsCodec.java | 43 ++++++ .../core/propagation/ptags/PTagsFactory.java | 131 +++++++++++++++++- .../core/propagation/ptags/W3CPTagsCodec.java | 26 +++- .../instrumentation/api/AgentSpanContext.java | 52 +++++++ 9 files changed, 381 insertions(+), 68 deletions(-) create mode 100644 dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/LLMObsTagValues.java diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/DDLLMObsPropagator.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/DDLLMObsPropagator.java index 473b123c3e0..4e4f2a2124b 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/DDLLMObsPropagator.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/DDLLMObsPropagator.java @@ -6,14 +6,12 @@ import datadog.trace.api.llmobs.LLMObs; import datadog.trace.api.llmobs.LLMObsContext; import datadog.trace.api.llmobs.LLMObsSpan; -import datadog.trace.api.llmobs.LLMObsTags; import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.llmobs.domain.DDLLMObsSpan; import java.io.Closeable; -import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import org.slf4j.Logger; @@ -25,17 +23,15 @@ * *

The standard APM trace context (trace id, parent id, sampling, {@code x-datadog-tags}, ...) is * injected/extracted via the normal {@link Propagators}. LLMObs-specific values (ml_app, - * session_id, agent attribution) ride as additional {@code _dd.p.llmobs_*} tags appended to the - * same {@code x-datadog-tags} carrier entry — the wire container dd-trace-py/js/go already use for - * these tags — so a mixed-language pipeline can still join a trace across this hop. + * session_id, agent attribution) are written onto the span's {@link AgentSpanContext} as dedicated + * propagation-tags fields before injection, so the same {@link Propagators} call serializes them as + * additional {@code _dd.p.llmobs_*} tags — via {@code x-datadog-tags} or {@code tracestate}, + * whichever the configured propagation style carries — the wire container dd-trace-py/js/go already + * use for these tags, so a mixed-language pipeline can still join a trace across this hop. */ public class DDLLMObsPropagator implements LLMObs.LLMObsPropagator { private static final Logger LOGGER = LoggerFactory.getLogger(DDLLMObsPropagator.class); - // Package-private (not exposed by dd-trace-core) but a long-stable wire header name; also - // listed in datadog.trace.util.PropagationUtils#KNOWN_PROPAGATION_HEADERS. - private static final String DATADOG_TAGS_HEADER = "x-datadog-tags"; - @Override public Map injectDistributedHeaders( LLMObsSpan span, Map headers) { @@ -50,22 +46,13 @@ public Map injectDistributedHeaders( DDLLMObsSpan llmObsSpan = (DDLLMObsSpan) span; AgentSpan agentSpan = llmObsSpan.getAgentSpan(); - Propagators.defaultPropagator().inject(agentSpan, headers, Map::put); - - StringBuilder llmObsTags = new StringBuilder(); - appendTag(llmObsTags, LLMObsTags.PROPAGATED_ML_APP, llmObsSpan.getMlApp()); - appendTag(llmObsTags, LLMObsTags.PROPAGATED_SESSION_ID, llmObsSpan.getSessionId()); - appendTag(llmObsTags, LLMObsTags.PROPAGATED_PAGENT_SPAN_ID, llmObsSpan.getParentAgentSpanId()); - appendTag(llmObsTags, LLMObsTags.PROPAGATED_PAGENT_NAME, llmObsSpan.getParentAgentName()); + AgentSpanContext spanContext = agentSpan.spanContext(); + spanContext.updateLLMObsMlApp(llmObsSpan.getMlApp()); + spanContext.updateLLMObsSessionId(llmObsSpan.getSessionId()); + spanContext.updateLLMObsParentAgentSpanId(llmObsSpan.getParentAgentSpanId()); + spanContext.updateLLMObsParentAgentName(llmObsSpan.getParentAgentName()); - if (llmObsTags.length() > 0) { - String existing = headers.get(DATADOG_TAGS_HEADER); - headers.put( - DATADOG_TAGS_HEADER, - existing == null || existing.isEmpty() - ? llmObsTags.toString() - : existing + "," + llmObsTags); - } + Propagators.defaultPropagator().inject(agentSpan, headers, Map::put); return headers; } @@ -83,47 +70,23 @@ public Closeable activateDistributedHeaders(Map headers) { return () -> {}; } - Map llmObsTags = parseLlmObsTags(headers.get(DATADOG_TAGS_HEADER)); - String sessionId = llmObsTags.get(LLMObsTags.PROPAGATED_SESSION_ID); - String pagentSpanId = llmObsTags.get(LLMObsTags.PROPAGATED_PAGENT_SPAN_ID); - String pagentName = llmObsTags.get(LLMObsTags.PROPAGATED_PAGENT_NAME); + AgentSpanContext extractedContext = extractedSpan.spanContext(); + CharSequence sessionId = extractedContext.getLLMObsSessionId(); + CharSequence pagentSpanId = extractedContext.getLLMObsParentAgentSpanId(); + CharSequence pagentName = extractedContext.getLLMObsParentAgentName(); AgentScope apmScope = AgentTracer.get().activateSpan(extractedSpan); - AgentSpanContext extractedContext = extractedSpan.spanContext(); ContextScope llmObsScope = - LLMObsContext.attach(extractedContext, sessionId, null, pagentSpanId, pagentName); + LLMObsContext.attach( + extractedContext, + sessionId == null ? null : sessionId.toString(), + null, + pagentSpanId == null ? null : pagentSpanId.toString(), + pagentName == null ? null : pagentName.toString()); return () -> { llmObsScope.close(); apmScope.close(); }; } - - private static void appendTag(StringBuilder sb, String key, String value) { - if (value == null || value.isEmpty()) { - return; - } - if (sb.length() > 0) { - sb.append(','); - } - sb.append(key).append('=').append(value); - } - - private static Map parseLlmObsTags(String xDatadogTags) { - Map tags = new LinkedHashMap<>(); - if (xDatadogTags == null || xDatadogTags.isEmpty()) { - return tags; - } - for (String pair : xDatadogTags.split(",")) { - int eq = pair.indexOf('='); - if (eq <= 0) { - continue; - } - String key = pair.substring(0, eq); - if (key.startsWith("_dd.p.llmobs_")) { - tags.put(key, pair.substring(eq + 1)); - } - } - return tags; - } } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java index adf4cd66156..73b3e2b24e1 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java @@ -1492,6 +1492,46 @@ public PropagationTags getPropagationTags() { return getRootSpanContextOrThis().propagationTags; } + @Override + public CharSequence getLLMObsMlApp() { + return getPropagationTags().getLLMObsMlApp(); + } + + @Override + public void updateLLMObsMlApp(CharSequence mlApp) { + getPropagationTags().updateLLMObsMlApp(mlApp); + } + + @Override + public CharSequence getLLMObsSessionId() { + return getPropagationTags().getLLMObsSessionId(); + } + + @Override + public void updateLLMObsSessionId(CharSequence sessionId) { + getPropagationTags().updateLLMObsSessionId(sessionId); + } + + @Override + public CharSequence getLLMObsParentAgentSpanId() { + return getPropagationTags().getLLMObsParentAgentSpanId(); + } + + @Override + public void updateLLMObsParentAgentSpanId(CharSequence parentAgentSpanId) { + getPropagationTags().updateLLMObsParentAgentSpanId(parentAgentSpanId); + } + + @Override + public CharSequence getLLMObsParentAgentName() { + return getPropagationTags().getLLMObsParentAgentName(); + } + + @Override + public void updateLLMObsParentAgentName(CharSequence parentAgentName) { + getPropagationTags().updateLLMObsParentAgentName(parentAgentName); + } + /** TraceSegment Implementation */ @Override public void setTagTop(String key, Object value, boolean sanitize) { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java index 3a0c57a4dd8..47161ef1276 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/PropagationTags.java @@ -169,6 +169,42 @@ public interface Factory { */ public abstract void updateOrgPropagationMarker(CharSequence opm); + /** + * Returns the LLM Observability {@code ml_app} currently propagated with this trace, encoded as + * {@code _dd.p.llmobs_ml_app}. Returns {@code null} if none is set. + */ + public abstract CharSequence getLLMObsMlApp(); + + /** Sets the LLM Observability {@code ml_app} to propagate with this trace. */ + public abstract void updateLLMObsMlApp(CharSequence mlApp); + + /** + * Returns the LLM Observability {@code session_id} currently propagated with this trace, encoded + * as {@code _dd.p.llmobs_sid}. Returns {@code null} if none is set. + */ + public abstract CharSequence getLLMObsSessionId(); + + /** Sets the LLM Observability {@code session_id} to propagate with this trace. */ + public abstract void updateLLMObsSessionId(CharSequence sessionId); + + /** + * Returns the span id of the parent LLM Observability agent span currently propagated with this + * trace, encoded as {@code _dd.p.llmobs_pagent_span_id}. Returns {@code null} if none is set. + */ + public abstract CharSequence getLLMObsParentAgentSpanId(); + + /** Sets the parent LLM Observability agent span id to propagate with this trace. */ + public abstract void updateLLMObsParentAgentSpanId(CharSequence parentAgentSpanId); + + /** + * Returns the name of the parent LLM Observability agent span currently propagated with this + * trace, encoded as {@code _dd.p.llmobs_pagent_name}. Returns {@code null} if none is set. + */ + public abstract CharSequence getLLMObsParentAgentName(); + + /** Sets the parent LLM Observability agent span name to propagate with this trace. */ + public abstract void updateLLMObsParentAgentName(CharSequence parentAgentName); + public HashMap createTagMap() { HashMap result = new HashMap<>(); fillTagMap(result); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java index 3ac0c7ad712..907fab25e36 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java @@ -64,6 +64,10 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { TagValue traceIdTagValue = null; int traceSource = 0; TagValue orgPropagationMarkerTagValue = null; + TagValue llmObsMlAppTagValue = null; + TagValue llmObsSessionIdTagValue = null; + TagValue llmObsParentAgentSpanIdTagValue = null; + TagValue llmObsParentAgentNameTagValue = null; while (tagPos < len) { int tagKeyEndsAt = validateCharsUntilSeparatorOrEnd( @@ -102,6 +106,14 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { traceSource = ProductTraceSource.parseBitfieldHex(tagValue.toString()); } else if (tagKey.equals(ORG_PROPAGATION_MARKER_TAG)) { orgPropagationMarkerTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_ML_APP_TAG)) { + llmObsMlAppTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_SESSION_ID_TAG)) { + llmObsSessionIdTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_PAGENT_SPAN_ID_TAG)) { + llmObsParentAgentSpanIdTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_PAGENT_NAME_TAG)) { + llmObsParentAgentNameTagValue = tagValue; } else { if (tagPairs == null) { // This is roughly the size of a two element linked list but can hold six @@ -119,7 +131,12 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { decisionMakerTagValue, traceIdTagValue, traceSource, - orgPropagationMarkerTagValue); + orgPropagationMarkerTagValue, + new LLMObsTagValues( + llmObsMlAppTagValue, + llmObsSessionIdTagValue, + llmObsParentAgentSpanIdTagValue, + llmObsParentAgentNameTagValue)); } @Override diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/LLMObsTagValues.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/LLMObsTagValues.java new file mode 100644 index 00000000000..7d34fdab011 --- /dev/null +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/LLMObsTagValues.java @@ -0,0 +1,23 @@ +package datadog.trace.core.propagation.ptags; + +/** + * Bundles the four LLM Observability propagation tag values ({@code ml_app}, {@code session_id}, + * parent agent span id, parent agent name) extracted from an incoming header, so they can be + * threaded through {@link PTagsFactory.PTags} construction as a single parameter. + */ +final class LLMObsTagValues { + static final LLMObsTagValues EMPTY = new LLMObsTagValues(null, null, null, null); + + final TagValue mlApp; + final TagValue sessionId; + final TagValue parentAgentSpanId; + final TagValue parentAgentName; + + LLMObsTagValues( + TagValue mlApp, TagValue sessionId, TagValue parentAgentSpanId, TagValue parentAgentName) { + this.mlApp = mlApp; + this.sessionId = sessionId; + this.parentAgentSpanId = parentAgentSpanId; + this.parentAgentName = parentAgentName; + } +} diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java index e2c0658a1d2..99875e7f0a3 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java @@ -23,6 +23,10 @@ abstract class PTagsCodec { protected static final String PROPAGATION_ERROR_MALFORMED_TID = "malformed_tid "; protected static final String PROPAGATION_ERROR_INCONSISTENT_TID = "inconsistent_tid "; protected static final TagKey UPSTREAM_SERVICES_DEPRECATED_TAG = TagKey.from("upstream_services"); + protected static final TagKey LLMOBS_ML_APP_TAG = TagKey.from("llmobs_ml_app"); + protected static final TagKey LLMOBS_SESSION_ID_TAG = TagKey.from("llmobs_sid"); + protected static final TagKey LLMOBS_PAGENT_SPAN_ID_TAG = TagKey.from("llmobs_pagent_span_id"); + protected static final TagKey LLMOBS_PAGENT_NAME_TAG = TagKey.from("llmobs_pagent_name"); static String headerValue(PTagsCodec codec, PTags ptags) { return headerValue(codec, ptags, null); @@ -65,6 +69,22 @@ static String headerValue(PTagsCodec codec, PTags ptags, CharSequence lastParent codec.appendTag( sb, ORG_PROPAGATION_MARKER_TAG, ptags.getOrgPropagationMarkerTagValue(), size); } + if (ptags.getLLMObsMlAppTagValue() != null) { + size = codec.appendTag(sb, LLMOBS_ML_APP_TAG, ptags.getLLMObsMlAppTagValue(), size); + } + if (ptags.getLLMObsSessionIdTagValue() != null) { + size = codec.appendTag(sb, LLMOBS_SESSION_ID_TAG, ptags.getLLMObsSessionIdTagValue(), size); + } + if (ptags.getLLMObsParentAgentSpanIdTagValue() != null) { + size = + codec.appendTag( + sb, LLMOBS_PAGENT_SPAN_ID_TAG, ptags.getLLMObsParentAgentSpanIdTagValue(), size); + } + if (ptags.getLLMObsParentAgentNameTagValue() != null) { + size = + codec.appendTag( + sb, LLMOBS_PAGENT_NAME_TAG, ptags.getLLMObsParentAgentNameTagValue(), size); + } Iterator it = ptags.getTagPairs().iterator(); while (it.hasNext() && !codec.isTooLarge(sb, size)) { TagElement tagKey = it.next(); @@ -137,6 +157,29 @@ static void fillTagMap(PTags propagationTags, Map tagMap) { .forType(Encoding.DATADOG) .toString()); } + if (propagationTags.getLLMObsMlAppTagValue() != null) { + tagMap.put( + LLMOBS_ML_APP_TAG.forType(Encoding.DATADOG).toString(), + propagationTags.getLLMObsMlAppTagValue().forType(Encoding.DATADOG).toString()); + } + if (propagationTags.getLLMObsSessionIdTagValue() != null) { + tagMap.put( + LLMOBS_SESSION_ID_TAG.forType(Encoding.DATADOG).toString(), + propagationTags.getLLMObsSessionIdTagValue().forType(Encoding.DATADOG).toString()); + } + if (propagationTags.getLLMObsParentAgentSpanIdTagValue() != null) { + tagMap.put( + LLMOBS_PAGENT_SPAN_ID_TAG.forType(Encoding.DATADOG).toString(), + propagationTags + .getLLMObsParentAgentSpanIdTagValue() + .forType(Encoding.DATADOG) + .toString()); + } + if (propagationTags.getLLMObsParentAgentNameTagValue() != null) { + tagMap.put( + LLMOBS_PAGENT_NAME_TAG.forType(Encoding.DATADOG).toString(), + propagationTags.getLLMObsParentAgentNameTagValue().forType(Encoding.DATADOG).toString()); + } if (propagationTags.getError() != null) { tagMap.put(PROPAGATION_ERROR_TAG_KEY, propagationTags.getError()); } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java index 0b5184d448a..a93fccc3bd0 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsFactory.java @@ -4,6 +4,10 @@ import static datadog.trace.core.propagation.PropagationTags.HeaderType.W3C; import static datadog.trace.core.propagation.ptags.PTagsCodec.DECISION_MAKER_TAG; import static datadog.trace.core.propagation.ptags.PTagsCodec.KNUTH_SAMPLING_RATE_TAG; +import static datadog.trace.core.propagation.ptags.PTagsCodec.LLMOBS_ML_APP_TAG; +import static datadog.trace.core.propagation.ptags.PTagsCodec.LLMOBS_PAGENT_NAME_TAG; +import static datadog.trace.core.propagation.ptags.PTagsCodec.LLMOBS_PAGENT_SPAN_ID_TAG; +import static datadog.trace.core.propagation.ptags.PTagsCodec.LLMOBS_SESSION_ID_TAG; import static datadog.trace.core.propagation.ptags.PTagsCodec.ORG_PROPAGATION_MARKER_TAG; import static datadog.trace.core.propagation.ptags.PTagsCodec.TRACE_ID_TAG; import static datadog.trace.core.propagation.ptags.PTagsCodec.TRACE_SOURCE_TAG; @@ -50,7 +54,7 @@ PTagsCodec getDecoderEncoder(@Nonnull HeaderType headerType) { @Override public final PropagationTags empty() { - return createValid(null, null, null, ProductTraceSource.UNSET, null); + return createValid(null, null, null, ProductTraceSource.UNSET, null, null); } @Override @@ -71,14 +75,16 @@ PropagationTags createValid( TagValue decisionMakerTagValue, TagValue traceIdTagValue, int productTraceSource, - TagValue orgPropagationMarkerTagValue) { + TagValue orgPropagationMarkerTagValue, + LLMObsTagValues llmObsTagValues) { return new PTags( this, tagPairs, decisionMakerTagValue, traceIdTagValue, productTraceSource, - orgPropagationMarkerTagValue); + orgPropagationMarkerTagValue, + llmObsTagValues); } PropagationTags createInvalid(String error) { @@ -112,6 +118,11 @@ static class PTags extends PropagationTags { private volatile TagValue orgPropagationMarkerTagValue; + private volatile TagValue llmObsMlAppTagValue; + private volatile TagValue llmObsSessionIdTagValue; + private volatile TagValue llmObsParentAgentSpanIdTagValue; + private volatile TagValue llmObsParentAgentNameTagValue; + // Static cache for the most-recently-seen rate → TagValue. In steady state a service uses one // rate, so this eliminates the char[] + String allocation on every new PTags instance. // Writes are benign-racy: two threads computing the same rate produce equal TagValues. @@ -158,7 +169,8 @@ static class PTags extends PropagationTags { TagValue decisionMakerTagValue, TagValue traceIdTagValue, int traceSource, - TagValue orgPropagationMarkerTagValue) { + TagValue orgPropagationMarkerTagValue, + LLMObsTagValues llmObsTagValues) { this( factory, tagPairs, @@ -168,7 +180,8 @@ static class PTags extends PropagationTags { PrioritySampling.UNSET, null, null, - orgPropagationMarkerTagValue); + orgPropagationMarkerTagValue, + llmObsTagValues); } PTags( @@ -180,7 +193,8 @@ static class PTags extends PropagationTags { int samplingPriority, CharSequence origin, CharSequence lastParentId, - TagValue orgPropagationMarkerTagValue) { + TagValue orgPropagationMarkerTagValue, + LLMObsTagValues llmObsTagValues) { assert tagPairs == null || tagPairs.size() % 2 == 0; this.factory = factory; this.tagPairs = tagPairs; @@ -191,6 +205,11 @@ static class PTags extends PropagationTags { this.origin = origin; this.lastParentId = lastParentId; this.orgPropagationMarkerTagValue = orgPropagationMarkerTagValue; + LLMObsTagValues lov = llmObsTagValues != null ? llmObsTagValues : LLMObsTagValues.EMPTY; + this.llmObsMlAppTagValue = lov.mlApp; + this.llmObsSessionIdTagValue = lov.sessionId; + this.llmObsParentAgentSpanIdTagValue = lov.parentAgentSpanId; + this.llmObsParentAgentNameTagValue = lov.parentAgentName; if (traceIdTagValue != null) { CharSequence traceIdHighOrderBitsHex = traceIdTagValue.forType(TagElement.Encoding.DATADOG); this.traceIdHighOrderBits = @@ -212,6 +231,7 @@ static PTags withError(PTagsFactory factory, String error) { PrioritySampling.UNSET, null, null, + null, null); pTags.error = error; return pTags; @@ -377,6 +397,96 @@ TagValue getOrgPropagationMarkerTagValue() { return orgPropagationMarkerTagValue; } + @Override + public CharSequence getLLMObsMlApp() { + return llmObsMlAppTagValue; + } + + @Override + public void updateLLMObsMlApp(CharSequence mlApp) { + TagValue newValue = toTagValue(mlApp); + if (!Objects.equals(this.llmObsMlAppTagValue, newValue)) { + clearCachedHeader(DATADOG); + clearCachedHeader(W3C); + this.llmObsMlAppTagValue = newValue; + } + } + + TagValue getLLMObsMlAppTagValue() { + return llmObsMlAppTagValue; + } + + @Override + public CharSequence getLLMObsSessionId() { + return llmObsSessionIdTagValue; + } + + @Override + public void updateLLMObsSessionId(CharSequence sessionId) { + TagValue newValue = toTagValue(sessionId); + if (!Objects.equals(this.llmObsSessionIdTagValue, newValue)) { + clearCachedHeader(DATADOG); + clearCachedHeader(W3C); + this.llmObsSessionIdTagValue = newValue; + } + } + + TagValue getLLMObsSessionIdTagValue() { + return llmObsSessionIdTagValue; + } + + @Override + public CharSequence getLLMObsParentAgentSpanId() { + return llmObsParentAgentSpanIdTagValue; + } + + @Override + public void updateLLMObsParentAgentSpanId(CharSequence parentAgentSpanId) { + TagValue newValue = toTagValue(parentAgentSpanId); + if (!Objects.equals(this.llmObsParentAgentSpanIdTagValue, newValue)) { + clearCachedHeader(DATADOG); + clearCachedHeader(W3C); + this.llmObsParentAgentSpanIdTagValue = newValue; + } + } + + TagValue getLLMObsParentAgentSpanIdTagValue() { + return llmObsParentAgentSpanIdTagValue; + } + + @Override + public CharSequence getLLMObsParentAgentName() { + return llmObsParentAgentNameTagValue; + } + + @Override + public void updateLLMObsParentAgentName(CharSequence parentAgentName) { + TagValue newValue = toTagValue(parentAgentName); + if (!Objects.equals(this.llmObsParentAgentNameTagValue, newValue)) { + clearCachedHeader(DATADOG); + clearCachedHeader(W3C); + this.llmObsParentAgentNameTagValue = newValue; + } + } + + TagValue getLLMObsParentAgentNameTagValue() { + return llmObsParentAgentNameTagValue; + } + + /** + * Wraps a non-empty value as a {@link TagValue}, or {@code null} if empty. No length capping is + * applied here — matching dd-trace-py, which writes these free-form values (ml_app, session_id, + * agent id/name) as-is and relies on the codecs' own overflow handling (dropping the whole + * {@code x-datadog-tags} header on the Datadog codec, or dropping individual overlong tags on + * the W3C codec) rather than a fixed per-field character limit. + */ + private static TagValue toTagValue(CharSequence value) { + if (value == null || value.length() == 0) { + return null; + } + return TagValue.from(value); + } + @Override public int getSamplingPriority() { return samplingPriority; @@ -512,6 +622,15 @@ int getXDatadogTagsSize() { size = PTagsCodec.calcXDatadogTagsSize( size, ORG_PROPAGATION_MARKER_TAG, getOrgPropagationMarkerTagValue()); + size = PTagsCodec.calcXDatadogTagsSize(size, LLMOBS_ML_APP_TAG, llmObsMlAppTagValue); + size = + PTagsCodec.calcXDatadogTagsSize(size, LLMOBS_SESSION_ID_TAG, llmObsSessionIdTagValue); + size = + PTagsCodec.calcXDatadogTagsSize( + size, LLMOBS_PAGENT_SPAN_ID_TAG, llmObsParentAgentSpanIdTagValue); + size = + PTagsCodec.calcXDatadogTagsSize( + size, LLMOBS_PAGENT_NAME_TAG, llmObsParentAgentNameTagValue); int currentProductTraceSource = traceSource; if (currentProductTraceSource != ProductTraceSource.UNSET) { size = diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java index c0018544188..0a6e18bd10a 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java @@ -99,6 +99,10 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { int maxUnknownSize = 0; CharSequence lastParentId = null; TagValue orgPropagationMarkerTagValue = null; + TagValue llmObsMlAppTagValue = null; + TagValue llmObsSessionIdTagValue = null; + TagValue llmObsParentAgentSpanIdTagValue = null; + TagValue llmObsParentAgentNameTagValue = null; while (tagPos < ddMemberValueEnd) { tagPos = skipEmptyElements(value, tagPos, ddMemberValueEnd); if (tagPos >= ddMemberValueEnd) { @@ -168,6 +172,14 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { traceSource = ProductTraceSource.parseBitfieldHex(tagValue.toString()); } else if (tagKey.equals(ORG_PROPAGATION_MARKER_TAG)) { orgPropagationMarkerTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_ML_APP_TAG)) { + llmObsMlAppTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_SESSION_ID_TAG)) { + llmObsSessionIdTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_PAGENT_SPAN_ID_TAG)) { + llmObsParentAgentSpanIdTagValue = tagValue; + } else if (tagKey.equals(LLMOBS_PAGENT_NAME_TAG)) { + llmObsParentAgentNameTagValue = tagValue; } else { if (tagPairs == null) { // This is roughly the size of a two element linked list but can hold six @@ -201,7 +213,12 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value) { ddMemberValueEnd, maxUnknownSize, lastParentId, - orgPropagationMarkerTagValue); + orgPropagationMarkerTagValue, + new LLMObsTagValues( + llmObsMlAppTagValue, + llmObsSessionIdTagValue, + llmObsParentAgentSpanIdTagValue, + llmObsParentAgentNameTagValue)); } @Override @@ -764,6 +781,7 @@ private static W3CPTags empty( ddMemberValueEnd, 0, null, + null, null); } @@ -799,7 +817,8 @@ public W3CPTags( int ddMemberValueEnd, int maxUnknownSize, CharSequence lastParentId, - TagValue orgPropagationMarkerTagValue) { + TagValue orgPropagationMarkerTagValue, + LLMObsTagValues llmObsTagValues) { super( factory, tagPairs, @@ -809,7 +828,8 @@ public W3CPTags( samplingPriority, origin, lastParentId, - orgPropagationMarkerTagValue); + orgPropagationMarkerTagValue, + llmObsTagValues); this.tracestate = original; this.firstMemberStart = firstMemberStart; this.ddMemberStart = ddMemberStart; diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java index 1dba9438168..77466ef92cd 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java @@ -55,6 +55,58 @@ default void mergePathwayContext(PathwayContext pathwayContext) {} default void setIntegrationName(CharSequence componentName) {} + /** + * Gets the LLM Observability {@code ml_app} propagated with this trace, or {@code null} if none + * is set or this context implementation doesn't have propagation-tags access. + */ + default CharSequence getLLMObsMlApp() { + return null; + } + + /** Sets the LLM Observability {@code ml_app} to propagate with this trace. No-op by default. */ + default void updateLLMObsMlApp(CharSequence mlApp) {} + + /** + * Gets the LLM Observability {@code session_id} propagated with this trace, or {@code null} if + * none is set or this context implementation doesn't have propagation-tags access. + */ + default CharSequence getLLMObsSessionId() { + return null; + } + + /** + * Sets the LLM Observability {@code session_id} to propagate with this trace. No-op by default. + */ + default void updateLLMObsSessionId(CharSequence sessionId) {} + + /** + * Gets the span id of the parent LLM Observability agent span propagated with this trace, or + * {@code null} if none is set or this context implementation doesn't have propagation-tags + * access. + */ + default CharSequence getLLMObsParentAgentSpanId() { + return null; + } + + /** + * Sets the parent LLM Observability agent span id to propagate with this trace. No-op by default. + */ + default void updateLLMObsParentAgentSpanId(CharSequence parentAgentSpanId) {} + + /** + * Gets the name of the parent LLM Observability agent span propagated with this trace, or {@code + * null} if none is set or this context implementation doesn't have propagation-tags access. + */ + default CharSequence getLLMObsParentAgentName() { + return null; + } + + /** + * Sets the parent LLM Observability agent span name to propagate with this trace. No-op by + * default. + */ + default void updateLLMObsParentAgentName(CharSequence parentAgentName) {} + /** * Gets whether the span context used is part of the local trace or from another service * From afcb644eab1b6d790042bddf51b6d66d001f45e4 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Fri, 4 Sep 2026 11:40:04 -0400 Subject: [PATCH 3/5] Fix LLMObs propagation-tag inheritance for extracted contexts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ExtractedContext didn't override the AgentSpanContext.getLLMObs*() getters, so they fell back to the null-returning interface defaults instead of reading its own PropagationTags field — session_id/ml_app/ agent-attribution set on the wire were silently lost when read back via activateDistributedHeaders(). --- .../core/propagation/ExtractedContext.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java index af503e6a6ed..52a40a94e4c 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java @@ -117,6 +117,26 @@ public PropagationTags getPropagationTags() { return propagationTags; } + @Override + public CharSequence getLLMObsMlApp() { + return propagationTags.getLLMObsMlApp(); + } + + @Override + public CharSequence getLLMObsSessionId() { + return propagationTags.getLLMObsSessionId(); + } + + @Override + public CharSequence getLLMObsParentAgentSpanId() { + return propagationTags.getLLMObsParentAgentSpanId(); + } + + @Override + public CharSequence getLLMObsParentAgentName() { + return propagationTags.getLLMObsParentAgentName(); + } + @Override public String toString() { StringBuilder builder = new StringBuilder("ExtractedContext{"); From 7a3da96daa54d43752340149c3f67cc1b34fcf6e Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Fri, 4 Sep 2026 12:06:03 -0400 Subject: [PATCH 4/5] Fix test-isolation flakes exposed by full agent-llmobs suite runs A pre-existing DDLLMObsSpanTest.groovy case never finished its span, leaking an open LLMObsContext scope into whichever test ran next in the same JVM. Add the missing finish(). Also make activateWithoutTraceContextIsNoOp compare against the ambient LLMObsContext going in rather than asserting a global null baseline, since it shouldn't assume it's the only test that ever touches it. --- .../datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy | 3 +++ .../java/datadog/trace/llmobs/DDLLMObsPropagatorTest.java | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy index 2f45f9268db..4dd70688efb 100644 --- a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy +++ b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy @@ -796,6 +796,9 @@ class DDLLMObsSpanTest extends DDSpecification{ def innerSpan = (AgentSpan) test.span innerSpan.getTag(LLMOBS_TAG_PREFIX + "team") == "backend" innerSpan.getTag(LLMOBS_TAG_PREFIX + "owner") == "ml-platform" + + cleanup: + test.finish() } def "agent manifest full annotation sets correct tag"() { diff --git a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/DDLLMObsPropagatorTest.java b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/DDLLMObsPropagatorTest.java index 92cd7f21a8f..309aa3ab6dc 100644 --- a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/DDLLMObsPropagatorTest.java +++ b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/DDLLMObsPropagatorTest.java @@ -1,7 +1,6 @@ package datadog.trace.llmobs; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -75,8 +74,12 @@ void activateRequiresNonNullHeaders() { @Test void activateWithoutTraceContextIsNoOp() throws Exception { + // Compare against whatever LLMObsContext happened to be ambient going in, rather than + // asserting a global null baseline — this test runs alongside many others in the same JVM + // and must not assume it is the only thing that has ever touched ambient context. + Object ambientBefore = LLMObsContext.current(); try (Closeable scope = propagator.activateDistributedHeaders(new HashMap<>())) { - assertNull(LLMObsContext.current()); + assertEquals(ambientBefore, LLMObsContext.current()); } } From fb3651c08b09e6cefa34bf00b7a289847cc97a4d Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Fri, 4 Sep 2026 13:03:42 -0400 Subject: [PATCH 5/5] Propagate LLMObs context automatically on every instrumented boundary Previously the _dd.p.llmobs_* propagation tags were only ever written onto a span context inside DDLLMObsPropagator.injectDistributedHeaders, so they reached the wire only where an application propagated them by hand. An LLMObs span followed by a plain auto-instrumented HTTP call carried no LLMObs context at all, unlike dd-trace-py. Register an LLMObsContextPropagator as a propagation concern instead. It contributes no headers of its own: it stages ml_app, session_id and agent attribution onto the span context ahead of the tracing propagator, which then serializes them like any other propagation tag. This mirrors dd-trace-py, where LLMObs subscribes to the generic http.span_inject hook rather than owning a wire format, and covers every boundary automatic instrumentation already reaches. Values are resolved from the ambient LLMObsContext at injection time, so the innermost active span wins and leaving a scope stops contributing tags without any save/restore bookkeeping. ml_app moves into LLMObsContext for this, having previously lived only as a span tag. On the receive side, DDLLMObsSpan now inherits session_id and agent attribution from the propagated context when no in-process LLMObs parent applies. DDLLMObsPropagator remains as the manual entry point for carriers no instrumentation covers, such as SQS message attributes. --- .../trace/llmobs/DDLLMObsPropagator.java | 10 +++ .../trace/llmobs/LLMObsContextPropagator.java | 62 +++++++++++++++++++ .../datadog/trace/llmobs/LLMObsSystem.java | 7 +++ .../trace/llmobs/domain/DDLLMObsSpan.java | 19 ++++++ .../trace/llmobs/DDLLMObsPropagatorTest.java | 48 ++++++++++++++ .../trace/api/llmobs/LLMObsContext.java | 25 ++++++++ .../instrumentation/api/AgentPropagation.java | 5 ++ 7 files changed, 176 insertions(+) create mode 100644 dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagator.java diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/DDLLMObsPropagator.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/DDLLMObsPropagator.java index 4e4f2a2124b..ef9b4416d36 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/DDLLMObsPropagator.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/DDLLMObsPropagator.java @@ -21,6 +21,12 @@ * Explicit, manual distributed tracing propagation for LLM Observability, for boundaries that * automatic instrumentation doesn't cover — e.g. an SQS worker reading its own message attributes. * + *

Boundaries that are auto-instrumented (HTTP, gRPC, ...) need none of this: {@link + * LLMObsContextPropagator} is registered as a propagation concern and stages the same {@code + * _dd.p.llmobs_*} tags on every injection. This class is the manual equivalent for carriers no + * instrumentation reaches, mirroring dd-trace-py's {@code inject_distributed_headers} / {@code + * activate_distributed_headers}. + * *

The standard APM trace context (trace id, parent id, sampling, {@code x-datadog-tags}, ...) is * injected/extracted via the normal {@link Propagators}. LLMObs-specific values (ml_app, * session_id, agent attribution) are written onto the span's {@link AgentSpanContext} as dedicated @@ -46,6 +52,8 @@ public Map injectDistributedHeaders( DDLLMObsSpan llmObsSpan = (DDLLMObsSpan) span; AgentSpan agentSpan = llmObsSpan.getAgentSpan(); + // Stage this span's own LLMObs values rather than relying on the ambient context: the caller + // may hand us a span that isn't the innermost active one, and may not even be inside its scope. AgentSpanContext spanContext = agentSpan.spanContext(); spanContext.updateLLMObsMlApp(llmObsSpan.getMlApp()); spanContext.updateLLMObsSessionId(llmObsSpan.getSessionId()); @@ -71,6 +79,7 @@ public Closeable activateDistributedHeaders(Map headers) { } AgentSpanContext extractedContext = extractedSpan.spanContext(); + CharSequence mlApp = extractedContext.getLLMObsMlApp(); CharSequence sessionId = extractedContext.getLLMObsSessionId(); CharSequence pagentSpanId = extractedContext.getLLMObsParentAgentSpanId(); CharSequence pagentName = extractedContext.getLLMObsParentAgentName(); @@ -79,6 +88,7 @@ public Closeable activateDistributedHeaders(Map headers) { ContextScope llmObsScope = LLMObsContext.attach( extractedContext, + mlApp == null ? null : mlApp.toString(), sessionId == null ? null : sessionId.toString(), null, pagentSpanId == null ? null : pagentSpanId.toString(), diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagator.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagator.java new file mode 100644 index 00000000000..92d6f3e0e26 --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagator.java @@ -0,0 +1,62 @@ +package datadog.trace.llmobs; + +import datadog.context.Context; +import datadog.context.propagation.CarrierSetter; +import datadog.context.propagation.CarrierVisitor; +import datadog.context.propagation.Propagator; +import datadog.trace.api.llmobs.LLMObsContext; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; + +/** + * Stages the LLM Observability propagation tags onto the span context being injected, so that every + * boundary already covered by automatic instrumentation — HTTP, gRPC, SQS, Kafka, ... — carries + * LLMObs context without the application having to propagate it by hand. + * + *

This propagator writes nothing to the carrier itself. It runs ahead of the tracing propagator + * (see {@code AgentPropagation.LLMOBS_CONCERN}) and only populates the {@code _dd.p.llmobs_*} + * fields on the span context; the tracing propagator then serializes them into {@code + * x-datadog-tags} / {@code tracestate} along with every other propagation tag. This mirrors + * dd-trace-py, where LLMObs subscribes to the generic {@code http.span_inject} hook that {@code + * HTTPPropagator.inject} fires on every outbound request, rather than owning a separate wire + * format. + * + *

Values are resolved from the ambient {@link LLMObsContext} at injection time rather than being + * written once when a span starts. That way the innermost active LLMObs span always wins, and + * leaving an LLMObs scope stops contributing its tags without any save/restore bookkeeping. + */ +public class LLMObsContextPropagator implements Propagator { + + @Override + public void inject(Context context, C carrier, CarrierSetter setter) { + AgentSpan span = AgentSpan.fromContext(context); + if (span == null) { + return; + } + AgentSpanContext spanContext = span.spanContext(); + if (spanContext == null) { + return; + } + + // Gate on trace-id consistency, the same way DDLLMObsSpan gates parent_id/session_id + // inheritance. An LLMObs context leaked across an async boundary must not tag an outbound + // request that belongs to an unrelated trace. + AgentSpanContext llmObsContext = LLMObsContext.current(); + if (llmObsContext == null || llmObsContext.getTraceId() != spanContext.getTraceId()) { + return; + } + + spanContext.updateLLMObsMlApp(LLMObsContext.currentMlApp()); + spanContext.updateLLMObsSessionId(LLMObsContext.currentSessionId()); + spanContext.updateLLMObsParentAgentSpanId(LLMObsContext.currentParentAgentSpanId()); + spanContext.updateLLMObsParentAgentName(LLMObsContext.currentParentAgentName()); + } + + @Override + public Context extract(Context context, C carrier, CarrierVisitor visitor) { + // Nothing to do: the tracing propagator's codecs already parse the _dd.p.llmobs_* tags back + // into the extracted context's propagation tags, and DDLLMObsSpan reads them from there when + // no in-process LLMObs parent applies. + return context; + } +} diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index a11f51c1b45..e9ea2a06723 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -1,6 +1,7 @@ package datadog.trace.llmobs; import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.context.propagation.Propagators; import datadog.trace.api.Config; import datadog.trace.api.WellKnownTags; import datadog.trace.api.llmobs.LLMObs; @@ -8,6 +9,7 @@ import datadog.trace.api.llmobs.LLMObsSpan; import datadog.trace.api.llmobs.LLMObsTags; import datadog.trace.api.telemetry.LLMObsMetricCollector; +import datadog.trace.bootstrap.instrumentation.api.AgentPropagation; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.llmobs.domain.DDLLMObsSpan; import datadog.trace.llmobs.domain.LLMObsEval; @@ -53,6 +55,11 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) { LLMObsInternal.setFeedbackProcessor(new LLMObsCustomFeedbackProcessor(mlApp, sco, config)); LLMObsInternal.setPropagator(new DDLLMObsPropagator()); + + // Automatic propagation: every boundary that injects trace context now carries the LLMObs + // propagation tags too, matching dd-trace-py. DDLLMObsPropagator stays as the manual entry + // point for carriers no instrumentation covers (e.g. SQS message attributes). + Propagators.register(AgentPropagation.LLMOBS_CONCERN, new LLMObsContextPropagator()); } private static class LLMObsCustomFeedbackProcessor implements LLMObs.LLMObsFeedbackProcessor { diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index 58362cf47d6..b0f1de9577b 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -162,6 +162,16 @@ public DDLLMObsSpan( } } + // No in-process LLMObs parent: this span may still be continuing a trace that arrived from + // another service, in which case the upstream session_id is on the span context's propagation + // tags (parsed back out of x-datadog-tags / tracestate by the tracing propagator). + if (null == parent && (sessionId == null || sessionId.isEmpty())) { + CharSequence propagated = span.spanContext().getLLMObsSessionId(); + if (propagated != null && propagated.length() > 0) { + sessionId = propagated.toString(); + } + } + this.hasSessionId = sessionId != null && !sessionId.isEmpty(); this.sessionId = this.hasSessionId ? sessionId : null; if (this.hasSessionId) { @@ -191,6 +201,14 @@ public DDLLMObsSpan( if (null != parent && parent.getTraceId() == span.getTraceId()) { resolvedParentAgentSpanId = LLMObsContext.currentParentAgentSpanId(); resolvedParentAgentName = LLMObsContext.currentParentAgentName(); + } else if (null == parent) { + // Continuing a distributed trace: attribute to the upstream agent carried on the wire. + CharSequence propagatedId = span.spanContext().getLLMObsParentAgentSpanId(); + if (propagatedId != null && propagatedId.length() > 0) { + resolvedParentAgentSpanId = propagatedId.toString(); + CharSequence propagatedName = span.spanContext().getLLMObsParentAgentName(); + resolvedParentAgentName = propagatedName == null ? null : propagatedName.toString(); + } } } @@ -208,6 +226,7 @@ public DDLLMObsSpan( scope = LLMObsContext.attach( span.spanContext(), + mlApp, sessionId, resolvedAgentVersion, resolvedParentAgentSpanId, diff --git a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/DDLLMObsPropagatorTest.java b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/DDLLMObsPropagatorTest.java index 309aa3ab6dc..67c63beac9c 100644 --- a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/DDLLMObsPropagatorTest.java +++ b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/DDLLMObsPropagatorTest.java @@ -1,12 +1,15 @@ package datadog.trace.llmobs; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import datadog.context.propagation.Propagators; import datadog.trace.agent.tooling.TracerInstaller; import datadog.trace.api.WellKnownTags; import datadog.trace.api.llmobs.LLMObsContext; +import datadog.trace.bootstrap.instrumentation.api.AgentPropagation; import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; @@ -109,6 +112,51 @@ void injectThenActivateJoinsSameTraceAndPropagatesLlmObsContext() throws Excepti } } + /** + * The automatic path: no LLMObs propagation API is called at all. Injecting the active span the + * way any auto-instrumented HTTP/gRPC client does must still carry the LLMObs context, matching + * dd-trace-py's {@code http.span_inject} hook. + */ + @Test + void autoInstrumentedInjectCarriesLlmObsContextWithoutManualPropagation() { + Propagators.register(AgentPropagation.LLMOBS_CONCERN, new LLMObsContextPropagator()); + Map headers = new HashMap<>(); + + try (AgentScope apmScope = startRootApmScope()) { + DDLLMObsSpan agent = newAgentSpan("producer-agent", "my-ml-app", "session-123"); + try { + Propagators.defaultPropagator().inject(agent.getAgentSpan(), headers, Map::put); + } finally { + agent.finish(); + } + } + + String xDatadogTags = headers.get("x-datadog-tags"); + assertNotNull(xDatadogTags, "expected x-datadog-tags to be injected"); + assertTrue( + xDatadogTags.contains("_dd.p.llmobs_ml_app=my-ml-app"), + () -> "ml_app missing from " + xDatadogTags); + assertTrue( + xDatadogTags.contains("_dd.p.llmobs_sid=session-123"), + () -> "session_id missing from " + xDatadogTags); + } + + /** An outbound call made with no LLMObs span active must not pick up LLMObs tags. */ + @Test + void autoInstrumentedInjectAddsNothingWithoutAnActiveLlmObsSpan() { + Propagators.register(AgentPropagation.LLMOBS_CONCERN, new LLMObsContextPropagator()); + Map headers = new HashMap<>(); + + try (AgentScope apmScope = startRootApmScope()) { + Propagators.defaultPropagator().inject(apmScope.span(), headers, Map::put); + } + + String xDatadogTags = headers.get("x-datadog-tags"); + assertTrue( + xDatadogTags == null || !xDatadogTags.contains("_dd.p.llmobs_"), + () -> "unexpected LLMObs tags in " + xDatadogTags); + } + @Test void injectAlwaysIncludesMlAppEvenWithoutSessionIdOrAttribution() { Map headers = new HashMap<>(); diff --git a/internal-api/src/main/java/datadog/trace/api/llmobs/LLMObsContext.java b/internal-api/src/main/java/datadog/trace/api/llmobs/LLMObsContext.java index e925a04f42f..c18dc51666e 100644 --- a/internal-api/src/main/java/datadog/trace/api/llmobs/LLMObsContext.java +++ b/internal-api/src/main/java/datadog/trace/api/llmobs/LLMObsContext.java @@ -13,6 +13,7 @@ private LLMObsContext() { } private static final ContextKey CONTEXT_KEY = ContextKey.named("llmobs_span"); + private static final ContextKey ML_APP_KEY = ContextKey.named("llmobs_ml_app"); private static final ContextKey SESSION_ID_KEY = ContextKey.named("llmobs_session_id"); private static final ContextKey AGENT_VERSION_KEY = ContextKey.named("llmobs_agent_version"); @@ -71,7 +72,26 @@ public static ContextScope attach( String agentVersion, String parentAgentSpanId, String parentAgentName) { + return attach(ctx, null, sessionId, agentVersion, parentAgentSpanId, parentAgentName); + } + + /** + * Attach an LLMObs span context, propagating ml_app alongside session_id, agent_version, and + * agent attribution. See {@link #attach(AgentSpanContext, String, String, String, String)}. + * + *

ml_app is carried here — rather than only as a span tag — so that distributed propagation + * can read the innermost active LLMObs span's ml_app at injection time, without needing a + * reference to the span itself. + */ + public static ContextScope attach( + AgentSpanContext ctx, + String mlApp, + String sessionId, + String agentVersion, + String parentAgentSpanId, + String parentAgentName) { Context updated = Context.current().with(CONTEXT_KEY, ctx); + updated = updated.with(ML_APP_KEY, mlApp != null && !mlApp.isEmpty() ? mlApp : null); if (sessionId != null && !sessionId.isEmpty()) { updated = updated.with(SESSION_ID_KEY, sessionId); } @@ -94,6 +114,11 @@ public static AgentSpanContext current() { return Context.current().get(CONTEXT_KEY); } + /** Return the ml_app of the innermost active LLMObs span, or null if none is active. */ + public static String currentMlApp() { + return Context.current().get(ML_APP_KEY); + } + /** * Return the session_id propagated from an enclosing LLMObs span, or null if no parent set one. */ diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentPropagation.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentPropagation.java index 46a01b3f70f..0cf43469604 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentPropagation.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentPropagation.java @@ -23,6 +23,11 @@ public final class AgentPropagation { // TODO DSM propagator should run after the other propagators as it stores the pathway context // TODO into the span context for now. Remove priority after the migration is complete. public static final Concern DSM_CONCERN = withPriority("data-stream-monitoring", 110); + // LLM Observability contributes no headers of its own: it stages the _dd.p.llmobs_* propagation + // tags onto the span context, which the tracing propagator then serializes into x-datadog-tags / + // tracestate. Composite injection runs in reverse priority order, so this must sort after + // TRACING_CONCERN to actually inject before it. + public static final Concern LLMOBS_CONCERN = withPriority("llm-observability", 115); private AgentPropagation() {}