From 4cbca562ecd6387f54710c7ad89cdc45b7c06331 Mon Sep 17 00:00:00 2001 From: "dd-octo-sts[bot]" <200755185+dd-octo-sts[bot]@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:26:59 -0400 Subject: [PATCH 01/32] Pin system tests for release branch (#12142) Co-authored-by: dd-octo-sts[bot] <200755185+dd-octo-sts[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .github/workflows/run-system-tests.yaml | 4 ++-- .gitlab-ci.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-system-tests.yaml b/.github/workflows/run-system-tests.yaml index 4b0ad663659..dbc99e4a1e2 100644 --- a/.github/workflows/run-system-tests.yaml +++ b/.github/workflows/run-system-tests.yaml @@ -62,7 +62,7 @@ jobs: needs: - build # If you change the following comment, update the pattern in the update_system_test_reference.sh script to match. - uses: DataDog/system-tests/.github/workflows/system-tests.yml@main # system tests are pinned on release branches only + uses: DataDog/system-tests/.github/workflows/system-tests.yml@20ec39676051d9eedf2fdcea17f716e7b89d3c10 # system tests are pinned on release branches only permissions: contents: read id-token: write @@ -70,7 +70,7 @@ jobs: with: library: java # If you change the following comment, update the pattern in the update_system_test_reference.sh script to match. - ref: "main" # system tests are pinned on release branches only + ref: "20ec39676051d9eedf2fdcea17f716e7b89d3c10" # system tests are pinned on release branches only binaries_artifact: binaries desired_execution_time: 900 # 15 minutes scenarios_groups: tracer-release diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fed9f25df92..df1aeb2708f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1475,7 +1475,7 @@ publishing-gate: configure_system_tests: variables: - SYSTEM_TESTS_REF: "main" # system tests are pinned on release branches only + SYSTEM_TESTS_REF: "20ec39676051d9eedf2fdcea17f716e7b89d3c10" # system tests are pinned on release branches only SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection" create_key: From db887be48fd4b44ba523eb768326f83fc13ad891 Mon Sep 17 00:00:00 2001 From: Paul Fournillon <112829622+kaahos@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:01:08 +0200 Subject: [PATCH 02/32] Skip latest broken Kafka Connect 7.x POMs (#12160) (#12172) Skip broken Kafka Connect 7.5.16 POMs Skip remaining broken Kafka Connect POMs (cherry picked from commit 19bad937a3f10e53dce2f80324af47d23645c9ef) Co-authored-by: Alexey Kuznetsov --- .../kafka/kafka-connect-0.11/build.gradle | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/build.gradle b/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/build.gradle index 57745940c19..f5540785beb 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/build.gradle +++ b/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/build.gradle @@ -17,30 +17,40 @@ muzzle { '7.5.14-ccs', '7.5.15-ce', '7.5.15-ccs', + '7.5.16-ce', + '7.5.16-ccs', '7.6.10-ce', '7.6.10-ccs', '7.6.11-ce', '7.6.11-ccs', '7.6.12-ce', '7.6.12-ccs', + '7.6.13-ce', + '7.6.13-ccs', '7.7.8-ce', '7.7.8-ccs', '7.7.9-ce', '7.7.9-ccs', '7.7.10-ce', '7.7.10-ccs', + '7.7.11-ce', + '7.7.11-ccs', '7.8.7-ce', '7.8.7-ccs', '7.8.8-ce', '7.8.8-ccs', '7.8.9-ce', '7.8.9-ccs', + '7.8.10-ce', + '7.8.10-ccs', '7.9.6-ce', '7.9.6-ccs', '7.9.7-ce', '7.9.7-ccs', '7.9.8-ce', - '7.9.8-ccs' + '7.9.8-ccs', + '7.9.9-ce', + '7.9.9-ccs' ] excludeDependency "io.confluent.cloud:*" excludeDependency "io.confluent.observability:*" From 5b1dc07ccf3eddde88ebffe29c158ec7d83b0541 Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Mon, 10 Aug 2026 14:02:00 +0200 Subject: [PATCH 03/32] ci: update one-pipeline to 1.2.0 (#12174) (cherry picked from commit c55cb16a8a8e7a18e4e0a925737df4bf3256661e) Co-authored-by: gh-worker-campaigns-3e9aa4[bot] <244854796+gh-worker-campaigns-3e9aa4[bot]@users.noreply.github.com> --- .gitlab/one-pipeline.locked.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/one-pipeline.locked.yml b/.gitlab/one-pipeline.locked.yml index e3ec0428cc7..08c4ee68544 100644 --- a/.gitlab/one-pipeline.locked.yml +++ b/.gitlab/one-pipeline.locked.yml @@ -1,4 +1,4 @@ # DO NOT EDIT THIS FILE MANUALLY # This file is auto-generated by automation. include: - - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/versions/1.1.0/one-pipeline.yml + - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/versions/1.2.0/one-pipeline.yml From 3277e8915901523c30100d3ce1c9123a09d32a1c Mon Sep 17 00:00:00 2001 From: Paul Fournillon <112829622+kaahos@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:38:47 +0200 Subject: [PATCH 04/32] Bump ddprof to 1.49.0 (#12168) (#12173) Bump ddprof to 1.49.0 (cherry picked from commit 960957e9d2f5f8892f9553717e3aa1392b3626bf) Co-authored-by: Jaroslav Bachorik --- dd-java-agent/ddprof-lib/gradle.lockfile | 2 +- gradle/libs.versions.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/ddprof-lib/gradle.lockfile b/dd-java-agent/ddprof-lib/gradle.lockfile index 3200a914dfc..d5bceac668b 100644 --- a/dd-java-agent/ddprof-lib/gradle.lockfile +++ b/dd-java-agent/ddprof-lib/gradle.lockfile @@ -5,7 +5,7 @@ ch.qos.logback:logback-classic:1.2.13=testCompileClasspath,testRuntimeClasspath ch.qos.logback:logback-core:1.2.13=testCompileClasspath,testRuntimeClasspath com.datadoghq:dd-javac-plugin-client:0.2.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:ddprof:1.48.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq:ddprof:1.49.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.github.javaparser:javaparser-core:3.25.6=codenarc com.github.spotbugs:spotbugs-annotations:4.9.8=compileClasspath,spotbugs com.github.spotbugs:spotbugs:4.9.8=spotbugs diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b7ec5949343..ad9f39d0740 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -10,7 +10,7 @@ shadow = "9.4.2" spotbugs_annotations = "4.9.8" # DataDog libs and forks -ddprof = "1.48.1" +ddprof = "1.49.0" dogstatsd = "4.4.5" okhttp = "3.12.15" # Datadog fork to support Java 7 From 83d0540286158857a1c26fa335a49acbff298dcf Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Mon, 10 Aug 2026 12:40:35 -0400 Subject: [PATCH 05/32] =?UTF-8?q?=F0=9F=8D=92=2011535=20-=20Handle=20grace?= =?UTF-8?q?fully=20malformed=20flag=20evaluations=20(#12176)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Handle gracefully malformed flag evaluations --- .../trace/api/openfeature/DDEvaluator.java | 18 ++- .../api/openfeature/DDEvaluatorTest.java | 38 ++++- .../api/featureflag/ufc/v1/Allocation.java | 52 +++++++ .../featureflag/ExposureWriterImpl.java | 14 +- .../UniversalFlagConfigParser.java | 102 +++++++++++-- .../featureflag/ExposureWriterTests.java | 28 +++- .../RemoteConfigServiceImplTest.java | 137 ++++++++++++++---- 7 files changed, 329 insertions(+), 60 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index 6d55c54a5b3..fe47bbab16f 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -26,8 +26,8 @@ import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.time.Instant; import java.util.AbstractMap; -import java.util.Date; import java.util.Deque; import java.util.HashMap; import java.util.HashSet; @@ -125,7 +125,7 @@ public ProviderEvaluation evaluate( return error(defaultValue, ErrorCode.GENERAL, "Missing allocations for flag " + key); } - final Date now = new Date(); + final Instant now = Instant.now(); final String targetingKey = context.getTargetingKey(); for (final Allocation allocation : flag.allocations) { @@ -208,14 +208,14 @@ private static boolean isEmpty(final List list) { return list == null || list.isEmpty(); } - private static boolean isAllocationActive(final Allocation allocation, final Date now) { - final Date startDate = allocation.startAt; - if (startDate != null && now.before(startDate)) { + static boolean isAllocationActive(final Allocation allocation, final Instant now) { + final Instant startDate = allocation.startAtInstant(); + if (startDate != null && now.isBefore(startDate)) { return false; } - final Date endDate = allocation.endAt; - if (endDate != null && now.after(endDate)) { + final Instant endDate = allocation.endAtInstant(); + if (endDate != null && now.isAfter(endDate)) { return false; } @@ -546,7 +546,9 @@ static AbstractMap flattenContext(final EvaluationContext contex new FlattenEntry(entry.key + "." + property, structure.getValue(property))); } } else { - result.put(entry.key, context.convertValue(value)); + result.put( + entry.key, + value.isInstant() ? value.asInstant().toString() : context.convertValue(value)); } } } diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java index 7b6cc01e021..155c5eb1ea8 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java @@ -24,6 +24,7 @@ import com.squareup.moshi.Moshi; import com.squareup.moshi.Types; import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.ufc.v1.Allocation; import datadog.trace.api.featureflag.ufc.v1.Flag; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import dev.openfeature.sdk.ErrorCode; @@ -37,7 +38,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.time.OffsetDateTime; +import java.time.Instant; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -123,7 +125,7 @@ private static Arguments[] valueMappingTestCases() { Arguments.of(Value.class, null, null), // Unsupported - Arguments.of(Date.class, "21-12-2023", IllegalArgumentException.class), + Arguments.of(Long.class, 42L, IllegalArgumentException.class), }; } @@ -212,6 +214,33 @@ public void testNoAllocations() { assertThat(details.getErrorCode(), nullValue()); } + @Test + public void testAllocationDateAbiAndInstantAccessors() throws Exception { + final Date startAt = Date.from(Instant.parse("2024-01-01T00:00:00Z")); + final Date endAt = Date.from(Instant.parse("2024-12-31T23:59:59Z")); + final Allocation allocation = + new Allocation("allocation", emptyList(), startAt, endAt, emptyList(), true); + + assertThat(Allocation.class.getField("startAt").getType(), equalTo(Date.class)); + assertThat(Allocation.class.getField("endAt").getType(), equalTo(Date.class)); + assertThat(allocation.startAtInstant(), equalTo(startAt.toInstant())); + assertThat(allocation.endAtInstant(), equalTo(endAt.toInstant())); + } + + @Test + public void testAllocationWindowHonorsMicrosecondPrecision() { + final Instant startAt = Instant.parse("2024-01-01T00:00:00.123456Z"); + final Instant endAt = Instant.parse("2024-01-01T00:00:00.987654Z"); + final Allocation allocation = + Allocation.fromInstants("allocation", emptyList(), startAt, endAt, emptyList(), true); + + assertThat( + DDEvaluator.isAllocationActive(allocation, startAt.minusNanos(1_000)), equalTo(false)); + assertThat(DDEvaluator.isAllocationActive(allocation, startAt), equalTo(true)); + assertThat(DDEvaluator.isAllocationActive(allocation, endAt), equalTo(true)); + assertThat(DDEvaluator.isAllocationActive(allocation, endAt.plusNanos(1_000)), equalTo(false)); + } + private static Arguments[] flatteningTestCases() { final List arguments = new ArrayList<>(); arguments.add(Arguments.of(emptyMap(), emptyMap())); @@ -227,6 +256,8 @@ private static Arguments[] flatteningTestCases() { Arguments.of( mapOf("map", mapOf("key1", 1, "key2", 2, "key3", mapOf("key4", 4))), mapOf("map.key1", 1, "map.key2", 2, "map.key3.key4", 4))); + final Instant instant = Instant.parse("2026-07-10T12:34:56Z"); + arguments.add(Arguments.of(mapOf("instant", instant), mapOf("instant", instant.toString()))); return arguments.toArray(new Arguments[0]); } @@ -410,7 +441,8 @@ public Date fromJson(final JsonReader reader) throws IOException { return reader.nextNull(); } try { - return Date.from(OffsetDateTime.parse(reader.nextString()).toInstant()); + return Date.from( + DateTimeFormatter.ISO_OFFSET_DATE_TIME.parse(reader.nextString(), Instant::from)); } catch (final Exception ignored) { return null; } diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/Allocation.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/Allocation.java index 2ce031243bb..b572e7e5aff 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/Allocation.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/Allocation.java @@ -1,5 +1,6 @@ package datadog.trace.api.featureflag.ufc.v1; +import java.time.Instant; import java.util.Date; import java.util.List; @@ -11,6 +12,9 @@ public class Allocation { public final List splits; public final Boolean doLog; + private final transient Instant preciseStartAt; + private final transient Instant preciseEndAt; + public Allocation( final String key, final List rules, @@ -18,11 +22,59 @@ public Allocation( final Date endAt, final List splits, final Boolean doLog) { + this( + key, + rules, + startAt, + endAt, + splits, + doLog, + startAt == null ? null : startAt.toInstant(), + endAt == null ? null : endAt.toInstant()); + } + + private Allocation( + final String key, + final List rules, + final Date startAt, + final Date endAt, + final List splits, + final Boolean doLog, + final Instant preciseStartAt, + final Instant preciseEndAt) { this.key = key; this.rules = rules; this.startAt = startAt; this.endAt = endAt; this.splits = splits; this.doLog = doLog; + this.preciseStartAt = preciseStartAt; + this.preciseEndAt = preciseEndAt; + } + + public static Allocation fromInstants( + final String key, + final List rules, + final Instant startAt, + final Instant endAt, + final List splits, + final Boolean doLog) { + return new Allocation( + key, + rules, + startAt == null ? null : Date.from(startAt), + endAt == null ? null : Date.from(endAt), + splits, + doLog, + startAt, + endAt); + } + + public Instant startAtInstant() { + return preciseStartAt != null ? preciseStartAt : startAt == null ? null : startAt.toInstant(); + } + + public Instant endAtInstant() { + return preciseEndAt != null ? preciseEndAt : endAt == null ? null : endAt.toInstant(); } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java index eddcd520f27..9932a20256b 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java @@ -1,5 +1,6 @@ package com.datadog.featureflag; +import static datadog.trace.api.telemetry.LogCollector.EXCLUDE_TELEMETRY; import static datadog.trace.util.AgentThreadFactory.AgentThread.FEATURE_FLAG_EXPOSURE_PROCESSOR; import static datadog.trace.util.AgentThreadFactory.newAgentThread; import static java.util.concurrent.TimeUnit.SECONDS; @@ -178,15 +179,22 @@ protected void flushIfNecessary() { return; } if (shouldFlush()) { + final String requestBodyJson; try { final ExposuresRequest exposures = new ExposuresRequest(this.context, this.buffer); - final String reqBod = jsonAdapter.toJson(exposures); + requestBodyJson = jsonAdapter.toJson(exposures); + } catch (RuntimeException e) { + LOGGER.error(EXCLUDE_TELEMETRY, "Could not serialize exposures; dropping batch", e); + this.buffer.clear(); + return; + } + try { final RequestBody requestBody = - RequestBody.create(okhttp3.MediaType.parse("application/json"), reqBod); + RequestBody.create(okhttp3.MediaType.parse("application/json"), requestBodyJson); evp.post("exposures", requestBody, stream -> null, null, false); this.buffer.clear(); } catch (Exception e) { - LOGGER.error("Could not submit exposures", e); + LOGGER.debug("Could not submit exposures", e); } } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java index ba5536601f7..8951f20f866 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java @@ -7,29 +7,40 @@ import com.squareup.moshi.Moshi; import com.squareup.moshi.Types; import datadog.remoteconfig.ConfigurationDeserializer; +import datadog.trace.api.featureflag.ufc.v1.Allocation; import datadog.trace.api.featureflag.ufc.v1.Flag; +import datadog.trace.api.featureflag.ufc.v1.Rule; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import datadog.trace.api.featureflag.ufc.v1.Split; import java.io.ByteArrayInputStream; import java.io.IOException; import java.lang.annotation.Annotation; import java.lang.reflect.Type; import java.time.Instant; import java.time.format.DateTimeFormatter; -import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import javax.annotation.Nonnull; import javax.annotation.Nullable; import okio.BufferedSource; import okio.Okio; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; final class UniversalFlagConfigParser implements ConfigurationDeserializer { + private static final Logger LOGGER = LoggerFactory.getLogger(UniversalFlagConfigParser.class); + static final UniversalFlagConfigParser INSTANCE = new UniversalFlagConfigParser(); private static final Moshi MOSHI = - new Moshi.Builder().add(Date.class, new DateAdapter()).add(FlagMapAdapter.FACTORY).build(); + new Moshi.Builder() + .add(Instant.class, new InstantAdapter()) + .add(AllocationAdapter.FACTORY) + .add(FlagMapAdapter.FACTORY) + .build(); private static final JsonAdapter V1_ADAPTER = MOSHI.adapter(ServerConfiguration.class); @@ -97,8 +108,11 @@ public Map fromJson(@Nonnull final JsonReader reader) throws IOExc if (flag != null) { flags.put(flagKey, flag); } - } catch (JsonDataException | IllegalArgumentException ignored) { - // A malformed flag must not prevent other flags in the same config from evaluating. + } catch (JsonDataException | IllegalArgumentException error) { + LOGGER.warn( + "Dropping malformed FFE flag {} during remote config deserialization: {}", + flagKey, + error.toString()); } } reader.endObject(); @@ -112,28 +126,86 @@ public void toJson(@Nonnull final JsonWriter writer, @Nullable final Map { + static final class InstantAdapter extends JsonAdapter { @Nullable @Override - public Date fromJson(@Nonnull final JsonReader reader) throws IOException { - final String date = reader.nextString(); - if (date == null) { - return null; + public Instant fromJson(@Nonnull final JsonReader reader) throws IOException { + if (reader.peek() == JsonReader.Token.NULL) { + return reader.nextNull(); } - try { - final Instant instant = DateTimeFormatter.ISO_OFFSET_DATE_TIME.parse(date, Instant::from); - return Date.from(instant); - } catch (Exception e) { - // ignore wrongly set dates + return parseInstant(reader.nextString()); + } + + @Override + public void toJson(@Nonnull final JsonWriter writer, @Nullable final Instant value) + throws IOException { + throw new UnsupportedOperationException("Reading only adapter"); + } + } + + static final class AllocationAdapter extends JsonAdapter { + + static final Factory FACTORY = + new Factory() { + @Nullable + @Override + public JsonAdapter create( + @Nonnull final Type type, + @Nonnull final Set annotations, + @Nonnull final Moshi moshi) { + if (!annotations.isEmpty() || !Types.equals(type, Allocation.class)) { + return null; + } + return new AllocationAdapter(moshi.adapter(AllocationJson.class)); + } + }; + + private final JsonAdapter delegate; + + AllocationAdapter(final JsonAdapter delegate) { + this.delegate = delegate; + } + + @Nullable + @Override + public Allocation fromJson(@Nonnull final JsonReader reader) throws IOException { + final AllocationJson allocation = delegate.fromJson(reader); + if (allocation == null) { return null; } + return Allocation.fromInstants( + allocation.key, + allocation.rules, + allocation.startAt, + allocation.endAt, + allocation.splits, + allocation.doLog); } @Override - public void toJson(@Nonnull final JsonWriter writer, @Nullable final Date value) + public void toJson(@Nonnull final JsonWriter writer, @Nullable final Allocation value) throws IOException { throw new UnsupportedOperationException("Reading only adapter"); } } + + static final class AllocationJson { + String key; + List rules; + Instant startAt; + Instant endAt; + List splits; + Boolean doLog; + } + + @Nullable + private static Instant parseInstant(final String date) { + try { + return DateTimeFormatter.ISO_OFFSET_DATE_TIME.parse(date, Instant::from); + } catch (Exception e) { + // ignore wrongly set dates + return null; + } + } } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java index 75c4b3b653b..76b9e2602d8 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java @@ -1,5 +1,6 @@ package com.datadog.featureflag; +import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -222,6 +223,23 @@ void testFailuresAreRetried(boolean finallyFail) throws Exception { } } + @Test + void testSerializationFailureDoesNotPoisonFollowingExposures() throws Exception { + Config config = mockConfig("test-service"); + ExposureEvent validExposure = buildExposure(); + + try (ExposureWriterImpl writer = + new ExposureWriterImpl(1 << 4, 100, MILLISECONDS, sharedCommunicationObjects, config)) { + writer.init(); + writer.accept(buildExposure(singletonMap("invalid", (Object) Double.NaN))); + + MILLISECONDS.sleep(300); // wait for the invalid batch to be dropped + writer.accept(validExposure); + + poll.eventually(() -> assertExposures(allExposures(), singletonList(validExposure))); + } + } + @Test void testWriterStopsReceivingExposuresIfEvpProxyIsNotAvailable() throws Exception { SharedCommunicationObjects sharedCommunicationObjects = sharedCommunicationObjects(false); @@ -398,11 +416,19 @@ private static List buildExposures(int count) { private static ExposureEvent buildExposure() { String id = UUID.randomUUID().toString(); + return buildExposure(id, singletonMap("key_" + id, (Object) ("value_" + id))); + } + + private static ExposureEvent buildExposure(Map attributes) { + return buildExposure(UUID.randomUUID().toString(), attributes); + } + + private static ExposureEvent buildExposure(String id, Map attributes) { return new ExposureEvent( System.currentTimeMillis(), new Allocation("Allocation_" + id), new Flag("Flag_" + id), new Variant("Variant_" + id), - new Subject("Subject_" + id, singletonMap("key_" + id, (Object) ("value_" + id)))); + new Subject("Subject_" + id, attributes)); } } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java index c1f5ef17b87..6d14a28f796 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java @@ -29,6 +29,7 @@ import datadog.remoteconfig.Product; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.ufc.v1.Allocation; import datadog.trace.api.featureflag.ufc.v1.Flag; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import java.io.IOException; @@ -138,6 +139,41 @@ void ignoresUnknownTopLevelFields() throws Exception { assertTrue(config.flags.isEmpty()); } + @Test + void parsesAllocationWindowDatesAsDateFieldsWithInstantAccessors() throws Exception { + final ServerConfiguration config = + deserialize( + "{" + + "\"createdAt\":\"2024-04-17T19:40:53.716Z\"," + + "\"format\":\"SERVER\"," + + "\"environment\":{\"name\":\"Test\"}," + + "\"flags\":{" + + "\"dated-flag\":{" + + "\"key\":\"dated-flag\"," + + "\"enabled\":true," + + "\"variationType\":\"STRING\"," + + "\"variations\":{\"expected\":{\"key\":\"expected\",\"value\":\"expected\"}}," + + "\"allocations\":[{" + + "\"key\":\"dated-allocation\"," + + "\"rules\":[]," + + "\"startAt\":\"2023-01-01T01:00:00.123456+01:00\"," + + "\"endAt\":\"2023-01-02T00:00:00.987654Z\"," + + "\"splits\":[{\"variationKey\":\"expected\",\"shards\":[]}]," + + "\"doLog\":true" + + "}]" + + "}" + + "}" + + "}"); + + final Allocation allocation = config.flags.get("dated-flag").allocations.get(0); + assertEquals(Date.class, Allocation.class.getField("startAt").getType()); + assertEquals(Date.class, Allocation.class.getField("endAt").getType()); + assertEquals(Instant.parse("2023-01-01T00:00:00.123Z"), allocation.startAt.toInstant()); + assertEquals(Instant.parse("2023-01-02T00:00:00.987Z"), allocation.endAt.toInstant()); + assertEquals(Instant.parse("2023-01-01T00:00:00.123456Z"), allocation.startAtInstant()); + assertEquals(Instant.parse("2023-01-02T00:00:00.987654Z"), allocation.endAtInstant()); + } + @Test void rejectsTrailingJson() { assertThrows(IOException.class, () -> deserialize(emptyConfig() + "{}")); @@ -262,50 +298,88 @@ void flagMapAdapterIsReadOnly() { () -> adapter.toJson(mock(JsonWriter.class), emptyMap())); } + @Test + void allocationAdapterFactoryOnlyCreatesAllocationAdapterForAllocationType() { + final Moshi moshi = moshi(); + + final JsonAdapter adapter = + UniversalFlagConfigParser.AllocationAdapter.FACTORY.create( + Allocation.class, emptySet(), moshi); + + assertNotNull(adapter); + assertTrue(adapter instanceof UniversalFlagConfigParser.AllocationAdapter); + assertNull( + UniversalFlagConfigParser.AllocationAdapter.FACTORY.create( + String.class, emptySet(), moshi)); + assertNull( + UniversalFlagConfigParser.AllocationAdapter.FACTORY.create( + Allocation.class, singleton(mock(Annotation.class)), moshi)); + } + + @Test + void allocationAdapterHandlesNullAndIsReadOnly() throws Exception { + final UniversalFlagConfigParser.AllocationAdapter adapter = + new UniversalFlagConfigParser.AllocationAdapter( + moshi().adapter(UniversalFlagConfigParser.AllocationJson.class)); + + assertNull(adapter.fromJson("null")); + assertThrows( + UnsupportedOperationException.class, () -> adapter.toJson(mock(JsonWriter.class), null)); + } + @TableTest({ - "scenario | value | expectedEpochMilli", - "utc second | '2023-01-01T00:00:00Z' | 1672531200000 ", - "utc end of year | '2023-12-31T23:59:59Z' | 1704067199000 ", - "leap day | '2024-02-29T12:00:00Z' | 1709208000000 ", - "millisecond precision | '2023-01-01T00:00:00.000Z' | 1672531200000 ", - "three fractional digits | '2023-06-15T14:30:45.123Z' | 1686839445123 ", - "six fractional digits truncate to millis | '2023-06-15T14:30:45.123456Z' | 1686839445123 ", - "six fractional digits preserve millis | '2023-06-15T14:30:45.235982Z' | 1686839445235 ", - "nine fractional digits truncate to millis | '2023-06-15T14:30:45.123456789Z' | 1686839445123 ", - "one fractional digit | '2023-06-15T14:30:45.1Z' | 1686839445100 ", - "two fractional digits | '2023-06-15T14:30:45.12Z' | 1686839445120 ", - "positive offset | '2023-01-01T01:00:00+01:00' | 1672531200000 ", - "negative offset | '2023-01-01T00:00:00-05:00' | 1672549200000 ", - "date only | '2023-01-01' | ", - "invalid | 'invalid-date' | ", - "empty string | '' | ", - "not a date | 'not-a-date' | ", - "slash date | '2023/01/01T00:00:00Z' | ", - "null | | " + "scenario | value | expectedInstant ", + "utc second | '2023-01-01T00:00:00Z' | '2023-01-01T00:00:00Z' ", + "utc end of year | '2023-12-31T23:59:59Z' | '2023-12-31T23:59:59Z' ", + "leap day | '2024-02-29T12:00:00Z' | '2024-02-29T12:00:00Z' ", + "millisecond precision | '2023-01-01T00:00:00.000Z' | '2023-01-01T00:00:00Z' ", + "three fractional digits | '2023-06-15T14:30:45.123Z' | '2023-06-15T14:30:45.123Z' ", + "six fractional digits | '2023-06-15T14:30:45.123456Z' | '2023-06-15T14:30:45.123456Z' ", + "six fractional digits distinct | '2023-06-15T14:30:45.235982Z' | '2023-06-15T14:30:45.235982Z' ", + "nine fractional digits | '2023-06-15T14:30:45.123456789Z' | '2023-06-15T14:30:45.123456789Z'", + "one fractional digit | '2023-06-15T14:30:45.1Z' | '2023-06-15T14:30:45.100Z' ", + "two fractional digits | '2023-06-15T14:30:45.12Z' | '2023-06-15T14:30:45.120Z' ", + "positive offset | '2023-01-01T01:00:00+01:00' | '2023-01-01T00:00:00Z' ", + "negative offset | '2023-01-01T00:00:00-05:00' | '2023-01-01T05:00:00Z' ", + "date only | '2023-01-01' | ", + "invalid | 'invalid-date' | ", + "empty string | '' | ", + "not a date | 'not-a-date' | ", + "slash date | '2023/01/01T00:00:00Z' | ", + "null | | " }) - void testDateParsing(final String value, final Long expectedEpochMilli) throws Exception { + void testInstantParsing(final String value, final String expectedInstant) throws Exception { final JsonReader reader = mock(JsonReader.class); - when(reader.nextString()).thenReturn(value); - final UniversalFlagConfigParser.DateAdapter adapter = - new UniversalFlagConfigParser.DateAdapter(); + if (value == null) { + when(reader.peek()).thenReturn(JsonReader.Token.NULL); + when(reader.nextNull()).thenReturn(null); + } else { + when(reader.peek()).thenReturn(JsonReader.Token.STRING); + when(reader.nextString()).thenReturn(value); + } + final UniversalFlagConfigParser.InstantAdapter adapter = + new UniversalFlagConfigParser.InstantAdapter(); - final Date parsed = adapter.fromJson(reader); - if (expectedEpochMilli == null) { + final Instant parsed = adapter.fromJson(reader); + if (value == null) { + verify(reader).nextNull(); + } + if (expectedInstant == null) { assertNull(parsed); } else { assertNotNull(parsed); - assertEquals(Instant.ofEpochMilli(expectedEpochMilli), parsed.toInstant()); + assertEquals(expectedInstant, parsed.toString()); } } @Test void testParsingOnlyAdapter() { - final UniversalFlagConfigParser.DateAdapter adapter = - new UniversalFlagConfigParser.DateAdapter(); + final UniversalFlagConfigParser.InstantAdapter adapter = + new UniversalFlagConfigParser.InstantAdapter(); assertThrows( UnsupportedOperationException.class, - () -> adapter.toJson(mock(JsonWriter.class), new Date())); + () -> adapter.toJson(mock(JsonWriter.class), Instant.EPOCH)); } @SuppressWarnings("unchecked") @@ -318,7 +392,10 @@ private static ServerConfiguration deserialize(final String json) throws Excepti } private static Moshi moshi() { - return new Moshi.Builder().add(Date.class, new UniversalFlagConfigParser.DateAdapter()).build(); + return new Moshi.Builder() + .add(Instant.class, new UniversalFlagConfigParser.InstantAdapter()) + .add(UniversalFlagConfigParser.AllocationAdapter.FACTORY) + .build(); } private static String emptyConfig() { From cd111ab72887dda5de31240d1391fcd4247e4c1c Mon Sep 17 00:00:00 2001 From: Mo Atie Date: Thu, 13 Aug 2026 13:55:17 -0400 Subject: [PATCH 06/32] =?UTF-8?q?=F0=9F=8D=92=2012203=20-=20ci:=20update?= =?UTF-8?q?=20one-pipeline=20to=201.3.1=20(#12206)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab/one-pipeline.locked.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/one-pipeline.locked.yml b/.gitlab/one-pipeline.locked.yml index 08c4ee68544..0d756c4f697 100644 --- a/.gitlab/one-pipeline.locked.yml +++ b/.gitlab/one-pipeline.locked.yml @@ -1,4 +1,4 @@ # DO NOT EDIT THIS FILE MANUALLY # This file is auto-generated by automation. include: - - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/versions/1.2.0/one-pipeline.yml + - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/versions/1.3.1/one-pipeline.yml From 93c577b72d11ef861f6385931a32c19aa830a2ea Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Wed, 26 Aug 2026 16:00:02 -0400 Subject: [PATCH 07/32] Add InstrumenterConfig.isDataStreamsEnabled() accessor Implements the pre-instrumentation config accessor for the data streams feature flag, mirroring the existing usmEnabled pattern. Reads the same data.streams.enabled key that Config.isDataStreamsEnabled() uses, enabling DSM to work independently of the general DD_INTEGRATIONS_ENABLED APM gate. Co-Authored-By: Claude Sonnet 5 --- .../datadog/trace/api/InstrumenterConfig.java | 10 ++++++++++ .../trace/api/InstrumenterConfigTest.groovy | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java b/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java index 74bff640024..dd32dff1c6f 100644 --- a/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java +++ b/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java @@ -7,6 +7,7 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_CIVISIBILITY_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_CODE_ORIGIN_FOR_SPANS_INTERFACE_SUPPORT; import static datadog.trace.api.ConfigDefaults.DEFAULT_DATA_JOBS_ENABLED; +import static datadog.trace.api.ConfigDefaults.DEFAULT_DATA_STREAMS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_IAST_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_INTEGRATIONS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_LLM_OBS_ENABLED; @@ -35,6 +36,7 @@ import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_ENABLED; import static datadog.trace.api.config.GeneralConfig.APP_LOGS_COLLECTION_ENABLED; import static datadog.trace.api.config.GeneralConfig.DATA_JOBS_ENABLED; +import static datadog.trace.api.config.GeneralConfig.DATA_STREAMS_ENABLED; import static datadog.trace.api.config.GeneralConfig.INTERNAL_EXIT_ON_FAILURE; import static datadog.trace.api.config.GeneralConfig.TELEMETRY_ENABLED; import static datadog.trace.api.config.GeneralConfig.TRACE_DEBUG; @@ -161,6 +163,7 @@ public class InstrumenterConfig { private final boolean appSecRaspEnabled; private final boolean iastFullyDisabled; private final boolean usmEnabled; + private final boolean dataStreamsEnabled; private final boolean telemetryEnabled; private final boolean llmObsEnabled; @@ -287,6 +290,8 @@ private InstrumenterConfig() { final Boolean iastEnabled = configProvider.getBoolean(IAST_ENABLED); iastFullyDisabled = iastEnabled != null && !iastEnabled; usmEnabled = configProvider.getBoolean(USM_ENABLED, DEFAULT_USM_ENABLED); + dataStreamsEnabled = + configProvider.getBoolean(DATA_STREAMS_ENABLED, DEFAULT_DATA_STREAMS_ENABLED); telemetryEnabled = configProvider.getBoolean(TELEMETRY_ENABLED, DEFAULT_TELEMETRY_ENABLED); llmObsEnabled = configProvider.getBoolean(LLMOBS_ENABLED, DEFAULT_LLM_OBS_ENABLED); } else { @@ -297,6 +302,7 @@ private InstrumenterConfig() { iastFullyDisabled = true; telemetryEnabled = false; usmEnabled = false; + dataStreamsEnabled = false; llmObsEnabled = false; } @@ -505,6 +511,10 @@ public boolean isUsmEnabled() { return usmEnabled; } + public boolean isDataStreamsEnabled() { + return dataStreamsEnabled; + } + public boolean isTelemetryEnabled() { return telemetryEnabled; } diff --git a/internal-api/src/test/groovy/datadog/trace/api/InstrumenterConfigTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/InstrumenterConfigTest.groovy index 15c15c1ef3d..f6b8bae42d4 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/InstrumenterConfigTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/InstrumenterConfigTest.groovy @@ -192,4 +192,22 @@ class InstrumenterConfigTest extends DDSpecification { "1" | ProductActivation.FULLY_ENABLED "inactive" | ProductActivation.ENABLED_INACTIVE } + + def "data streams enabled = #input"() { + setup: + if (input != null) { + injectSysConfig("data.streams.enabled", input) + } + + expect: + InstrumenterConfig.get().isDataStreamsEnabled() == expected + + where: + input | expected + null | false + "false" | false + "true" | true + "1" | true + "0" | false + } } From e0c0c2d56f645ec6920bc2135ac3605a7f73ccda Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 09:21:52 -0400 Subject: [PATCH 08/32] Add TargetSystem.DATA_STREAMS and InstrumenterModule.DataStreams base class --- .../agent/tooling/InstrumenterModule.java | 20 +++++++ .../agent/tooling/InstrumenterModuleTest.java | 53 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterModule.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterModule.java index d2abbc265e5..57e11f6bcfe 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterModule.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterModule.java @@ -41,6 +41,7 @@ public abstract class InstrumenterModule implements Instrumenter { *
  • {@link TargetSystem#IAST iast} *
  • {@link TargetSystem#CIVISIBILITY ci-visibility} *
  • {@link TargetSystem#USM usm} + *
  • {@link TargetSystem#DATA_STREAMS data-streams} *
  • {@link TargetSystem#CONTEXT_TRACKING context-tracking} *
  • {@link TargetSystem#RASP rasp} * @@ -53,6 +54,7 @@ public enum TargetSystem { CIVISIBILITY, USM, LLMOBS, + DATA_STREAMS, CONTEXT_TRACKING, RASP, } @@ -320,6 +322,24 @@ public final boolean isApplicable(Set enabledSystems) { } } + /** Parent class for instrumentations that support both tracing and Data Streams Monitoring */ + public abstract static class DataStreams extends InstrumenterModule { + public DataStreams(String instrumentationName, String... additionalNames) { + super(instrumentationName, additionalNames); + } + + @Override + public final boolean isApplicable(Set enabledSystems) { + return enabledSystems.contains(TargetSystem.TRACING) + || enabledSystems.contains(TargetSystem.DATA_STREAMS); + } + + @Override + public boolean isEnabled() { + return super.isEnabled() || InstrumenterConfig.get().isDataStreamsEnabled(); + } + } + /** Parent class for all CI related instrumentations */ public abstract static class CiVisibility extends InstrumenterModule { public CiVisibility(String instrumentationName, String... additionalNames) { diff --git a/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java b/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java new file mode 100644 index 00000000000..b4c52ded871 --- /dev/null +++ b/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java @@ -0,0 +1,53 @@ +package datadog.trace.agent.tooling; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.agent.tooling.InstrumenterModule.TargetSystem; +import java.util.HashSet; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class InstrumenterModuleTest { + + @Test + void testDataStreamsIsApplicableWithTracing() { + Set enabledSystems = new HashSet<>(); + enabledSystems.add(TargetSystem.TRACING); + + InstrumenterModule.DataStreams module = new InstrumenterModule.DataStreams("test-module") {}; + + assertTrue(module.isApplicable(enabledSystems)); + } + + @Test + void testDataStreamsIsApplicableWithDataStreams() { + Set enabledSystems = new HashSet<>(); + enabledSystems.add(TargetSystem.DATA_STREAMS); + + InstrumenterModule.DataStreams module = new InstrumenterModule.DataStreams("test-module") {}; + + assertTrue(module.isApplicable(enabledSystems)); + } + + @Test + void testDataStreamsIsApplicableWithBoth() { + Set enabledSystems = new HashSet<>(); + enabledSystems.add(TargetSystem.TRACING); + enabledSystems.add(TargetSystem.DATA_STREAMS); + + InstrumenterModule.DataStreams module = new InstrumenterModule.DataStreams("test-module") {}; + + assertTrue(module.isApplicable(enabledSystems)); + } + + @Test + void testDataStreamsIsApplicableWithNeither() { + Set enabledSystems = new HashSet<>(); + enabledSystems.add(TargetSystem.APPSEC); + + InstrumenterModule.DataStreams module = new InstrumenterModule.DataStreams("test-module") {}; + + assertFalse(module.isApplicable(enabledSystems)); + } +} From fc02467d38cb72ee26f11b946371920f76fdd5e3 Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 09:28:25 -0400 Subject: [PATCH 09/32] Add isEnabled() test coverage for DataStreams --- .../agent/tooling/InstrumenterModuleTest.java | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java b/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java index b4c52ded871..09acd7fb6b8 100644 --- a/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java +++ b/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java @@ -4,12 +4,27 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.trace.agent.tooling.InstrumenterModule.TargetSystem; +import datadog.trace.api.InstrumenterConfig; +import java.lang.reflect.Field; import java.util.HashSet; import java.util.Set; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; class InstrumenterModuleTest { + private static final Field ENABLED_FIELD = getEnabledField(); + + private static Field getEnabledField() { + try { + Field field = InstrumenterModule.class.getDeclaredField("enabled"); + field.setAccessible(true); + return field; + } catch (NoSuchFieldException e) { + throw new ExceptionInInitializerError(e); + } + } + @Test void testDataStreamsIsApplicableWithTracing() { Set enabledSystems = new HashSet<>(); @@ -50,4 +65,95 @@ void testDataStreamsIsApplicableWithNeither() { assertFalse(module.isApplicable(enabledSystems)); } + + @Test + void testDataStreamsIsEnabledWhenDataStreamsEnabledOverridesFalse() + throws IllegalAccessException { + // When DSM is enabled, isEnabled() should return true even if super.isEnabled() is false + // This tests the edge case where trace.kafka.enabled=false but DSM is explicitly enabled + InstrumenterModule.DataStreams module = + new InstrumenterModule.DataStreams("test-kafka-module") {}; + + // Set the enabled field to false to simulate disabled tracing + ENABLED_FIELD.setBoolean(module, false); + + // Override InstrumenterConfig to return true for isDataStreamsEnabled() + InstrumenterConfig originalConfig = InstrumenterConfig.get(); + setFieldInConfig(originalConfig, "dataStreamsEnabled", true); + + try { + assertTrue(module.isEnabled()); + } finally { + setFieldInConfig(originalConfig, "dataStreamsEnabled", false); + } + } + + @Test + void testDataStreamsIsEnabledWhenSuperEnabledIsTrue() throws IllegalAccessException { + // When super.isEnabled() is true, isEnabled() should return true regardless of DSM state + InstrumenterModule.DataStreams module = + new InstrumenterModule.DataStreams("test-kafka-module") {}; + + // Set the enabled field to true to simulate enabled tracing + ENABLED_FIELD.setBoolean(module, true); + + // Ensure DSM is disabled + InstrumenterConfig originalConfig = InstrumenterConfig.get(); + boolean originalDataStreamsEnabled = originalConfig.isDataStreamsEnabled(); + setFieldInConfig(originalConfig, "dataStreamsEnabled", false); + + try { + assertTrue(module.isEnabled()); + } finally { + setFieldInConfig(originalConfig, "dataStreamsEnabled", originalDataStreamsEnabled); + } + } + + @Test + void testDataStreamsIsEnabledWhenBothEnabled() throws IllegalAccessException { + // When both super.isEnabled() and DSM are enabled, isEnabled() should return true + InstrumenterModule.DataStreams module = + new InstrumenterModule.DataStreams("test-kafka-module") {}; + + ENABLED_FIELD.setBoolean(module, true); + + InstrumenterConfig originalConfig = InstrumenterConfig.get(); + boolean originalDataStreamsEnabled = originalConfig.isDataStreamsEnabled(); + setFieldInConfig(originalConfig, "dataStreamsEnabled", true); + + try { + assertTrue(module.isEnabled()); + } finally { + setFieldInConfig(originalConfig, "dataStreamsEnabled", originalDataStreamsEnabled); + } + } + + @Test + void testDataStreamsIsEnabledWhenBothDisabled() throws IllegalAccessException { + // When both super.isEnabled() and DSM are disabled, isEnabled() should return false + InstrumenterModule.DataStreams module = + new InstrumenterModule.DataStreams("test-kafka-module") {}; + + ENABLED_FIELD.setBoolean(module, false); + + InstrumenterConfig originalConfig = InstrumenterConfig.get(); + boolean originalDataStreamsEnabled = originalConfig.isDataStreamsEnabled(); + setFieldInConfig(originalConfig, "dataStreamsEnabled", false); + + try { + assertFalse(module.isEnabled()); + } finally { + setFieldInConfig(originalConfig, "dataStreamsEnabled", originalDataStreamsEnabled); + } + } + + private static void setFieldInConfig(Object target, String fieldName, Object value) { + try { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } } From c78e86ebc4a3607c716bc14cf63c6563513b2749 Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 09:34:07 -0400 Subject: [PATCH 10/32] Fix test isolation bug in testDataStreamsIsEnabledWhenDataStreamsEnabledOverridesFalse Save and restore the original dataStreamsEnabled value instead of hardcoding false in the finally block. This prevents test pollution where an incorrect state could leak to the next test. --- .../datadog/trace/agent/tooling/InstrumenterModuleTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java b/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java index 09acd7fb6b8..dfc2af602fc 100644 --- a/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java +++ b/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java @@ -79,12 +79,13 @@ void testDataStreamsIsEnabledWhenDataStreamsEnabledOverridesFalse() // Override InstrumenterConfig to return true for isDataStreamsEnabled() InstrumenterConfig originalConfig = InstrumenterConfig.get(); + boolean originalDataStreamsEnabled = originalConfig.isDataStreamsEnabled(); setFieldInConfig(originalConfig, "dataStreamsEnabled", true); try { assertTrue(module.isEnabled()); } finally { - setFieldInConfig(originalConfig, "dataStreamsEnabled", false); + setFieldInConfig(originalConfig, "dataStreamsEnabled", originalDataStreamsEnabled); } } From 7c3ddfe18156d0725572bb26cd469fcd1a9da9d4 Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 12:58:47 -0400 Subject: [PATCH 11/32] Remove unused AfterEach import from InstrumenterModuleTest Co-Authored-By: Claude Sonnet 5 --- .../java/datadog/trace/agent/tooling/InstrumenterModuleTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java b/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java index dfc2af602fc..7037acccd09 100644 --- a/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java +++ b/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java @@ -8,7 +8,6 @@ import java.lang.reflect.Field; import java.util.HashSet; import java.util.Set; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; class InstrumenterModuleTest { From 09bd612443f2f72dde080bd749782d255e97d2cc Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 12:58:53 -0400 Subject: [PATCH 12/32] Wire AgentInstaller.getEnabledSystems() to DATA_STREAMS target system Co-Authored-By: Claude Sonnet 5 --- .../main/java/datadog/trace/agent/tooling/AgentInstaller.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dd-java-agent/agent-installer/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java b/dd-java-agent/agent-installer/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java index 3a8c7065362..d4fd57d270d 100644 --- a/dd-java-agent/agent-installer/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java +++ b/dd-java-agent/agent-installer/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java @@ -331,6 +331,9 @@ public static Set getEnabledSystems() { if (cfg.isUsmEnabled()) { enabledSystems.add(InstrumenterModule.TargetSystem.USM); } + if (cfg.isDataStreamsEnabled()) { + enabledSystems.add(InstrumenterModule.TargetSystem.DATA_STREAMS); + } if (cfg.isLlmObsEnabled()) { enabledSystems.add(InstrumenterModule.TargetSystem.LLMOBS); } From 9e82290a3657ca7b633ad70ace0339d15ae76021 Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 13:02:37 -0400 Subject: [PATCH 13/32] Add SamplingMechanism.DATA_STREAMS constant Add DATA_STREAMS sampling mechanism (byte value 14) to support decoupling Kafka DSM from APM tracing. This mechanism enforces USER_DROP priority for DSM-only spans and allows overriding locked priority when DSM is enabled. Co-Authored-By: Claude Sonnet 5 --- .../java/datadog/trace/api/sampling/SamplingMechanism.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java b/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java index 40ac7bf4990..83404075a83 100644 --- a/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java +++ b/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java @@ -38,6 +38,7 @@ public class SamplingMechanism { public static final byte REMOTE_USER_RULE = 11; public static final byte REMOTE_ADAPTIVE_RULE = 12; public static final byte AI_GUARD = 13; + public static final byte DATA_STREAMS = 14; /** Force override sampling decision from external source, like W3C traceparent. */ public static final byte EXTERNAL_OVERRIDE = Byte.MIN_VALUE; @@ -65,6 +66,9 @@ public static boolean validateWithSamplingPriority(int mechanism, int priority) case DATA_JOBS: return priority == PrioritySampling.USER_KEEP; + case DATA_STREAMS: + return priority == USER_DROP; + case EXTERNAL_OVERRIDE: return false; } @@ -80,7 +84,8 @@ public static boolean validateWithSamplingPriority(int mechanism, int priority) */ public static boolean canAvoidSamplingPriorityLock(int priority, int mechanism) { return (!Config.get().isApmTracingEnabled() && mechanism == SamplingMechanism.APPSEC) - || (Config.get().isDataJobsEnabled() && mechanism == DATA_JOBS); + || (Config.get().isDataJobsEnabled() && mechanism == DATA_JOBS) + || (Config.get().isDataStreamsEnabled() && mechanism == DATA_STREAMS); } private SamplingMechanism() {} From 7f98b64563902c761a7010bf8226881bf0278d41 Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 13:04:49 -0400 Subject: [PATCH 14/32] Fix DATA_STREAMS case formatting to match DATA_JOBS style Use qualified PrioritySampling.USER_DROP form in validateWithSamplingPriority to match the DATA_JOBS case style (line 70) which uses qualified form. Co-Authored-By: Claude Sonnet 5 --- .../main/java/datadog/trace/api/sampling/SamplingMechanism.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java b/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java index 83404075a83..43654412ece 100644 --- a/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java +++ b/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java @@ -67,7 +67,7 @@ public static boolean validateWithSamplingPriority(int mechanism, int priority) return priority == PrioritySampling.USER_KEEP; case DATA_STREAMS: - return priority == USER_DROP; + return priority == PrioritySampling.USER_DROP; case EXTERNAL_OVERRIDE: return false; From 76a263f9159270c51fbe17033de8a14ddad87e9a Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 13:08:45 -0400 Subject: [PATCH 15/32] Swap kafka-clients-0.11 instrumentations to InstrumenterModule.DataStreams base class Co-Authored-By: Claude Sonnet 5 --- .../kafka_clients/ConsumerCoordinatorInstrumentation.java | 2 +- .../kafka_clients/KafkaConsumerInfoInstrumentation.java | 2 +- .../kafka_clients/KafkaConsumerInstrumentation.java | 2 +- .../kafka_clients/KafkaProducerInstrumentation.java | 2 +- .../instrumentation/kafka_clients/MetadataInstrumentation.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/ConsumerCoordinatorInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/ConsumerCoordinatorInstrumentation.java index a2d0b28799c..83fee16be9b 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/ConsumerCoordinatorInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/ConsumerCoordinatorInstrumentation.java @@ -23,7 +23,7 @@ import org.apache.kafka.common.TopicPartition; @AutoService(InstrumenterModule.class) -public final class ConsumerCoordinatorInstrumentation extends InstrumenterModule.Tracing +public final class ConsumerCoordinatorInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public ConsumerCoordinatorInstrumentation() { diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java index a2fa481491c..18f62fa8e34 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java @@ -44,7 +44,7 @@ * and cluster ID, in the context store for later use. */ @AutoService(InstrumenterModule.class) -public final class KafkaConsumerInfoInstrumentation extends InstrumenterModule.Tracing +public final class KafkaConsumerInfoInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public KafkaConsumerInfoInstrumentation() { diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInstrumentation.java index 756f59aad4b..7fcc8a31aa9 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInstrumentation.java @@ -27,7 +27,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecords; @AutoService(InstrumenterModule.class) -public final class KafkaConsumerInstrumentation extends InstrumenterModule.Tracing +public final class KafkaConsumerInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public KafkaConsumerInstrumentation() { diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java index 227d8872648..c3206017eb9 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java @@ -58,7 +58,7 @@ import org.apache.kafka.common.record.RecordBatch; @AutoService(InstrumenterModule.class) -public final class KafkaProducerInstrumentation extends InstrumenterModule.Tracing +public final class KafkaProducerInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public KafkaProducerInstrumentation() { diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/MetadataInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/MetadataInstrumentation.java index d6acfe30369..d2557db92e7 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/MetadataInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/MetadataInstrumentation.java @@ -24,7 +24,7 @@ import org.apache.kafka.common.requests.MetadataResponse; @AutoService(InstrumenterModule.class) -public class MetadataInstrumentation extends InstrumenterModule.Tracing +public class MetadataInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { public MetadataInstrumentation() { From 5b346fb54a74099f1ee9acc352ae7d7959924dde Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 13:11:52 -0400 Subject: [PATCH 16/32] Suppress APM billing for DSM-only Kafka produce spans When Kafka tracing is disabled but Data Streams Monitoring is enabled, produce-path spans created without a propagated trace context should be marked with PrioritySampling.USER_DROP to avoid billing as APM spans. This implements the 'local-root spans only' scoping from the spec: spans with an extracted parent context (genuine distributed traces) are left completely untouched. Co-Authored-By: Claude Sonnet 5 --- .../instrumentation/kafka_clients/KafkaDecorator.java | 7 +++++++ .../kafka_clients/KafkaProducerInstrumentation.java | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDecorator.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDecorator.java index 53c579a4dee..2f0ab40a643 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDecorator.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDecorator.java @@ -12,6 +12,7 @@ import datadog.trace.api.Config; import datadog.trace.api.Functions; +import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.cache.DDCache; import datadog.trace.api.cache.DDCaches; import datadog.trace.api.naming.SpanNaming; @@ -20,6 +21,7 @@ import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.bootstrap.instrumentation.decorator.MessagingClientDecorator; +import java.util.Arrays; import java.util.function.Function; import java.util.function.Supplier; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -42,6 +44,11 @@ public class KafkaDecorator extends MessagingClientDecorator { public static final boolean KAFKA_LEGACY_TRACING = Config.get().isKafkaLegacyTracingEnabled(); public static final boolean TIME_IN_QUEUE_ENABLED = Config.get().isTimeInQueueEnabled(!KAFKA_LEGACY_TRACING, KAFKA); + public static final boolean TRACING_ENABLED = + InstrumenterConfig.get() + .isIntegrationEnabled( + Arrays.asList("kafka", "kafka-0.11"), + InstrumenterConfig.get().isIntegrationsEnabled()); public static final String KAFKA_PRODUCED_KEY = "x_datadog_kafka_produced"; private final String spanKind; private final CharSequence spanType; diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java index c3206017eb9..0850d586667 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java @@ -37,6 +37,8 @@ import datadog.trace.api.datastreams.DataStreamsTags; import datadog.trace.api.datastreams.DataStreamsTransactionExtractor; import datadog.trace.api.datastreams.StatsPoint; +import datadog.trace.api.sampling.PrioritySampling; +import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -165,6 +167,9 @@ public static AgentScope onEnter( } else { span = startSpan(JAVA_KAFKA.toString(), KAFKA_PRODUCE); callbackParentSpan = localActiveSpan; + if (!KafkaDecorator.TRACING_ENABLED && Config.get().isDataStreamsEnabled()) { + span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); + } } PRODUCER_DECORATE.afterStart(span); PRODUCER_DECORATE.onProduce(span, record, producerConfig, clusterId); From bc554912f624dea5855164bffde3205daae8178e Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 13:16:50 -0400 Subject: [PATCH 17/32] Swap kafka-clients-3.8 instrumentations to InstrumenterModule.DataStreams base class Change extends InstrumenterModule.Tracing to extends InstrumenterModule.DataStreams for all 8 instrumentation classes in the kafka-clients-3.8 module. This decouples Data Streams Monitoring from APM tracing and prepares for billing-suppression logic. Co-Authored-By: Claude Sonnet 5 --- .../kafka_clients38/ConsumerCoordinatorInstrumentation.java | 2 +- .../kafka_clients38/KafkaConsumerInfoInstrumentation.java | 2 +- .../kafka_clients38/KafkaConsumerInstrumentation.java | 2 +- .../kafka_clients38/KafkaProducerInstrumentation.java | 2 +- .../kafka_clients38/LegacyKafkaConsumerInfoInstrumentation.java | 2 +- .../kafka_clients38/MessageListenerInstrumentation.java | 2 +- .../kafka_clients38/MetadataInstrumentation.java | 2 +- .../OffsetCommitCallbackInvokerInstrumentation.java | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/ConsumerCoordinatorInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/ConsumerCoordinatorInstrumentation.java index 3bf531a703d..9c14f94cb0c 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/ConsumerCoordinatorInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/ConsumerCoordinatorInstrumentation.java @@ -12,7 +12,7 @@ import net.bytebuddy.matcher.ElementMatcher; @AutoService(InstrumenterModule.class) -public final class ConsumerCoordinatorInstrumentation extends InstrumenterModule.Tracing +public final class ConsumerCoordinatorInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public ConsumerCoordinatorInstrumentation() { diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfoInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfoInstrumentation.java index c94c49369ed..d37315d37b4 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfoInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfoInstrumentation.java @@ -24,7 +24,7 @@ * and cluster ID, in the context store for later use. */ @AutoService(InstrumenterModule.class) -public final class KafkaConsumerInfoInstrumentation extends InstrumenterModule.Tracing +public final class KafkaConsumerInfoInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice, Instrumenter.WithTypeStructure { diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentation.java index 70f3f6dbc92..21931394117 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentation.java @@ -19,7 +19,7 @@ import net.bytebuddy.matcher.ElementMatcher; @AutoService(InstrumenterModule.class) -public final class KafkaConsumerInstrumentation extends InstrumenterModule.Tracing +public final class KafkaConsumerInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public KafkaConsumerInstrumentation() { diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaProducerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaProducerInstrumentation.java index 0680c757e37..ce1c4735d9c 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaProducerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaProducerInstrumentation.java @@ -16,7 +16,7 @@ import net.bytebuddy.matcher.ElementMatcher; @AutoService(InstrumenterModule.class) -public final class KafkaProducerInstrumentation extends InstrumenterModule.Tracing +public final class KafkaProducerInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public KafkaProducerInstrumentation() { diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/LegacyKafkaConsumerInfoInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/LegacyKafkaConsumerInfoInstrumentation.java index dd36ff1d934..a3f661fcc8f 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/LegacyKafkaConsumerInfoInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/LegacyKafkaConsumerInfoInstrumentation.java @@ -24,7 +24,7 @@ * and cluster ID, in the context store for later use. */ @AutoService(InstrumenterModule.class) -public final class LegacyKafkaConsumerInfoInstrumentation extends InstrumenterModule.Tracing +public final class LegacyKafkaConsumerInfoInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice, Instrumenter.WithTypeStructure { diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MessageListenerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MessageListenerInstrumentation.java index da17835a1a2..ba28a6eb969 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MessageListenerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MessageListenerInstrumentation.java @@ -13,7 +13,7 @@ import net.bytebuddy.matcher.ElementMatcher; @AutoService(InstrumenterModule.class) -public class MessageListenerInstrumentation extends InstrumenterModule.Tracing +public class MessageListenerInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { public MessageListenerInstrumentation() { diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MetadataInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MetadataInstrumentation.java index 3907ad0c18c..f2fbab71c87 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MetadataInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MetadataInstrumentation.java @@ -15,7 +15,7 @@ import net.bytebuddy.matcher.ElementMatcher; @AutoService(InstrumenterModule.class) -public class MetadataInstrumentation extends InstrumenterModule.Tracing +public class MetadataInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { public MetadataInstrumentation() { diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/OffsetCommitCallbackInvokerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/OffsetCommitCallbackInvokerInstrumentation.java index e64beeb1ebd..5a450b29ad3 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/OffsetCommitCallbackInvokerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/OffsetCommitCallbackInvokerInstrumentation.java @@ -10,7 +10,7 @@ // new - this instrumentation is completely new. // the purpose of this class is to provide us with information on consumer group and cluster ID -public class OffsetCommitCallbackInvokerInstrumentation extends InstrumenterModule.Tracing +public class OffsetCommitCallbackInvokerInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public OffsetCommitCallbackInvokerInstrumentation() { super("kafka", "kafka-3.8"); From 4ef423829e6ff7682ddd48c2f1d7546d00967aa5 Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 13:22:02 -0400 Subject: [PATCH 18/32] Decouple DSM from APM tracing in kafka-clients-3.8 (produce and consume paths) Add TRACING_ENABLED constant and billing-suppression logic for produce and consume paths to suppress unnecessary billing when APM tracing is disabled but DSM is enabled. The suppression only applies to local-root spans (no extracted parent context) per the DSM-Kafka decoupling plan. Co-Authored-By: Claude Sonnet 5 --- .../kafka_clients38/KafkaDecorator.java | 7 +++++++ .../kafka_clients38/ProducerAdvice.java | 6 ++++++ .../kafka_clients38/TracingIterator.java | 10 ++++++++++ 3 files changed, 23 insertions(+) diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaDecorator.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaDecorator.java index d2d6f53b8a9..2b16ad34af0 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaDecorator.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaDecorator.java @@ -12,6 +12,7 @@ import datadog.trace.api.Config; import datadog.trace.api.Functions; +import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.cache.DDCache; import datadog.trace.api.cache.DDCaches; import datadog.trace.api.naming.SpanNaming; @@ -20,6 +21,7 @@ import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.bootstrap.instrumentation.decorator.MessagingClientDecorator; +import java.util.Arrays; import java.util.function.Function; import java.util.function.Supplier; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -42,6 +44,11 @@ public class KafkaDecorator extends MessagingClientDecorator { public static final boolean KAFKA_LEGACY_TRACING = Config.get().isKafkaLegacyTracingEnabled(); public static final boolean TIME_IN_QUEUE_ENABLED = Config.get().isTimeInQueueEnabled(!KAFKA_LEGACY_TRACING, KAFKA); + public static final boolean TRACING_ENABLED = + InstrumenterConfig.get() + .isIntegrationEnabled( + Arrays.asList("kafka", "kafka-3.8"), + InstrumenterConfig.get().isIntegrationsEnabled()); public static final String KAFKA_PRODUCED_KEY = "x_datadog_kafka_produced"; private final String spanKind; private final CharSequence spanType; diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java index 01905ee65e4..637c5b19000 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java @@ -8,6 +8,9 @@ import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.KAFKA_PRODUCE; import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.PRODUCER_DECORATE; +import datadog.trace.api.Config; +import datadog.trace.api.sampling.PrioritySampling; +import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -55,6 +58,9 @@ public static AgentScope onEnter( } else { span = startSpan(JAVA_KAFKA.toString(), KAFKA_PRODUCE); callbackParentSpan = localActiveSpan; + if (!KafkaDecorator.TRACING_ENABLED && Config.get().isDataStreamsEnabled()) { + span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); + } } PRODUCER_DECORATE.afterStart(span); PRODUCER_DECORATE.onProduce(span, record, producerConfig, clusterId); diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterator.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterator.java index 7beba848473..a14c3da29d6 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterator.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterator.java @@ -22,6 +22,8 @@ import datadog.trace.api.datastreams.DataStreamsContext; import datadog.trace.api.datastreams.DataStreamsTags; import datadog.trace.api.datastreams.DataStreamsTransactionExtractor; +import datadog.trace.api.sampling.PrioritySampling; +import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; @@ -116,6 +118,11 @@ protected void startNewRecordSpan(ConsumerRecord val) { // spans are written out together by TraceStructureWriter when running in strict mode } + if (spanContext == null + && !KafkaDecorator.TRACING_ENABLED + && traceConfig().isDataStreamsEnabled()) { + span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); + } DataStreamsTags tags = create("kafka", INBOUND, val.topic(), group, clusterId); final long payloadSize = traceConfig().isDataStreamsEnabled() ? Utils.computePayloadSizeBytes(val) : 0; @@ -137,6 +144,9 @@ protected void startNewRecordSpan(ConsumerRecord val) { } } else { span = startSpan(JAVA_KAFKA.toString(), operationName, null); + if (!KafkaDecorator.TRACING_ENABLED && traceConfig().isDataStreamsEnabled()) { + span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); + } } if (val.value() == null) { span.setTag(InstrumentationTags.TOMBSTONE, true); From a4b9ad51cba632c2736658752c4a37a1f07d8dfc Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 13:26:07 -0400 Subject: [PATCH 19/32] chore: kafka-streams-0.11 DSM decoupling (base-class swap + suppression) --- .../kafka_streams/KafkaStreamTaskInstrumentation.java | 10 +++++++++- .../kafka_streams/KafkaStreamsDecorator.java | 7 +++++++ ...amsSourceNodeRecordDeserializerInstrumentation.java | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java index 81ddfd4202f..d3df5f7e8ed 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java @@ -41,6 +41,8 @@ import datadog.trace.api.Config; import datadog.trace.api.datastreams.DataStreamsContext; import datadog.trace.api.datastreams.DataStreamsTags; +import datadog.trace.api.sampling.PrioritySampling; +import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -57,7 +59,7 @@ import org.apache.kafka.streams.processor.internals.StreamTask; @AutoService(InstrumenterModule.class) -public class KafkaStreamTaskInstrumentation extends InstrumenterModule.Tracing +public class KafkaStreamTaskInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public KafkaStreamTaskInstrumentation() { @@ -294,6 +296,9 @@ public static void start( // spans are written out together by TraceStructureWriter when running in strict mode } + if (!KafkaStreamsDecorator.TRACING_ENABLED && traceConfig().isDataStreamsEnabled()) { + span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); + } String applicationId = null; if (streamTaskContext != null) { applicationId = streamTaskContext.getApplicationId(); @@ -360,6 +365,9 @@ public static void start( // spans are written out together by TraceStructureWriter when running in strict mode } + if (!KafkaStreamsDecorator.TRACING_ENABLED && traceConfig().isDataStreamsEnabled()) { + span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); + } String applicationId = null; if (streamTaskContext != null) { applicationId = streamTaskContext.getApplicationId(); diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsDecorator.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsDecorator.java index 97f52bae52c..3bb5832450d 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsDecorator.java +++ b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsDecorator.java @@ -7,6 +7,7 @@ import datadog.trace.api.Config; import datadog.trace.api.Functions; +import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.cache.DDCache; import datadog.trace.api.cache.DDCaches; import datadog.trace.api.naming.SpanNaming; @@ -15,6 +16,7 @@ import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.bootstrap.instrumentation.decorator.MessagingClientDecorator; +import java.util.Arrays; import java.util.function.Supplier; import org.apache.kafka.streams.processor.internals.ProcessorNode; import org.apache.kafka.streams.processor.internals.ProcessorRecordContext; @@ -31,6 +33,11 @@ public class KafkaStreamsDecorator extends MessagingClientDecorator { public static final boolean KAFKA_LEGACY_TRACING = Config.get().isKafkaLegacyTracingEnabled(); public static final boolean TIME_IN_QUEUE_ENABLED = Config.get().isTimeInQueueEnabled(!KAFKA_LEGACY_TRACING, KAFKA); + public static final boolean TRACING_ENABLED = + InstrumenterConfig.get() + .isIntegrationEnabled( + Arrays.asList("kafka", "kafka-streams"), + InstrumenterConfig.get().isIntegrationsEnabled()); public static final String KAFKA_PRODUCED_KEY = "x_datadog_kafka_produced"; private final String spanKind; diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsSourceNodeRecordDeserializerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsSourceNodeRecordDeserializerInstrumentation.java index 81348490676..3f47e9cdf6e 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsSourceNodeRecordDeserializerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsSourceNodeRecordDeserializerInstrumentation.java @@ -16,7 +16,7 @@ // This is necessary because SourceNodeRecordDeserializer drops the headers. :-( @AutoService(InstrumenterModule.class) public class KafkaStreamsSourceNodeRecordDeserializerInstrumentation - extends InstrumenterModule.Tracing + extends InstrumenterModule.DataStreams implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public KafkaStreamsSourceNodeRecordDeserializerInstrumentation() { From bda034a008ea15fed5c982982a668ebd36622252 Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 13:29:19 -0400 Subject: [PATCH 20/32] refactor: change kafka-streams-1.0 InternalTopologyBuilderInstrumentation to DataStreams This class populates StreamingContextUpdater's source/disabled-topic registry via BuildAdvice.exit, which KafkaStreamTaskInstrumentation's DSM checkpoint logic reads. Switching from InstrumenterModule.Tracing to DataStreams ensures the registry is populated when DSM is the only reason Kafka is instrumented, allowing DSM's STREAMING_CONTEXT.isSourceTopic()/isDisabledForTopic() checks to work correctly instead of silently falling back to defaults. Co-Authored-By: Claude Sonnet 5 --- .../kafka_streams10/InternalTopologyBuilderInstrumentation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-1.0/src/main/java/datadog/trace/instrumentation/kafka_streams10/InternalTopologyBuilderInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-streams-1.0/src/main/java/datadog/trace/instrumentation/kafka_streams10/InternalTopologyBuilderInstrumentation.java index 07271bb7904..a32e48ac094 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-1.0/src/main/java/datadog/trace/instrumentation/kafka_streams10/InternalTopologyBuilderInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-streams-1.0/src/main/java/datadog/trace/instrumentation/kafka_streams10/InternalTopologyBuilderInstrumentation.java @@ -12,7 +12,7 @@ import org.apache.kafka.streams.processor.internals.ProcessorTopology; @AutoService(InstrumenterModule.class) -public class InternalTopologyBuilderInstrumentation extends InstrumenterModule.Tracing +public class InternalTopologyBuilderInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public InternalTopologyBuilderInstrumentation() { From 481223e7b0df65dc84b921caba79a8e0d7d5707e Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 13:31:34 -0400 Subject: [PATCH 21/32] refactor: decouple kafka-connect-0.11 instrumentation from APM tracing Change ConnectWorkerInstrumentation to extend InstrumenterModule.DataStreams instead of InstrumenterModule.Tracing. This class only wraps TaskStatus.Listener for forwarding task status callbacks and does not create spans, making it appropriate for the DataStreams module instead of the Tracing module. Task 11 of DSMS-161 (kafka-connect-0.11: base-class swap) Co-Authored-By: Claude Haiku 4.5 --- .../kafka_connect/ConnectWorkerInstrumentation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/src/main/java/datadog/trace/instrumentation/kafka_connect/ConnectWorkerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/src/main/java/datadog/trace/instrumentation/kafka_connect/ConnectWorkerInstrumentation.java index 8983fd46fc8..85d546fa587 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/src/main/java/datadog/trace/instrumentation/kafka_connect/ConnectWorkerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/src/main/java/datadog/trace/instrumentation/kafka_connect/ConnectWorkerInstrumentation.java @@ -15,7 +15,7 @@ import org.apache.kafka.connect.util.ConnectorTaskId; @AutoService(InstrumenterModule.class) -public final class ConnectWorkerInstrumentation extends InstrumenterModule.Tracing +public final class ConnectWorkerInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { static final String TARGET_TYPE = "org.apache.kafka.connect.runtime.WorkerTask"; From c3eef89097bcc763ad8ac49fb7a293b4766368c2 Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 13:45:49 -0400 Subject: [PATCH 22/32] Migrate InstrumenterConfigTest to JUnit 5 and add isDataStreamsEnabled tests Converts the Groovy/Spock InstrumenterConfigTest to a JUnit 5 Java test (per repo workflow), preserving all 44 existing cases, and adds explicit coverage for InstrumenterConfig.isDataStreamsEnabled(): default false, and true via DD_DATA_STREAMS_ENABLED. Part of the DSM/Kafka decoupling test task (Task 12a). Co-Authored-By: Claude Sonnet 5 --- .../trace/api/InstrumenterConfigTest.groovy | 213 ------------------ .../trace/api/InstrumenterConfigTest.java | 198 ++++++++++++++++ 2 files changed, 198 insertions(+), 213 deletions(-) delete mode 100644 internal-api/src/test/groovy/datadog/trace/api/InstrumenterConfigTest.groovy create mode 100644 internal-api/src/test/java/datadog/trace/api/InstrumenterConfigTest.java diff --git a/internal-api/src/test/groovy/datadog/trace/api/InstrumenterConfigTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/InstrumenterConfigTest.groovy deleted file mode 100644 index f6b8bae42d4..00000000000 --- a/internal-api/src/test/groovy/datadog/trace/api/InstrumenterConfigTest.groovy +++ /dev/null @@ -1,213 +0,0 @@ -package datadog.trace.api - -import datadog.trace.config.inversion.ConfigHelper -import datadog.trace.test.util.DDSpecification - -class InstrumenterConfigTest extends DDSpecification { - - def strictness - - def setup(){ - strictness = ConfigHelper.get().configInversionStrictFlag() - ConfigHelper.get().setConfigInversionStrict(ConfigHelper.StrictnessPolicy.TEST) - } - - def cleanup(){ - ConfigHelper.get().setConfigInversionStrict(strictness) - } - - def "verify integration config"() { - setup: - environmentVariables.set("DD_INTEGRATION_ORDER_ENABLED", "false") - environmentVariables.set("DD_INTEGRATION_TEST_ENV_ENABLED", "true") - environmentVariables.set("DD_TRACE_NEW_ENV_ENABLED", "false") - environmentVariables.set("DD_INTEGRATION_DISABLED_ENV_ENABLED", "false") - - System.setProperty("dd.integration.order.enabled", "true") - System.setProperty("dd.integration.test-prop.enabled", "true") - System.setProperty("dd.integration.disabled-prop.enabled", "false") - - environmentVariables.set("DD_INTEGRATION_ORDER_MATCHING_SHORTCUT_ENABLED", "false") - environmentVariables.set("DD_INTEGRATION_TEST_ENV_MATCHING_SHORTCUT_ENABLED", "true") - environmentVariables.set("DD_INTEGRATION_NEW_ENV_MATCHING_SHORTCUT_ENABLED", "false") - environmentVariables.set("DD_INTEGRATION_DISABLED_ENV_MATCHING_SHORTCUT_ENABLED", "false") - - System.setProperty("dd.integration.order.matching.shortcut.enabled", "true") - System.setProperty("dd.integration.test-prop.matching.shortcut.enabled", "true") - System.setProperty("dd.integration.disabled-prop.matching.shortcut.enabled", "false") - - expect: - InstrumenterConfig.get().isIntegrationEnabled(integrationNames, defaultEnabled) == expected - InstrumenterConfig.get().isIntegrationShortcutMatchingEnabled(integrationNames, defaultEnabled) == expected - - where: - // spotless:off - names | defaultEnabled | expected - [] | true | true - [] | false | false - ["invalid"] | true | true - ["invalid"] | false | false - ["test-prop"] | false | true - ["test-env"] | false | true - ["disabled-prop"] | true | false - ["disabled-env"] | true | false - ["other", "test-prop"] | false | true - ["other", "test-env"] | false | true - ["order"] | false | true - ["test-prop", "disabled-prop"] | false | true - ["disabled-env", "test-env"] | false | true - ["test-prop", "disabled-prop"] | true | false - ["disabled-env", "test-env"] | true | false - ["new-env"] | true | false - // spotless:on - - integrationNames = new TreeSet<>(names) - } - - def setEnv(String key, String value) { - environmentVariables.set(key, value) - } - - def setSysProp(String key, String value) { - System.setProperty(key, value) - } - - def randomIntegrationEnabled() { - return InstrumenterConfig.get().isIntegrationEnabled(["random"], true) - } - - def "verify integration enabled hierarchy"() { - when: - // the below should have no effect - setEnv("DD_RANDOM_ENABLED", "false") - setSysProp("dd.random.enabled", "false") - - then: - randomIntegrationEnabled() == true - - when: - setEnv("DD_INTEGRATION_RANDOM_ENABLED", "false") - - then: - randomIntegrationEnabled() == false - - when: - setEnv("DD_TRACE_INTEGRATION_RANDOM_ENABLED", "true") - - then: - randomIntegrationEnabled() == true - - when: - setEnv("DD_TRACE_RANDOM_ENABLED", "false") - - then: - randomIntegrationEnabled() == false - - // assert all system properties take precedence over all env vars - when: - setSysProp("dd.integration.random.enabled", "true") - - then: - randomIntegrationEnabled() == true - - when: - setSysProp("dd.trace.integration.random.enabled", "false") - - then: - randomIntegrationEnabled() == false - - when: - setSysProp("dd.trace.random.enabled", "true") - - then: - randomIntegrationEnabled() == true - } - - def "valid resolver presets"() { - setup: - injectSysConfig("resolver.cache.config", preset) - - expect: - InstrumenterConfig.get().resolverOutliningEnabled == outlining - - where: - // spotless:off - preset | outlining - 'LARGE' | true - 'SMALL' | true - 'DEFAULT' | true - 'LEGACY' | false - // spotless:on - } - - def "invalid resolver presets"() { - setup: - injectSysConfig("resolver.cache.config", preset) - - expect: - InstrumenterConfig.get().resolverOutliningEnabled - - where: - preset << ['INVALID', ''] - } - - def "appsec enabled = #input"() { - setup: - if (input != null) { - injectSysConfig("appsec.enabled", input) - } - - expect: - InstrumenterConfig.get().getAppSecActivation() == expected - - where: - input | expected - null | ProductActivation.ENABLED_INACTIVE - "" | ProductActivation.ENABLED_INACTIVE - "bad" | ProductActivation.FULLY_DISABLED - "false" | ProductActivation.FULLY_DISABLED - "0" | ProductActivation.FULLY_DISABLED - "true" | ProductActivation.FULLY_ENABLED - "1" | ProductActivation.FULLY_ENABLED - "inactive" | ProductActivation.ENABLED_INACTIVE - } - - def "iast enabled = #input"() { - setup: - if (input != null) { - injectSysConfig("iast.enabled", input) - } - - expect: - InstrumenterConfig.get().getIastActivation() == expected - - where: - input | expected - null | ProductActivation.FULLY_DISABLED - "" | ProductActivation.FULLY_DISABLED - "bad" | ProductActivation.FULLY_DISABLED - "false" | ProductActivation.FULLY_DISABLED - "0" | ProductActivation.FULLY_DISABLED - "true" | ProductActivation.FULLY_ENABLED - "1" | ProductActivation.FULLY_ENABLED - "inactive" | ProductActivation.ENABLED_INACTIVE - } - - def "data streams enabled = #input"() { - setup: - if (input != null) { - injectSysConfig("data.streams.enabled", input) - } - - expect: - InstrumenterConfig.get().isDataStreamsEnabled() == expected - - where: - input | expected - null | false - "false" | false - "true" | true - "1" | true - "0" | false - } -} diff --git a/internal-api/src/test/java/datadog/trace/api/InstrumenterConfigTest.java b/internal-api/src/test/java/datadog/trace/api/InstrumenterConfigTest.java new file mode 100644 index 00000000000..0916b2ca9f3 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/InstrumenterConfigTest.java @@ -0,0 +1,198 @@ +package datadog.trace.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.config.inversion.ConfigHelper; +import datadog.trace.config.inversion.ConfigHelper.StrictnessPolicy; +import datadog.trace.test.junit.utils.config.WithConfig; +import datadog.trace.test.junit.utils.config.WithConfigExtension; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.tabletest.junit.TableTest; + +@ExtendWith(WithConfigExtension.class) +class InstrumenterConfigTest { + + private StrictnessPolicy strictness; + + @BeforeEach + void setup() { + strictness = ConfigHelper.get().configInversionStrictFlag(); + ConfigHelper.get().setConfigInversionStrict(StrictnessPolicy.TEST); + } + + @AfterEach + void cleanup() { + ConfigHelper.get().setConfigInversionStrict(strictness); + } + + @TableTest({ + "scenario | names | defaultEnabled | expected", + "empty names, default enabled | [] | true | true ", + "empty names, default disabled | [] | false | false ", + "invalid name, default enabled | [invalid] | true | true ", + "invalid name, default disabled | [invalid] | false | false ", + "test-prop env var enabled overrides default off | [test-prop] | false | true ", + "test-env env var enabled overrides default off | [test-env] | false | true ", + "disabled-prop sys prop overrides default on | [disabled-prop] | true | false ", + "disabled-env env var overrides default on | [disabled-env] | true | false ", + "mixed names, test-prop wins | [other, test-prop] | false | true ", + "mixed names, test-env wins | [other, test-env] | false | true ", + "order enabled by both sys prop and env var | [order] | false | true ", + "test-prop and disabled-prop, default off | [test-prop, disabled-prop] | false | true ", + "disabled-env and test-env, default off | [disabled-env, test-env] | false | true ", + "test-prop and disabled-prop, default on | [test-prop, disabled-prop] | true | false ", + "disabled-env and test-env, default on | [disabled-env, test-env] | true | false ", + "new-env disabled overrides default on | [new-env] | true | false " + }) + @WithConfig(key = "INTEGRATION_ORDER_ENABLED", value = "false", env = true) + @WithConfig(key = "INTEGRATION_TEST_ENV_ENABLED", value = "true", env = true) + @WithConfig(key = "TRACE_NEW_ENV_ENABLED", value = "false", env = true) + @WithConfig(key = "INTEGRATION_DISABLED_ENV_ENABLED", value = "false", env = true) + @WithConfig(key = "INTEGRATION_ORDER_MATCHING_SHORTCUT_ENABLED", value = "false", env = true) + @WithConfig(key = "INTEGRATION_TEST_ENV_MATCHING_SHORTCUT_ENABLED", value = "true", env = true) + @WithConfig(key = "INTEGRATION_NEW_ENV_MATCHING_SHORTCUT_ENABLED", value = "false", env = true) + @WithConfig( + key = "INTEGRATION_DISABLED_ENV_MATCHING_SHORTCUT_ENABLED", + value = "false", + env = true) + @WithConfig(key = "integration.order.enabled", value = "true") + @WithConfig(key = "integration.test-prop.enabled", value = "true") + @WithConfig(key = "integration.disabled-prop.enabled", value = "false") + @WithConfig(key = "integration.order.matching.shortcut.enabled", value = "true") + @WithConfig(key = "integration.test-prop.matching.shortcut.enabled", value = "true") + @WithConfig(key = "integration.disabled-prop.matching.shortcut.enabled", value = "false") + void verifyIntegrationConfig(List names, boolean defaultEnabled, boolean expected) { + Set integrationNames = new TreeSet<>(names); + assertEquals( + expected, InstrumenterConfig.get().isIntegrationEnabled(integrationNames, defaultEnabled)); + assertEquals( + expected, + InstrumenterConfig.get() + .isIntegrationShortcutMatchingEnabled(integrationNames, defaultEnabled)); + } + + private static boolean randomIntegrationEnabled() { + return InstrumenterConfig.get().isIntegrationEnabled(Collections.singletonList("random"), true); + } + + @Test + void verifyIntegrationEnabledHierarchy() { + // the below should have no effect + WithConfigExtension.injectEnvConfig("RANDOM_ENABLED", "false"); + WithConfigExtension.injectSysConfig("random.enabled", "false"); + assertTrue(randomIntegrationEnabled()); + + WithConfigExtension.injectEnvConfig("INTEGRATION_RANDOM_ENABLED", "false"); + assertFalse(randomIntegrationEnabled()); + + WithConfigExtension.injectEnvConfig("TRACE_INTEGRATION_RANDOM_ENABLED", "true"); + assertTrue(randomIntegrationEnabled()); + + WithConfigExtension.injectEnvConfig("TRACE_RANDOM_ENABLED", "false"); + assertFalse(randomIntegrationEnabled()); + + // assert all system properties take precedence over all env vars + WithConfigExtension.injectSysConfig("integration.random.enabled", "true"); + assertTrue(randomIntegrationEnabled()); + + WithConfigExtension.injectSysConfig("trace.integration.random.enabled", "false"); + assertFalse(randomIntegrationEnabled()); + + WithConfigExtension.injectSysConfig("trace.random.enabled", "true"); + assertTrue(randomIntegrationEnabled()); + } + + @TableTest({ + "scenario | preset | outlining", + "large preset | LARGE | true ", + "small preset | SMALL | true ", + "default preset | DEFAULT | true ", + "legacy preset | LEGACY | false " + }) + void validResolverPresets(String preset, boolean outlining) { + WithConfigExtension.injectSysConfig("resolver.cache.config", preset); + + assertEquals(outlining, InstrumenterConfig.get().isResolverOutliningEnabled()); + } + + @TableTest({"scenario | preset ", "invalid preset | INVALID", "empty preset | '' "}) + void invalidResolverPresets(String preset) { + WithConfigExtension.injectSysConfig("resolver.cache.config", preset); + + assertTrue(InstrumenterConfig.get().isResolverOutliningEnabled()); + } + + @TableTest({ + "scenario | input | expected ", + "unset defaults to inactive | | ENABLED_INACTIVE", + "empty string is inactive | '' | ENABLED_INACTIVE", + "unparseable value disables | bad | FULLY_DISABLED ", + "explicit false disables | false | FULLY_DISABLED ", + "zero disables | 0 | FULLY_DISABLED ", + "explicit true enables | true | FULLY_ENABLED ", + "one enables | 1 | FULLY_ENABLED ", + "inactive keyword enables inactive | inactive | ENABLED_INACTIVE" + }) + void appsecEnabled(String input, ProductActivation expected) { + if (input != null) { + WithConfigExtension.injectSysConfig("appsec.enabled", input); + } + + assertEquals(expected, InstrumenterConfig.get().getAppSecActivation()); + } + + @TableTest({ + "scenario | input | expected ", + "unset disables | | FULLY_DISABLED ", + "empty string disables | '' | FULLY_DISABLED ", + "unparseable value disables | bad | FULLY_DISABLED ", + "explicit false disables | false | FULLY_DISABLED ", + "zero disables | 0 | FULLY_DISABLED ", + "explicit true enables | true | FULLY_ENABLED ", + "one enables | 1 | FULLY_ENABLED ", + "inactive keyword enables inactive | inactive | ENABLED_INACTIVE" + }) + void iastEnabled(String input, ProductActivation expected) { + if (input != null) { + WithConfigExtension.injectSysConfig("iast.enabled", input); + } + + assertEquals(expected, InstrumenterConfig.get().getIastActivation()); + } + + @TableTest({ + "scenario | input | expected", + "unset defaults to false | | false ", + "explicit false disables | false | false ", + "explicit true enables | true | true ", + "one enables | 1 | true ", + "zero disables | 0 | false " + }) + void dataStreamsEnabled(String input, boolean expected) { + if (input != null) { + WithConfigExtension.injectSysConfig("data.streams.enabled", input); + } + + assertEquals(expected, InstrumenterConfig.get().isDataStreamsEnabled()); + } + + @Test + void dataStreamsEnabledDefaultsToFalse() { + assertFalse(InstrumenterConfig.get().isDataStreamsEnabled()); + } + + @Test + @WithConfig(key = "DATA_STREAMS_ENABLED", value = "true", env = true) + void dataStreamsEnabledViaEnvVar() { + assertTrue(InstrumenterConfig.get().isDataStreamsEnabled()); + } +} From e75142830c32850e9bc602bf07acd7a701e0230e Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 14:01:32 -0400 Subject: [PATCH 23/32] Migrate SamplingMechanismTest to JUnit 5 and add DATA_STREAMS cases Convert the Spock SamplingMechanismTest to a parameterized JUnit 5 Java test, and add DATA_STREAMS cases mirroring the existing DATA_JOBS coverage for validateWithSamplingPriority() and canAvoidSamplingPriorityLock(), per DSMS-161 task 12. Co-Authored-By: Claude Sonnet 5 --- .../api/sampling/SamplingMechanismTest.groovy | 120 -------------- .../api/sampling/SamplingMechanismTest.java | 148 ++++++++++++++++++ 2 files changed, 148 insertions(+), 120 deletions(-) delete mode 100644 internal-api/src/test/groovy/datadog/trace/api/sampling/SamplingMechanismTest.groovy create mode 100644 internal-api/src/test/java/datadog/trace/api/sampling/SamplingMechanismTest.java diff --git a/internal-api/src/test/groovy/datadog/trace/api/sampling/SamplingMechanismTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/sampling/SamplingMechanismTest.groovy deleted file mode 100644 index 4a4890435c1..00000000000 --- a/internal-api/src/test/groovy/datadog/trace/api/sampling/SamplingMechanismTest.groovy +++ /dev/null @@ -1,120 +0,0 @@ -package datadog.trace.api.sampling - -import datadog.trace.test.util.DDSpecification -import static datadog.trace.api.sampling.PrioritySampling.* -import static datadog.trace.api.sampling.SamplingMechanism.* - -class SamplingMechanismTest extends DDSpecification { - - static userDropX = USER_DROP - 1 - static userKeepX = USER_KEEP + 1 - - def "test validation"() { - expect: - validateWithSamplingPriority(mechanism, priority) == valid - - where: - mechanism | priority | valid - UNKNOWN | UNSET | true - UNKNOWN | SAMPLER_DROP | true - UNKNOWN | SAMPLER_KEEP | true - UNKNOWN | USER_DROP | true - UNKNOWN | USER_KEEP | true - UNKNOWN | userDropX | true - UNKNOWN | userKeepX | true - - DEFAULT | UNSET | false - DEFAULT | SAMPLER_DROP | true - DEFAULT | SAMPLER_KEEP | true - DEFAULT | USER_DROP | false - DEFAULT | USER_KEEP | false - DEFAULT | userDropX | false - DEFAULT | userKeepX | false - - AGENT_RATE | UNSET | false - AGENT_RATE | SAMPLER_DROP | true - AGENT_RATE | SAMPLER_KEEP | true - AGENT_RATE | USER_DROP | false - AGENT_RATE | USER_KEEP | false - AGENT_RATE | userDropX | false - AGENT_RATE | userKeepX | false - - REMOTE_AUTO_RATE | UNSET | false - REMOTE_AUTO_RATE | SAMPLER_DROP | true - REMOTE_AUTO_RATE | SAMPLER_KEEP | true - REMOTE_AUTO_RATE | USER_DROP | false - REMOTE_AUTO_RATE | USER_KEEP | false - REMOTE_AUTO_RATE | userDropX | false - REMOTE_AUTO_RATE | userKeepX | false - - LOCAL_USER_RULE | UNSET | false - LOCAL_USER_RULE | SAMPLER_DROP | false - LOCAL_USER_RULE | SAMPLER_KEEP | false - LOCAL_USER_RULE | USER_DROP | true - LOCAL_USER_RULE | USER_KEEP | true - LOCAL_USER_RULE | userDropX | false - LOCAL_USER_RULE | userKeepX | false - - MANUAL | UNSET | false - MANUAL | SAMPLER_DROP | false - MANUAL | SAMPLER_KEEP | false - MANUAL | USER_DROP | true - MANUAL | USER_KEEP | true - MANUAL | userDropX | false - MANUAL | userKeepX | false - - REMOTE_USER_RATE | UNSET | false - REMOTE_USER_RATE | SAMPLER_DROP | false - REMOTE_USER_RATE | SAMPLER_KEEP | false - REMOTE_USER_RATE | USER_DROP | true - REMOTE_USER_RATE | USER_KEEP | true - REMOTE_USER_RATE | userDropX | false - REMOTE_USER_RATE | userKeepX | false - - APPSEC | UNSET | false - APPSEC | SAMPLER_DROP | true - APPSEC | SAMPLER_KEEP | true - APPSEC | USER_DROP | false - APPSEC | USER_KEEP | true - APPSEC | userDropX | false - APPSEC | userKeepX | false - - DATA_JOBS | UNSET | false - DATA_JOBS | SAMPLER_DROP | false - DATA_JOBS | SAMPLER_KEEP | false - DATA_JOBS | USER_DROP | false - DATA_JOBS | USER_KEEP | true - DATA_JOBS | userDropX | false - DATA_JOBS | userKeepX | false - - EXTERNAL_OVERRIDE | UNSET | false - EXTERNAL_OVERRIDE | SAMPLER_DROP | false - EXTERNAL_OVERRIDE | SAMPLER_KEEP | false - EXTERNAL_OVERRIDE | USER_DROP | false - EXTERNAL_OVERRIDE | USER_KEEP | false - EXTERNAL_OVERRIDE | userDropX | false - EXTERNAL_OVERRIDE | userKeepX | false - } - - void 'Test canAvoidSamplingPriorityLock'(){ - setup: - injectSysConfig("dd.apm.tracing.enabled", "false") - - expect: - canAvoidSamplingPriorityLock(priority, mechanism) == valid - - where: - mechanism | priority | valid - APPSEC | UNSET | true - APPSEC | SAMPLER_KEEP | true - UNKNOWN | SAMPLER_KEEP | false - DEFAULT | SAMPLER_KEEP | false - AGENT_RATE | SAMPLER_KEEP | false - REMOTE_AUTO_RATE | SAMPLER_KEEP | false - LOCAL_USER_RULE | SAMPLER_KEEP | false - MANUAL | SAMPLER_KEEP | false - REMOTE_USER_RATE | SAMPLER_KEEP | false - DATA_JOBS | SAMPLER_KEEP | false - EXTERNAL_OVERRIDE | SAMPLER_KEEP | false - } -} diff --git a/internal-api/src/test/java/datadog/trace/api/sampling/SamplingMechanismTest.java b/internal-api/src/test/java/datadog/trace/api/sampling/SamplingMechanismTest.java new file mode 100644 index 00000000000..86ee27f2f55 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/sampling/SamplingMechanismTest.java @@ -0,0 +1,148 @@ +package datadog.trace.api.sampling; + +import static datadog.trace.api.config.GeneralConfig.APM_TRACING_ENABLED; +import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP; +import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP; +import static datadog.trace.api.sampling.PrioritySampling.UNSET; +import static datadog.trace.api.sampling.PrioritySampling.USER_DROP; +import static datadog.trace.api.sampling.PrioritySampling.USER_KEEP; +import static datadog.trace.api.sampling.SamplingMechanism.AGENT_RATE; +import static datadog.trace.api.sampling.SamplingMechanism.APPSEC; +import static datadog.trace.api.sampling.SamplingMechanism.DATA_JOBS; +import static datadog.trace.api.sampling.SamplingMechanism.DATA_STREAMS; +import static datadog.trace.api.sampling.SamplingMechanism.DEFAULT; +import static datadog.trace.api.sampling.SamplingMechanism.EXTERNAL_OVERRIDE; +import static datadog.trace.api.sampling.SamplingMechanism.LOCAL_USER_RULE; +import static datadog.trace.api.sampling.SamplingMechanism.MANUAL; +import static datadog.trace.api.sampling.SamplingMechanism.REMOTE_AUTO_RATE; +import static datadog.trace.api.sampling.SamplingMechanism.REMOTE_USER_RATE; +import static datadog.trace.api.sampling.SamplingMechanism.UNKNOWN; +import static datadog.trace.api.sampling.SamplingMechanism.canAvoidSamplingPriorityLock; +import static datadog.trace.api.sampling.SamplingMechanism.validateWithSamplingPriority; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import datadog.trace.test.junit.utils.config.WithConfig; +import datadog.trace.test.junit.utils.config.WithConfigExtension; +import java.util.stream.Stream; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +@ExtendWith(WithConfigExtension.class) +class SamplingMechanismTest { + + // one below USER_DROP / one above USER_KEEP: neither a valid sampler nor user priority value + private static final byte USER_DROP_X = (byte) (USER_DROP - 1); + private static final byte USER_KEEP_X = (byte) (USER_KEEP + 1); + + @ParameterizedTest + @MethodSource("testValidationArguments") + void testValidation(byte mechanism, byte priority, boolean valid) { + assertEquals(valid, validateWithSamplingPriority(mechanism, priority)); + } + + private static Stream testValidationArguments() { + return Stream.of( + Arguments.of(UNKNOWN, UNSET, true), + Arguments.of(UNKNOWN, SAMPLER_DROP, true), + Arguments.of(UNKNOWN, SAMPLER_KEEP, true), + Arguments.of(UNKNOWN, USER_DROP, true), + Arguments.of(UNKNOWN, USER_KEEP, true), + Arguments.of(UNKNOWN, USER_DROP_X, true), + Arguments.of(UNKNOWN, USER_KEEP_X, true), + Arguments.of(DEFAULT, UNSET, false), + Arguments.of(DEFAULT, SAMPLER_DROP, true), + Arguments.of(DEFAULT, SAMPLER_KEEP, true), + Arguments.of(DEFAULT, USER_DROP, false), + Arguments.of(DEFAULT, USER_KEEP, false), + Arguments.of(DEFAULT, USER_DROP_X, false), + Arguments.of(DEFAULT, USER_KEEP_X, false), + Arguments.of(AGENT_RATE, UNSET, false), + Arguments.of(AGENT_RATE, SAMPLER_DROP, true), + Arguments.of(AGENT_RATE, SAMPLER_KEEP, true), + Arguments.of(AGENT_RATE, USER_DROP, false), + Arguments.of(AGENT_RATE, USER_KEEP, false), + Arguments.of(AGENT_RATE, USER_DROP_X, false), + Arguments.of(AGENT_RATE, USER_KEEP_X, false), + Arguments.of(REMOTE_AUTO_RATE, UNSET, false), + Arguments.of(REMOTE_AUTO_RATE, SAMPLER_DROP, true), + Arguments.of(REMOTE_AUTO_RATE, SAMPLER_KEEP, true), + Arguments.of(REMOTE_AUTO_RATE, USER_DROP, false), + Arguments.of(REMOTE_AUTO_RATE, USER_KEEP, false), + Arguments.of(REMOTE_AUTO_RATE, USER_DROP_X, false), + Arguments.of(REMOTE_AUTO_RATE, USER_KEEP_X, false), + Arguments.of(LOCAL_USER_RULE, UNSET, false), + Arguments.of(LOCAL_USER_RULE, SAMPLER_DROP, false), + Arguments.of(LOCAL_USER_RULE, SAMPLER_KEEP, false), + Arguments.of(LOCAL_USER_RULE, USER_DROP, true), + Arguments.of(LOCAL_USER_RULE, USER_KEEP, true), + Arguments.of(LOCAL_USER_RULE, USER_DROP_X, false), + Arguments.of(LOCAL_USER_RULE, USER_KEEP_X, false), + Arguments.of(MANUAL, UNSET, false), + Arguments.of(MANUAL, SAMPLER_DROP, false), + Arguments.of(MANUAL, SAMPLER_KEEP, false), + Arguments.of(MANUAL, USER_DROP, true), + Arguments.of(MANUAL, USER_KEEP, true), + Arguments.of(MANUAL, USER_DROP_X, false), + Arguments.of(MANUAL, USER_KEEP_X, false), + Arguments.of(REMOTE_USER_RATE, UNSET, false), + Arguments.of(REMOTE_USER_RATE, SAMPLER_DROP, false), + Arguments.of(REMOTE_USER_RATE, SAMPLER_KEEP, false), + Arguments.of(REMOTE_USER_RATE, USER_DROP, true), + Arguments.of(REMOTE_USER_RATE, USER_KEEP, true), + Arguments.of(REMOTE_USER_RATE, USER_DROP_X, false), + Arguments.of(REMOTE_USER_RATE, USER_KEEP_X, false), + Arguments.of(APPSEC, UNSET, false), + Arguments.of(APPSEC, SAMPLER_DROP, true), + Arguments.of(APPSEC, SAMPLER_KEEP, true), + Arguments.of(APPSEC, USER_DROP, false), + Arguments.of(APPSEC, USER_KEEP, true), + Arguments.of(APPSEC, USER_DROP_X, false), + Arguments.of(APPSEC, USER_KEEP_X, false), + Arguments.of(DATA_JOBS, UNSET, false), + Arguments.of(DATA_JOBS, SAMPLER_DROP, false), + Arguments.of(DATA_JOBS, SAMPLER_KEEP, false), + Arguments.of(DATA_JOBS, USER_DROP, false), + Arguments.of(DATA_JOBS, USER_KEEP, true), + Arguments.of(DATA_JOBS, USER_DROP_X, false), + Arguments.of(DATA_JOBS, USER_KEEP_X, false), + Arguments.of(DATA_STREAMS, UNSET, false), + Arguments.of(DATA_STREAMS, SAMPLER_DROP, false), + Arguments.of(DATA_STREAMS, SAMPLER_KEEP, false), + Arguments.of(DATA_STREAMS, USER_DROP, true), + Arguments.of(DATA_STREAMS, USER_KEEP, false), + Arguments.of(DATA_STREAMS, USER_DROP_X, false), + Arguments.of(DATA_STREAMS, USER_KEEP_X, false), + Arguments.of(EXTERNAL_OVERRIDE, UNSET, false), + Arguments.of(EXTERNAL_OVERRIDE, SAMPLER_DROP, false), + Arguments.of(EXTERNAL_OVERRIDE, SAMPLER_KEEP, false), + Arguments.of(EXTERNAL_OVERRIDE, USER_DROP, false), + Arguments.of(EXTERNAL_OVERRIDE, USER_KEEP, false), + Arguments.of(EXTERNAL_OVERRIDE, USER_DROP_X, false), + Arguments.of(EXTERNAL_OVERRIDE, USER_KEEP_X, false)); + } + + @ParameterizedTest + @MethodSource("testCanAvoidSamplingPriorityLockArguments") + @WithConfig(key = APM_TRACING_ENABLED, value = "false") + void testCanAvoidSamplingPriorityLock(byte mechanism, byte priority, boolean valid) { + assertEquals(valid, canAvoidSamplingPriorityLock(priority, mechanism)); + } + + private static Stream testCanAvoidSamplingPriorityLockArguments() { + return Stream.of( + Arguments.of(APPSEC, UNSET, true), + Arguments.of(APPSEC, SAMPLER_KEEP, true), + Arguments.of(UNKNOWN, SAMPLER_KEEP, false), + Arguments.of(DEFAULT, SAMPLER_KEEP, false), + Arguments.of(AGENT_RATE, SAMPLER_KEEP, false), + Arguments.of(REMOTE_AUTO_RATE, SAMPLER_KEEP, false), + Arguments.of(LOCAL_USER_RULE, SAMPLER_KEEP, false), + Arguments.of(MANUAL, SAMPLER_KEEP, false), + Arguments.of(REMOTE_USER_RATE, SAMPLER_KEEP, false), + Arguments.of(DATA_JOBS, SAMPLER_KEEP, false), + Arguments.of(DATA_STREAMS, SAMPLER_KEEP, false), + Arguments.of(EXTERNAL_OVERRIDE, SAMPLER_KEEP, false)); + } +} From 5f060a3d9d7af3f8a0d47567f4ac4118c9c91f3e Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 14:10:24 -0400 Subject: [PATCH 24/32] test: add JUnit 5 tests for AgentInstaller.getEnabledSystems() DATA_STREAMS flag Test that AgentInstaller.getEnabledSystems() correctly includes or excludes TargetSystem.DATA_STREAMS based on the data.streams.enabled configuration flag. Also test USM and LLMOBS for consistency. Uses @WithConfig extension to manage configuration state across test cases. Tests verify: - DATA_STREAMS not included when data.streams.enabled is false (default) - DATA_STREAMS included when data.streams.enabled is true - USM not included when usm.enabled is false (default) - USM included when usm.enabled is true - LLMOBS not included when llmobs.enabled is false (default) - LLMOBS included when llmobs.enabled is true Co-Authored-By: Claude Sonnet 5 --- dd-java-agent/agent-installer/build.gradle | 1 + .../AgentInstallerGetEnabledSystemsTest.java | 75 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 dd-java-agent/agent-installer/src/test/java/datadog/trace/agent/tooling/AgentInstallerGetEnabledSystemsTest.java diff --git a/dd-java-agent/agent-installer/build.gradle b/dd-java-agent/agent-installer/build.gradle index 766e55af799..6a63ac6e57e 100644 --- a/dd-java-agent/agent-installer/build.gradle +++ b/dd-java-agent/agent-installer/build.gradle @@ -29,6 +29,7 @@ dependencies { compileOnly project(':products:metrics:metrics-lib') testImplementation project(':dd-java-agent:testing') + testImplementation project(':utils:test-junit-utils') } tasks.named("compileMain_java11Java", JavaCompile) { diff --git a/dd-java-agent/agent-installer/src/test/java/datadog/trace/agent/tooling/AgentInstallerGetEnabledSystemsTest.java b/dd-java-agent/agent-installer/src/test/java/datadog/trace/agent/tooling/AgentInstallerGetEnabledSystemsTest.java new file mode 100644 index 00000000000..8352fc8d3cb --- /dev/null +++ b/dd-java-agent/agent-installer/src/test/java/datadog/trace/agent/tooling/AgentInstallerGetEnabledSystemsTest.java @@ -0,0 +1,75 @@ +package datadog.trace.agent.tooling; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.agent.tooling.InstrumenterModule.TargetSystem; +import datadog.trace.test.junit.utils.config.WithConfig; +import datadog.trace.test.junit.utils.config.WithConfigExtension; +import java.util.Set; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Tests for {@link AgentInstaller#getEnabledSystems()} to verify that it correctly includes target + * systems based on their corresponding configuration flags. + */ +@ExtendWith(WithConfigExtension.class) +class AgentInstallerGetEnabledSystemsTest { + + /** + * Verifies that DATA_STREAMS target system is not included when data.streams.enabled is false + * (default). + */ + @Test + void dataStreamsNotIncludedWhenDisabled() { + Set enabledSystems = AgentInstaller.getEnabledSystems(); + assertFalse( + enabledSystems.contains(TargetSystem.DATA_STREAMS), + "DATA_STREAMS should not be included when disabled"); + } + + /** Verifies that DATA_STREAMS target system is included when data.streams.enabled is true. */ + @Test + @WithConfig(key = "data.streams.enabled", value = "true") + void dataStreamsIncludedWhenEnabled() { + Set enabledSystems = AgentInstaller.getEnabledSystems(); + assertTrue( + enabledSystems.contains(TargetSystem.DATA_STREAMS), + "DATA_STREAMS should be included when enabled"); + } + + /** Verifies that USM target system is not included when usm.enabled is false (default). */ + @Test + void usmNotIncludedWhenDisabled() { + Set enabledSystems = AgentInstaller.getEnabledSystems(); + assertFalse( + enabledSystems.contains(TargetSystem.USM), "USM should not be included when disabled"); + } + + /** Verifies that USM target system is included when usm.enabled is true. */ + @Test + @WithConfig(key = "usm.enabled", value = "true") + void usmIncludedWhenEnabled() { + Set enabledSystems = AgentInstaller.getEnabledSystems(); + assertTrue(enabledSystems.contains(TargetSystem.USM), "USM should be included when enabled"); + } + + /** Verifies that LLMOBS target system is not included when llmobs.enabled is false (default). */ + @Test + void llmobsNotIncludedWhenDisabled() { + Set enabledSystems = AgentInstaller.getEnabledSystems(); + assertFalse( + enabledSystems.contains(TargetSystem.LLMOBS), + "LLMOBS should not be included when disabled"); + } + + /** Verifies that LLMOBS target system is included when llmobs.enabled is true. */ + @Test + @WithConfig(key = "llmobs.enabled", value = "true") + void llmobsIncludedWhenEnabled() { + Set enabledSystems = AgentInstaller.getEnabledSystems(); + assertTrue( + enabledSystems.contains(TargetSystem.LLMOBS), "LLMOBS should be included when enabled"); + } +} From 914974d26b41d249e89b8ce62387596d018b416c Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 14:18:47 -0400 Subject: [PATCH 25/32] fix: suppress APM billing for DSM-only Kafka consume spans in kafka-clients-0.11 Mirrors the fix already applied to kafka-clients-3.8's TracingIterator: when Kafka APM tracing is disabled but Data Streams Monitoring is enabled, and no genuine propagated trace context was extracted from the record headers, the local-root consume span is marked USER_DROP so it is not billed as a normal APM span on kafka-clients < 3.8.0. --- .../instrumentation/kafka_clients/TracingIterator.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/TracingIterator.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/TracingIterator.java index 93179e4e3f2..cd242c152ec 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/TracingIterator.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/TracingIterator.java @@ -28,6 +28,8 @@ import datadog.trace.api.datastreams.DataStreamsContext; import datadog.trace.api.datastreams.DataStreamsTags; import datadog.trace.api.datastreams.DataStreamsTransactionExtractor; +import datadog.trace.api.sampling.PrioritySampling; +import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; @@ -120,6 +122,11 @@ protected void startNewRecordSpan(ConsumerRecord val) { // spans are written out together by TraceStructureWriter when running in strict mode } + if (spanContext == null + && !KafkaDecorator.TRACING_ENABLED + && traceConfig().isDataStreamsEnabled()) { + span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); + } DataStreamsTags tags = create("kafka", INBOUND, val.topic(), group, clusterId); final long payloadSize = traceConfig().isDataStreamsEnabled() ? computePayloadSizeBytes(val) : 0; @@ -141,6 +148,9 @@ protected void startNewRecordSpan(ConsumerRecord val) { } } else { span = startSpan(JAVA_KAFKA.toString(), operationName, null); + if (!KafkaDecorator.TRACING_ENABLED && traceConfig().isDataStreamsEnabled()) { + span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); + } } if (val.value() == null) { span.setTag(InstrumentationTags.TOMBSTONE, true); From 54f1b8008f588e24f8d88d1fb5970da307ebddd7 Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 14:42:24 -0400 Subject: [PATCH 26/32] Add tests for DSM billing suppression on kafka produce/consume spans Cover the local-root sampling-priority suppression (USER_DROP) that ProducerAdvice and TracingIterator apply when kafka tracing is disabled (via integrations.enabled or the per-integration trace.kafka.enabled override) but DSM is enabled, plus a regression guard confirming spans that join a real propagated trace are not suppressed. Co-Authored-By: Claude Sonnet 5 --- .../test/groovy/KafkaClientTestBase.groovy | 215 ++++++++++++++++++ 1 file changed, 215 insertions(+) diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy index 6d311a1b8d5..3342ce0a3dc 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy @@ -14,6 +14,7 @@ import datadog.trace.agent.test.asserts.TraceAssert import datadog.trace.agent.test.naming.VersionedNamingTestBase import datadog.trace.api.Config import datadog.trace.api.DDTags +import datadog.trace.api.sampling.PrioritySampling import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.common.writer.ListWriter @@ -1544,3 +1545,217 @@ class KafkaClientBadBase64HeaderForkedTest extends InstrumentationSpecification producer?.close() } } + +// DSM billing-suppression coverage: when tracing is disabled for kafka (globally, via +// integrations.enabled) but DSM is enabled, a genuinely local-root produce/consume span (no +// extracted/real parent trace context) must have its sampling priority forced to USER_DROP so it +// does not count towards APM billing, while a span that joins a real propagated trace must not be. +class KafkaClientDataStreamsOnlyLocalRootForkedTest extends KafkaClientTestBase { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig("integrations.enabled", "false") + injectSysConfig("data.streams.enabled", "true") + } + + @Override + String service() { + return "kafka" + } + + @Override + boolean hasQueueSpan() { + return false + } + + @Override + boolean splitByDestination() { + return false + } + + @Override + boolean isDataStreamsEnabled() { + return true + } + + def "local-root produce and consume spans are forced to USER_DROP when kafka tracing is disabled and DSM is enabled"() { + setup: + def kafkaPartition = 0 + def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) + consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") + def consumer = new KafkaConsumer(consumerProperties) + def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) + def producer = new KafkaProducer<>(senderProps, new StringSerializer(), new StringSerializer()) + consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) + + when: "a message is produced with no propagated trace headers, i.e. a genuine local root" + def record = new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, "local-root-message") + producer.send(record).get() + + then: "the produce span's trace is forced to USER_DROP to suppress APM billing" + TEST_WRITER.waitForTraces(1) + def producedSpan = TEST_WRITER[0][0] + producedSpan.getSamplingPriority() == PrioritySampling.USER_DROP + + when: "the message is consumed" + def pollResult = KafkaTestUtils.getRecords(consumer) + def recs = pollResult.records(new TopicPartition(SHARED_TOPIC, kafkaPartition)).iterator() + + then: "the consume span's trace is also forced to USER_DROP" + recs.hasNext() + recs.next().value() == "local-root-message" + !recs.hasNext() + TEST_WRITER.waitForTraces(2) + def consumedSpan = TEST_WRITER[1][0] + consumedSpan.getSamplingPriority() == PrioritySampling.USER_DROP + + cleanup: + consumer?.close() + producer?.close() + } +} + +// Regression guard: a span that joins a real, externally-propagated Datadog trace (extracted +// x-datadog-trace-id/x-datadog-parent-id headers) must NOT be forced to USER_DROP, even under the +// same "kafka tracing disabled + DSM enabled" configuration as above, since it is not a local root. +class KafkaClientDataStreamsOnlyExtractedParentForkedTest extends KafkaClientTestBase { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig("integrations.enabled", "false") + injectSysConfig("data.streams.enabled", "true") + } + + @Override + String service() { + return "kafka" + } + + @Override + boolean hasQueueSpan() { + return false + } + + @Override + boolean splitByDestination() { + return false + } + + @Override + boolean isDataStreamsEnabled() { + return true + } + + def "produce and consume spans with an extracted parent trace context are not forced to USER_DROP"() { + setup: + def kafkaPartition = 0 + def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) + consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") + def consumer = new KafkaConsumer(consumerProperties) + def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) + def producer = new KafkaProducer<>(senderProps, new StringSerializer(), new StringSerializer()) + consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) + + def existingTraceId = 1234567890123456L + def existingSpanId = 9876543210987654L + def headers = new RecordHeaders() + headers.add(new RecordHeader("x-datadog-trace-id", + String.valueOf(existingTraceId).getBytes(StandardCharsets.UTF_8))) + headers.add(new RecordHeader("x-datadog-parent-id", + String.valueOf(existingSpanId).getBytes(StandardCharsets.UTF_8))) + + when: "a message carrying a real, externally-propagated Datadog trace context is produced" + def record = new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, "propagated-trace-message", headers) + producer.send(record).get() + + then: "the produce span joins the propagated trace and is NOT forced to USER_DROP" + TEST_WRITER.waitForTraces(1) + def producedSpan = TEST_WRITER[0][0] + producedSpan.traceId.toLong() == existingTraceId + producedSpan.parentId == existingSpanId + producedSpan.getSamplingPriority() != PrioritySampling.USER_DROP + + when: "the message is consumed" + def pollResult = KafkaTestUtils.getRecords(consumer) + def recs = pollResult.records(new TopicPartition(SHARED_TOPIC, kafkaPartition)).iterator() + + then: "the consume span also joins the propagated trace and is NOT forced to USER_DROP" + recs.hasNext() + recs.next().value() == "propagated-trace-message" + !recs.hasNext() + TEST_WRITER.waitForTraces(2) + def consumedSpan = TEST_WRITER[1][0] + consumedSpan.getSamplingPriority() != PrioritySampling.USER_DROP + + cleanup: + consumer?.close() + producer?.close() + } +} + +// Confirms the per-integration override (trace.kafka.enabled=false) suppresses billing identically +// to the global integrations.enabled=false toggle used above, for a genuine local-root span. +class KafkaClientDataStreamsOnlyIntegrationOverrideForkedTest extends KafkaClientTestBase { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig("trace.kafka.enabled", "false") + injectSysConfig("data.streams.enabled", "true") + } + + @Override + String service() { + return "kafka" + } + + @Override + boolean hasQueueSpan() { + return false + } + + @Override + boolean splitByDestination() { + return false + } + + @Override + boolean isDataStreamsEnabled() { + return true + } + + def "local-root produce and consume spans are forced to USER_DROP when trace.kafka.enabled=false and DSM is enabled"() { + setup: + def kafkaPartition = 0 + def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) + consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") + def consumer = new KafkaConsumer(consumerProperties) + def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) + def producer = new KafkaProducer<>(senderProps, new StringSerializer(), new StringSerializer()) + consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) + + when: "a message is produced with no propagated trace headers, i.e. a genuine local root" + def record = new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, "local-root-message") + producer.send(record).get() + + then: "the produce span's trace is forced to USER_DROP to suppress APM billing" + TEST_WRITER.waitForTraces(1) + def producedSpan = TEST_WRITER[0][0] + producedSpan.getSamplingPriority() == PrioritySampling.USER_DROP + + when: "the message is consumed" + def pollResult = KafkaTestUtils.getRecords(consumer) + def recs = pollResult.records(new TopicPartition(SHARED_TOPIC, kafkaPartition)).iterator() + + then: "the consume span's trace is also forced to USER_DROP" + recs.hasNext() + recs.next().value() == "local-root-message" + !recs.hasNext() + TEST_WRITER.waitForTraces(2) + def consumedSpan = TEST_WRITER[1][0] + consumedSpan.getSamplingPriority() == PrioritySampling.USER_DROP + + cleanup: + consumer?.close() + producer?.close() + } +} From fcb30920774ec7efaf8a3cf1466be4bb8a2b12c3 Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 15:43:11 -0400 Subject: [PATCH 27/32] fix: only force DSM-only USER_DROP when the Kafka span owns the local trace `AgentSpan.setSamplingPriority` is trace-level: it resolves to the local root span. The DSM billing-suppression guard was gated on "no context was extracted from the record headers", which is not the same predicate as "this span is the local root". Because the kafka-clients producer and the kafka-streams StreamTask advices create their spans with scope-honouring `startSpan` overloads (USE_SCOPE), the created span can inherit the locally active span -- and since the new DATA_STREAMS mechanism bypasses the sampling priority lock, the guard could force-drop a genuine customer trace (e.g. producing a Kafka message inside an instrumented HTTP request). Producer sites (kafka-clients-0.11 and kafka-clients-3.8) now additionally require `span.getLocalRootSpan() == span`. Entering the else branch already rules out a header-extracted parent, so the local root check is exactly the remaining condition. The kafka-streams sites need a stronger predicate: - `TIME_IN_QUEUE_ENABLED` (on by default) parents the consume span onto the queue span, so `getLocalRootSpan() == span` is never true there; the root candidate is the queue span when one exists. - An `ExtractedContext` parent allocates a fresh TraceCollector, so a span continuing a header-propagated trace is still its own local root. The sibling ContextPropagationAdvice attaches that extracted context to the scope before this advice runs, so the active span captured on entry is what detects it. Both sites therefore require no active span on entry and that the local root is the first span the advice created. Also switches the producer sites from the static `Config.get()` to the remote-config-aware `traceConfig()`, matching the consumer/streams sites so a dynamic DSM toggle cannot make them disagree. Co-Authored-By: Claude Sonnet 5 --- .../KafkaProducerInstrumentation.java | 9 ++++- .../kafka_clients38/ProducerAdvice.java | 10 ++++-- .../KafkaStreamTaskInstrumentation.java | 33 +++++++++++++++++-- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java index 0850d586667..95f1eaf5532 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java @@ -12,6 +12,7 @@ import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.traceConfig; import static datadog.trace.instrumentation.kafka_clients.KafkaDecorator.JAVA_KAFKA; import static datadog.trace.instrumentation.kafka_clients.KafkaDecorator.KAFKA_PRODUCE; import static datadog.trace.instrumentation.kafka_clients.KafkaDecorator.PRODUCER_DECORATE; @@ -167,7 +168,13 @@ public static AgentScope onEnter( } else { span = startSpan(JAVA_KAFKA.toString(), KAFKA_PRODUCE); callbackParentSpan = localActiveSpan; - if (!KafkaDecorator.TRACING_ENABLED && Config.get().isDataStreamsEnabled()) { + // setSamplingPriority is trace-level: it resolves to the local root span. This 2-arg + // startSpan honours the active scope, so `span` may be a child of a customer trace + // (localActiveSpan above). Only force the DSM-only drop when `span` is the local root, + // otherwise we would silently drop that whole customer trace. + if (!KafkaDecorator.TRACING_ENABLED + && traceConfig().isDataStreamsEnabled() + && span.getLocalRootSpan() == span) { span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); } } diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java index 637c5b19000..5a1c2f05989 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java @@ -4,11 +4,11 @@ import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.traceConfig; import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.JAVA_KAFKA; import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.KAFKA_PRODUCE; import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.PRODUCER_DECORATE; -import datadog.trace.api.Config; import datadog.trace.api.sampling.PrioritySampling; import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.InstrumentationContext; @@ -58,7 +58,13 @@ public static AgentScope onEnter( } else { span = startSpan(JAVA_KAFKA.toString(), KAFKA_PRODUCE); callbackParentSpan = localActiveSpan; - if (!KafkaDecorator.TRACING_ENABLED && Config.get().isDataStreamsEnabled()) { + // setSamplingPriority is trace-level: it resolves to the local root span. This 2-arg + // startSpan honours the active scope, so `span` may be a child of a customer trace + // (localActiveSpan above). Only force the DSM-only drop when `span` is the local root, + // otherwise we would silently drop that whole customer trace. + if (!KafkaDecorator.TRACING_ENABLED + && traceConfig().isDataStreamsEnabled() + && span.getLocalRootSpan() == span) { span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); } } diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java index d3df5f7e8ed..ef546e3ed59 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java @@ -8,6 +8,7 @@ import static datadog.trace.api.datastreams.DataStreamsTags.createWithGroup; import static datadog.trace.bootstrap.instrumentation.api.AgentPropagation.DSM_CONCERN; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.traceConfig; import static datadog.trace.bootstrap.instrumentation.api.Java8BytecodeBridge.rootContext; @@ -278,6 +279,12 @@ public static void start( return; } + // Captured before any span is created. A non-null value here means this record is being + // consumed either inside a locally active trace, or under a context that the sibling + // ContextPropagationAdvice extracted from the record headers and attached to the scope + // (it is registered on the same method and runs first). Either way the spans created + // below will not own the resulting trace. + final AgentSpan localActiveSpan = activeSpan(); AgentSpan span, queueSpan = null; StreamTaskContext streamTaskContext = InstrumentationContext.get(StreamTask.class, StreamTaskContext.class).get(task); @@ -296,7 +303,15 @@ public static void start( // spans are written out together by TraceStructureWriter when running in strict mode } - if (!KafkaStreamsDecorator.TRACING_ENABLED && traceConfig().isDataStreamsEnabled()) { + // setSamplingPriority is trace-level: it resolves to the local root span. Only force the + // DSM-only drop when this instrumentation owns the whole local trace, i.e. nothing was + // active when we started (no local parent, no header-extracted parent) and the local root + // really is the first span we created here. + final AgentSpan ourLocalRoot = queueSpan == null ? span : queueSpan; + if (!KafkaStreamsDecorator.TRACING_ENABLED + && traceConfig().isDataStreamsEnabled() + && localActiveSpan == null + && span.getLocalRootSpan() == ourLocalRoot) { span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); } String applicationId = null; @@ -347,6 +362,12 @@ public static void start( return; } + // Captured before any span is created. A non-null value here means this record is being + // consumed either inside a locally active trace, or under a context that the sibling + // ContextPropagationAdvice extracted from the record headers and attached to the scope + // (it is registered on the same method and runs first). Either way the spans created + // below will not own the resulting trace. + final AgentSpan localActiveSpan = activeSpan(); AgentSpan span, queueSpan = null; StreamTaskContext streamTaskContext = InstrumentationContext.get(StreamTask.class, StreamTaskContext.class).get(task); @@ -365,7 +386,15 @@ public static void start( // spans are written out together by TraceStructureWriter when running in strict mode } - if (!KafkaStreamsDecorator.TRACING_ENABLED && traceConfig().isDataStreamsEnabled()) { + // setSamplingPriority is trace-level: it resolves to the local root span. Only force the + // DSM-only drop when this instrumentation owns the whole local trace, i.e. nothing was + // active when we started (no local parent, no header-extracted parent) and the local root + // really is the first span we created here. + final AgentSpan ourLocalRoot = queueSpan == null ? span : queueSpan; + if (!KafkaStreamsDecorator.TRACING_ENABLED + && traceConfig().isDataStreamsEnabled() + && localActiveSpan == null + && span.getLocalRootSpan() == ourLocalRoot) { span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); } String applicationId = null; From 3b2994c97de0d6589a0fda8339d5bac9412f7055 Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 15:43:21 -0400 Subject: [PATCH 28/32] fix: keep Kafka Code Origin instrumentation on the Tracing base class `MessageListenerInstrumentation` applies Code Origin (span origin) advice to Spring Kafka message listeners -- a purely-APM concern with no Data Streams behaviour. Swapping it to `InstrumenterModule.DataStreams` ORed `isDataStreamsEnabled()` into `isEnabled()`, so with `code.origin.enabled=true` and `data.streams.enabled=true` it installed in DSM-only deployments where Kafka tracing is off. Revert this one module to `InstrumenterModule.Tracing` and record why in a class comment. Co-Authored-By: Claude Sonnet 5 --- .../kafka_clients38/MessageListenerInstrumentation.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MessageListenerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MessageListenerInstrumentation.java index ba28a6eb969..2124f853645 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MessageListenerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MessageListenerInstrumentation.java @@ -12,8 +12,15 @@ import net.bytebuddy.description.type.TypeDescription; import net.bytebuddy.matcher.ElementMatcher; +/** + * Applies Code Origin (span origin) advice to Spring Kafka message listeners. This is a purely-APM + * concern with no Data Streams behaviour, so it deliberately stays on the {@link + * InstrumenterModule.Tracing} base class: {@link InstrumenterModule.DataStreams} ORs {@code + * isDataStreamsEnabled()} into {@code isEnabled()}, which would install this instrumentation in + * DSM-only deployments where Kafka tracing is off. + */ @AutoService(InstrumenterModule.class) -public class MessageListenerInstrumentation extends InstrumenterModule.DataStreams +public class MessageListenerInstrumentation extends InstrumenterModule.Tracing implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { public MessageListenerInstrumentation() { From feba908ca2a671f324fc69a6cda03d6579bddc05 Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 28 Aug 2026 15:43:41 -0400 Subject: [PATCH 29/32] test: cover DSM-only sampling suppression for Kafka produce, consume and streams Regression guards for the two billing-suppression bugs, plus the coverage gaps the final review flagged. All four new specs were confirmed red against a temporarily reverted guard. kafka-streams-0.11 (new spec, module previously had no working DSM coverage -- its only end-to-end test is @Ignore'd): - a local-root streams consume span is still forced to USER_DROP - a streams consume span continuing a header-propagated trace is not. The local-root case needs kafka.client.propagation.disabled.topics, since an in-JVM producer would otherwise inject its own context into the headers. kafka-clients-0.11: - producing inside an active local trace does not force that trace to USER_DROP -- the one-line test that would have caught the producer bug - strengthen the extracted-parent scenario from "!= USER_DROP" to the concrete expected SAMPLER_KEEP, and assert the consume span's trace id too kafka-clients-3.8 (new spec, module had no suppression coverage at all): the local-root USER_DROP scenario and the active-local-trace regression guard. These extend InstrumentationSpecification rather than KafkaClientTestBase, whose inherited test asserts Code Origin tags that correctly no longer apply once Kafka tracing is off. SamplingMechanismTest: the only DATA_STREAMS row in the parameterized lock-bypass test left DSM at its disabled default, so it passed for a config reason rather than a mechanism one. Add dedicated tests with data.streams.enabled set both ways, including that enabling DSM does not unlock any other mechanism. Co-Authored-By: Claude Sonnet 5 --- .../test/groovy/KafkaClientTestBase.groovy | 65 ++++++- ...afkaClientDataStreamsOnlyForkedTest.groovy | 141 ++++++++++++++ ...fkaStreamsDataStreamsOnlyForkedTest.groovy | 173 ++++++++++++++++++ .../api/sampling/SamplingMechanismTest.java | 26 +++ 4 files changed, 403 insertions(+), 2 deletions(-) create mode 100644 dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientDataStreamsOnlyForkedTest.groovy create mode 100644 dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/test/groovy/KafkaStreamsDataStreamsOnlyForkedTest.groovy diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy index 3342ce0a3dc..88a55b8b4ce 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy @@ -1673,7 +1673,11 @@ class KafkaClientDataStreamsOnlyExtractedParentForkedTest extends KafkaClientTes def producedSpan = TEST_WRITER[0][0] producedSpan.traceId.toLong() == existingTraceId producedSpan.parentId == existingSpanId - producedSpan.getSamplingPriority() != PrioritySampling.USER_DROP + // The injected headers carry no x-datadog-sampling-priority, so the extracted context's + // priority is UNSET and the normal sampler decides. Assert the concrete resulting value + // rather than just "not USER_DROP", so any regression that lands on a different-but-also + // wrong priority is caught too. + producedSpan.getSamplingPriority() == PrioritySampling.SAMPLER_KEEP when: "the message is consumed" def pollResult = KafkaTestUtils.getRecords(consumer) @@ -1685,7 +1689,8 @@ class KafkaClientDataStreamsOnlyExtractedParentForkedTest extends KafkaClientTes !recs.hasNext() TEST_WRITER.waitForTraces(2) def consumedSpan = TEST_WRITER[1][0] - consumedSpan.getSamplingPriority() != PrioritySampling.USER_DROP + consumedSpan.traceId.toLong() == existingTraceId + consumedSpan.getSamplingPriority() == PrioritySampling.SAMPLER_KEEP cleanup: consumer?.close() @@ -1759,3 +1764,59 @@ class KafkaClientDataStreamsOnlyIntegrationOverrideForkedTest extends KafkaClien producer?.close() } } + +// Regression guard for the producer suppression site: producing a message from inside an already +// active local trace (e.g. an instrumented HTTP request) must NOT force that customer trace to +// USER_DROP. The produce span is created with a scope-honouring startSpan overload, so it becomes +// a child of the active span, and setSamplingPriority is trace-level - without the local-root +// check on the suppression guard the whole surrounding trace would be silently dropped. +class KafkaClientDataStreamsOnlyActiveLocalTraceForkedTest extends KafkaClientTestBase { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig("integrations.enabled", "false") + injectSysConfig("data.streams.enabled", "true") + } + + @Override + String service() { + return "kafka" + } + + @Override + boolean hasQueueSpan() { + return false + } + + @Override + boolean splitByDestination() { + return false + } + + @Override + boolean isDataStreamsEnabled() { + return true + } + + def "producing inside an active local trace does not force that trace to USER_DROP"() { + setup: + def kafkaPartition = 0 + def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) + def producer = new KafkaProducer<>(senderProps, new StringSerializer(), new StringSerializer()) + + when: "a message is produced from within an already active local trace" + runUnderTrace("parent") { + producer.send(new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, "in-active-trace")).get() + } + + then: "the surrounding customer trace is not force-dropped" + TEST_WRITER.waitForTraces(1) + def trace = TEST_WRITER[0] + def localRoot = trace[0].localRootSpan + localRoot.operationName.toString() == "parent" + localRoot.getSamplingPriority() != PrioritySampling.USER_DROP + + cleanup: + producer?.close() + } +} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientDataStreamsOnlyForkedTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientDataStreamsOnlyForkedTest.groovy new file mode 100644 index 00000000000..c3b6734cac3 --- /dev/null +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientDataStreamsOnlyForkedTest.groovy @@ -0,0 +1,141 @@ +import datadog.trace.agent.test.InstrumentationSpecification +import datadog.trace.api.sampling.PrioritySampling +import org.apache.kafka.clients.consumer.ConsumerConfig +import org.apache.kafka.clients.consumer.KafkaConsumer +import org.apache.kafka.clients.producer.KafkaProducer +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.common.TopicPartition +import org.apache.kafka.common.serialization.StringSerializer +import org.springframework.kafka.test.EmbeddedKafkaBroker +import org.springframework.kafka.test.EmbeddedKafkaKraftBroker +import org.springframework.kafka.test.utils.KafkaTestUtils + +import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace + +/** + * DSM billing-suppression coverage for kafka-clients-3.8, mirroring the kafka-clients-0.11 suite. + * + *

    Both scenarios run under "kafka tracing disabled (integrations.enabled=false) + DSM enabled", + * the configuration in which the suppression guard is active. These specs deliberately do not + * extend {@code KafkaClientTestBase}: that base asserts Code Origin tags, which are correctly + * absent once Kafka tracing is off. + */ +abstract class KafkaClientDataStreamsOnlyForkedTest extends InstrumentationSpecification { + static final SHARED_TOPIC = "shared.topic" + + EmbeddedKafkaBroker embeddedKafka + + def setup() { + embeddedKafka = new EmbeddedKafkaKraftBroker(1, 2, SHARED_TOPIC) + embeddedKafka.afterPropertiesSet() + } + + def cleanup() { + embeddedKafka.destroy() + } + + @Override + boolean useStrictTraceWrites() { + return false + } + + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig("integrations.enabled", "false") + injectSysConfig("data.streams.enabled", "true") + } + + /** + * Locates a written span by operation name. The consumer's kafka.poll spans interleave + * unpredictably with the produce/consume traces, so indexing into TEST_WRITER is unreliable. + */ + protected findSpan(String operationName) { + for (int i = 0; i < 100; i++) { + def span = TEST_WRITER.flatten().find { it.operationName.toString() == operationName } + if (span != null) { + return span + } + Thread.sleep(100) + } + return null + } + + protected KafkaProducer newProducer() { + return new KafkaProducer( + KafkaTestUtils.producerProps(embeddedKafka.getBrokersAsString()), + new StringSerializer(), + new StringSerializer()) + } +} + +/** + * A genuinely local-root produce/consume span must have its sampling priority forced to USER_DROP + * so it does not count towards APM billing. + */ +class KafkaClientDataStreamsOnlyLocalRootForkedTest extends KafkaClientDataStreamsOnlyForkedTest { + + def "local-root produce and consume spans are forced to USER_DROP"() { + setup: + def kafkaPartition = 0 + def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) + consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") + def consumer = new KafkaConsumer(consumerProperties) + def producer = newProducer() + consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) + + when: "a message is produced with no propagated trace headers, i.e. a genuine local root" + producer.send(new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, "local-root-message")).get() + + then: "the produce span's trace is forced to USER_DROP to suppress APM billing" + TEST_WRITER.waitForTraces(1) + def produceSpan = findSpan("kafka.produce") + produceSpan != null + produceSpan.getSamplingPriority() == PrioritySampling.USER_DROP + + when: "the message is consumed" + def recs = KafkaTestUtils.getRecords(consumer) + .records(new TopicPartition(SHARED_TOPIC, kafkaPartition)).iterator() + + then: "the consume span's trace is also forced to USER_DROP" + recs.hasNext() + recs.next().value() == "local-root-message" + !recs.hasNext() + TEST_WRITER.waitForTraces(2) + def consumeSpan = findSpan("kafka.consume") + consumeSpan != null + consumeSpan.getSamplingPriority() == PrioritySampling.USER_DROP + + cleanup: + consumer?.close() + producer?.close() + } +} + +/** + * Regression guard for the producer suppression site: producing a message from inside an already + * active local trace must NOT force that customer trace to USER_DROP. The produce span is created + * with a scope-honouring startSpan overload, so it becomes a child of the active span, and + * setSamplingPriority is trace-level. + */ +class KafkaClientDataStreamsOnlyActiveLocalTraceForkedTest extends KafkaClientDataStreamsOnlyForkedTest { + + def "producing inside an active local trace does not force that trace to USER_DROP"() { + setup: + def producer = newProducer() + + when: "a message is produced from within an already active local trace" + runUnderTrace("parent") { + producer.send(new ProducerRecord(SHARED_TOPIC, 0, null, "in-active-trace")).get() + } + + then: "the surrounding customer trace is not force-dropped" + TEST_WRITER.waitForTraces(1) + def localRoot = TEST_WRITER[0][0].localRootSpan + localRoot.operationName.toString() == "parent" + localRoot.getSamplingPriority() != PrioritySampling.USER_DROP + + cleanup: + producer?.close() + } +} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/test/groovy/KafkaStreamsDataStreamsOnlyForkedTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/test/groovy/KafkaStreamsDataStreamsOnlyForkedTest.groovy new file mode 100644 index 00000000000..185121d447b --- /dev/null +++ b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/test/groovy/KafkaStreamsDataStreamsOnlyForkedTest.groovy @@ -0,0 +1,173 @@ +import datadog.trace.agent.test.InstrumentationSpecification +import datadog.trace.api.config.TraceInstrumentationConfig +import datadog.trace.api.sampling.PrioritySampling +import datadog.trace.bootstrap.instrumentation.api.Tags +import org.apache.kafka.clients.producer.KafkaProducer +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.common.header.internals.RecordHeader +import org.apache.kafka.common.header.internals.RecordHeaders +import org.apache.kafka.common.serialization.Serdes +import org.apache.kafka.common.serialization.StringSerializer +import org.apache.kafka.streams.KafkaStreams +import org.apache.kafka.streams.StreamsConfig +import org.apache.kafka.streams.kstream.KStream +import org.apache.kafka.streams.kstream.KStreamBuilder +import org.apache.kafka.streams.kstream.ValueMapper +import org.springframework.kafka.test.rule.KafkaEmbedded +import org.springframework.kafka.test.utils.KafkaTestUtils +import spock.lang.Shared + +import java.nio.charset.StandardCharsets + +/** + * DSM billing-suppression coverage for the kafka-streams StreamTask consume spans. + * + *

    Both scenarios run under "kafka tracing disabled (integrations.enabled=false) + DSM enabled", + * the configuration in which the suppression guard is active. + */ +abstract class KafkaStreamsDataStreamsOnlyForkedTest extends InstrumentationSpecification { + static final STREAM_PENDING = "test.pending" + static final STREAM_PROCESSED = "test.processed" + + @Shared + protected KafkaEmbedded embeddedKafka + + def setupSpec() { + embeddedKafka = new KafkaEmbedded(1, true, 1, STREAM_PENDING, STREAM_PROCESSED) + embeddedKafka.before() + } + + def cleanupSpec() { + embeddedKafka?.after() + } + + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig("integrations.enabled", "false") + injectSysConfig("data.streams.enabled", "true") + } + + @Override + boolean useStrictTraceWrites() { + return false + } + + protected KafkaStreams startLowercasingTopology() { + def config = new Properties() + config.putAll(KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString())) + config.put(StreamsConfig.APPLICATION_ID_CONFIG, "dsm-only-test-application") + config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()) + config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()) + + def builder = new KStreamBuilder() + KStream textLines = builder.stream(STREAM_PENDING) + textLines + .mapValues(new ValueMapper() { + @Override + String apply(String textLine) { + return textLine.toLowerCase() + } + }) + .to(Serdes.String(), Serdes.String(), STREAM_PROCESSED) + + def streams = new KafkaStreams(builder, config) + streams.start() + return streams + } + + protected KafkaProducer newProducer() { + return new KafkaProducer( + KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()), + new StringSerializer(), + new StringSerializer()) + } + + /** + * Polls the test writer until a kafka-streams consume span shows up, so the assertions do not + * depend on how many other traces (produce, poll, downstream produce) are flushed first. + */ + protected findStreamsConsumeSpan() { + for (int i = 0; i < 100; i++) { + def span = TEST_WRITER.flatten().find { + it.operationName.toString() == "kafka.consume" && + it.getTag(Tags.COMPONENT)?.toString() == "java-kafka-streams" + } + if (span != null) { + return span + } + Thread.sleep(100) + } + return null + } +} + +/** + * A record with no propagated Datadog trace context produces a genuinely local-root streams + * consume span, which must be forced to USER_DROP so it does not count towards APM billing. + */ +class KafkaStreamsDataStreamsOnlyLocalRootForkedTest extends KafkaStreamsDataStreamsOnlyForkedTest { + + @Override + void configurePreAgent() { + super.configurePreAgent() + // The in-JVM producer would otherwise inject its own trace context into the record headers, + // which the streams ContextPropagationAdvice would then extract - so there would be no way to + // exercise the genuinely-local-root path. Disabling client propagation for the topic stops + // both the injection and the extraction. + injectSysConfig(TraceInstrumentationConfig.KAFKA_CLIENT_PROPAGATION_DISABLED_TOPICS, STREAM_PENDING) + } + + def "a local-root streams consume span is forced to USER_DROP"() { + setup: + def streams = startLowercasingTopology() + def producer = newProducer() + + when: + producer.send(new ProducerRecord(STREAM_PENDING, "LOCAL ROOT")).get() + + then: + def consumeSpan = findStreamsConsumeSpan() + consumeSpan != null + consumeSpan.getSamplingPriority() == PrioritySampling.USER_DROP + + cleanup: + producer?.close() + streams?.close() + } +} + +/** + * Regression guard: a record carrying a real, externally-propagated Datadog trace context must NOT + * have its trace force-dropped. The sibling ContextPropagationAdvice attaches that extracted + * context to the scope before the span-starting advice runs, and setSamplingPriority is + * trace-level, so without the guard the whole propagated trace would be silently dropped. + */ +class KafkaStreamsDataStreamsOnlyExtractedParentForkedTest extends KafkaStreamsDataStreamsOnlyForkedTest { + + def "a streams consume span continuing a propagated trace is not forced to USER_DROP"() { + setup: + def streams = startLowercasingTopology() + def producer = newProducer() + def existingTraceId = 1234567890123456L + def existingSpanId = 9876543210987654L + def headers = new RecordHeaders() + headers.add(new RecordHeader("x-datadog-trace-id", + String.valueOf(existingTraceId).getBytes(StandardCharsets.UTF_8))) + headers.add(new RecordHeader("x-datadog-parent-id", + String.valueOf(existingSpanId).getBytes(StandardCharsets.UTF_8))) + + when: + producer.send(new ProducerRecord(STREAM_PENDING, null, null, "PROPAGATED", headers)).get() + + then: + def consumeSpan = findStreamsConsumeSpan() + consumeSpan != null + consumeSpan.traceId.toLong() == existingTraceId + consumeSpan.getSamplingPriority() != PrioritySampling.USER_DROP + + cleanup: + producer?.close() + streams?.close() + } +} diff --git a/internal-api/src/test/java/datadog/trace/api/sampling/SamplingMechanismTest.java b/internal-api/src/test/java/datadog/trace/api/sampling/SamplingMechanismTest.java index 86ee27f2f55..cd07af4fe78 100644 --- a/internal-api/src/test/java/datadog/trace/api/sampling/SamplingMechanismTest.java +++ b/internal-api/src/test/java/datadog/trace/api/sampling/SamplingMechanismTest.java @@ -1,6 +1,7 @@ package datadog.trace.api.sampling; import static datadog.trace.api.config.GeneralConfig.APM_TRACING_ENABLED; +import static datadog.trace.api.config.GeneralConfig.DATA_STREAMS_ENABLED; import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP; import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP; import static datadog.trace.api.sampling.PrioritySampling.UNSET; @@ -20,10 +21,13 @@ import static datadog.trace.api.sampling.SamplingMechanism.canAvoidSamplingPriorityLock; import static datadog.trace.api.sampling.SamplingMechanism.validateWithSamplingPriority; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.trace.test.junit.utils.config.WithConfig; import datadog.trace.test.junit.utils.config.WithConfigExtension; import java.util.stream.Stream; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -142,7 +146,29 @@ private static Stream testCanAvoidSamplingPriorityLockArguments() { Arguments.of(MANUAL, SAMPLER_KEEP, false), Arguments.of(REMOTE_USER_RATE, SAMPLER_KEEP, false), Arguments.of(DATA_JOBS, SAMPLER_KEEP, false), + // DSM is left at its config default (disabled) for this parameterized run, so the + // DATA_STREAMS case is false here for a config reason. The two dedicated tests below + // cover the mechanism itself with data.streams.enabled explicitly set both ways. Arguments.of(DATA_STREAMS, SAMPLER_KEEP, false), Arguments.of(EXTERNAL_OVERRIDE, SAMPLER_KEEP, false)); } + + @Test + @WithConfig(key = DATA_STREAMS_ENABLED, value = "true") + void dataStreamsMechanismCanAvoidSamplingPriorityLockWhenDataStreamsEnabled() { + // The DATA_STREAMS case is priority-independent: the mechanism alone unlocks the priority. + assertTrue(canAvoidSamplingPriorityLock(USER_DROP, DATA_STREAMS)); + assertTrue(canAvoidSamplingPriorityLock(SAMPLER_KEEP, DATA_STREAMS)); + assertTrue(canAvoidSamplingPriorityLock(UNSET, DATA_STREAMS)); + // Enabling DSM must not unlock any other mechanism. + assertFalse(canAvoidSamplingPriorityLock(USER_DROP, MANUAL)); + assertFalse(canAvoidSamplingPriorityLock(USER_DROP, DEFAULT)); + } + + @Test + @WithConfig(key = DATA_STREAMS_ENABLED, value = "false") + void dataStreamsMechanismCannotAvoidSamplingPriorityLockWhenDataStreamsDisabled() { + assertFalse(canAvoidSamplingPriorityLock(USER_DROP, DATA_STREAMS)); + assertFalse(canAvoidSamplingPriorityLock(SAMPLER_KEEP, DATA_STREAMS)); + } } From c06736551298adc7ba2ba1e9bc1470d50127e796 Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Mon, 31 Aug 2026 14:41:19 -0400 Subject: [PATCH 30/32] fix: suppress DSM-only Kafka poll-span billing gap and address code review findings Fixes a billing gap where the standalone kafka.poll span/trace created for DSM was never forced to USER_DROP, so it would be billed as APM even with Kafka tracing disabled. Also delegates Config.isDataStreamsEnabled() to InstrumenterConfig, dedupes each KafkaDecorator's hardcoded integration-name list into shared constants used by TRACING_ENABLED and every module constructor, extracts the duplicated DSM-drop guard in KafkaStreamTaskInstrumentation into a helper, and rewrites InstrumenterModuleTest to use @WithConfig instead of raw reflection. Co-Authored-By: Claude Sonnet 5 --- dd-java-agent/agent-tooling/build.gradle | 1 + .../agent/tooling/InstrumenterModuleTest.java | 107 ++++-------------- .../ConsumerCoordinatorInstrumentation.java | 2 +- .../KafkaConsumerInfoInstrumentation.java | 11 +- .../KafkaConsumerInstrumentation.java | 2 +- .../kafka_clients/KafkaDecorator.java | 6 +- .../KafkaProducerInstrumentation.java | 2 +- .../MetadataInstrumentation.java | 2 +- .../test/groovy/KafkaClientTestBase.groovy | 67 +++++++++++ .../ConsumerCoordinatorInstrumentation.java | 2 +- .../KafkaConsumerInfoInstrumentation.java | 2 +- .../KafkaConsumerInstrumentation.java | 2 +- .../KafkaProducerInstrumentation.java | 2 +- ...egacyKafkaConsumerInfoInstrumentation.java | 2 +- .../MessageListenerInstrumentation.java | 2 +- .../MetadataInstrumentation.java | 2 +- ...tCommitCallbackInvokerInstrumentation.java | 2 +- .../kafka_clients38/KafkaDecorator.java | 6 +- .../kafka_clients38/RecordsAdvice.java | 9 ++ ...afkaClientDataStreamsOnlyForkedTest.groovy | 27 +++++ .../KafkaStreamTaskInstrumentation.java | 38 +++---- .../kafka_streams/KafkaStreamsDecorator.java | 6 +- ...NodeRecordDeserializerInstrumentation.java | 2 +- .../main/java/datadog/trace/api/Config.java | 7 +- 24 files changed, 185 insertions(+), 126 deletions(-) diff --git a/dd-java-agent/agent-tooling/build.gradle b/dd-java-agent/agent-tooling/build.gradle index c32d4824c34..ea2ee3f42f5 100644 --- a/dd-java-agent/agent-tooling/build.gradle +++ b/dd-java-agent/agent-tooling/build.gradle @@ -48,6 +48,7 @@ dependencies { api libs.bytebuddyagent testImplementation project(':dd-java-agent:testing') + testImplementation project(':utils:test-junit-utils') testImplementation libs.bytebuddy testImplementation group: 'com.google.guava', name: 'guava-testlib', version: '20.0' diff --git a/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java b/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java index 7037acccd09..1fb55ea9103 100644 --- a/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java +++ b/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/InstrumenterModuleTest.java @@ -4,26 +4,16 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.trace.agent.tooling.InstrumenterModule.TargetSystem; -import datadog.trace.api.InstrumenterConfig; -import java.lang.reflect.Field; +import datadog.trace.test.junit.utils.config.WithConfig; +import datadog.trace.test.junit.utils.config.WithConfigExtension; import java.util.HashSet; import java.util.Set; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +@ExtendWith(WithConfigExtension.class) class InstrumenterModuleTest { - private static final Field ENABLED_FIELD = getEnabledField(); - - private static Field getEnabledField() { - try { - Field field = InstrumenterModule.class.getDeclaredField("enabled"); - field.setAccessible(true); - return field; - } catch (NoSuchFieldException e) { - throw new ExceptionInInitializerError(e); - } - } - @Test void testDataStreamsIsApplicableWithTracing() { Set enabledSystems = new HashSet<>(); @@ -66,94 +56,47 @@ void testDataStreamsIsApplicableWithNeither() { } @Test - void testDataStreamsIsEnabledWhenDataStreamsEnabledOverridesFalse() - throws IllegalAccessException { - // When DSM is enabled, isEnabled() should return true even if super.isEnabled() is false - // This tests the edge case where trace.kafka.enabled=false but DSM is explicitly enabled + @WithConfig(key = "trace.test-kafka-module.enabled", value = "false") + @WithConfig(key = "data.streams.enabled", value = "true") + void testDataStreamsIsEnabledWhenDataStreamsEnabledOverridesFalse() { + // When tracing for this integration is disabled but DSM is explicitly enabled, + // isEnabled() should still return true. InstrumenterModule.DataStreams module = new InstrumenterModule.DataStreams("test-kafka-module") {}; - // Set the enabled field to false to simulate disabled tracing - ENABLED_FIELD.setBoolean(module, false); - - // Override InstrumenterConfig to return true for isDataStreamsEnabled() - InstrumenterConfig originalConfig = InstrumenterConfig.get(); - boolean originalDataStreamsEnabled = originalConfig.isDataStreamsEnabled(); - setFieldInConfig(originalConfig, "dataStreamsEnabled", true); - - try { - assertTrue(module.isEnabled()); - } finally { - setFieldInConfig(originalConfig, "dataStreamsEnabled", originalDataStreamsEnabled); - } + assertTrue(module.isEnabled()); } @Test - void testDataStreamsIsEnabledWhenSuperEnabledIsTrue() throws IllegalAccessException { - // When super.isEnabled() is true, isEnabled() should return true regardless of DSM state + @WithConfig(key = "trace.test-kafka-module.enabled", value = "true") + @WithConfig(key = "data.streams.enabled", value = "false") + void testDataStreamsIsEnabledWhenSuperEnabledIsTrue() { + // When super.isEnabled() is true, isEnabled() should return true regardless of DSM state. InstrumenterModule.DataStreams module = new InstrumenterModule.DataStreams("test-kafka-module") {}; - // Set the enabled field to true to simulate enabled tracing - ENABLED_FIELD.setBoolean(module, true); - - // Ensure DSM is disabled - InstrumenterConfig originalConfig = InstrumenterConfig.get(); - boolean originalDataStreamsEnabled = originalConfig.isDataStreamsEnabled(); - setFieldInConfig(originalConfig, "dataStreamsEnabled", false); - - try { - assertTrue(module.isEnabled()); - } finally { - setFieldInConfig(originalConfig, "dataStreamsEnabled", originalDataStreamsEnabled); - } + assertTrue(module.isEnabled()); } @Test - void testDataStreamsIsEnabledWhenBothEnabled() throws IllegalAccessException { - // When both super.isEnabled() and DSM are enabled, isEnabled() should return true + @WithConfig(key = "trace.test-kafka-module.enabled", value = "true") + @WithConfig(key = "data.streams.enabled", value = "true") + void testDataStreamsIsEnabledWhenBothEnabled() { + // When both super.isEnabled() and DSM are enabled, isEnabled() should return true. InstrumenterModule.DataStreams module = new InstrumenterModule.DataStreams("test-kafka-module") {}; - ENABLED_FIELD.setBoolean(module, true); - - InstrumenterConfig originalConfig = InstrumenterConfig.get(); - boolean originalDataStreamsEnabled = originalConfig.isDataStreamsEnabled(); - setFieldInConfig(originalConfig, "dataStreamsEnabled", true); - - try { - assertTrue(module.isEnabled()); - } finally { - setFieldInConfig(originalConfig, "dataStreamsEnabled", originalDataStreamsEnabled); - } + assertTrue(module.isEnabled()); } @Test - void testDataStreamsIsEnabledWhenBothDisabled() throws IllegalAccessException { - // When both super.isEnabled() and DSM are disabled, isEnabled() should return false + @WithConfig(key = "trace.test-kafka-module.enabled", value = "false") + @WithConfig(key = "data.streams.enabled", value = "false") + void testDataStreamsIsEnabledWhenBothDisabled() { + // When both super.isEnabled() and DSM are disabled, isEnabled() should return false. InstrumenterModule.DataStreams module = new InstrumenterModule.DataStreams("test-kafka-module") {}; - ENABLED_FIELD.setBoolean(module, false); - - InstrumenterConfig originalConfig = InstrumenterConfig.get(); - boolean originalDataStreamsEnabled = originalConfig.isDataStreamsEnabled(); - setFieldInConfig(originalConfig, "dataStreamsEnabled", false); - - try { - assertFalse(module.isEnabled()); - } finally { - setFieldInConfig(originalConfig, "dataStreamsEnabled", originalDataStreamsEnabled); - } - } - - private static void setFieldInConfig(Object target, String fieldName, Object value) { - try { - Field field = target.getClass().getDeclaredField(fieldName); - field.setAccessible(true); - field.set(target, value); - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new RuntimeException(e); - } + assertFalse(module.isEnabled()); } } diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/ConsumerCoordinatorInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/ConsumerCoordinatorInstrumentation.java index 83fee16be9b..e787036d762 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/ConsumerCoordinatorInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/ConsumerCoordinatorInstrumentation.java @@ -27,7 +27,7 @@ public final class ConsumerCoordinatorInstrumentation extends InstrumenterModule implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public ConsumerCoordinatorInstrumentation() { - super("kafka", "kafka-0.11"); + super(KafkaDecorator.INTEGRATION_NAME, KafkaDecorator.LEGACY_INTEGRATION_NAME); } @Override diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java index 18f62fa8e34..7b1aa90cd82 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java @@ -20,6 +20,8 @@ import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.api.Config; +import datadog.trace.api.sampling.PrioritySampling; +import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -48,7 +50,7 @@ public final class KafkaConsumerInfoInstrumentation extends InstrumenterModule.D implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public KafkaConsumerInfoInstrumentation() { - super("kafka", "kafka-0.11"); + super(KafkaDecorator.INTEGRATION_NAME, KafkaDecorator.LEGACY_INTEGRATION_NAME); } @Override @@ -254,6 +256,13 @@ public static AgentScope onEnter(@Advice.This KafkaConsumer consumer) { if (traceConfig().isDataStreamsEnabled()) { final AgentSpan span = startSpan(JAVA_KAFKA.toString(), KAFKA_POLL); + // setSamplingPriority is trace-level: it resolves to the local root span. This 2-arg + // startSpan honours the active scope, so `span` may be a child of a customer trace. Only + // force the DSM-only drop when `span` is the local root, otherwise we would silently drop + // that whole customer trace. + if (!KafkaDecorator.TRACING_ENABLED && span.getLocalRootSpan() == span) { + span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); + } return activateSpan(span); } return null; diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInstrumentation.java index 7fcc8a31aa9..d516d48a169 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInstrumentation.java @@ -31,7 +31,7 @@ public final class KafkaConsumerInstrumentation extends InstrumenterModule.DataS implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public KafkaConsumerInstrumentation() { - super("kafka", "kafka-0.11"); + super(KafkaDecorator.INTEGRATION_NAME, KafkaDecorator.LEGACY_INTEGRATION_NAME); } @Override diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDecorator.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDecorator.java index 2f0ab40a643..eef3c71103e 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDecorator.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDecorator.java @@ -31,6 +31,10 @@ public class KafkaDecorator extends MessagingClientDecorator { private static final String KAFKA = "kafka"; + // Kept in sync with the names each kafka-clients-0.11 instrumentation module passes to its own + // super(...) constructor call, so TRACING_ENABLED can't drift from what is actually registered. + public static final String INTEGRATION_NAME = KAFKA; + public static final String LEGACY_INTEGRATION_NAME = "kafka-0.11"; public static final CharSequence JAVA_KAFKA = UTF8BytesString.create("java-kafka"); public static final CharSequence KAFKA_CONSUME = UTF8BytesString.create( @@ -47,7 +51,7 @@ public class KafkaDecorator extends MessagingClientDecorator { public static final boolean TRACING_ENABLED = InstrumenterConfig.get() .isIntegrationEnabled( - Arrays.asList("kafka", "kafka-0.11"), + Arrays.asList(INTEGRATION_NAME, LEGACY_INTEGRATION_NAME), InstrumenterConfig.get().isIntegrationsEnabled()); public static final String KAFKA_PRODUCED_KEY = "x_datadog_kafka_produced"; private final String spanKind; diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java index 95f1eaf5532..fe232fea06d 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java @@ -65,7 +65,7 @@ public final class KafkaProducerInstrumentation extends InstrumenterModule.DataS implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public KafkaProducerInstrumentation() { - super("kafka", "kafka-0.11"); + super(KafkaDecorator.INTEGRATION_NAME, KafkaDecorator.LEGACY_INTEGRATION_NAME); } @Override diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/MetadataInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/MetadataInstrumentation.java index d2557db92e7..871e82cc6b5 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/MetadataInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/MetadataInstrumentation.java @@ -28,7 +28,7 @@ public class MetadataInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { public MetadataInstrumentation() { - super("kafka", "kafka-0.11"); + super(KafkaDecorator.INTEGRATION_NAME, KafkaDecorator.LEGACY_INTEGRATION_NAME); } @Override diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy index 88a55b8b4ce..7bf4553d867 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy @@ -1820,3 +1820,70 @@ class KafkaClientDataStreamsOnlyActiveLocalTraceForkedTest extends KafkaClientTe producer?.close() } } + +// Regression guard for the poll-span suppression site: KafkaConsumerInfoInstrumentation's +// RecordsAdvice creates a standalone "kafka.poll" span/trace around every consumer.poll() call +// whenever DSM is enabled, regardless of whether Kafka APM tracing itself is enabled. Unlike the +// other cases above, this test intentionally keeps the "kafka.poll" trace instead of relying on +// the base class's DROP_KAFKA_POLL filter, so the assertion actually observes what gets written. +class KafkaClientDataStreamsOnlyPollSpanForkedTest extends KafkaClientTestBase { + static final ListWriter.Filter ACCEPT_ALL = new ListWriter.Filter() { + @Override + boolean accept(List trace) { + return true + } + } + + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig("integrations.enabled", "false") + injectSysConfig("data.streams.enabled", "true") + } + + def setup() { + // Undo the base class's DROP_KAFKA_POLL filter: this test needs to see the "kafka.poll" trace. + TEST_WRITER.setFilter(ACCEPT_ALL) + } + + @Override + String service() { + return "kafka" + } + + @Override + boolean hasQueueSpan() { + return false + } + + @Override + boolean splitByDestination() { + return false + } + + @Override + boolean isDataStreamsEnabled() { + return true + } + + def "poll span is forced to USER_DROP when kafka tracing is disabled and DSM is enabled"() { + setup: + def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) + consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") + def consumer = new KafkaConsumer(consumerProperties) + consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, 0))) + + when: "the consumer polls with no active local trace and no records to consume" + KafkaTestUtils.getRecords(consumer) + + then: "the standalone kafka.poll trace is forced to USER_DROP, so it is not billed as APM" + TEST_WRITER.waitForTraces(1) + def trace = TEST_WRITER[0] + trace.size() == 1 + trace[0].getResourceName().toString() == "kafka.poll" + trace[0].getSamplingPriority() == PrioritySampling.USER_DROP + + cleanup: + consumer?.close() + } +} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/ConsumerCoordinatorInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/ConsumerCoordinatorInstrumentation.java index 9c14f94cb0c..0a2457261fa 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/ConsumerCoordinatorInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/ConsumerCoordinatorInstrumentation.java @@ -16,7 +16,7 @@ public final class ConsumerCoordinatorInstrumentation extends InstrumenterModule implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public ConsumerCoordinatorInstrumentation() { - super("kafka", "kafka-3.8"); + super(KafkaDecorator.INTEGRATION_NAME, KafkaDecorator.LEGACY_INTEGRATION_NAME); } @Override diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfoInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfoInstrumentation.java index d37315d37b4..e824b1af9ae 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfoInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfoInstrumentation.java @@ -30,7 +30,7 @@ public final class KafkaConsumerInfoInstrumentation extends InstrumenterModule.D Instrumenter.WithTypeStructure { public KafkaConsumerInfoInstrumentation() { - super("kafka", "kafka-3.8"); + super(KafkaDecorator.INTEGRATION_NAME, KafkaDecorator.LEGACY_INTEGRATION_NAME); } @Override diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentation.java index 21931394117..202fcd37770 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentation.java @@ -23,7 +23,7 @@ public final class KafkaConsumerInstrumentation extends InstrumenterModule.DataS implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public KafkaConsumerInstrumentation() { - super("kafka", "kafka-3.8"); + super(KafkaDecorator.INTEGRATION_NAME, KafkaDecorator.LEGACY_INTEGRATION_NAME); } @Override diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaProducerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaProducerInstrumentation.java index ce1c4735d9c..a5a25128f3a 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaProducerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaProducerInstrumentation.java @@ -20,7 +20,7 @@ public final class KafkaProducerInstrumentation extends InstrumenterModule.DataS implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public KafkaProducerInstrumentation() { - super("kafka", "kafka-3.8"); + super(KafkaDecorator.INTEGRATION_NAME, KafkaDecorator.LEGACY_INTEGRATION_NAME); } @Override diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/LegacyKafkaConsumerInfoInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/LegacyKafkaConsumerInfoInstrumentation.java index a3f661fcc8f..67833947dab 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/LegacyKafkaConsumerInfoInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/LegacyKafkaConsumerInfoInstrumentation.java @@ -30,7 +30,7 @@ public final class LegacyKafkaConsumerInfoInstrumentation extends InstrumenterMo Instrumenter.WithTypeStructure { public LegacyKafkaConsumerInfoInstrumentation() { - super("kafka", "kafka-3.8"); + super(KafkaDecorator.INTEGRATION_NAME, KafkaDecorator.LEGACY_INTEGRATION_NAME); } @Override diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MessageListenerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MessageListenerInstrumentation.java index 2124f853645..9da06603724 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MessageListenerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MessageListenerInstrumentation.java @@ -24,7 +24,7 @@ public class MessageListenerInstrumentation extends InstrumenterModule.Tracing implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { public MessageListenerInstrumentation() { - super("kafka", "kafka-3.8"); + super(KafkaDecorator.INTEGRATION_NAME, KafkaDecorator.LEGACY_INTEGRATION_NAME); } @Override diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MetadataInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MetadataInstrumentation.java index f2fbab71c87..7bab5497904 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MetadataInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MetadataInstrumentation.java @@ -19,7 +19,7 @@ public class MetadataInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { public MetadataInstrumentation() { - super("kafka", "kafka-3.8"); + super(KafkaDecorator.INTEGRATION_NAME, KafkaDecorator.LEGACY_INTEGRATION_NAME); } @Override diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/OffsetCommitCallbackInvokerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/OffsetCommitCallbackInvokerInstrumentation.java index 5a450b29ad3..62dd9416b85 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/OffsetCommitCallbackInvokerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/OffsetCommitCallbackInvokerInstrumentation.java @@ -13,7 +13,7 @@ public class OffsetCommitCallbackInvokerInstrumentation extends InstrumenterModule.DataStreams implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public OffsetCommitCallbackInvokerInstrumentation() { - super("kafka", "kafka-3.8"); + super(KafkaDecorator.INTEGRATION_NAME, KafkaDecorator.LEGACY_INTEGRATION_NAME); } @Override diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaDecorator.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaDecorator.java index 2b16ad34af0..1a1172d264d 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaDecorator.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaDecorator.java @@ -31,6 +31,10 @@ public class KafkaDecorator extends MessagingClientDecorator { private static final String KAFKA = "kafka"; + // Kept in sync with the names each kafka-clients-3.8 instrumentation module passes to its own + // super(...) constructor call, so TRACING_ENABLED can't drift from what is actually registered. + public static final String INTEGRATION_NAME = KAFKA; + public static final String LEGACY_INTEGRATION_NAME = "kafka-3.8"; public static final CharSequence JAVA_KAFKA = UTF8BytesString.create("java-kafka"); public static final CharSequence KAFKA_CONSUME = UTF8BytesString.create( @@ -47,7 +51,7 @@ public class KafkaDecorator extends MessagingClientDecorator { public static final boolean TRACING_ENABLED = InstrumenterConfig.get() .isIntegrationEnabled( - Arrays.asList("kafka", "kafka-3.8"), + Arrays.asList(INTEGRATION_NAME, LEGACY_INTEGRATION_NAME), InstrumenterConfig.get().isIntegrationsEnabled()); public static final String KAFKA_PRODUCED_KEY = "x_datadog_kafka_produced"; private final String spanKind; diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/RecordsAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/RecordsAdvice.java index b8f3dff049a..a3c6b964c24 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/RecordsAdvice.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/RecordsAdvice.java @@ -8,6 +8,8 @@ import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.KAFKA_POLL; import datadog.trace.api.Config; +import datadog.trace.api.sampling.PrioritySampling; +import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -41,6 +43,13 @@ public static AgentScope onEnter(@Advice.This ConsumerDelegate consumer) { if (traceConfig().isDataStreamsEnabled()) { final AgentSpan span = startSpan(JAVA_KAFKA.toString(), KAFKA_POLL); + // setSamplingPriority is trace-level: it resolves to the local root span. This 2-arg + // startSpan honours the active scope, so `span` may be a child of a customer trace. Only + // force the DSM-only drop when `span` is the local root, otherwise we would silently drop + // that whole customer trace. + if (!KafkaDecorator.TRACING_ENABLED && span.getLocalRootSpan() == span) { + span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); + } return activateSpan(span); } return null; diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientDataStreamsOnlyForkedTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientDataStreamsOnlyForkedTest.groovy index c3b6734cac3..98c47a98e37 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientDataStreamsOnlyForkedTest.groovy +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientDataStreamsOnlyForkedTest.groovy @@ -139,3 +139,30 @@ class KafkaClientDataStreamsOnlyActiveLocalTraceForkedTest extends KafkaClientDa producer?.close() } } + +/** + * Regression guard for the poll-span suppression site: KafkaConsumerInfoInstrumentation's + * RecordsAdvice creates a standalone "kafka.poll" span/trace around every consumer.poll() call + * whenever DSM is enabled, regardless of whether Kafka APM tracing itself is enabled. + */ +class KafkaClientDataStreamsOnlyPollSpanForkedTest extends KafkaClientDataStreamsOnlyForkedTest { + + def "poll span is forced to USER_DROP"() { + setup: + def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) + consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") + def consumer = new KafkaConsumer(consumerProperties) + consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, 0))) + + when: "the consumer polls with no active local trace and no records to consume" + KafkaTestUtils.getRecords(consumer) + + then: "the standalone kafka.poll trace is forced to USER_DROP, so it is not billed as APM" + def pollSpan = findSpan("kafka.poll") + pollSpan != null + pollSpan.getSamplingPriority() == PrioritySampling.USER_DROP + + cleanup: + consumer?.close() + } +} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java index ef546e3ed59..e2842543d98 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java @@ -64,7 +64,21 @@ public class KafkaStreamTaskInstrumentation extends InstrumenterModule.DataStrea implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public KafkaStreamTaskInstrumentation() { - super("kafka", "kafka-streams"); + super(KafkaStreamsDecorator.INTEGRATION_NAME, KafkaStreamsDecorator.LEGACY_INTEGRATION_NAME); + } + + // setSamplingPriority is trace-level: it resolves to the local root span. Only force the + // DSM-only drop when this instrumentation owns the whole local trace, i.e. nothing was + // active when we started (no local parent, no header-extracted parent) and the local root + // really is the first span we created here. + private static void maybeDropForDataStreamsOnly( + final AgentSpan span, final AgentSpan localActiveSpan, final AgentSpan ourLocalRoot) { + if (!KafkaStreamsDecorator.TRACING_ENABLED + && traceConfig().isDataStreamsEnabled() + && localActiveSpan == null + && span.getLocalRootSpan() == ourLocalRoot) { + span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); + } } @Override @@ -303,17 +317,8 @@ public static void start( // spans are written out together by TraceStructureWriter when running in strict mode } - // setSamplingPriority is trace-level: it resolves to the local root span. Only force the - // DSM-only drop when this instrumentation owns the whole local trace, i.e. nothing was - // active when we started (no local parent, no header-extracted parent) and the local root - // really is the first span we created here. final AgentSpan ourLocalRoot = queueSpan == null ? span : queueSpan; - if (!KafkaStreamsDecorator.TRACING_ENABLED - && traceConfig().isDataStreamsEnabled() - && localActiveSpan == null - && span.getLocalRootSpan() == ourLocalRoot) { - span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); - } + maybeDropForDataStreamsOnly(span, localActiveSpan, ourLocalRoot); String applicationId = null; if (streamTaskContext != null) { applicationId = streamTaskContext.getApplicationId(); @@ -386,17 +391,8 @@ public static void start( // spans are written out together by TraceStructureWriter when running in strict mode } - // setSamplingPriority is trace-level: it resolves to the local root span. Only force the - // DSM-only drop when this instrumentation owns the whole local trace, i.e. nothing was - // active when we started (no local parent, no header-extracted parent) and the local root - // really is the first span we created here. final AgentSpan ourLocalRoot = queueSpan == null ? span : queueSpan; - if (!KafkaStreamsDecorator.TRACING_ENABLED - && traceConfig().isDataStreamsEnabled() - && localActiveSpan == null - && span.getLocalRootSpan() == ourLocalRoot) { - span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); - } + maybeDropForDataStreamsOnly(span, localActiveSpan, ourLocalRoot); String applicationId = null; if (streamTaskContext != null) { applicationId = streamTaskContext.getApplicationId(); diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsDecorator.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsDecorator.java index 3bb5832450d..99a505a0d63 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsDecorator.java +++ b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsDecorator.java @@ -24,6 +24,10 @@ public class KafkaStreamsDecorator extends MessagingClientDecorator { private static final String KAFKA = "kafka"; + // Kept in sync with the names each kafka-streams-0.11 instrumentation module passes to its own + // super(...) constructor call, so TRACING_ENABLED can't drift from what is actually registered. + public static final String INTEGRATION_NAME = KAFKA; + public static final String LEGACY_INTEGRATION_NAME = "kafka-streams"; public static final CharSequence JAVA_KAFKA = UTF8BytesString.create("java-kafka-streams"); public static final CharSequence KAFKA_CONSUME = UTF8BytesString.create( @@ -36,7 +40,7 @@ public class KafkaStreamsDecorator extends MessagingClientDecorator { public static final boolean TRACING_ENABLED = InstrumenterConfig.get() .isIntegrationEnabled( - Arrays.asList("kafka", "kafka-streams"), + Arrays.asList(INTEGRATION_NAME, LEGACY_INTEGRATION_NAME), InstrumenterConfig.get().isIntegrationsEnabled()); public static final String KAFKA_PRODUCED_KEY = "x_datadog_kafka_produced"; diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsSourceNodeRecordDeserializerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsSourceNodeRecordDeserializerInstrumentation.java index 3f47e9cdf6e..f2c473591d1 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsSourceNodeRecordDeserializerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsSourceNodeRecordDeserializerInstrumentation.java @@ -20,7 +20,7 @@ public class KafkaStreamsSourceNodeRecordDeserializerInstrumentation implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public KafkaStreamsSourceNodeRecordDeserializerInstrumentation() { - super("kafka", "kafka-streams"); + super(KafkaStreamsDecorator.INTEGRATION_NAME, KafkaStreamsDecorator.LEGACY_INTEGRATION_NAME); } @Override diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index fade2b4c417..7af8f65207d 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -53,7 +53,6 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_DATA_JOBS_OPENLINEAGE_TIMEOUT_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_DATA_JOBS_PARSE_SPARK_PLAN_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_DATA_STREAMS_BUCKET_DURATION; -import static datadog.trace.api.ConfigDefaults.DEFAULT_DATA_STREAMS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_DB_CLIENT_HOST_SPLIT_BY_HOST; import static datadog.trace.api.ConfigDefaults.DEFAULT_DB_CLIENT_HOST_SPLIT_BY_INSTANCE; import static datadog.trace.api.ConfigDefaults.DEFAULT_DB_CLIENT_HOST_SPLIT_BY_INSTANCE_TYPE_SUFFIX; @@ -382,7 +381,6 @@ import static datadog.trace.api.config.GeneralConfig.DATA_JOBS_OPENLINEAGE_TIMEOUT_ENABLED; import static datadog.trace.api.config.GeneralConfig.DATA_JOBS_PARSE_SPARK_PLAN_ENABLED; import static datadog.trace.api.config.GeneralConfig.DATA_STREAMS_BUCKET_DURATION_SECONDS; -import static datadog.trace.api.config.GeneralConfig.DATA_STREAMS_ENABLED; import static datadog.trace.api.config.GeneralConfig.DATA_STREAMS_TRANSACTION_EXTRACTORS; import static datadog.trace.api.config.GeneralConfig.DOGSTATSD_ARGS; import static datadog.trace.api.config.GeneralConfig.DOGSTATSD_HOST; @@ -1367,7 +1365,6 @@ public static String getHostName() { private final boolean dataJobsParseSparkPlanEnabled; private final boolean dataJobsExperimentalFeaturesEnabled; - private final boolean dataStreamsEnabled; private final float dataStreamsBucketDurationSeconds; private final String dataStreamsTransactionExtractors; @@ -3192,8 +3189,6 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) DATA_JOBS_EXPERIMENTAL_FEATURES_ENABLED, DEFAULT_DATA_JOBS_EXPERIMENTAL_FEATURES_ENABLED); - dataStreamsEnabled = - configProvider.getBoolean(DATA_STREAMS_ENABLED, DEFAULT_DATA_STREAMS_ENABLED); dataStreamsBucketDurationSeconds = configProvider.getFloat( DATA_STREAMS_BUCKET_DURATION_SECONDS, DEFAULT_DATA_STREAMS_BUCKET_DURATION); @@ -5169,7 +5164,7 @@ public boolean isAwsServerless() { } public boolean isDataStreamsEnabled() { - return dataStreamsEnabled; + return instrumenterConfig.isDataStreamsEnabled(); } public float getDataStreamsBucketDurationSeconds() { From 9043a04afde2561cc8db405b04c38ce464700958 Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 4 Sep 2026 12:59:41 -0400 Subject: [PATCH 31/32] fix: widen maybeDropForDataStreamsOnly visibility to satisfy Muzzle Muzzle's static-reference check expects public_or_protected static for a method invoked across nested advice classes, since those classes may be injected into a different classloader than their enclosing class. The package-private helper caused KafkaStreamTaskInstrumentation to be muzzled (blocked) at runtime; making it public resolves the mismatch. --- .../kafka_streams/KafkaStreamTaskInstrumentation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java index e2842543d98..41e1447180c 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java @@ -71,7 +71,7 @@ public KafkaStreamTaskInstrumentation() { // DSM-only drop when this instrumentation owns the whole local trace, i.e. nothing was // active when we started (no local parent, no header-extracted parent) and the local root // really is the first span we created here. - private static void maybeDropForDataStreamsOnly( + public static void maybeDropForDataStreamsOnly( final AgentSpan span, final AgentSpan localActiveSpan, final AgentSpan ourLocalRoot) { if (!KafkaStreamsDecorator.TRACING_ENABLED && traceConfig().isDataStreamsEnabled() From 81a34b530f847a5e03ca9b1dd67bd20c58ae9491 Mon Sep 17 00:00:00 2001 From: Eric Firth Date: Fri, 4 Sep 2026 12:44:14 -0400 Subject: [PATCH 32/32] fix: never create a real span for DSM-only Kafka consume/produce paths Ensures the cross-language contract holds: when APM tracing is disabled and DSM is enabled, no span is ever written to the agent for Kafka instrumentations. Replaces AgentTracer.startSpan with a lightweight Utils.newPathwayOnlySpan shim across kafka-clients-0.11, kafka-clients-3.8, and kafka-streams-0.11, and extracts shared DSM checkpoint/inject and transaction-tracking logic into helper methods to remove duplication. Co-Authored-By: Claude Sonnet 5 --- .../KafkaConsumerInfoInstrumentation.java | 14 +- .../KafkaProducerInstrumentation.java | 38 +- .../kafka_clients/TracingIterator.java | 171 +++++---- ...afkaClientDataStreamsOnlyForkedTest.groovy | 253 +++++++++++++ .../test/groovy/KafkaClientTestBase.groovy | 342 ------------------ .../kafka_clients38/ProducerAdvice.java | 38 +- .../kafka_clients38/RecordsAdvice.java | 14 +- .../kafka_clients38/TracingIterator.java | 170 +++++---- ...afkaClientDataStreamsOnlyForkedTest.groovy | 92 ++--- .../instrumentation/kafka_common/Utils.java | 20 + .../KafkaStreamTaskInstrumentation.java | 223 +++++++----- ...fkaStreamsDataStreamsOnlyForkedTest.groovy | 58 +-- 12 files changed, 712 insertions(+), 721 deletions(-) create mode 100644 dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientDataStreamsOnlyForkedTest.groovy diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java index 7b1aa90cd82..9446334ce24 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java @@ -20,8 +20,6 @@ import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.api.Config; -import datadog.trace.api.sampling.PrioritySampling; -import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -254,15 +252,11 @@ public static AgentScope onEnter(@Advice.This KafkaConsumer consumer) { } } - if (traceConfig().isDataStreamsEnabled()) { + if (traceConfig().isDataStreamsEnabled() && KafkaDecorator.TRACING_ENABLED) { + // DSM-only mode (tracing disabled) never creates a real poll span: TracingIterator + // carries its pathway context on a lightweight, never-collected span shim instead, so + // there's nothing here that needs wrapping/protecting from being force-dropped. final AgentSpan span = startSpan(JAVA_KAFKA.toString(), KAFKA_POLL); - // setSamplingPriority is trace-level: it resolves to the local root span. This 2-arg - // startSpan honours the active scope, so `span` may be a child of a customer trace. Only - // force the DSM-only drop when `span` is the local root, otherwise we would silently drop - // that whole customer trace. - if (!KafkaDecorator.TRACING_ENABLED && span.getLocalRootSpan() == span) { - span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); - } return activateSpan(span); } return null; diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java index fe232fea06d..580ce12d19b 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java @@ -19,6 +19,7 @@ import static datadog.trace.instrumentation.kafka_clients.KafkaDecorator.TIME_IN_QUEUE_ENABLED; import static datadog.trace.instrumentation.kafka_common.StreamingContext.STREAMING_CONTEXT; import static datadog.trace.instrumentation.kafka_common.Utils.DSM_TRANSACTION_SOURCE_READER; +import static datadog.trace.instrumentation.kafka_common.Utils.newPathwayOnlySpan; import static java.util.Collections.singletonMap; import static net.bytebuddy.matcher.ElementMatchers.isConstructor; import static net.bytebuddy.matcher.ElementMatchers.isMethod; @@ -38,8 +39,6 @@ import datadog.trace.api.datastreams.DataStreamsTags; import datadog.trace.api.datastreams.DataStreamsTransactionExtractor; import datadog.trace.api.datastreams.StatsPoint; -import datadog.trace.api.sampling.PrioritySampling; -import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -157,29 +156,32 @@ public static AgentScope onEnter( final AgentSpanContext extractedContext = extractContextAndGetSpanContext(record.headers(), TextMapExtractAdapter.GETTER); - final AgentSpan localActiveSpan = activeSpan(); - final AgentSpan span; final AgentSpan callbackParentSpan; - if (extractedContext != null) { - span = startSpan(JAVA_KAFKA.toString(), KAFKA_PRODUCE, extractedContext); + if (!KafkaDecorator.TRACING_ENABLED && traceConfig().isDataStreamsEnabled()) { + // DSM-only mode: never create a real span, so nothing for this integration is ever + // written to the agent. The pathway is carried on a lightweight, never-collected span + // shim instead, falling back to whatever pathway the currently active span (if any) + // is carrying so a consume->produce chain keeps propagating the same pathway. + final AgentSpan localActiveSpan = activeSpan(); + final AgentSpanContext pathwaySource = + extractedContext != null + ? extractedContext + : localActiveSpan == null ? null : localActiveSpan.spanContext(); + span = newPathwayOnlySpan(pathwaySource); callbackParentSpan = span; } else { - span = startSpan(JAVA_KAFKA.toString(), KAFKA_PRODUCE); - callbackParentSpan = localActiveSpan; - // setSamplingPriority is trace-level: it resolves to the local root span. This 2-arg - // startSpan honours the active scope, so `span` may be a child of a customer trace - // (localActiveSpan above). Only force the DSM-only drop when `span` is the local root, - // otherwise we would silently drop that whole customer trace. - if (!KafkaDecorator.TRACING_ENABLED - && traceConfig().isDataStreamsEnabled() - && span.getLocalRootSpan() == span) { - span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); + if (extractedContext != null) { + span = startSpan(JAVA_KAFKA.toString(), KAFKA_PRODUCE, extractedContext); + callbackParentSpan = span; + } else { + span = startSpan(JAVA_KAFKA.toString(), KAFKA_PRODUCE); + callbackParentSpan = activeSpan(); } + PRODUCER_DECORATE.afterStart(span); + PRODUCER_DECORATE.onProduce(span, record, producerConfig, clusterId); } - PRODUCER_DECORATE.afterStart(span); - PRODUCER_DECORATE.onProduce(span, record, producerConfig, clusterId); callback = new KafkaProducerCallback(callback, callbackParentSpan, span, clusterId); diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/TracingIterator.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/TracingIterator.java index cd242c152ec..c53d56a4ea5 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/TracingIterator.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/TracingIterator.java @@ -18,6 +18,7 @@ import static datadog.trace.instrumentation.kafka_common.StreamingContext.STREAMING_CONTEXT; import static datadog.trace.instrumentation.kafka_common.Utils.DSM_TRANSACTION_SOURCE_READER; import static datadog.trace.instrumentation.kafka_common.Utils.computePayloadSizeBytes; +import static datadog.trace.instrumentation.kafka_common.Utils.newPathwayOnlySpan; import static java.util.concurrent.TimeUnit.MILLISECONDS; import datadog.context.Context; @@ -28,8 +29,6 @@ import datadog.trace.api.datastreams.DataStreamsContext; import datadog.trace.api.datastreams.DataStreamsTags; import datadog.trace.api.datastreams.DataStreamsTransactionExtractor; -import datadog.trace.api.sampling.PrioritySampling; -import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; @@ -99,64 +98,11 @@ protected void startNewRecordSpan(ConsumerRecord val) { previousSpan.finishWithEndToEnd(); } } - AgentSpan span, queueSpan = null; if (val != null) { - if (!Config.get().isKafkaClientPropagationDisabledForTopic(val.topic())) { - final AgentSpanContext spanContext = - extractContextAndGetSpanContext(val.headers(), GETTER); - long timeInQueueStart = GETTER.extractTimeInQueueStart(val.headers()); - if (timeInQueueStart == 0 || !TIME_IN_QUEUE_ENABLED) { - span = startSpan(JAVA_KAFKA.toString(), operationName, spanContext); - } else { - queueSpan = - startSpan( - JAVA_KAFKA.toString(), - KAFKA_DELIVER, - spanContext, - MILLISECONDS.toMicros(timeInQueueStart)); - BROKER_DECORATE.afterStart(queueSpan); - BROKER_DECORATE.onTimeInQueue(queueSpan, val); - span = startSpan(JAVA_KAFKA.toString(), operationName, queueSpan.spanContext()); - BROKER_DECORATE.beforeFinish(queueSpan); - // The queueSpan will be finished after inner span has been activated to ensure that - // spans are written out together by TraceStructureWriter when running in strict mode - } - - if (spanContext == null - && !KafkaDecorator.TRACING_ENABLED - && traceConfig().isDataStreamsEnabled()) { - span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); - } - DataStreamsTags tags = create("kafka", INBOUND, val.topic(), group, clusterId); - final long payloadSize = - traceConfig().isDataStreamsEnabled() ? computePayloadSizeBytes(val) : 0; - if (STREAMING_CONTEXT.isDisabledForTopic(val.topic())) { - AgentTracer.get() - .getDataStreamsMonitoring() - .setCheckpoint(span, create(tags, val.timestamp(), payloadSize)); - } else { - // when we're in a streaming context we want to consume only from source topics - if (STREAMING_CONTEXT.isSourceTopic(val.topic())) { - // We have to inject the context to headers here, - // since the data received from the source may leave the topology on - // some other instance of the application, breaking the context propagation - // for DSM users - Propagator dsmPropagator = Propagators.forConcern(DSM_CONCERN); - DataStreamsContext dsmContext = create(tags, val.timestamp(), payloadSize); - dsmPropagator.inject(span.with(dsmContext), val.headers(), SETTER); - } - } - } else { - span = startSpan(JAVA_KAFKA.toString(), operationName, null); - if (!KafkaDecorator.TRACING_ENABLED && traceConfig().isDataStreamsEnabled()) { - span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); - } - } - if (val.value() == null) { - span.setTag(InstrumentationTags.TOMBSTONE, true); - } - decorator.afterStart(span); - decorator.onConsume(span, val, group, clusterId, bootstrapServers); + final AgentSpan span = + !KafkaDecorator.TRACING_ENABLED && traceConfig().isDataStreamsEnabled() + ? startDsmOnlyPathwaySpan(val) + : startTracedConsumeSpan(val); if (InstrumenterConfig.get().isLegacyContextManagerEnabled()) { activateNext(span); } else { @@ -165,23 +111,108 @@ && traceConfig().isDataStreamsEnabled()) { previousSpan.finishWithEndToEnd(); } } - if (null != queueSpan) { - queueSpan.finish(); - } - - AgentTracer.get() - .getDataStreamsMonitoring() - .trackTransaction( - span, - DataStreamsTransactionExtractor.Type.KAFKA_CONSUME_HEADERS, - val.headers(), - DSM_TRANSACTION_SOURCE_READER); } } catch (final Exception e) { log.debug("Error starting new record span", e); } } + /** + * Creates and activates the real APM consume span (and, when time-in-queue is enabled, its broker + * parent), tags it, and reports DSM checkpoints/transactions off of it. + */ + private AgentSpan startTracedConsumeSpan(ConsumerRecord val) { + AgentSpan span, queueSpan = null; + if (!Config.get().isKafkaClientPropagationDisabledForTopic(val.topic())) { + final AgentSpanContext spanContext = extractContextAndGetSpanContext(val.headers(), GETTER); + long timeInQueueStart = GETTER.extractTimeInQueueStart(val.headers()); + if (timeInQueueStart == 0 || !TIME_IN_QUEUE_ENABLED) { + span = startSpan(JAVA_KAFKA.toString(), operationName, spanContext); + } else { + queueSpan = + startSpan( + JAVA_KAFKA.toString(), + KAFKA_DELIVER, + spanContext, + MILLISECONDS.toMicros(timeInQueueStart)); + BROKER_DECORATE.afterStart(queueSpan); + BROKER_DECORATE.onTimeInQueue(queueSpan, val); + span = startSpan(JAVA_KAFKA.toString(), operationName, queueSpan.spanContext()); + BROKER_DECORATE.beforeFinish(queueSpan); + // The queueSpan will be finished after inner span has been activated to ensure that + // spans are written out together by TraceStructureWriter when running in strict mode + } + + DataStreamsTags tags = create("kafka", INBOUND, val.topic(), group, clusterId); + final long payloadSize = + traceConfig().isDataStreamsEnabled() ? computePayloadSizeBytes(val) : 0; + reportDsmCheckpointOrInject(span, val, tags, payloadSize); + } else { + span = startSpan(JAVA_KAFKA.toString(), operationName, null); + } + if (val.value() == null) { + span.setTag(InstrumentationTags.TOMBSTONE, true); + } + decorator.afterStart(span); + decorator.onConsume(span, val, group, clusterId, bootstrapServers); + if (null != queueSpan) { + queueSpan.finish(); + } + + trackDsmConsumeTransaction(span, val); + return span; + } + + /** + * DSM-only mode (tracing disabled for kafka, DSM enabled): never creates a real span, so no span + * is ever written to the agent for this integration. Only the pathway checkpoint/injection and + * transaction tracking happen, carried by a lightweight, never-collected span shim. + */ + private AgentSpan startDsmOnlyPathwaySpan(ConsumerRecord val) { + AgentSpan span; + if (!Config.get().isKafkaClientPropagationDisabledForTopic(val.topic())) { + final AgentSpanContext extractedContext = + extractContextAndGetSpanContext(val.headers(), GETTER); + span = newPathwayOnlySpan(extractedContext); + DataStreamsTags tags = create("kafka", INBOUND, val.topic(), group, clusterId); + final long payloadSize = computePayloadSizeBytes(val); + reportDsmCheckpointOrInject(span, val, tags, payloadSize); + } else { + span = newPathwayOnlySpan(null); + } + trackDsmConsumeTransaction(span, val); + return span; + } + + /** + * Reports a DSM checkpoint for {@code val}'s topic, or - when in a streaming context and {@code + * val}'s topic is a source topic - injects the pathway context into its headers so it survives + * leaving the topology on another instance of the application. + */ + private void reportDsmCheckpointOrInject( + AgentSpan span, ConsumerRecord val, DataStreamsTags tags, long payloadSize) { + if (STREAMING_CONTEXT.isDisabledForTopic(val.topic())) { + AgentTracer.get() + .getDataStreamsMonitoring() + .setCheckpoint(span, create(tags, val.timestamp(), payloadSize)); + } else if (STREAMING_CONTEXT.isSourceTopic(val.topic())) { + // when we're in a streaming context we want to consume only from source topics + Propagator dsmPropagator = Propagators.forConcern(DSM_CONCERN); + DataStreamsContext dsmContext = create(tags, val.timestamp(), payloadSize); + dsmPropagator.inject(span.with(dsmContext), val.headers(), SETTER); + } + } + + private void trackDsmConsumeTransaction(AgentSpan span, ConsumerRecord val) { + AgentTracer.get() + .getDataStreamsMonitoring() + .trackTransaction( + span, + DataStreamsTransactionExtractor.Type.KAFKA_CONSUME_HEADERS, + val.headers(), + DSM_TRANSACTION_SOURCE_READER); + } + @Override public void remove() { delegateIterator.remove(); diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientDataStreamsOnlyForkedTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientDataStreamsOnlyForkedTest.groovy new file mode 100644 index 00000000000..77a6b18c51a --- /dev/null +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientDataStreamsOnlyForkedTest.groovy @@ -0,0 +1,253 @@ +import datadog.trace.agent.test.InstrumentationSpecification +import datadog.trace.common.writer.ListWriter +import datadog.trace.core.DDSpan +import org.apache.kafka.clients.consumer.ConsumerConfig +import org.apache.kafka.clients.consumer.KafkaConsumer +import org.apache.kafka.clients.producer.KafkaProducer +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.common.TopicPartition +import org.apache.kafka.common.header.internals.RecordHeader +import org.apache.kafka.common.header.internals.RecordHeaders +import org.apache.kafka.common.serialization.StringSerializer +import org.springframework.kafka.test.rule.KafkaEmbedded +import org.springframework.kafka.test.utils.KafkaTestUtils + +import java.nio.charset.StandardCharsets + +import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace + +/** + * DSM-only coverage for kafka-clients-0.11: when Kafka APM tracing is disabled + * (integrations.enabled=false, or the per-integration trace.kafka.enabled=false override) but DSM + * is enabled, this integration must never create or write a real span/trace, while pathway + * checkpoints must still be tracked. These specs deliberately extend + * {@code InstrumentationSpecification} directly rather than {@code KafkaClientTestBase}: that base + * carries many concrete tests which assert on APM spans that are correctly absent once Kafka + * tracing is off, and Spock would run them (and fail) as inherited tests on every subclass. + */ +abstract class KafkaClientDataStreamsOnlyForkedTest extends InstrumentationSpecification { + static final SHARED_TOPIC = "shared.topic" + + KafkaEmbedded embeddedKafka + + def setup() { + embeddedKafka = new KafkaEmbedded(1, true, SHARED_TOPIC) + embeddedKafka.before() + } + + def cleanup() { + embeddedKafka?.after() + } + + @Override + boolean useStrictTraceWrites() { + return false + } + + @Override + protected boolean isDataStreamsEnabled() { + return true + } + + protected KafkaProducer newProducer() { + return new KafkaProducer( + KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()), + new StringSerializer(), + new StringSerializer()) + } +} + +/** + * Regression guard for the contract that "integration disabled => zero spans of that type ever + * reach the agent": producing and consuming a message in DSM-only mode must not write any trace, + * even though DSM checkpoints for the same produce/consume are still tracked. + */ +class KafkaClientDataStreamsOnlyLocalRootForkedTest extends KafkaClientDataStreamsOnlyForkedTest { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig("integrations.enabled", "false") + } + + def "local-root produce and consume write no spans when kafka tracing is disabled and DSM is enabled"() { + setup: + def kafkaPartition = 0 + def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) + consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") + def consumer = new KafkaConsumer(consumerProperties) + def producer = newProducer() + consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) + + when: "a message is produced with no propagated trace headers, i.e. a genuine local root" + producer.send(new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, "local-root-message")).get() + def recs = KafkaTestUtils.getRecords(consumer) + .records(new TopicPartition(SHARED_TOPIC, kafkaPartition)).iterator() + + then: "the message is delivered and DSM checkpoints are tracked, but no span is ever written" + recs.hasNext() + recs.next().value() == "local-root-message" + !recs.hasNext() + TEST_DATA_STREAMS_WRITER.waitForGroups(2) + TEST_WRITER.isEmpty() + + cleanup: + consumer?.close() + producer?.close() + } +} + +/** + * Regression guard: a message carrying a real, externally-propagated Datadog trace context must + * also write no span, under the same "kafka tracing disabled + DSM enabled" configuration - the + * DSM-only decision is based purely on the tracing/DSM config flags, not on the record's headers. + */ +class KafkaClientDataStreamsOnlyExtractedParentForkedTest extends KafkaClientDataStreamsOnlyForkedTest { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig("integrations.enabled", "false") + } + + def "produce and consume with an extracted parent trace context still write no spans"() { + setup: + def kafkaPartition = 0 + def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) + consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") + def consumer = new KafkaConsumer(consumerProperties) + def producer = newProducer() + consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) + + def existingTraceId = 1234567890123456L + def existingSpanId = 9876543210987654L + def headers = new RecordHeaders() + headers.add(new RecordHeader("x-datadog-trace-id", + String.valueOf(existingTraceId).getBytes(StandardCharsets.UTF_8))) + headers.add(new RecordHeader("x-datadog-parent-id", + String.valueOf(existingSpanId).getBytes(StandardCharsets.UTF_8))) + + when: "a message carrying a real, externally-propagated Datadog trace context is produced" + producer.send(new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, "propagated-trace-message", headers)).get() + def recs = KafkaTestUtils.getRecords(consumer) + .records(new TopicPartition(SHARED_TOPIC, kafkaPartition)).iterator() + + then: "the message is delivered and DSM checkpoints are tracked, but no span is ever written" + recs.hasNext() + recs.next().value() == "propagated-trace-message" + !recs.hasNext() + TEST_DATA_STREAMS_WRITER.waitForGroups(2) + TEST_WRITER.isEmpty() + + cleanup: + consumer?.close() + producer?.close() + } +} + +/** + * Confirms the per-integration override (trace.kafka.enabled=false) suppresses span creation + * identically to the global integrations.enabled=false toggle used above. + */ +class KafkaClientDataStreamsOnlyIntegrationOverrideForkedTest extends KafkaClientDataStreamsOnlyForkedTest { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig("trace.kafka.enabled", "false") + } + + def "local-root produce and consume write no spans when trace.kafka.enabled=false and DSM is enabled"() { + setup: + def kafkaPartition = 0 + def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) + consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") + def consumer = new KafkaConsumer(consumerProperties) + def producer = newProducer() + consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) + + when: "a message is produced with no propagated trace headers, i.e. a genuine local root" + producer.send(new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, "local-root-message")).get() + def recs = KafkaTestUtils.getRecords(consumer) + .records(new TopicPartition(SHARED_TOPIC, kafkaPartition)).iterator() + + then: "the message is delivered and DSM checkpoints are tracked, but no span is ever written" + recs.hasNext() + recs.next().value() == "local-root-message" + !recs.hasNext() + TEST_DATA_STREAMS_WRITER.waitForGroups(2) + TEST_WRITER.isEmpty() + + cleanup: + consumer?.close() + producer?.close() + } +} + +/** + * Regression guard: producing from inside an already active local trace in DSM-only mode must not + * create any additional span either - the surrounding customer trace is untouched. + */ +class KafkaClientDataStreamsOnlyActiveLocalTraceForkedTest extends KafkaClientDataStreamsOnlyForkedTest { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig("integrations.enabled", "false") + } + + def "producing inside an active local trace adds no span to that trace"() { + setup: + def producer = newProducer() + + when: "a message is produced from within an already active local trace" + runUnderTrace("parent") { + producer.send(new ProducerRecord(SHARED_TOPIC, 0, null, "in-active-trace")).get() + } + + then: "the surrounding customer trace contains only its own span, no kafka.produce span" + TEST_WRITER.waitForTraces(1) + TEST_WRITER[0].size() == 1 + TEST_WRITER[0][0].operationName.toString() == "parent" + + cleanup: + producer?.close() + } +} + +/** + * Regression guard for the poll-span suppression site: KafkaConsumerInfoInstrumentation's + * RecordsAdvice used to create a standalone "kafka.poll" span/trace around every consumer.poll() + * call whenever DSM is enabled, regardless of whether Kafka APM tracing itself was enabled. In + * DSM-only mode that span must no longer be created at all. + */ +class KafkaClientDataStreamsOnlyPollSpanForkedTest extends KafkaClientDataStreamsOnlyForkedTest { + static final ListWriter.Filter ACCEPT_ALL = new ListWriter.Filter() { + @Override + boolean accept(List trace) { + return true + } + } + + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig("integrations.enabled", "false") + } + + def "no poll span is created when kafka tracing is disabled and DSM is enabled"() { + setup: + // Undo any default poll-trace filter: this test needs to see the "kafka.poll" trace, if one + // were (incorrectly) written. + TEST_WRITER.setFilter(ACCEPT_ALL) + def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) + consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") + def consumer = new KafkaConsumer(consumerProperties) + consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, 0))) + + when: "the consumer polls with no active local trace and no records to consume" + KafkaTestUtils.getRecords(consumer) + + then: "no standalone kafka.poll trace was written" + TEST_WRITER.isEmpty() + + cleanup: + consumer?.close() + } +} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy index 7bf4553d867..17c3e665f67 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy @@ -14,7 +14,6 @@ import datadog.trace.agent.test.asserts.TraceAssert import datadog.trace.agent.test.naming.VersionedNamingTestBase import datadog.trace.api.Config import datadog.trace.api.DDTags -import datadog.trace.api.sampling.PrioritySampling import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.common.writer.ListWriter @@ -1546,344 +1545,3 @@ class KafkaClientBadBase64HeaderForkedTest extends InstrumentationSpecification } } -// DSM billing-suppression coverage: when tracing is disabled for kafka (globally, via -// integrations.enabled) but DSM is enabled, a genuinely local-root produce/consume span (no -// extracted/real parent trace context) must have its sampling priority forced to USER_DROP so it -// does not count towards APM billing, while a span that joins a real propagated trace must not be. -class KafkaClientDataStreamsOnlyLocalRootForkedTest extends KafkaClientTestBase { - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig("integrations.enabled", "false") - injectSysConfig("data.streams.enabled", "true") - } - - @Override - String service() { - return "kafka" - } - - @Override - boolean hasQueueSpan() { - return false - } - - @Override - boolean splitByDestination() { - return false - } - - @Override - boolean isDataStreamsEnabled() { - return true - } - - def "local-root produce and consume spans are forced to USER_DROP when kafka tracing is disabled and DSM is enabled"() { - setup: - def kafkaPartition = 0 - def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") - def consumer = new KafkaConsumer(consumerProperties) - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - def producer = new KafkaProducer<>(senderProps, new StringSerializer(), new StringSerializer()) - consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) - - when: "a message is produced with no propagated trace headers, i.e. a genuine local root" - def record = new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, "local-root-message") - producer.send(record).get() - - then: "the produce span's trace is forced to USER_DROP to suppress APM billing" - TEST_WRITER.waitForTraces(1) - def producedSpan = TEST_WRITER[0][0] - producedSpan.getSamplingPriority() == PrioritySampling.USER_DROP - - when: "the message is consumed" - def pollResult = KafkaTestUtils.getRecords(consumer) - def recs = pollResult.records(new TopicPartition(SHARED_TOPIC, kafkaPartition)).iterator() - - then: "the consume span's trace is also forced to USER_DROP" - recs.hasNext() - recs.next().value() == "local-root-message" - !recs.hasNext() - TEST_WRITER.waitForTraces(2) - def consumedSpan = TEST_WRITER[1][0] - consumedSpan.getSamplingPriority() == PrioritySampling.USER_DROP - - cleanup: - consumer?.close() - producer?.close() - } -} - -// Regression guard: a span that joins a real, externally-propagated Datadog trace (extracted -// x-datadog-trace-id/x-datadog-parent-id headers) must NOT be forced to USER_DROP, even under the -// same "kafka tracing disabled + DSM enabled" configuration as above, since it is not a local root. -class KafkaClientDataStreamsOnlyExtractedParentForkedTest extends KafkaClientTestBase { - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig("integrations.enabled", "false") - injectSysConfig("data.streams.enabled", "true") - } - - @Override - String service() { - return "kafka" - } - - @Override - boolean hasQueueSpan() { - return false - } - - @Override - boolean splitByDestination() { - return false - } - - @Override - boolean isDataStreamsEnabled() { - return true - } - - def "produce and consume spans with an extracted parent trace context are not forced to USER_DROP"() { - setup: - def kafkaPartition = 0 - def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") - def consumer = new KafkaConsumer(consumerProperties) - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - def producer = new KafkaProducer<>(senderProps, new StringSerializer(), new StringSerializer()) - consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) - - def existingTraceId = 1234567890123456L - def existingSpanId = 9876543210987654L - def headers = new RecordHeaders() - headers.add(new RecordHeader("x-datadog-trace-id", - String.valueOf(existingTraceId).getBytes(StandardCharsets.UTF_8))) - headers.add(new RecordHeader("x-datadog-parent-id", - String.valueOf(existingSpanId).getBytes(StandardCharsets.UTF_8))) - - when: "a message carrying a real, externally-propagated Datadog trace context is produced" - def record = new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, "propagated-trace-message", headers) - producer.send(record).get() - - then: "the produce span joins the propagated trace and is NOT forced to USER_DROP" - TEST_WRITER.waitForTraces(1) - def producedSpan = TEST_WRITER[0][0] - producedSpan.traceId.toLong() == existingTraceId - producedSpan.parentId == existingSpanId - // The injected headers carry no x-datadog-sampling-priority, so the extracted context's - // priority is UNSET and the normal sampler decides. Assert the concrete resulting value - // rather than just "not USER_DROP", so any regression that lands on a different-but-also - // wrong priority is caught too. - producedSpan.getSamplingPriority() == PrioritySampling.SAMPLER_KEEP - - when: "the message is consumed" - def pollResult = KafkaTestUtils.getRecords(consumer) - def recs = pollResult.records(new TopicPartition(SHARED_TOPIC, kafkaPartition)).iterator() - - then: "the consume span also joins the propagated trace and is NOT forced to USER_DROP" - recs.hasNext() - recs.next().value() == "propagated-trace-message" - !recs.hasNext() - TEST_WRITER.waitForTraces(2) - def consumedSpan = TEST_WRITER[1][0] - consumedSpan.traceId.toLong() == existingTraceId - consumedSpan.getSamplingPriority() == PrioritySampling.SAMPLER_KEEP - - cleanup: - consumer?.close() - producer?.close() - } -} - -// Confirms the per-integration override (trace.kafka.enabled=false) suppresses billing identically -// to the global integrations.enabled=false toggle used above, for a genuine local-root span. -class KafkaClientDataStreamsOnlyIntegrationOverrideForkedTest extends KafkaClientTestBase { - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig("trace.kafka.enabled", "false") - injectSysConfig("data.streams.enabled", "true") - } - - @Override - String service() { - return "kafka" - } - - @Override - boolean hasQueueSpan() { - return false - } - - @Override - boolean splitByDestination() { - return false - } - - @Override - boolean isDataStreamsEnabled() { - return true - } - - def "local-root produce and consume spans are forced to USER_DROP when trace.kafka.enabled=false and DSM is enabled"() { - setup: - def kafkaPartition = 0 - def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") - def consumer = new KafkaConsumer(consumerProperties) - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - def producer = new KafkaProducer<>(senderProps, new StringSerializer(), new StringSerializer()) - consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) - - when: "a message is produced with no propagated trace headers, i.e. a genuine local root" - def record = new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, "local-root-message") - producer.send(record).get() - - then: "the produce span's trace is forced to USER_DROP to suppress APM billing" - TEST_WRITER.waitForTraces(1) - def producedSpan = TEST_WRITER[0][0] - producedSpan.getSamplingPriority() == PrioritySampling.USER_DROP - - when: "the message is consumed" - def pollResult = KafkaTestUtils.getRecords(consumer) - def recs = pollResult.records(new TopicPartition(SHARED_TOPIC, kafkaPartition)).iterator() - - then: "the consume span's trace is also forced to USER_DROP" - recs.hasNext() - recs.next().value() == "local-root-message" - !recs.hasNext() - TEST_WRITER.waitForTraces(2) - def consumedSpan = TEST_WRITER[1][0] - consumedSpan.getSamplingPriority() == PrioritySampling.USER_DROP - - cleanup: - consumer?.close() - producer?.close() - } -} - -// Regression guard for the producer suppression site: producing a message from inside an already -// active local trace (e.g. an instrumented HTTP request) must NOT force that customer trace to -// USER_DROP. The produce span is created with a scope-honouring startSpan overload, so it becomes -// a child of the active span, and setSamplingPriority is trace-level - without the local-root -// check on the suppression guard the whole surrounding trace would be silently dropped. -class KafkaClientDataStreamsOnlyActiveLocalTraceForkedTest extends KafkaClientTestBase { - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig("integrations.enabled", "false") - injectSysConfig("data.streams.enabled", "true") - } - - @Override - String service() { - return "kafka" - } - - @Override - boolean hasQueueSpan() { - return false - } - - @Override - boolean splitByDestination() { - return false - } - - @Override - boolean isDataStreamsEnabled() { - return true - } - - def "producing inside an active local trace does not force that trace to USER_DROP"() { - setup: - def kafkaPartition = 0 - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - def producer = new KafkaProducer<>(senderProps, new StringSerializer(), new StringSerializer()) - - when: "a message is produced from within an already active local trace" - runUnderTrace("parent") { - producer.send(new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, "in-active-trace")).get() - } - - then: "the surrounding customer trace is not force-dropped" - TEST_WRITER.waitForTraces(1) - def trace = TEST_WRITER[0] - def localRoot = trace[0].localRootSpan - localRoot.operationName.toString() == "parent" - localRoot.getSamplingPriority() != PrioritySampling.USER_DROP - - cleanup: - producer?.close() - } -} - -// Regression guard for the poll-span suppression site: KafkaConsumerInfoInstrumentation's -// RecordsAdvice creates a standalone "kafka.poll" span/trace around every consumer.poll() call -// whenever DSM is enabled, regardless of whether Kafka APM tracing itself is enabled. Unlike the -// other cases above, this test intentionally keeps the "kafka.poll" trace instead of relying on -// the base class's DROP_KAFKA_POLL filter, so the assertion actually observes what gets written. -class KafkaClientDataStreamsOnlyPollSpanForkedTest extends KafkaClientTestBase { - static final ListWriter.Filter ACCEPT_ALL = new ListWriter.Filter() { - @Override - boolean accept(List trace) { - return true - } - } - - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig("integrations.enabled", "false") - injectSysConfig("data.streams.enabled", "true") - } - - def setup() { - // Undo the base class's DROP_KAFKA_POLL filter: this test needs to see the "kafka.poll" trace. - TEST_WRITER.setFilter(ACCEPT_ALL) - } - - @Override - String service() { - return "kafka" - } - - @Override - boolean hasQueueSpan() { - return false - } - - @Override - boolean splitByDestination() { - return false - } - - @Override - boolean isDataStreamsEnabled() { - return true - } - - def "poll span is forced to USER_DROP when kafka tracing is disabled and DSM is enabled"() { - setup: - def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") - def consumer = new KafkaConsumer(consumerProperties) - consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, 0))) - - when: "the consumer polls with no active local trace and no records to consume" - KafkaTestUtils.getRecords(consumer) - - then: "the standalone kafka.poll trace is forced to USER_DROP, so it is not billed as APM" - TEST_WRITER.waitForTraces(1) - def trace = TEST_WRITER[0] - trace.size() == 1 - trace[0].getResourceName().toString() == "kafka.poll" - trace[0].getSamplingPriority() == PrioritySampling.USER_DROP - - cleanup: - consumer?.close() - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java index 5a1c2f05989..c2b2fc4725d 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java @@ -9,8 +9,6 @@ import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.KAFKA_PRODUCE; import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.PRODUCER_DECORATE; -import datadog.trace.api.sampling.PrioritySampling; -import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -18,6 +16,7 @@ import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags; import datadog.trace.instrumentation.kafka_common.ClusterIdHolder; import datadog.trace.instrumentation.kafka_common.MetadataState; +import datadog.trace.instrumentation.kafka_common.Utils; import net.bytebuddy.asm.Advice; import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.producer.Callback; @@ -47,29 +46,32 @@ public static AgentScope onEnter( final AgentSpanContext extractedContext = extractContextAndGetSpanContext(record.headers(), TextMapExtractAdapter.GETTER); - final AgentSpan localActiveSpan = activeSpan(); - final AgentSpan span; final AgentSpan callbackParentSpan; - if (extractedContext != null) { - span = startSpan(JAVA_KAFKA.toString(), KAFKA_PRODUCE, extractedContext); + if (!KafkaDecorator.TRACING_ENABLED && traceConfig().isDataStreamsEnabled()) { + // DSM-only mode: never create a real span, so nothing for this integration is ever + // written to the agent. The pathway is carried on a lightweight, never-collected span + // shim instead, falling back to whatever pathway the currently active span (if any) + // is carrying so a consume->produce chain keeps propagating the same pathway. + final AgentSpan localActiveSpan = activeSpan(); + final AgentSpanContext pathwaySource = + extractedContext != null + ? extractedContext + : localActiveSpan == null ? null : localActiveSpan.spanContext(); + span = Utils.newPathwayOnlySpan(pathwaySource); callbackParentSpan = span; } else { - span = startSpan(JAVA_KAFKA.toString(), KAFKA_PRODUCE); - callbackParentSpan = localActiveSpan; - // setSamplingPriority is trace-level: it resolves to the local root span. This 2-arg - // startSpan honours the active scope, so `span` may be a child of a customer trace - // (localActiveSpan above). Only force the DSM-only drop when `span` is the local root, - // otherwise we would silently drop that whole customer trace. - if (!KafkaDecorator.TRACING_ENABLED - && traceConfig().isDataStreamsEnabled() - && span.getLocalRootSpan() == span) { - span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); + if (extractedContext != null) { + span = startSpan(JAVA_KAFKA.toString(), KAFKA_PRODUCE, extractedContext); + callbackParentSpan = span; + } else { + span = startSpan(JAVA_KAFKA.toString(), KAFKA_PRODUCE); + callbackParentSpan = activeSpan(); } + PRODUCER_DECORATE.afterStart(span); + PRODUCER_DECORATE.onProduce(span, record, producerConfig, clusterId); } - PRODUCER_DECORATE.afterStart(span); - PRODUCER_DECORATE.onProduce(span, record, producerConfig, clusterId); callback = new KafkaProducerCallback(callback, callbackParentSpan, span, clusterId); diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/RecordsAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/RecordsAdvice.java index a3c6b964c24..2884eaecf9d 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/RecordsAdvice.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/RecordsAdvice.java @@ -8,8 +8,6 @@ import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.KAFKA_POLL; import datadog.trace.api.Config; -import datadog.trace.api.sampling.PrioritySampling; -import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -41,15 +39,11 @@ public static AgentScope onEnter(@Advice.This ConsumerDelegate consumer) { } } - if (traceConfig().isDataStreamsEnabled()) { + if (traceConfig().isDataStreamsEnabled() && KafkaDecorator.TRACING_ENABLED) { + // DSM-only mode (tracing disabled) never creates a real poll span: TracingIterator carries + // its pathway context on a lightweight, never-collected span shim instead, so there's + // nothing here that needs wrapping/protecting from being force-dropped. final AgentSpan span = startSpan(JAVA_KAFKA.toString(), KAFKA_POLL); - // setSamplingPriority is trace-level: it resolves to the local root span. This 2-arg - // startSpan honours the active scope, so `span` may be a child of a customer trace. Only - // force the DSM-only drop when `span` is the local root, otherwise we would silently drop - // that whole customer trace. - if (!KafkaDecorator.TRACING_ENABLED && span.getLocalRootSpan() == span) { - span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); - } return activateSpan(span); } return null; diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterator.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterator.java index a14c3da29d6..524accbe86f 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterator.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterator.java @@ -22,8 +22,6 @@ import datadog.trace.api.datastreams.DataStreamsContext; import datadog.trace.api.datastreams.DataStreamsTags; import datadog.trace.api.datastreams.DataStreamsTransactionExtractor; -import datadog.trace.api.sampling.PrioritySampling; -import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; @@ -95,64 +93,11 @@ protected void startNewRecordSpan(ConsumerRecord val) { previousSpan.finishWithEndToEnd(); } } - AgentSpan span, queueSpan = null; if (val != null) { - if (!Config.get().isKafkaClientPropagationDisabledForTopic(val.topic())) { - final AgentSpanContext spanContext = - extractContextAndGetSpanContext(val.headers(), GETTER); - long timeInQueueStart = GETTER.extractTimeInQueueStart(val.headers()); - if (timeInQueueStart == 0 || !KafkaDecorator.TIME_IN_QUEUE_ENABLED) { - span = startSpan(JAVA_KAFKA.toString(), operationName, spanContext); - } else { - queueSpan = - startSpan( - JAVA_KAFKA.toString(), - KafkaDecorator.KAFKA_DELIVER, - spanContext, - MILLISECONDS.toMicros(timeInQueueStart)); - KafkaDecorator.BROKER_DECORATE.afterStart(queueSpan); - KafkaDecorator.BROKER_DECORATE.onTimeInQueue(queueSpan, val); - span = startSpan(JAVA_KAFKA.toString(), operationName, queueSpan.spanContext()); - KafkaDecorator.BROKER_DECORATE.beforeFinish(queueSpan); - // The queueSpan will be finished after inner span has been activated to ensure that - // spans are written out together by TraceStructureWriter when running in strict mode - } - - if (spanContext == null - && !KafkaDecorator.TRACING_ENABLED - && traceConfig().isDataStreamsEnabled()) { - span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); - } - DataStreamsTags tags = create("kafka", INBOUND, val.topic(), group, clusterId); - final long payloadSize = - traceConfig().isDataStreamsEnabled() ? Utils.computePayloadSizeBytes(val) : 0; - if (StreamingContext.STREAMING_CONTEXT.isDisabledForTopic(val.topic())) { - AgentTracer.get() - .getDataStreamsMonitoring() - .setCheckpoint(span, create(tags, val.timestamp(), payloadSize)); - } else { - // when we're in a streaming context we want to consume only from source topics - if (StreamingContext.STREAMING_CONTEXT.isSourceTopic(val.topic())) { - // We have to inject the context to headers here, - // since the data received from the source may leave the topology on - // some other instance of the application, breaking the context propagation - // for DSM users - Propagator dsmPropagator = Propagators.forConcern(DSM_CONCERN); - DataStreamsContext dsmContext = create(tags, val.timestamp(), payloadSize); - dsmPropagator.inject(span.with(dsmContext), val.headers(), SETTER); - } - } - } else { - span = startSpan(JAVA_KAFKA.toString(), operationName, null); - if (!KafkaDecorator.TRACING_ENABLED && traceConfig().isDataStreamsEnabled()) { - span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); - } - } - if (val.value() == null) { - span.setTag(InstrumentationTags.TOMBSTONE, true); - } - decorator.afterStart(span); - decorator.onConsume(span, val, group, clusterId, bootstrapServers); + final AgentSpan span = + !KafkaDecorator.TRACING_ENABLED && traceConfig().isDataStreamsEnabled() + ? startDsmOnlyPathwaySpan(val) + : startTracedConsumeSpan(val); if (InstrumenterConfig.get().isLegacyContextManagerEnabled()) { activateNext(span); } else { @@ -161,23 +106,108 @@ && traceConfig().isDataStreamsEnabled()) { previousSpan.finishWithEndToEnd(); } } - if (null != queueSpan) { - queueSpan.finish(); - } - - AgentTracer.get() - .getDataStreamsMonitoring() - .trackTransaction( - span, - DataStreamsTransactionExtractor.Type.KAFKA_CONSUME_HEADERS, - val.headers(), - Utils.DSM_TRANSACTION_SOURCE_READER); } } catch (final Exception e) { log.debug("Error starting new record span", e); } } + /** + * Creates and activates the real APM consume span (and, when time-in-queue is enabled, its broker + * parent), tags it, and reports DSM checkpoints/transactions off of it. + */ + private AgentSpan startTracedConsumeSpan(ConsumerRecord val) { + AgentSpan span, queueSpan = null; + if (!Config.get().isKafkaClientPropagationDisabledForTopic(val.topic())) { + final AgentSpanContext spanContext = extractContextAndGetSpanContext(val.headers(), GETTER); + long timeInQueueStart = GETTER.extractTimeInQueueStart(val.headers()); + if (timeInQueueStart == 0 || !KafkaDecorator.TIME_IN_QUEUE_ENABLED) { + span = startSpan(JAVA_KAFKA.toString(), operationName, spanContext); + } else { + queueSpan = + startSpan( + JAVA_KAFKA.toString(), + KafkaDecorator.KAFKA_DELIVER, + spanContext, + MILLISECONDS.toMicros(timeInQueueStart)); + KafkaDecorator.BROKER_DECORATE.afterStart(queueSpan); + KafkaDecorator.BROKER_DECORATE.onTimeInQueue(queueSpan, val); + span = startSpan(JAVA_KAFKA.toString(), operationName, queueSpan.spanContext()); + KafkaDecorator.BROKER_DECORATE.beforeFinish(queueSpan); + // The queueSpan will be finished after inner span has been activated to ensure that + // spans are written out together by TraceStructureWriter when running in strict mode + } + + DataStreamsTags tags = create("kafka", INBOUND, val.topic(), group, clusterId); + final long payloadSize = + traceConfig().isDataStreamsEnabled() ? Utils.computePayloadSizeBytes(val) : 0; + reportDsmCheckpointOrInject(span, val, tags, payloadSize); + } else { + span = startSpan(JAVA_KAFKA.toString(), operationName, null); + } + if (val.value() == null) { + span.setTag(InstrumentationTags.TOMBSTONE, true); + } + decorator.afterStart(span); + decorator.onConsume(span, val, group, clusterId, bootstrapServers); + if (null != queueSpan) { + queueSpan.finish(); + } + + trackDsmConsumeTransaction(span, val); + return span; + } + + /** + * DSM-only mode (tracing disabled for kafka, DSM enabled): never creates a real span, so no span + * is ever written to the agent for this integration. Only the pathway checkpoint/injection and + * transaction tracking happen, carried by a lightweight, never-collected span shim. + */ + private AgentSpan startDsmOnlyPathwaySpan(ConsumerRecord val) { + AgentSpan span; + if (!Config.get().isKafkaClientPropagationDisabledForTopic(val.topic())) { + final AgentSpanContext extractedContext = + extractContextAndGetSpanContext(val.headers(), GETTER); + span = Utils.newPathwayOnlySpan(extractedContext); + DataStreamsTags tags = create("kafka", INBOUND, val.topic(), group, clusterId); + final long payloadSize = Utils.computePayloadSizeBytes(val); + reportDsmCheckpointOrInject(span, val, tags, payloadSize); + } else { + span = Utils.newPathwayOnlySpan(null); + } + trackDsmConsumeTransaction(span, val); + return span; + } + + /** + * Reports a DSM checkpoint for {@code val}'s topic, or - when in a streaming context and {@code + * val}'s topic is a source topic - injects the pathway context into its headers so it survives + * leaving the topology on another instance of the application. + */ + private void reportDsmCheckpointOrInject( + AgentSpan span, ConsumerRecord val, DataStreamsTags tags, long payloadSize) { + if (StreamingContext.STREAMING_CONTEXT.isDisabledForTopic(val.topic())) { + AgentTracer.get() + .getDataStreamsMonitoring() + .setCheckpoint(span, create(tags, val.timestamp(), payloadSize)); + } else if (StreamingContext.STREAMING_CONTEXT.isSourceTopic(val.topic())) { + // when we're in a streaming context we want to consume only from source topics + Propagator dsmPropagator = Propagators.forConcern(DSM_CONCERN); + DataStreamsContext dsmContext = create(tags, val.timestamp(), payloadSize); + dsmPropagator.inject(span.with(dsmContext), val.headers(), SETTER); + } + } + + private void trackDsmConsumeTransaction(AgentSpan span, ConsumerRecord val) { + AgentTracer.get() + .getDataStreamsMonitoring() + .trackTransaction( + span, + DataStreamsTransactionExtractor.Type.KAFKA_CONSUME_HEADERS, + val.headers(), + Utils.DSM_TRANSACTION_SOURCE_READER); + } + @Override public void remove() { delegateIterator.remove(); diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientDataStreamsOnlyForkedTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientDataStreamsOnlyForkedTest.groovy index 98c47a98e37..0bf9b64a94a 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientDataStreamsOnlyForkedTest.groovy +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientDataStreamsOnlyForkedTest.groovy @@ -1,5 +1,4 @@ import datadog.trace.agent.test.InstrumentationSpecification -import datadog.trace.api.sampling.PrioritySampling import org.apache.kafka.clients.consumer.ConsumerConfig import org.apache.kafka.clients.consumer.KafkaConsumer import org.apache.kafka.clients.producer.KafkaProducer @@ -13,11 +12,10 @@ import org.springframework.kafka.test.utils.KafkaTestUtils import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace /** - * DSM billing-suppression coverage for kafka-clients-3.8, mirroring the kafka-clients-0.11 suite. - * - *

    Both scenarios run under "kafka tracing disabled (integrations.enabled=false) + DSM enabled", - * the configuration in which the suppression guard is active. These specs deliberately do not - * extend {@code KafkaClientTestBase}: that base asserts Code Origin tags, which are correctly + * DSM-only coverage for kafka-clients-3.8: when Kafka APM tracing is disabled + * (integrations.enabled=false) but DSM is enabled, this integration must never create or write a + * real span/trace, while pathway checkpoints must still be tracked. These specs deliberately do + * not extend {@code KafkaClientTestBase}: that base asserts on APM spans, which are correctly * absent once Kafka tracing is off. */ abstract class KafkaClientDataStreamsOnlyForkedTest extends InstrumentationSpecification { @@ -39,26 +37,15 @@ abstract class KafkaClientDataStreamsOnlyForkedTest extends InstrumentationSpeci return false } + @Override + protected boolean isDataStreamsEnabled() { + return true + } + @Override void configurePreAgent() { super.configurePreAgent() injectSysConfig("integrations.enabled", "false") - injectSysConfig("data.streams.enabled", "true") - } - - /** - * Locates a written span by operation name. The consumer's kafka.poll spans interleave - * unpredictably with the produce/consume traces, so indexing into TEST_WRITER is unreliable. - */ - protected findSpan(String operationName) { - for (int i = 0; i < 100; i++) { - def span = TEST_WRITER.flatten().find { it.operationName.toString() == operationName } - if (span != null) { - return span - } - Thread.sleep(100) - } - return null } protected KafkaProducer newProducer() { @@ -70,12 +57,13 @@ abstract class KafkaClientDataStreamsOnlyForkedTest extends InstrumentationSpeci } /** - * A genuinely local-root produce/consume span must have its sampling priority forced to USER_DROP - * so it does not count towards APM billing. + * Regression guard for the contract that "integration disabled => zero spans of that type ever + * reach the agent": producing and consuming a message in DSM-only mode must not write any trace, + * even though DSM checkpoints for the same produce/consume are still tracked. */ -class KafkaClientDataStreamsOnlyLocalRootForkedTest extends KafkaClientDataStreamsOnlyForkedTest { +class KafkaClientDataStreamsOnlyNoSpansForkedTest extends KafkaClientDataStreamsOnlyForkedTest { - def "local-root produce and consume spans are forced to USER_DROP"() { + def "produce and consume in DSM-only mode write no spans, but still track DSM checkpoints"() { setup: def kafkaPartition = 0 def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) @@ -84,27 +72,19 @@ class KafkaClientDataStreamsOnlyLocalRootForkedTest extends KafkaClientDataStrea def producer = newProducer() consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) - when: "a message is produced with no propagated trace headers, i.e. a genuine local root" - producer.send(new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, "local-root-message")).get() - - then: "the produce span's trace is forced to USER_DROP to suppress APM billing" - TEST_WRITER.waitForTraces(1) - def produceSpan = findSpan("kafka.produce") - produceSpan != null - produceSpan.getSamplingPriority() == PrioritySampling.USER_DROP - - when: "the message is consumed" + when: "a message is produced and consumed with no active trace" + producer.send(new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, "dsm-only-message")).get() def recs = KafkaTestUtils.getRecords(consumer) .records(new TopicPartition(SHARED_TOPIC, kafkaPartition)).iterator() - then: "the consume span's trace is also forced to USER_DROP" + then: "the message is delivered and DSM checkpoints are tracked for both hops" recs.hasNext() - recs.next().value() == "local-root-message" + recs.next().value() == "dsm-only-message" !recs.hasNext() - TEST_WRITER.waitForTraces(2) - def consumeSpan = findSpan("kafka.consume") - consumeSpan != null - consumeSpan.getSamplingPriority() == PrioritySampling.USER_DROP + TEST_DATA_STREAMS_WRITER.waitForGroups(2, 15000) + + and: "no span was ever created or written for this integration" + TEST_WRITER.isEmpty() cleanup: consumer?.close() @@ -113,14 +93,12 @@ class KafkaClientDataStreamsOnlyLocalRootForkedTest extends KafkaClientDataStrea } /** - * Regression guard for the producer suppression site: producing a message from inside an already - * active local trace must NOT force that customer trace to USER_DROP. The produce span is created - * with a scope-honouring startSpan overload, so it becomes a child of the active span, and - * setSamplingPriority is trace-level. + * Regression guard: producing from inside an already active local trace in DSM-only mode must not + * create any additional span either - the surrounding customer trace is untouched. */ class KafkaClientDataStreamsOnlyActiveLocalTraceForkedTest extends KafkaClientDataStreamsOnlyForkedTest { - def "producing inside an active local trace does not force that trace to USER_DROP"() { + def "producing inside an active local trace in DSM-only mode adds no span to that trace"() { setup: def producer = newProducer() @@ -129,11 +107,10 @@ class KafkaClientDataStreamsOnlyActiveLocalTraceForkedTest extends KafkaClientDa producer.send(new ProducerRecord(SHARED_TOPIC, 0, null, "in-active-trace")).get() } - then: "the surrounding customer trace is not force-dropped" + then: "the surrounding customer trace contains only its own span, no kafka.produce span" TEST_WRITER.waitForTraces(1) - def localRoot = TEST_WRITER[0][0].localRootSpan - localRoot.operationName.toString() == "parent" - localRoot.getSamplingPriority() != PrioritySampling.USER_DROP + TEST_WRITER[0].size() == 1 + TEST_WRITER[0][0].operationName.toString() == "parent" cleanup: producer?.close() @@ -142,12 +119,13 @@ class KafkaClientDataStreamsOnlyActiveLocalTraceForkedTest extends KafkaClientDa /** * Regression guard for the poll-span suppression site: KafkaConsumerInfoInstrumentation's - * RecordsAdvice creates a standalone "kafka.poll" span/trace around every consumer.poll() call - * whenever DSM is enabled, regardless of whether Kafka APM tracing itself is enabled. + * RecordsAdvice used to create a standalone "kafka.poll" span/trace around every consumer.poll() + * call whenever DSM is enabled, regardless of whether Kafka APM tracing itself was enabled. In + * DSM-only mode that span must no longer be created at all. */ class KafkaClientDataStreamsOnlyPollSpanForkedTest extends KafkaClientDataStreamsOnlyForkedTest { - def "poll span is forced to USER_DROP"() { + def "no poll span is created in DSM-only mode"() { setup: def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") @@ -157,10 +135,8 @@ class KafkaClientDataStreamsOnlyPollSpanForkedTest extends KafkaClientDataStream when: "the consumer polls with no active local trace and no records to consume" KafkaTestUtils.getRecords(consumer) - then: "the standalone kafka.poll trace is forced to USER_DROP, so it is not billed as APM" - def pollSpan = findSpan("kafka.poll") - pollSpan != null - pollSpan.getSamplingPriority() == PrioritySampling.USER_DROP + then: "no standalone kafka.poll trace was written" + TEST_WRITER.isEmpty() cleanup: consumer?.close() diff --git a/dd-java-agent/instrumentation/kafka/kafka-common/src/main/java/datadog/trace/instrumentation/kafka_common/Utils.java b/dd-java-agent/instrumentation/kafka/kafka-common/src/main/java/datadog/trace/instrumentation/kafka_common/Utils.java index d0f7eb4fcea..51648f299af 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-common/src/main/java/datadog/trace/instrumentation/kafka_common/Utils.java +++ b/dd-java-agent/instrumentation/kafka/kafka-common/src/main/java/datadog/trace/instrumentation/kafka_common/Utils.java @@ -1,6 +1,11 @@ package datadog.trace.instrumentation.kafka_common; import datadog.trace.api.datastreams.DataStreamsTransactionTracker; +import datadog.trace.api.datastreams.PathwayContext; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.TagContext; import java.nio.charset.StandardCharsets; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.header.Header; @@ -9,6 +14,21 @@ public final class Utils { private Utils() {} // prevent instantiation + /** + * Builds a span-shaped carrier for a {@link PathwayContext} only, without creating a real trace: + * no sampling, no trace-collector registration, never written to the agent. Used when APM tracing + * is disabled for the integration but DSM is enabled, so a pathway can still be + * propagated/checkpointed through the normal active-span-based DSM APIs. + */ + public static AgentSpan newPathwayOnlySpan(AgentSpanContext extractedContext) { + PathwayContext pathwayContext = + extractedContext == null ? null : extractedContext.getPathwayContext(); + if (pathwayContext == null) { + pathwayContext = AgentTracer.get().getDataStreamsMonitoring().newPathwayContext(); + } + return AgentSpan.fromSpanContext(new TagContext().withPathwayContext(pathwayContext)); + } + public static DataStreamsTransactionTracker.TransactionSourceReader DSM_TRANSACTION_SOURCE_READER = (source, headerName) -> { diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java index 41e1447180c..20598e3a66f 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java @@ -14,6 +14,7 @@ import static datadog.trace.bootstrap.instrumentation.api.Java8BytecodeBridge.rootContext; import static datadog.trace.instrumentation.kafka_common.StreamingContext.STREAMING_CONTEXT; import static datadog.trace.instrumentation.kafka_common.Utils.computePayloadSizeBytes; +import static datadog.trace.instrumentation.kafka_common.Utils.newPathwayOnlySpan; import static datadog.trace.instrumentation.kafka_streams.KafkaStreamsDecorator.BROKER_DECORATE; import static datadog.trace.instrumentation.kafka_streams.KafkaStreamsDecorator.CONSUMER_DECORATE; import static datadog.trace.instrumentation.kafka_streams.KafkaStreamsDecorator.JAVA_KAFKA; @@ -42,8 +43,6 @@ import datadog.trace.api.Config; import datadog.trace.api.datastreams.DataStreamsContext; import datadog.trace.api.datastreams.DataStreamsTags; -import datadog.trace.api.sampling.PrioritySampling; -import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -67,20 +66,6 @@ public KafkaStreamTaskInstrumentation() { super(KafkaStreamsDecorator.INTEGRATION_NAME, KafkaStreamsDecorator.LEGACY_INTEGRATION_NAME); } - // setSamplingPriority is trace-level: it resolves to the local root span. Only force the - // DSM-only drop when this instrumentation owns the whole local trace, i.e. nothing was - // active when we started (no local parent, no header-extracted parent) and the local root - // really is the first span we created here. - public static void maybeDropForDataStreamsOnly( - final AgentSpan span, final AgentSpan localActiveSpan, final AgentSpan ourLocalRoot) { - if (!KafkaStreamsDecorator.TRACING_ENABLED - && traceConfig().isDataStreamsEnabled() - && localActiveSpan == null - && span.getLocalRootSpan() == ourLocalRoot) { - span.setSamplingPriority(PrioritySampling.USER_DROP, SamplingMechanism.DATA_STREAMS); - } - } - @Override public String instrumentedType() { return "org.apache.kafka.streams.processor.internals.StreamTask"; @@ -293,15 +278,33 @@ public static void start( return; } - // Captured before any span is created. A non-null value here means this record is being - // consumed either inside a locally active trace, or under a context that the sibling - // ContextPropagationAdvice extracted from the record headers and attached to the scope - // (it is registered on the same method and runs first). Either way the spans created - // below will not own the resulting trace. - final AgentSpan localActiveSpan = activeSpan(); - AgentSpan span, queueSpan = null; StreamTaskContext streamTaskContext = InstrumentationContext.get(StreamTask.class, StreamTaskContext.class).get(task); + String applicationId = + streamTaskContext != null ? streamTaskContext.getApplicationId() : null; + + final AgentSpan span = + !KafkaStreamsDecorator.TRACING_ENABLED && traceConfig().isDataStreamsEnabled() + ? startDsmOnlyPathwaySpan(record, applicationId) + : startTracedConsumeSpan(record, node, applicationId); + + AgentScope agentScope = activateSpan(span); + + if (streamTaskContext == null) { + streamTaskContext = new StreamTaskContext(); + } + streamTaskContext.setAgentScope(agentScope); + InstrumentationContext.get(StreamTask.class, StreamTaskContext.class) + .put(task, streamTaskContext); + } + + /** + * Creates and activates the real APM consume span (and, when time-in-queue is enabled, its + * broker parent), tags it, and reports DSM checkpoints/transactions off of it. + */ + private static AgentSpan startTracedConsumeSpan( + final StampedRecord record, final ProcessorNode node, final String applicationId) { + AgentSpan span, queueSpan = null; long timeInQueueStart = SR_GETTER.extractTimeInQueueStart(record); if (timeInQueueStart == 0 || !TIME_IN_QUEUE_ENABLED) { span = startSpan(JAVA_KAFKA.toString(), KAFKA_CONSUME); @@ -317,41 +320,56 @@ public static void start( // spans are written out together by TraceStructureWriter when running in strict mode } - final AgentSpan ourLocalRoot = queueSpan == null ? span : queueSpan; - maybeDropForDataStreamsOnly(span, localActiveSpan, ourLocalRoot); - String applicationId = null; - if (streamTaskContext != null) { - applicationId = streamTaskContext.getApplicationId(); - } DataStreamsTags tags = createWithGroup("kafka", INBOUND, applicationId, record.topic()); final long payloadSize = traceConfig().isDataStreamsEnabled() ? computePayloadSizeBytes(record.value) : 0; - if (STREAMING_CONTEXT.isDisabledForTopic(record.topic())) { - AgentTracer.get() - .getDataStreamsMonitoring() - .setCheckpoint(span, create(tags, record.timestamp, payloadSize)); - } else { - if (STREAMING_CONTEXT.isSourceTopic(record.topic())) { - Propagator dsmPropagator = Propagators.forConcern(DSM_CONCERN); - DataStreamsContext dsmContext = create(tags, record.timestamp, payloadSize); - dsmPropagator.inject(span.with(dsmContext), record, SR_SETTER); - } - } + reportDsmCheckpointOrInject(span, record, tags, payloadSize); CONSUMER_DECORATE.afterStart(span); CONSUMER_DECORATE.onConsume(span, record, node); - AgentScope agentScope = activateSpan(span); if (null != queueSpan) { queueSpan.finish(); } + return span; + } - if (streamTaskContext == null) { - streamTaskContext = new StreamTaskContext(); + /** + * DSM-only mode (tracing disabled for kafka-streams, DSM enabled): never creates a real span, + * so no span is ever written to the agent for this integration. Only the pathway + * checkpoint/injection happens, carried by a lightweight, never-collected span shim. + */ + private static AgentSpan startDsmOnlyPathwaySpan( + final StampedRecord record, final String applicationId) { + final AgentSpan localActiveSpan = activeSpan(); + final AgentSpan span = + newPathwayOnlySpan(localActiveSpan == null ? null : localActiveSpan.spanContext()); + + DataStreamsTags tags = createWithGroup("kafka", INBOUND, applicationId, record.topic()); + final long payloadSize = computePayloadSizeBytes(record.value); + reportDsmCheckpointOrInject(span, record, tags, payloadSize); + return span; + } + + /** + * Reports a DSM checkpoint for {@code record}'s topic, or - when in a streaming context and + * {@code record}'s topic is a source topic - injects the pathway context so it survives leaving + * the topology on another instance of the application. + */ + private static void reportDsmCheckpointOrInject( + final AgentSpan span, + final StampedRecord record, + final DataStreamsTags tags, + final long payloadSize) { + if (STREAMING_CONTEXT.isDisabledForTopic(record.topic())) { + AgentTracer.get() + .getDataStreamsMonitoring() + .setCheckpoint(span, create(tags, record.timestamp, payloadSize)); + } else if (STREAMING_CONTEXT.isSourceTopic(record.topic())) { + Propagator dsmPropagator = Propagators.forConcern(DSM_CONCERN); + DataStreamsContext dsmContext = create(tags, record.timestamp, payloadSize); + dsmPropagator.inject(span.with(dsmContext), record, SR_SETTER); } - streamTaskContext.setAgentScope(agentScope); - InstrumentationContext.get(StreamTask.class, StreamTaskContext.class) - .put(task, streamTaskContext); } } @@ -367,15 +385,43 @@ public static void start( return; } - // Captured before any span is created. A non-null value here means this record is being - // consumed either inside a locally active trace, or under a context that the sibling - // ContextPropagationAdvice extracted from the record headers and attached to the scope - // (it is registered on the same method and runs first). Either way the spans created - // below will not own the resulting trace. - final AgentSpan localActiveSpan = activeSpan(); - AgentSpan span, queueSpan = null; StreamTaskContext streamTaskContext = InstrumentationContext.get(StreamTask.class, StreamTaskContext.class).get(task); + String applicationId = + streamTaskContext != null ? streamTaskContext.getApplicationId() : null; + + final AgentSpan span = + !KafkaStreamsDecorator.TRACING_ENABLED && traceConfig().isDataStreamsEnabled() + ? startDsmOnlyPathwaySpan(record, applicationId) + : startTracedConsumeSpan(record, node, applicationId); + + AgentScope agentScope = activateSpan(span); + + if (streamTaskContext == null) { + streamTaskContext = new StreamTaskContext(); + } + streamTaskContext.setAgentScope(agentScope); + InstrumentationContext.get(StreamTask.class, StreamTaskContext.class) + .put(task, streamTaskContext); + } + + private static long payloadSizeBytes(final ProcessorRecordContext record) { + // we have to go through Object to get the RecordMetadata here because the class of `record` + // only implements it after 2.7 (and this class is only used if v >= 2.7) + if ((Object) record instanceof RecordMetadata) { // should always be true + RecordMetadata metadata = (RecordMetadata) (Object) record; + return metadata.serializedKeySize() + metadata.serializedValueSize(); + } + return 0; + } + + /** + * Creates and activates the real APM consume span (and, when time-in-queue is enabled, its + * broker parent), tags it, and reports DSM checkpoints/transactions off of it. + */ + private static AgentSpan startTracedConsumeSpan( + final ProcessorRecordContext record, final ProcessorNode node, final String applicationId) { + AgentSpan span, queueSpan = null; long timeInQueueStart = PR_GETTER.extractTimeInQueueStart(record); if (timeInQueueStart == 0 || !TIME_IN_QUEUE_ENABLED) { span = startSpan(JAVA_KAFKA.toString(), KAFKA_CONSUME); @@ -391,47 +437,54 @@ public static void start( // spans are written out together by TraceStructureWriter when running in strict mode } - final AgentSpan ourLocalRoot = queueSpan == null ? span : queueSpan; - maybeDropForDataStreamsOnly(span, localActiveSpan, ourLocalRoot); - String applicationId = null; - if (streamTaskContext != null) { - applicationId = streamTaskContext.getApplicationId(); - } DataStreamsTags tags = createWithGroup("kafka", INBOUND, applicationId, record.topic()); - - long payloadSize = 0; - // we have to go through Object to get the RecordMetadata here because the class of `record` - // only implements it after 2.7 (and this class is only used if v >= 2.7) - if ((Object) record instanceof RecordMetadata) { // should always be true - RecordMetadata metadata = (RecordMetadata) (Object) record; - payloadSize = metadata.serializedKeySize() + metadata.serializedValueSize(); - } - - if (STREAMING_CONTEXT.isDisabledForTopic(record.topic())) { - AgentTracer.get() - .getDataStreamsMonitoring() - .setCheckpoint(span, create(tags, record.timestamp(), payloadSize)); - } else { - if (STREAMING_CONTEXT.isSourceTopic(record.topic())) { - Propagator dsmPropagator = Propagators.forConcern(DSM_CONCERN); - DataStreamsContext dsmContext = create(tags, record.timestamp(), payloadSize); - dsmPropagator.inject(span.with(dsmContext), record, PR_SETTER); - } - } + long payloadSize = payloadSizeBytes(record); + reportDsmCheckpointOrInject(span, record, tags, payloadSize); CONSUMER_DECORATE.afterStart(span); CONSUMER_DECORATE.onConsume(span, record, node); - AgentScope agentScope = activateSpan(span); if (null != queueSpan) { queueSpan.finish(); } + return span; + } - if (streamTaskContext == null) { - streamTaskContext = new StreamTaskContext(); + /** + * DSM-only mode (tracing disabled for kafka-streams, DSM enabled): never creates a real span, + * so no span is ever written to the agent for this integration. Only the pathway + * checkpoint/injection happens, carried by a lightweight, never-collected span shim. + */ + private static AgentSpan startDsmOnlyPathwaySpan( + final ProcessorRecordContext record, final String applicationId) { + final AgentSpan localActiveSpan = activeSpan(); + final AgentSpan span = + newPathwayOnlySpan(localActiveSpan == null ? null : localActiveSpan.spanContext()); + + DataStreamsTags tags = createWithGroup("kafka", INBOUND, applicationId, record.topic()); + long payloadSize = payloadSizeBytes(record); + reportDsmCheckpointOrInject(span, record, tags, payloadSize); + return span; + } + + /** + * Reports a DSM checkpoint for {@code record}'s topic, or - when in a streaming context and + * {@code record}'s topic is a source topic - injects the pathway context so it survives leaving + * the topology on another instance of the application. + */ + private static void reportDsmCheckpointOrInject( + final AgentSpan span, + final ProcessorRecordContext record, + final DataStreamsTags tags, + final long payloadSize) { + if (STREAMING_CONTEXT.isDisabledForTopic(record.topic())) { + AgentTracer.get() + .getDataStreamsMonitoring() + .setCheckpoint(span, create(tags, record.timestamp(), payloadSize)); + } else if (STREAMING_CONTEXT.isSourceTopic(record.topic())) { + Propagator dsmPropagator = Propagators.forConcern(DSM_CONCERN); + DataStreamsContext dsmContext = create(tags, record.timestamp(), payloadSize); + dsmPropagator.inject(span.with(dsmContext), record, PR_SETTER); } - streamTaskContext.setAgentScope(agentScope); - InstrumentationContext.get(StreamTask.class, StreamTaskContext.class) - .put(task, streamTaskContext); } } diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/test/groovy/KafkaStreamsDataStreamsOnlyForkedTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/test/groovy/KafkaStreamsDataStreamsOnlyForkedTest.groovy index 185121d447b..ad3d2ff0f27 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/test/groovy/KafkaStreamsDataStreamsOnlyForkedTest.groovy +++ b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/test/groovy/KafkaStreamsDataStreamsOnlyForkedTest.groovy @@ -1,7 +1,5 @@ import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.api.config.TraceInstrumentationConfig -import datadog.trace.api.sampling.PrioritySampling -import datadog.trace.bootstrap.instrumentation.api.Tags import org.apache.kafka.clients.producer.KafkaProducer import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.common.header.internals.RecordHeader @@ -20,10 +18,9 @@ import spock.lang.Shared import java.nio.charset.StandardCharsets /** - * DSM billing-suppression coverage for the kafka-streams StreamTask consume spans. - * - *

    Both scenarios run under "kafka tracing disabled (integrations.enabled=false) + DSM enabled", - * the configuration in which the suppression guard is active. + * DSM-only coverage for the kafka-streams StreamTask consume path: when Kafka APM tracing is + * disabled (integrations.enabled=false) but DSM is enabled, this integration must never create or + * write a real span/trace, whether or not the record carries a propagated trace context. */ abstract class KafkaStreamsDataStreamsOnlyForkedTest extends InstrumentationSpecification { static final STREAM_PENDING = "test.pending" @@ -45,7 +42,6 @@ abstract class KafkaStreamsDataStreamsOnlyForkedTest extends InstrumentationSpec void configurePreAgent() { super.configurePreAgent() injectSysConfig("integrations.enabled", "false") - injectSysConfig("data.streams.enabled", "true") } @Override @@ -53,6 +49,11 @@ abstract class KafkaStreamsDataStreamsOnlyForkedTest extends InstrumentationSpec return false } + @Override + protected boolean isDataStreamsEnabled() { + return true + } + protected KafkaStreams startLowercasingTopology() { def config = new Properties() config.putAll(KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString())) @@ -82,29 +83,11 @@ abstract class KafkaStreamsDataStreamsOnlyForkedTest extends InstrumentationSpec new StringSerializer(), new StringSerializer()) } - - /** - * Polls the test writer until a kafka-streams consume span shows up, so the assertions do not - * depend on how many other traces (produce, poll, downstream produce) are flushed first. - */ - protected findStreamsConsumeSpan() { - for (int i = 0; i < 100; i++) { - def span = TEST_WRITER.flatten().find { - it.operationName.toString() == "kafka.consume" && - it.getTag(Tags.COMPONENT)?.toString() == "java-kafka-streams" - } - if (span != null) { - return span - } - Thread.sleep(100) - } - return null - } } /** - * A record with no propagated Datadog trace context produces a genuinely local-root streams - * consume span, which must be forced to USER_DROP so it does not count towards APM billing. + * A record with no propagated Datadog trace context must not create any streams consume span in + * DSM-only mode - only a DSM checkpoint, tracked via the lightweight pathway-only span shim. */ class KafkaStreamsDataStreamsOnlyLocalRootForkedTest extends KafkaStreamsDataStreamsOnlyForkedTest { @@ -118,7 +101,7 @@ class KafkaStreamsDataStreamsOnlyLocalRootForkedTest extends KafkaStreamsDataStr injectSysConfig(TraceInstrumentationConfig.KAFKA_CLIENT_PROPAGATION_DISABLED_TOPICS, STREAM_PENDING) } - def "a local-root streams consume span is forced to USER_DROP"() { + def "a local-root record does not create a streams consume span"() { setup: def streams = startLowercasingTopology() def producer = newProducer() @@ -127,9 +110,8 @@ class KafkaStreamsDataStreamsOnlyLocalRootForkedTest extends KafkaStreamsDataStr producer.send(new ProducerRecord(STREAM_PENDING, "LOCAL ROOT")).get() then: - def consumeSpan = findStreamsConsumeSpan() - consumeSpan != null - consumeSpan.getSamplingPriority() == PrioritySampling.USER_DROP + TEST_DATA_STREAMS_WRITER.waitForGroups(1) + TEST_WRITER.isEmpty() cleanup: producer?.close() @@ -138,14 +120,12 @@ class KafkaStreamsDataStreamsOnlyLocalRootForkedTest extends KafkaStreamsDataStr } /** - * Regression guard: a record carrying a real, externally-propagated Datadog trace context must NOT - * have its trace force-dropped. The sibling ContextPropagationAdvice attaches that extracted - * context to the scope before the span-starting advice runs, and setSamplingPriority is - * trace-level, so without the guard the whole propagated trace would be silently dropped. + * Regression guard: a record carrying a real, externally-propagated Datadog trace context must + * still not create any streams consume span in DSM-only mode. */ class KafkaStreamsDataStreamsOnlyExtractedParentForkedTest extends KafkaStreamsDataStreamsOnlyForkedTest { - def "a streams consume span continuing a propagated trace is not forced to USER_DROP"() { + def "a record continuing a propagated trace does not create a streams consume span either"() { setup: def streams = startLowercasingTopology() def producer = newProducer() @@ -161,10 +141,8 @@ class KafkaStreamsDataStreamsOnlyExtractedParentForkedTest extends KafkaStreamsD producer.send(new ProducerRecord(STREAM_PENDING, null, null, "PROPAGATED", headers)).get() then: - def consumeSpan = findStreamsConsumeSpan() - consumeSpan != null - consumeSpan.traceId.toLong() == existingTraceId - consumeSpan.getSamplingPriority() != PrioritySampling.USER_DROP + TEST_DATA_STREAMS_WRITER.waitForGroups(1) + TEST_WRITER.isEmpty() cleanup: producer?.close()