diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/GeneralConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/GeneralConfig.java index c9963516f1a..aa87269f964 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/GeneralConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/GeneralConfig.java @@ -70,6 +70,14 @@ public final class GeneralConfig { public static final String TRACE_STATS_COMPUTATION_IGNORE_AGENT_VERSION = "trace.stats.computation.ignore.agent.version"; + public static final String TRACE_STATS_CARDINALITY_LIMIT = "trace.stats.cardinality.limit"; + public static final String TRACE_STATS_RESOURCE_CARDINALITY_LIMIT = + "trace.stats.resource.cardinality.limit"; + public static final String TRACE_STATS_HTTP_ENDPOINT_CARDINALITY_LIMIT = + "trace.stats.http_endpoint.cardinality.limit"; + public static final String TRACE_STATS_PEER_TAGS_CARDINALITY_LIMIT = + "trace.stats.peer_tags.cardinality.limit"; + public static final String TRACE_OTEL_SEMANTICS_ENABLED = "trace.otel.semantics.enabled"; public static final String TRACER_METRICS_ENABLED = "trace.tracer.metrics.enabled"; @@ -79,6 +87,7 @@ public final class GeneralConfig { public static final String TRACER_METRICS_MAX_PENDING = "trace.tracer.metrics.max.pending"; public static final String TRACER_METRICS_IGNORED_RESOURCES = "trace.tracer.metrics.ignored.resources"; + public static final String AZURE_APP_SERVICES = "azure.app.services"; public static final String INTERNAL_EXIT_ON_FAILURE = "trace.internal.exit.on.failure"; diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java index b18fce272a4..dbfd436b60a 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java @@ -1,6 +1,7 @@ package datadog.trace.common.metrics; import datadog.metrics.api.Histogram; +import datadog.trace.api.Config; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.util.Hashtable; import datadog.trace.util.LongHashingUtils; @@ -28,36 +29,33 @@ final class AggregateEntry extends Hashtable.Entry { private static final UTF8BytesString[] EMPTY_TAGS = new UTF8BytesString[0]; - // Sentinel substitution is disabled until per-component config is wired in a follow-up PR. - // Tests that need sentinel mode should pass useBlockedSentinel=true explicitly. - static final boolean LIMITS_ENABLED = false; - - // Per-field cardinality handlers. Limits live on MetricCardinalityLimits -- see that class for - // per-field rationale. + // Configurable per-field cardinality handlers — tunable via env var. static final PropertyCardinalityHandler RESOURCE_HANDLER = - new PropertyCardinalityHandler("resource", MetricCardinalityLimits.RESOURCE, LIMITS_ENABLED); + new PropertyCardinalityHandler( + "resource", + Config.get().getTraceStatsCardinalityLimit("resource", MetricCardinalityLimits.RESOURCE)); + static final PropertyCardinalityHandler HTTP_ENDPOINT_HANDLER = + new PropertyCardinalityHandler( + "http_endpoint", + Config.get() + .getTraceStatsCardinalityLimit( + "http_endpoint", MetricCardinalityLimits.HTTP_ENDPOINT)); + + // Fixed per-field cardinality handlers — hardcoded, not user-configurable. static final PropertyCardinalityHandler SERVICE_HANDLER = - new PropertyCardinalityHandler("service", MetricCardinalityLimits.SERVICE, LIMITS_ENABLED); + new PropertyCardinalityHandler("service", MetricCardinalityLimits.SERVICE); static final PropertyCardinalityHandler OPERATION_HANDLER = - new PropertyCardinalityHandler( - "operation", MetricCardinalityLimits.OPERATION, LIMITS_ENABLED); + new PropertyCardinalityHandler("operation", MetricCardinalityLimits.OPERATION); static final PropertyCardinalityHandler SERVICE_SOURCE_HANDLER = - new PropertyCardinalityHandler( - "service_source", MetricCardinalityLimits.SERVICE_SOURCE, LIMITS_ENABLED); + new PropertyCardinalityHandler("service_source", MetricCardinalityLimits.SERVICE_SOURCE); static final PropertyCardinalityHandler TYPE_HANDLER = - new PropertyCardinalityHandler("type", MetricCardinalityLimits.TYPE, LIMITS_ENABLED); + new PropertyCardinalityHandler("type", MetricCardinalityLimits.TYPE); static final PropertyCardinalityHandler SPAN_KIND_HANDLER = - new PropertyCardinalityHandler( - "span_kind", MetricCardinalityLimits.SPAN_KIND, LIMITS_ENABLED); + new PropertyCardinalityHandler("span_kind", MetricCardinalityLimits.SPAN_KIND); static final PropertyCardinalityHandler HTTP_METHOD_HANDLER = - new PropertyCardinalityHandler( - "http_method", MetricCardinalityLimits.HTTP_METHOD, LIMITS_ENABLED); - static final PropertyCardinalityHandler HTTP_ENDPOINT_HANDLER = - new PropertyCardinalityHandler( - "http_endpoint", MetricCardinalityLimits.HTTP_ENDPOINT, LIMITS_ENABLED); + new PropertyCardinalityHandler("http_method", MetricCardinalityLimits.HTTP_METHOD); static final PropertyCardinalityHandler GRPC_STATUS_CODE_HANDLER = - new PropertyCardinalityHandler( - "grpc_status_code", MetricCardinalityLimits.GRPC_STATUS_CODE, LIMITS_ENABLED); + new PropertyCardinalityHandler("grpc_status_code", MetricCardinalityLimits.GRPC_STATUS_CODE); // Single authoritative list used by resetCardinalityHandlers(). populateFrom() and hashOf() keep // named access for readability and to avoid per-span iteration overhead; this array ensures the diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java index fcd6c5fcdb4..6f2072f6495 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java @@ -83,11 +83,11 @@ AggregateEntry findOrInsert(SpanSnapshot snapshot) { * {@code onStatsAggregateDropped}) rather than evicting an established one. Cap is sized to the * steady-state working set, so eviction is rare in the common case. * - *

How often this fires depends on {@link AggregateEntry#LIMITS_ENABLED}. With limits enabled, - * over-cap values for a given field collapse into a shared {@code blocked_by_tracer} bucket, so - * the table itself rarely reaches {@code maxAggregates}. With limits disabled (the default), - * over-cap values flow to distinct buckets and {@code maxAggregates} becomes the load-bearing - * backstop -- the cursor-resumed scan was added specifically for this regime. + *

With per-field cardinality limits enabled, over-cap values for a given field collapse into a + * shared {@code tracer_blocked_value} bucket, so the table itself rarely reaches {@code + * maxAggregates}. Without per-field limits, over-cap values flow to distinct buckets and {@code + * maxAggregates} becomes the load-bearing backstop -- the cursor-resumed scan was added + * specifically for this regime. */ private boolean evictOneStale() { // Two passes -- [cursor, length) then [0, cursor) -- using the half-open-range iterator. The diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricCardinalityLimits.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricCardinalityLimits.java index 58dcd3a70a1..6a679e98133 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricCardinalityLimits.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricCardinalityLimits.java @@ -14,9 +14,10 @@ private MetricCardinalityLimits() {} /** * Distinct {@code resource.name} values per cycle. Highest-cardinality field by far: DB-query - * obfuscations, HTTP route templates, custom resources. Typical service: 30-200 unique. + * obfuscations, HTTP route templates, custom resources. Typical service: 30-200 unique; 1024 + * leaves headroom for high-cardinality SQL/HTTP workloads without risking premature collapse. */ - static final int RESOURCE = 256; + static final int RESOURCE = 1024; /** * Distinct {@code service.name} values per cycle. Local service plus downstream peer-service @@ -57,7 +58,7 @@ private MetricCardinalityLimits() {} * Distinct {@code http.endpoint} values per cycle. Path templates -- same shape as {@code * RESOURCE} for HTTP-heavy services. Only used when {@code includeEndpointInMetrics} is enabled. */ - static final int HTTP_ENDPOINT = 64; + static final int HTTP_ENDPOINT = 512; /** * Distinct gRPC status code values per cycle. gRPC spec defines exactly 17 codes (0-16); 24 diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/PeerTagSchema.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/PeerTagSchema.java index eca5c6c6a1a..1e0d51e84c8 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/PeerTagSchema.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/PeerTagSchema.java @@ -3,6 +3,7 @@ import static datadog.trace.api.DDTags.BASE_SERVICE; import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.trace.api.Config; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import java.util.Set; @@ -74,7 +75,10 @@ static PeerTagSchema of(Set names, String state) { for (int i = 0; i < names.length; i++) { this.handlers[i] = new TagCardinalityHandler( - names[i], MetricCardinalityLimits.PEER_TAG_VALUE, AggregateEntry.LIMITS_ENABLED); + names[i], + Config.get() + .getTraceStatsCardinalityLimit( + "peer_tags", MetricCardinalityLimits.PEER_TAG_VALUE)); } } diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 70ff04932e7..c74f276cff7 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -426,6 +426,7 @@ import static datadog.trace.api.config.GeneralConfig.TRACE_DEBUG; import static datadog.trace.api.config.GeneralConfig.TRACE_LOG_LEVEL; import static datadog.trace.api.config.GeneralConfig.TRACE_OTEL_SEMANTICS_ENABLED; +import static datadog.trace.api.config.GeneralConfig.TRACE_STATS_CARDINALITY_LIMIT; import static datadog.trace.api.config.GeneralConfig.TRACE_STATS_COMPUTATION_ENABLED; import static datadog.trace.api.config.GeneralConfig.TRACE_STATS_COMPUTATION_IGNORE_AGENT_VERSION; import static datadog.trace.api.config.GeneralConfig.TRACE_TAGS; @@ -2260,7 +2261,8 @@ && isMetricsOtelEnabled() final int defaultMaxPending = tightHeap ? 64 : 128; tracerMetricsMaxAggregates = - configProvider.getInteger(TRACER_METRICS_MAX_AGGREGATES, defaultMaxAggregates); + configProvider.getInteger( + TRACE_STATS_CARDINALITY_LIMIT, defaultMaxAggregates, TRACER_METRICS_MAX_AGGREGATES); /* * TRACER_METRICS_MAX_PENDING historically counted conflating Batch slots (~64 spans per batch * via Batch.MAX_BATCH_SIZE). The inbox now holds 1 SpanSnapshot per metrics-eligible span, so @@ -3882,6 +3884,16 @@ public int getTraceStatsInterval() { return traceStatsInterval; } + /** + * Returns the per-cycle cardinality limit for the named stats field, following the RFC naming + * pattern {@code DD_TRACE_STATS_{tagName}_CARDINALITY_LIMIT} (e.g. {@code + * DD_TRACE_STATS_RESOURCE_CARDINALITY_LIMIT}). The caller supplies the default from {@code + * MetricCardinalityLimits} so per-field rationale stays co-located with the defaults. + */ + public int getTraceStatsCardinalityLimit(String tagName, int defaultLimit) { + return configProvider.getInteger("trace.stats." + tagName + ".cardinality.limit", defaultLimit); + } + public boolean isLogsInjectionEnabled() { return logsInjectionEnabled; } diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 389b5b1392f..20edcb4a513 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -10689,6 +10689,14 @@ "aliases": ["DD_TRACE_TRACER_METRICS_ENABLED"] } ], + "DD_TRACE_STATS_CARDINALITY_LIMIT": [ + { + "version": "B", + "type": "int", + "default": "2048", + "aliases": ["DD_TRACE_TRACER_METRICS_MAX_AGGREGATES"] + } + ], "DD_TRACE_STATS_COMPUTATION_IGNORE_AGENT_VERSION": [ { "version": "A", @@ -10697,6 +10705,30 @@ "aliases": [] } ], + "DD_TRACE_STATS_RESOURCE_CARDINALITY_LIMIT": [ + { + "version": "A", + "type": "int", + "default": "1024", + "aliases": [] + } + ], + "DD_TRACE_STATS_HTTP_ENDPOINT_CARDINALITY_LIMIT": [ + { + "version": "A", + "type": "int", + "default": "512", + "aliases": [] + } + ], + "DD_TRACE_STATS_PEER_TAGS_CARDINALITY_LIMIT": [ + { + "version": "A", + "type": "int", + "default": "512", + "aliases": [] + } + ], "DD_TRACE_STATUS404DECORATOR_ENABLED": [ { "version": "A",