From d453468f522884262e3d0d6b3acf0f4f80622a04 Mon Sep 17 00:00:00 2001 From: Takwa Date: Wed, 28 Oct 2026 15:26:05 +0100 Subject: [PATCH] Trace 5% of route-origin's requests route-origin's Envoy sends an OpenTelemetry span for 5% of requests, and one for its call to needle, to the trace collector on the box's loopback (ipni/infra#163). Span names come from route decorators, never the path. Clients cannot force or continue a trace: x-client-trace-id is ignored and traceparent/tracestate are dropped before Envoy reads them. The access log carries the trace ID, which Grafana links to Tempo. Co-Authored-By: Claude Opus 5.5 --- k8s/route-origin/envoy.yaml.j2 | 65 ++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/k8s/route-origin/envoy.yaml.j2 b/k8s/route-origin/envoy.yaml.j2 index a070474..7e3ded0 100644 --- a/k8s/route-origin/envoy.yaml.j2 +++ b/k8s/route-origin/envoy.yaml.j2 @@ -78,6 +78,44 @@ static_resources: common_http_protocol_options: idle_timeout: 300s headers_with_underscores_action: REJECT_REQUEST + # Traces (ipni/infra#163), to the trace collector on the box's + # loopback (ipni/infra kubernetes/platform/alloy-traces, + # 127.0.0.1:4317), which turns them into latency metrics, keeps + # the failed and slow ones and a tenth of the rest, and sends + # those to Tempo on lith-1. 5% of requests: ~7 traces/s per box + # at the fleet's ~424 req/s. Each trace is Envoy's span for the + # request and, with spawn_upstream_span, one for the call to + # needle (which instance, how long). The span names are the + # route decorators below, never the path: a path carries a CID, + # and every span name becomes a metric series. needle does not + # join these traces yet: v1.0.0 drops the traceparent Envoy + # sends (NEEDLE_TRACING_AUTH is unset), and its own sampling + # never fires (ipni/infra#163). + # + # Only this sampling decides. A client cannot ask to be traced: + # client_sampling ignores x-client-trace-id, and the early + # header mutation drops a traceparent or tracestate from the + # request before Envoy reads it, so an outside trace cannot + # be continued (or force a sample) here either. + tracing: + random_sampling: { value: 5 } + client_sampling: { value: 0 } + spawn_upstream_span: true + provider: + name: envoy.tracers.opentelemetry + typed_config: + "@type": type.googleapis.com/envoy.config.trace.v3.OpenTelemetryConfig + service_name: route-origin + grpc_service: + envoy_grpc: { cluster_name: otel_collector } + timeout: 1s + early_header_mutation_extensions: + - name: envoy.http.early_header_mutation.header_mutation + typed_config: + "@type": type.googleapis.com/envoy.extensions.http.early_header_mutation.header_mutation.v3.HeaderMutation + mutations: + - remove: traceparent + - remove: tracestate access_log: - name: envoy.access_loggers.stdout typed_config: @@ -100,6 +138,10 @@ static_resources: # instance after a rollout looks exactly like a slow # box until you can group by this. upstream: "%UPSTREAM_HOST%" + # The request's trace in Tempo; Grafana's Loki data + # source links it. Set on every request, but only + # the sampled ones (see tracing above) are stored. + trace_id: "%TRACE_ID%" http_filters: # Per-box backstop behind Cloudflare's own rate limiting rule. # No bucket is configured here: it only acts on routes that @@ -156,6 +198,7 @@ static_resources: route: cluster: needle timeout: 60s + decorator: { operation: ipns-put } typed_per_filter_config: envoy.filters.http.local_ratelimit: "@type": type.googleapis.com/envoy.extensions.filters.http.local_ratelimit.v3.LocalRateLimit @@ -196,6 +239,7 @@ static_resources: # Above the 25s routing timeout, below Cloudflare's # 100s proxy read timeout. timeout: 60s + decorator: { operation: lookup } typed_per_filter_config: envoy.filters.http.local_ratelimit: "@type": type.googleapis.com/envoy.extensions.filters.http.local_ratelimit.v3.LocalRateLimit @@ -214,10 +258,12 @@ static_resources: route: cluster: needle timeout: 5s + decorator: { operation: version } - match: { prefix: "/" } direct_response: status: 404 body: { inline_string: "not found\n" } + decorator: { operation: not-found } # Every other Host. 421 Misdirected Request tells a # misconfigured caller exactly what is wrong. @@ -228,8 +274,27 @@ static_resources: direct_response: status: 421 body: { inline_string: "misdirected request\n" } + decorator: { operation: misdirected } clusters: + # The trace collector (see tracing above). OTLP/gRPC, so HTTP/2. If it is + # down, Envoy drops the spans; requests are not affected. + - name: otel_collector + type: STATIC + connect_timeout: 1s + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: {} + load_assignment: + cluster_name: otel_collector + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: { address: 127.0.0.1, port_value: 4317 } + - name: needle type: STATIC connect_timeout: 2s