diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 3ee3c339..5ab10ade 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -70,7 +70,8 @@ sha2 = { workspace = true } sha1 = "0.11" # Nexus imports: the Nexus REST API and asset downloads, through forge-core's # upstream client. -reqwest = { workspace = true } +# `blocking`: the client of the OTLP exporters (telemetry.rs). +reqwest = { workspace = true, features = ["blocking"] } subtle = { workspace = true } hex = { workspace = true } ipnet = { workspace = true } diff --git a/crates/server/src/telemetry.rs b/crates/server/src/telemetry.rs index 5ba11b67..494026d9 100644 --- a/crates/server/src/telemetry.rs +++ b/crates/server/src/telemetry.rs @@ -48,6 +48,7 @@ use opentelemetry::metrics::MeterProvider as _; use opentelemetry::propagation::Extractor; use opentelemetry::trace::TracerProvider as _; use opentelemetry::KeyValue; +use opentelemetry_otlp::WithHttpConfig as _; use opentelemetry_sdk::logs::SdkLoggerProvider; use opentelemetry_sdk::metrics::SdkMeterProvider; use opentelemetry_sdk::propagation::TraceContextPropagator; @@ -90,6 +91,9 @@ impl Telemetry { let meter = if config.metrics { let exporter = opentelemetry_otlp::MetricExporter::builder() .with_http() + .with_http_client(direct_client( + opentelemetry_otlp::OTEL_EXPORTER_OTLP_METRICS_TIMEOUT, + )?) .build() .context("build the OTLP metric exporter")?; Some( @@ -104,6 +108,9 @@ impl Telemetry { let tracer = if config.traces { let exporter = opentelemetry_otlp::SpanExporter::builder() .with_http() + .with_http_client(direct_client( + opentelemetry_otlp::OTEL_EXPORTER_OTLP_TRACES_TIMEOUT, + )?) .build() .context("build the OTLP span exporter")?; Some( @@ -118,6 +125,9 @@ impl Telemetry { let logger = if config.logs { let exporter = opentelemetry_otlp::LogExporter::builder() .with_http() + .with_http_client(direct_client( + opentelemetry_otlp::OTEL_EXPORTER_OTLP_LOGS_TIMEOUT, + )?) .build() .context("build the OTLP log exporter")?; Some( @@ -220,6 +230,42 @@ impl Telemetry { } } +/// The HTTP client of one signal's exporter. It connects to the endpoint +/// itself, whatever `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY` and the +/// platform's proxy settings say: those are for the upstream traffic of proxy +/// repositories, while the OTLP endpoint is the operator's own Collector, +/// beside Forge. The exporter's default client would send the exports, and +/// `OTEL_EXPORTER_OTLP_HEADERS` with them, to the outbound proxy (#558). +/// +/// The timeout is the one the exporter gives its default client: +/// `signal_timeout_var`, then `OTEL_EXPORTER_OTLP_TIMEOUT`, in milliseconds. +fn direct_client(signal_timeout_var: &str) -> anyhow::Result { + let timeout = [ + signal_timeout_var, + opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT, + ] + .iter() + .find_map(|var| std::env::var(var).ok()?.parse().ok()) + .map_or( + opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT, + Duration::from_millis, + ); + // A blocking client starts a runtime of its own, which panics on an + // async worker: build it on a thread, like the exporter builds its own. + std::thread::Builder::new() + .name("otlp-client".into()) + .spawn(move || { + reqwest::blocking::Client::builder() + .no_proxy() + .timeout(timeout) + .build() + }) + .context("spawn the thread building the OTLP HTTP client")? + .join() + .map_err(|_| anyhow::anyhow!("building the OTLP HTTP client panicked"))? + .context("build the OTLP HTTP client") +} + /// The filter of the console log: `RUST_LOG` (default `info`), without the /// spans meant for export. pub fn console_log_filter() -> EnvFilter { diff --git a/crates/server/tests/otlp_export.rs b/crates/server/tests/otlp_export.rs index bb126fe2..f9d48674 100644 --- a/crates/server/tests/otlp_export.rs +++ b/crates/server/tests/otlp_export.rs @@ -24,8 +24,40 @@ use opentelemetry_proto::tonic::trace::v1::span::SpanKind; use prost::Message as _; use tracing_subscriber::layer::SubscriberExt; -/// What the receiver was sent: the signal, the content type, the body. -type Received = Arc>>; +/// What the receiver was sent: the signal, the content type, the `api-key` +/// header, the body. +type Received = Arc>>; + +/// What the outbound proxy was sent: the first bytes of every connection. +type Proxied = Arc>>>; + +/// A stand-in for an outbound HTTP proxy: it keeps what every connection +/// sends first and answers 502, as an egress proxy that cannot reach the +/// Collector would. +async fn start_proxy() -> (String, Proxied) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let proxied: Proxied = Arc::default(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let seen = proxied.clone(); + tokio::spawn(async move { + loop { + let (mut stream, _) = listener.accept().await.unwrap(); + let seen = seen.clone(); + tokio::spawn(async move { + let mut request = vec![0; 16 * 1024]; + let read = stream.read(&mut request).await.unwrap_or(0); + request.truncate(read); + seen.lock().unwrap().push(request); + let _ = stream + .write_all(b"HTTP/1.1 502 Bad Gateway\r\ncontent-length: 0\r\nconnection: close\r\n\r\n") + .await; + }); + } + }); + (url, proxied) +} /// A mock OTLP/HTTP receiver keeping every export it is sent. async fn start_receiver() -> (String, Received) { @@ -42,7 +74,15 @@ async fn start_receiver() -> (String, Received) { .and_then(|v| v.to_str().ok()) .unwrap_or_default() .to_string(); - received.lock().unwrap().push((signal, content_type, body)); + let api_key = headers + .get("api-key") + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .to_string(); + received + .lock() + .unwrap() + .push((signal, content_type, api_key, body)); // An empty Export*ServiceResponse: everything accepted. ( [(header::CONTENT_TYPE, "application/x-protobuf")], @@ -75,6 +115,23 @@ fn resource_attribute<'a>(resource: Option<&'a Resource>, key: &str) -> Option<& #[tokio::test(flavor = "multi_thread")] async fn signals_are_exported_over_otlp_http() { let (endpoint, received) = start_receiver().await; + // An outbound proxy is configured, as for the proxy repositories' + // upstream traffic, and nothing exempts the Collector: the export goes + // to its endpoint all the same, and nothing of it to the proxy (#558). + let (proxy, proxied) = start_proxy().await; + for var in [ + "HTTP_PROXY", + "http_proxy", + "HTTPS_PROXY", + "https_proxy", + "ALL_PROXY", + "all_proxy", + ] { + std::env::set_var(var, &proxy); + } + std::env::remove_var("NO_PROXY"); + std::env::remove_var("no_proxy"); + std::env::set_var("OTEL_EXPORTER_OTLP_HEADERS", "api-key=s3cret"); std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", &endpoint); std::env::set_var( "OTEL_RESOURCE_ATTRIBUTES", @@ -111,15 +168,26 @@ async fn signals_are_exported_over_otlp_http() { .await .unwrap(); + let proxied = proxied.lock().unwrap().clone(); + assert!( + proxied.is_empty(), + "the outbound proxy was sent {} request(s): {:?}", + proxied.len(), + proxied + .iter() + .map(|request| String::from_utf8_lossy(&request[..request.len().min(200)]).into_owned()) + .collect::>() + ); let received = received.lock().unwrap().clone(); - for (signal, content_type, _) in &received { + for (signal, content_type, api_key, _) in &received { assert_eq!(content_type, "application/x-protobuf", "{signal}"); + assert_eq!(api_key, "s3cret", "{signal}"); } let bodies = |signal: &str| -> Vec { received .iter() - .filter(|(s, _, _)| s == signal) - .map(|(_, _, body)| body.clone()) + .filter(|(s, _, _, _)| s == signal) + .map(|(_, _, _, body)| body.clone()) .collect() }; diff --git a/docs/changes/2.2.0/558.fixed.md b/docs/changes/2.2.0/558.fixed.md new file mode 100644 index 00000000..a2c837de --- /dev/null +++ b/docs/changes/2.2.0/558.fixed.md @@ -0,0 +1,9 @@ +With an outbound proxy set for the proxy repositories (`HTTPS_PROXY`, +`HTTP_PROXY` or `ALL_PROXY`) and no `NO_PROXY` entry for the Collector, the +telemetry export went to that proxy, also for the default +`http://localhost:4318`: the exports failed, and to a plain-HTTP endpoint the +proxy was sent the telemetry and `OTEL_EXPORTER_OTLP_HEADERS` (a vendor's API +key, say) in the clear. Forge now connects to the OTLP endpoint directly, +whatever the proxy variables say. An installation that reached its telemetry +backend through the outbound proxy exports to a Collector instead, which +forwards through the proxy. diff --git a/docs/deployment.md b/docs/deployment.md index 05d1c7c0..4c108f89 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1463,6 +1463,13 @@ variables, read by the SDK: | `OTEL_TRACES_SAMPLER`, `OTEL_TRACES_SAMPLER_ARG` | Which traces are kept (default `parentbased_always_on`: all, and a caller's decision); e.g. `parentbased_traceidratio` and `0.1`. | | `OTEL_SDK_DISABLED` | `true` turns every signal off, whatever `[telemetry]` says. | +Forge connects to the OTLP endpoint directly, never through an outbound +proxy: `HTTPS_PROXY`, `HTTP_PROXY` and `ALL_PROXY` are for the upstream +traffic of proxy repositories, and the exports ignore them, so a Collector +beside Forge or in the cluster needs no `NO_PROXY` entry. Where a telemetry backend is reachable through the +outbound proxy alone, export to a Collector that Forge reaches directly and +let the Collector forward through the proxy. + An export that fails is logged and never fails a request: metrics are cumulative, so the next export carries what a failed one held, while a failed batch of spans or log records is dropped. Each instance exports its @@ -1707,7 +1714,8 @@ notices within about 15 s, drops its in-memory cache and resubscribes. requests and tokens. - **Outbound proxy.** Proxy repositories send upstream traffic through `HTTPS_PROXY` / `HTTP_PROXY` / `ALL_PROXY` unless `NO_PROXY` exempts the - host. Forge never sends an upstream request, or follows a redirect, to a + host. Those variables route nothing else: the [telemetry](#telemetry) + export connects to its OTLP endpoint directly. Forge never sends an upstream request, or follows a redirect, to a link-local or cloud-metadata address (`169.254.0.0/16`, `fe80::/10`, `metadata.google.internal`, …). Through an outbound proxy the proxy resolves the host name, so Forge resolves it first and refuses it when any