Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion crates/server/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,8 @@ sha2 = { workspace = true }
sha1 = "0.11"
# Nexus imports: the Nexus REST API and asset downloads, through forge-core's
# upstream client.
reqwest = { workspace = true }
# `blocking`: the client of the OTLP exporters (telemetry.rs).
reqwest = { workspace = true, features = ["blocking"] }
subtle = { workspace = true }
hex = { workspace = true }
ipnet = { workspace = true }
Expand Down
46 changes: 46 additions & 0 deletions crates/server/src/telemetry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ use opentelemetry::metrics::MeterProvider as _;
use opentelemetry::propagation::Extractor;
use opentelemetry::trace::TracerProvider as _;
use opentelemetry::KeyValue;
use opentelemetry_otlp::WithHttpConfig as _;
use opentelemetry_sdk::logs::SdkLoggerProvider;
use opentelemetry_sdk::metrics::SdkMeterProvider;
use opentelemetry_sdk::propagation::TraceContextPropagator;
Expand Down Expand Up @@ -90,6 +91,9 @@ impl Telemetry {
let meter = if config.metrics {
let exporter = opentelemetry_otlp::MetricExporter::builder()
.with_http()
.with_http_client(direct_client(
opentelemetry_otlp::OTEL_EXPORTER_OTLP_METRICS_TIMEOUT,
)?)
.build()
.context("build the OTLP metric exporter")?;
Some(
Expand All @@ -104,6 +108,9 @@ impl Telemetry {
let tracer = if config.traces {
let exporter = opentelemetry_otlp::SpanExporter::builder()
.with_http()
.with_http_client(direct_client(
opentelemetry_otlp::OTEL_EXPORTER_OTLP_TRACES_TIMEOUT,
)?)
.build()
.context("build the OTLP span exporter")?;
Some(
Expand All @@ -118,6 +125,9 @@ impl Telemetry {
let logger = if config.logs {
let exporter = opentelemetry_otlp::LogExporter::builder()
.with_http()
.with_http_client(direct_client(
opentelemetry_otlp::OTEL_EXPORTER_OTLP_LOGS_TIMEOUT,
)?)
.build()
.context("build the OTLP log exporter")?;
Some(
Expand Down Expand Up @@ -220,6 +230,42 @@ impl Telemetry {
}
}

/// The HTTP client of one signal's exporter. It connects to the endpoint
/// itself, whatever `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY` and the
/// platform's proxy settings say: those are for the upstream traffic of proxy
/// repositories, while the OTLP endpoint is the operator's own Collector,
/// beside Forge. The exporter's default client would send the exports, and
/// `OTEL_EXPORTER_OTLP_HEADERS` with them, to the outbound proxy (#558).
///
/// The timeout is the one the exporter gives its default client:
/// `signal_timeout_var`, then `OTEL_EXPORTER_OTLP_TIMEOUT`, in milliseconds.
fn direct_client(signal_timeout_var: &str) -> anyhow::Result<reqwest::blocking::Client> {
let timeout = [
signal_timeout_var,
opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT,
]
.iter()
.find_map(|var| std::env::var(var).ok()?.parse().ok())
.map_or(
opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT,
Duration::from_millis,
);
// A blocking client starts a runtime of its own, which panics on an
// async worker: build it on a thread, like the exporter builds its own.
std::thread::Builder::new()
.name("otlp-client".into())
.spawn(move || {
reqwest::blocking::Client::builder()
.no_proxy()
.timeout(timeout)
.build()
})
.context("spawn the thread building the OTLP HTTP client")?
.join()
.map_err(|_| anyhow::anyhow!("building the OTLP HTTP client panicked"))?
.context("build the OTLP HTTP client")
}

/// The filter of the console log: `RUST_LOG` (default `info`), without the
/// spans meant for export.
pub fn console_log_filter() -> EnvFilter {
Expand Down
80 changes: 74 additions & 6 deletions crates/server/tests/otlp_export.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,40 @@ use opentelemetry_proto::tonic::trace::v1::span::SpanKind;
use prost::Message as _;
use tracing_subscriber::layer::SubscriberExt;

/// What the receiver was sent: the signal, the content type, the body.
type Received = Arc<Mutex<Vec<(String, String, Bytes)>>>;
/// What the receiver was sent: the signal, the content type, the `api-key`
/// header, the body.
type Received = Arc<Mutex<Vec<(String, String, String, Bytes)>>>;

/// What the outbound proxy was sent: the first bytes of every connection.
type Proxied = Arc<Mutex<Vec<Vec<u8>>>>;

/// A stand-in for an outbound HTTP proxy: it keeps what every connection
/// sends first and answers 502, as an egress proxy that cannot reach the
/// Collector would.
async fn start_proxy() -> (String, Proxied) {
use tokio::io::{AsyncReadExt, AsyncWriteExt};

let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let proxied: Proxied = Arc::default();
let url = format!("http://{}", listener.local_addr().unwrap());
let seen = proxied.clone();
tokio::spawn(async move {
loop {
let (mut stream, _) = listener.accept().await.unwrap();
let seen = seen.clone();
tokio::spawn(async move {
let mut request = vec![0; 16 * 1024];
let read = stream.read(&mut request).await.unwrap_or(0);
request.truncate(read);
seen.lock().unwrap().push(request);
let _ = stream
.write_all(b"HTTP/1.1 502 Bad Gateway\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
.await;
});
}
});
(url, proxied)
}

/// A mock OTLP/HTTP receiver keeping every export it is sent.
async fn start_receiver() -> (String, Received) {
Expand All @@ -42,7 +74,15 @@ async fn start_receiver() -> (String, Received) {
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string();
received.lock().unwrap().push((signal, content_type, body));
let api_key = headers
.get("api-key")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string();
received
.lock()
.unwrap()
.push((signal, content_type, api_key, body));
// An empty Export*ServiceResponse: everything accepted.
(
[(header::CONTENT_TYPE, "application/x-protobuf")],
Expand Down Expand Up @@ -75,6 +115,23 @@ fn resource_attribute<'a>(resource: Option<&'a Resource>, key: &str) -> Option<&
#[tokio::test(flavor = "multi_thread")]
async fn signals_are_exported_over_otlp_http() {
let (endpoint, received) = start_receiver().await;
// An outbound proxy is configured, as for the proxy repositories'
// upstream traffic, and nothing exempts the Collector: the export goes
// to its endpoint all the same, and nothing of it to the proxy (#558).
let (proxy, proxied) = start_proxy().await;
for var in [
"HTTP_PROXY",
"http_proxy",
"HTTPS_PROXY",
"https_proxy",
"ALL_PROXY",
"all_proxy",
] {
std::env::set_var(var, &proxy);
}
std::env::remove_var("NO_PROXY");
std::env::remove_var("no_proxy");
std::env::set_var("OTEL_EXPORTER_OTLP_HEADERS", "api-key=s3cret");
std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", &endpoint);
std::env::set_var(
"OTEL_RESOURCE_ATTRIBUTES",
Expand Down Expand Up @@ -111,15 +168,26 @@ async fn signals_are_exported_over_otlp_http() {
.await
.unwrap();

let proxied = proxied.lock().unwrap().clone();
assert!(
proxied.is_empty(),
"the outbound proxy was sent {} request(s): {:?}",
proxied.len(),
proxied
.iter()
.map(|request| String::from_utf8_lossy(&request[..request.len().min(200)]).into_owned())
.collect::<Vec<_>>()
);
let received = received.lock().unwrap().clone();
for (signal, content_type, _) in &received {
for (signal, content_type, api_key, _) in &received {
assert_eq!(content_type, "application/x-protobuf", "{signal}");
assert_eq!(api_key, "s3cret", "{signal}");
}
let bodies = |signal: &str| -> Vec<Bytes> {
received
.iter()
.filter(|(s, _, _)| s == signal)
.map(|(_, _, body)| body.clone())
.filter(|(s, _, _, _)| s == signal)
.map(|(_, _, _, body)| body.clone())
.collect()
};

Expand Down
9 changes: 9 additions & 0 deletions docs/changes/2.2.0/558.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
With an outbound proxy set for the proxy repositories (`HTTPS_PROXY`,
`HTTP_PROXY` or `ALL_PROXY`) and no `NO_PROXY` entry for the Collector, the
telemetry export went to that proxy, also for the default
`http://localhost:4318`: the exports failed, and to a plain-HTTP endpoint the
proxy was sent the telemetry and `OTEL_EXPORTER_OTLP_HEADERS` (a vendor's API
key, say) in the clear. Forge now connects to the OTLP endpoint directly,
whatever the proxy variables say. An installation that reached its telemetry
backend through the outbound proxy exports to a Collector instead, which
forwards through the proxy.
10 changes: 9 additions & 1 deletion docs/deployment.md
Original file line number Diff line number Diff line change
Expand Up @@ -1463,6 +1463,13 @@ variables, read by the SDK:
| `OTEL_TRACES_SAMPLER`, `OTEL_TRACES_SAMPLER_ARG` | Which traces are kept (default `parentbased_always_on`: all, and a caller's decision); e.g. `parentbased_traceidratio` and `0.1`. |
| `OTEL_SDK_DISABLED` | `true` turns every signal off, whatever `[telemetry]` says. |

Forge connects to the OTLP endpoint directly, never through an outbound
proxy: `HTTPS_PROXY`, `HTTP_PROXY` and `ALL_PROXY` are for the upstream
traffic of proxy repositories, and the exports ignore them, so a Collector
beside Forge or in the cluster needs no `NO_PROXY` entry. Where a telemetry backend is reachable through the
outbound proxy alone, export to a Collector that Forge reaches directly and
let the Collector forward through the proxy.

An export that fails is logged and never fails a request: metrics are
cumulative, so the next export carries what a failed one held, while a
failed batch of spans or log records is dropped. Each instance exports its
Expand Down Expand Up @@ -1707,7 +1714,8 @@ notices within about 15 s, drops its in-memory cache and resubscribes.
requests and tokens.
- **Outbound proxy.** Proxy repositories send upstream traffic through
`HTTPS_PROXY` / `HTTP_PROXY` / `ALL_PROXY` unless `NO_PROXY` exempts the
host. Forge never sends an upstream request, or follows a redirect, to a
host. Those variables route nothing else: the [telemetry](#telemetry)
export connects to its OTLP endpoint directly. Forge never sends an upstream request, or follows a redirect, to a
link-local or cloud-metadata address (`169.254.0.0/16`, `fe80::/10`,
`metadata.google.internal`, …). Through an outbound proxy the proxy
resolves the host name, so Forge resolves it first and refuses it when any
Expand Down
Loading