From 80660ca678c8c1d390fb420c47f354b3efb6ab11 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 09:22:09 +0000 Subject: [PATCH] Telemetry: log the SDK's startup warnings and a failed start Telemetry::new ran while no tracing subscriber was installed, so the OpenTelemetry SDK's warnings about the OTEL_* values it replaces with a default went nowhere: an OTEL_TRACES_SAMPLER_ARG that is no number silently sampled every trace. It is now built under the console log, as the configuration is loaded (telemetry::with_console_log). A start that failed after the subscriber was installed returned from main as Err: printed to stderr only, with telemetry.shutdown() skipped. main now logs it with tracing::error! and flushes the telemetry before it returns the error, so the log export carries it. Fixes #559 Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_012e6DJaXfLrW5Fz4BhRngM8 --- crates/server/src/main.rs | 33 ++++-- crates/server/src/telemetry.rs | 16 +++ crates/server/tests/startup_logs.rs | 178 ++++++++++++++++++++++++++++ docs/changes/2.2.0/559.fixed.md | 9 ++ 4 files changed, 225 insertions(+), 11 deletions(-) create mode 100644 crates/server/tests/startup_logs.rs create mode 100644 docs/changes/2.2.0/559.fixed.md diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index f58b138a..72a79018 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -35,15 +35,14 @@ async fn main() -> anyhow::Result<()> { let default = PathBuf::from("forge.toml"); default.exists().then_some(default) }); - let config = tracing::subscriber::with_default( - tracing_subscriber::registry() - .with(tracing_subscriber::fmt::layer().with_filter(telemetry::console_log_filter())), - || Config::load(config_path.as_deref()), - ) - .context("load configuration")?; + let config = telemetry::with_console_log(|| Config::load(config_path.as_deref())) + .context("load configuration")?; // OpenTelemetry: the signals `[telemetry]` turns on, exported over OTLP. - let telemetry = Telemetry::new(&config.telemetry).context("set up OpenTelemetry")?; + // Built with the console log too: the SDK reads its `OTEL_*` variables + // here and warns about the values it replaces with a default. + let telemetry = telemetry::with_console_log(|| Telemetry::new(&config.telemetry)) + .context("set up OpenTelemetry")?; telemetry.install(); forge_telemetry::set_per_repository(config.telemetry.per_repository); @@ -89,6 +88,22 @@ async fn main() -> anyhow::Result<()> { tracing::warn!("{warning}"); } + // A failed start is logged before it is returned, which only prints it + // to stderr: the exported log says why an instance does not come up. + let result = run(config).await; + if let Err(e) = &result { + tracing::error!(error = %format_args!("{e:#}"), "forge-server failed to start"); + } + // Export what the telemetry still buffers, that error included. + let flushed = tokio::task::spawn_blocking(move || telemetry.shutdown()) + .await + .context("flush the telemetry"); + result.and(flushed) +} + +/// Start the server and serve until the shutdown signal. An error is a +/// failed start: nothing fails once the listener is bound. +async fn run(config: Config) -> anyhow::Result<()> { // Database. let pool = forge_db::connect_with( &config.database.url, @@ -202,10 +217,6 @@ async fn main() -> anyhow::Result<()> { // So would the traffic counted since the last minute's flush. forge_server::traffic::flush_on_shutdown(&traffic, &pool).await; tracing::info!("forge-server stopped"); - // Export what the telemetry still buffers. - tokio::task::spawn_blocking(move || telemetry.shutdown()) - .await - .context("flush the telemetry")?; Ok(()) } diff --git a/crates/server/src/telemetry.rs b/crates/server/src/telemetry.rs index 5ba11b67..432d1f50 100644 --- a/crates/server/src/telemetry.rs +++ b/crates/server/src/telemetry.rs @@ -230,6 +230,22 @@ pub fn console_log_filter() -> EnvFilter { ) } +/// Run `f` with the console log alone as this thread's `tracing` subscriber: +/// what starts before the process's subscriber exists (the configuration, +/// then [`Telemetry::new`], which the subscriber is built from) logs through +/// it. Without one, the warnings of that time went nowhere: the +/// OpenTelemetry SDK's about an `OTEL_*` value it cannot use, such as an +/// `OTEL_TRACES_SAMPLER_ARG` that is no number, which it replaces with its +/// default. +pub fn with_console_log(f: impl FnOnce() -> T) -> T { + use tracing_subscriber::layer::SubscriberExt; + tracing::subscriber::with_default( + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().with_filter(console_log_filter())), + f, + ) +} + /// The filter of the exported logs: what the console log prints, but never /// the events of the export itself (OpenTelemetry, whose OTLP exporter logs /// as `opentelemetry-otlp`, and the HTTP client it sends with), which would diff --git a/crates/server/tests/startup_logs.rs b/crates/server/tests/startup_logs.rs new file mode 100644 index 00000000..c6da9c8c --- /dev/null +++ b/crates/server/tests/startup_logs.rs @@ -0,0 +1,178 @@ +//! What the `forge-server` binary logs while it starts (#559), run as a +//! process with the trace and log export on, against a database that +//! refuses the connection: +//! +//! * the OpenTelemetry SDK's warning about an `OTEL_TRACES_SAMPLER_ARG` it +//! cannot use is printed on the console, where it went nowhere before the +//! process's subscriber was installed; +//! * the failed start is an `ERROR` of the log, on the console and in the +//! log export, which is flushed before the process ends: the batch delay +//! is far longer than the test, so only the shutdown sends the record. +//! +//! Needs no database and no Docker. + +use std::process::{Command, Stdio}; +use std::sync::{Arc, Mutex}; + +use axum::body::Bytes; +use axum::extract::{Path, State}; +use axum::routing::post; +use axum::Router; +use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; +use opentelemetry_proto::tonic::common::v1::any_value::Value as AnyValue; +use opentelemetry_proto::tonic::common::v1::AnyValue as Any; +use opentelemetry_proto::tonic::logs::v1::SeverityNumber; +use prost::Message as _; + +/// What the receiver was sent: the signal and the body. +type Received = Arc>>; + +/// A mock OTLP/HTTP receiver keeping every export it is sent. +async fn start_receiver() -> (String, Received) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let received: Received = Arc::default(); + async fn export( + State(received): State, + Path(signal): Path, + body: Bytes, + ) -> Vec { + received.lock().unwrap().push((signal, body)); + // An empty Export*ServiceResponse: everything accepted. + Vec::new() + } + let app = Router::new() + .route("/v1/{signal}", post(export)) + .with_state(received.clone()); + let base = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + (base, received) +} + +fn text(value: Option<&Any>) -> Option<&str> { + match value?.value.as_ref()? { + AnyValue::StringValue(s) => Some(s), + _ => None, + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_failed_start_and_the_sdks_warnings_reach_the_logs() { + let (endpoint, received) = start_receiver().await; + // A port nothing listens on: the database refuses the connection. + let closed = { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.local_addr().unwrap().port() + }; + let dir = tempfile::tempdir().unwrap(); + let config = dir.path().join("forge.toml"); + std::fs::write( + &config, + format!( + r#" +[server] +bind = "127.0.0.1:0" +public_base_url = "http://localhost:8421" +allow_insecure_defaults = true + +[database] +url = "postgres://forge:forge@127.0.0.1:{closed}/forge" +acquire_timeout_seconds = 2 + +[s3] +endpoint = "http://127.0.0.1:{closed}" +region = "us-east-1" +bucket = "forge" +access_key = "minioadmin" +secret_key = "minioadmin" +force_path_style = true + +[crypto.root] +provider = "env" + +[telemetry] +traces = true +logs = true +"# + ), + ) + .unwrap(); + + let mut command = Command::new(env!("CARGO_BIN_EXE_forge-server")); + // Nothing of the environment the tests run in configures the server. + for (name, _) in std::env::vars_os() { + let name_str = name.to_string_lossy(); + if name_str.starts_with("FORGE_") || name_str.starts_with("OTEL_") || name_str == "RUST_LOG" + { + command.env_remove(&name); + } + } + command + .current_dir(dir.path()) + .env("FORGE_CONFIG", &config) + .env( + "FORGE_ROOT_KEY", + "ZGV2LXJvb3Qta2V5LWZvci1sb2NhbC1mb3JnZS0wMDE=", + ) + .env("NO_COLOR", "1") + .env("OTEL_EXPORTER_OTLP_ENDPOINT", &endpoint) + // Helm's `telemetry.samplerArg: "10%"`: no number. + .env("OTEL_TRACES_SAMPLER", "parentbased_traceidratio") + .env("OTEL_TRACES_SAMPLER_ARG", "10%") + // No batch is sent on its own within the test: what arrives was + // flushed by the shutdown. + .env("OTEL_BLRP_SCHEDULE_DELAY", "600000") + .env("OTEL_BSP_SCHEDULE_DELAY", "600000") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let output = tokio::task::spawn_blocking(move || command.output()) + .await + .unwrap() + .expect("run forge-server"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let said = format!("stdout:\n{stdout}\nstderr:\n{stderr}"); + + assert!(!output.status.success(), "the start fails\n{said}"); + + // The SDK's warning, on the console. + let warning = stdout + .lines() + .find(|line| line.contains("OTEL_TRACES_SAMPLER_ARG")) + .unwrap_or_else(|| panic!("the SDK's warning is printed\n{said}")); + assert!(warning.contains("WARN"), "{warning}"); + assert!(warning.contains("10%"), "{warning}"); + + // The failed start, on the console as a log line... + let error = stdout + .lines() + .find(|line| line.contains("forge-server failed to start")) + .unwrap_or_else(|| panic!("the failed start is logged\n{said}")); + assert!(error.contains("ERROR"), "{error}"); + assert!(error.contains("connect database"), "{error}"); + // ...and still what the process ends with. + assert!(stderr.contains("connect database"), "{said}"); + + // ...and in the log export, which the shutdown flushed. + let received = received.lock().unwrap().clone(); + let records: Vec<_> = received + .iter() + .filter(|(signal, _)| signal == "logs") + .map(|(_, body)| ExportLogsServiceRequest::decode(body.as_ref()).expect("OTLP logs")) + .flat_map(|request| request.resource_logs) + .flat_map(|resource| resource.scope_logs) + .flat_map(|scope| scope.log_records) + .collect(); + let record = records + .iter() + .find(|record| text(record.body.as_ref()) == Some("forge-server failed to start")) + .unwrap_or_else(|| panic!("the failed start is exported: {records:?}\n{said}")); + assert_eq!(record.severity_number, SeverityNumber::Error as i32); + let cause = record + .attributes + .iter() + .find(|kv| kv.key == "error") + .and_then(|kv| text(kv.value.as_ref())) + .expect("the error's cause"); + assert!(cause.contains("connect database"), "{cause}"); +} diff --git a/docs/changes/2.2.0/559.fixed.md b/docs/changes/2.2.0/559.fixed.md new file mode 100644 index 00000000..129affd1 --- /dev/null +++ b/docs/changes/2.2.0/559.fixed.md @@ -0,0 +1,9 @@ +With `[telemetry]` on, the OpenTelemetry SDK's warnings about an `OTEL_*` +value it cannot use were lost, because it reads them before the log is set +up: a `telemetry.samplerArg: "10%"` in the Helm values, or any +`OTEL_TRACES_SAMPLER_ARG` that is no number, silently sampled every trace. +The warnings are now printed on the console. A start that fails (the +database, the migrations, the bucket) was only printed to stderr and the +telemetry was not flushed, so with `telemetry.logs` the log backend never +showed why an instance was crash-looping; the failure is now logged as an +error, `forge-server failed to start`, and exported before the process ends.