From a451f4df61056891c717d41e3df3952a61ebc750 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 08:51:40 +0000 Subject: [PATCH 1/2] Telemetry off: no repository size reads around retention runs, no pool acquire events Two measurements cost work with the metrics off (#549): - The retention worker read the repository's size before every claimed run and again after one that deleted something, only to feed `forge.retention.freed`. It now reads the size only when the metrics record (`forge_telemetry::enabled()`), and then not for a dry run or a disabled policy, which free nothing. - `forge_db::connect_with` had sqlx emit a `TRACE` event per acquire, and `main` always installed the `AcquireTimes` layer for it. The pool now emits the event, and `main` installs the layer (`telemetry::acquire_times()`), only with the metrics on. A slow acquire still logs its `WARN`. Tests: `retention_metrics_off.rs` and `telemetry_off.rs` are binaries without the metrics, where a run sends no size query and the server's pool emits no acquire event; `retention_metrics.rs` and `telemetry.rs` show the two size reads with the freed bytes, and the acquire times in `db.client.connection.wait_time`, with the metrics on. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_012e6DJYg356rtc9YTDE6RrF --- Cargo.lock | 1 + crates/core/Cargo.toml | 5 +- crates/core/src/retention/worker.rs | 39 +++-- crates/core/tests/retention_metrics.rs | 130 ++++++++++++++ crates/core/tests/retention_metrics_off.rs | 36 ++++ crates/core/tests/retention_size/mod.rs | 194 +++++++++++++++++++++ crates/db/src/pool.rs | 20 ++- crates/server/src/main.rs | 28 ++- crates/server/src/telemetry.rs | 23 ++- crates/server/tests/telemetry.rs | 29 ++- crates/server/tests/telemetry_off.rs | 81 +++++++++ docs/changes/2.2.0/549.fixed.md | 6 + 12 files changed, 553 insertions(+), 39 deletions(-) create mode 100644 crates/core/tests/retention_metrics.rs create mode 100644 crates/core/tests/retention_metrics_off.rs create mode 100644 crates/core/tests/retention_size/mod.rs create mode 100644 crates/server/tests/telemetry_off.rs create mode 100644 docs/changes/2.2.0/549.fixed.md diff --git a/Cargo.lock b/Cargo.lock index aef12e58..40519e74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2206,6 +2206,7 @@ dependencies = [ "tokio", "tokio-util", "tracing", + "tracing-subscriber", "url", "uuid", "wiremock", diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 55ac502b..0397d92a 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -81,9 +81,12 @@ zip = { version = "9.0", default-features = false, features = ["deflate"] } testcontainers = "0.27" testcontainers-modules = { version = "0.15", features = ["postgres"] } axum = { workspace = true } -# Read back the metrics recorded (tests/auth_pool_timeout.rs). +# Read back the metrics recorded (tests/auth_pool_timeout.rs, +# tests/retention_metrics.rs). opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true, features = ["testing"] } +# Record the statements sqlx logs (tests/retention_size). +tracing-subscriber = { workspace = true } # Mock HTTP server for discovery/JWKS unit tests (no real network). wiremock = "0.6" # Generate a throwaway RSA keypair in-test: sign a self-signed ID token and diff --git a/crates/core/src/retention/worker.rs b/crates/core/src/retention/worker.rs index 3537a456..72e13af3 100644 --- a/crates/core/src/retention/worker.rs +++ b/crates/core/src/retention/worker.rs @@ -194,9 +194,15 @@ impl RetentionWorker { run_id, self.config.lease, )); - // The size before the run, for the bytes it frees (metrics only: a - // failed read records none). - let size_before = self.repository_size(&repo).await.ok(); + // The size before the run, for the bytes it frees. Only the metrics + // use it: without them nothing reads the size, and with them neither + // does a dry run or a disabled policy, which frees nothing. A failed + // read records no freed bytes. + let size_before = if forge_telemetry::enabled() && frees_bytes(&repo.policy) { + self.repository_size(&repo).await.ok() + } else { + None + }; // The policy's `max_run_seconds`, counted from the claim. let deadline = schedule::max_run_duration(&repo.policy).map(|limit| std::time::Instant::now() + limit); @@ -241,7 +247,8 @@ impl RetentionWorker { } } - /// The logical size of a claimed repository, as its format stores it. + /// The logical size of a claimed repository, as its format stores it: a + /// `SUM` over the repository's rows, read for the metrics only. async fn repository_size(&self, repo: &retention_dao::ClaimedRepo) -> sqlx::Result { let format = self.formats.resolve(&repo.format); if format.as_ref().is_some_and(|f| f.stores_images()) { @@ -262,15 +269,11 @@ impl RetentionWorker { summary: &RunSummary, size_before: Option, ) { - let dry_run = repo - .policy - .get("dry_run_only") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); - if dry_run { + if flag(&repo.policy, "dry_run_only", false) { forge_telemetry::retention::run(&repo.name, true, 0, 0, 0); return; } + // `size_before` is `None` without the metrics: no second read then. let freed = match (size_before, summary.deleted > 0) { (Some(before), true) => self .repository_size(repo) @@ -741,6 +744,22 @@ fn panic_message(payload: &(dyn std::any::Any + Send)) -> String { .unwrap_or_else(|| "non-string panic payload".to_string()) } +/// A boolean of a stored policy (`dry_run_only`, `enabled`), `default` +/// when the policy does not set it: the three policy shapes (packages, +/// images, files) share these. +fn flag(policy: &serde_json::Value, name: &str, default: bool) -> bool { + policy + .get(name) + .and_then(serde_json::Value::as_bool) + .unwrap_or(default) +} + +/// Whether a run of `policy` can delete anything: not a dry run, and not a +/// disabled policy. +fn frees_bytes(policy: &serde_json::Value) -> bool { + flag(policy, "enabled", true) && !flag(policy, "dry_run_only", false) +} + /// Renew the run's lease every third of its length until aborted; stops on /// its own once the lease is lost. fn spawn_lease_heartbeat( diff --git a/crates/core/tests/retention_metrics.rs b/crates/core/tests/retention_metrics.rs new file mode 100644 index 00000000..eab46f08 --- /dev/null +++ b/crates/core/tests/retention_metrics.rs @@ -0,0 +1,130 @@ +//! With the metrics on, a retention run reads the repository's size before +//! and after it deleted something, and `forge.retention.freed` counts the +//! difference; a dry run and a disabled policy, which free nothing, read no +//! size (#549). `retention_metrics_off.rs` is the run with the metrics off. +//! +//! The tests of this binary share the metrics and the statement recorder, +//! as a process does, so they run one at a time. Postgres only. + +mod common; +mod retention_size; + +use common::{pool, unique}; +use opentelemetry::metrics::MeterProvider as _; +use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData}; +use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader, SdkMeterProvider}; +use retention_size::{Statements, ASSET_SIZE}; +use serde_json::json; +use std::sync::OnceLock; + +/// The metrics, recorded to memory. +struct Metrics { + provider: SdkMeterProvider, + exporter: InMemoryMetricExporter, +} + +/// Turn the metrics on, once per process. +fn metrics() -> &'static Metrics { + static METRICS: OnceLock = OnceLock::new(); + METRICS.get_or_init(|| { + let exporter = InMemoryMetricExporter::default(); + let provider = SdkMeterProvider::builder() + .with_reader(PeriodicReader::builder(exporter.clone()).build()) + .build(); + forge_telemetry::init(&provider.meter("forge")); + Metrics { provider, exporter } + }) +} + +impl Metrics { + /// The cumulative value of counter `name` for `repository`, `None` + /// when it has no such series. + fn counter(&self, name: &str, repository: &str) -> Option { + self.exporter.reset(); + self.provider.force_flush().expect("flush"); + let finished = self.exporter.get_finished_metrics().expect("metrics"); + let latest = finished.last()?; + let mut total = None; + for scope in latest.scope_metrics() { + for metric in scope.metrics().filter(|m| m.name() == name) { + let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else { + panic!("{name} is a u64 counter"); + }; + for point in sum.data_points() { + if point.attributes().any(|kv| kv.value.as_str() == repository) { + *total.get_or_insert(0) += point.value(); + } + } + } + } + total + } +} + +/// One test at a time: they read the shared statement recorder. +static ONE_AT_A_TIME: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +#[tokio::test] +async fn a_run_that_deletes_reads_the_size_twice_and_records_the_freed_bytes() { + let _one = ONE_AT_A_TIME.lock().await; + let statements = Statements::install(); + let metrics = metrics(); + assert!(forge_telemetry::enabled()); + let pool = pool().await; + let name = format!("rsize-on-{}", unique()); + let repo = retention_size::seed(&pool, &name, json!({})).await; + + let mark = statements.len(); + let summary = retention_size::run(&pool, repo).await; + + assert_eq!(summary.deleted, 1); + assert_eq!( + statements.size_reads_since(mark).len(), + 2, + "the size before and after the run: {:?}", + statements.size_reads_since(mark) + ); + assert_eq!( + metrics.counter("forge.retention.freed", &name), + Some(ASSET_SIZE), + "the deleted version's bytes" + ); + assert_eq!(metrics.counter("forge.retention.runs", &name), Some(1)); +} + +#[tokio::test] +async fn a_dry_run_and_a_disabled_policy_read_no_size() { + let _one = ONE_AT_A_TIME.lock().await; + let statements = Statements::install(); + let metrics = metrics(); + let pool = pool().await; + + for (what, extra, would_delete) in [ + ("dry", json!({ "dry_run_only": true }), 1), + ("disabled", json!({ "enabled": false }), 0), + ] { + let name = format!("rsize-{what}-{}", unique()); + let repo = retention_size::seed(&pool, &name, extra).await; + + let mark = statements.len(); + let summary = retention_size::run(&pool, repo).await; + + assert_eq!(summary.deleted, would_delete, "{what}"); + assert_eq!( + statements.size_reads_since(mark), + Vec::::new(), + "{what}: a run that frees nothing reads no size" + ); + // The run still counts, with nothing freed. + assert_eq!( + metrics.counter("forge.retention.runs", &name), + Some(1), + "{what}" + ); + assert_eq!( + metrics.counter("forge.retention.freed", &name), + Some(0), + "{what}" + ); + } +} diff --git a/crates/core/tests/retention_metrics_off.rs b/crates/core/tests/retention_metrics_off.rs new file mode 100644 index 00000000..df7d3c5e --- /dev/null +++ b/crates/core/tests/retention_metrics_off.rs @@ -0,0 +1,36 @@ +//! With the metrics off (the default), a retention run never reads the +//! repository's size: the read only feeds `forge.retention.freed` (#549). +//! +//! A test binary of its own: the metrics are a process's, and this one never +//! turns them on. `retention_metrics.rs` is the same run with them on. +//! Postgres only. + +mod common; +mod retention_size; + +use common::{pool, unique}; +use retention_size::Statements; +use serde_json::json; + +#[tokio::test] +async fn a_run_reads_no_repository_size_without_the_metrics() { + let statements = Statements::install(); + assert!(!forge_telemetry::enabled(), "this binary has no metrics"); + let pool = pool().await; + let repo = retention_size::seed(&pool, &format!("rsize-off-{}", unique()), json!({})).await; + + let mark = statements.len(); + let summary = retention_size::run(&pool, repo).await; + + // The run did its work, and the recorder saw it. + assert_eq!(summary.deleted, 1); + assert!( + statements.len() > mark, + "the run's statements were recorded" + ); + assert_eq!( + statements.size_reads_since(mark), + Vec::::new(), + "no size read around a run without the metrics" + ); +} diff --git a/crates/core/tests/retention_size/mod.rs b/crates/core/tests/retention_size/mod.rs new file mode 100644 index 00000000..01bbb7e8 --- /dev/null +++ b/crates/core/tests/retention_size/mod.rs @@ -0,0 +1,194 @@ +//! What `retention_metrics.rs` and `retention_metrics_off.rs` share: a +//! repository a retention run deletes from, and the statements the run sent +//! to the database, to count its size reads (#549). + +use std::sync::{Arc, Mutex, OnceLock}; +use std::time::Duration as StdDuration; + +use chrono::Duration; +use forge_core::repo::NoopCacheInvalidator; +use forge_core::retention::worker::{RetentionConfig, RetentionWorker}; +use forge_core::Format; +use forge_db::asset_dao::{self, NewAsset}; +use forge_db::blob_dao; +use forge_db::component_dao::{self, NewComponent}; +use forge_db::repository_dao::{self, NewRepository}; +use forge_db::retention_dao::{self, RunSummary}; +use serde_json::{json, Value}; +use sqlx::PgPool; +use tracing_subscriber::layer::SubscriberExt as _; +use tracing_subscriber::Layer as _; + +use crate::common::unique; + +/// The size of each seeded version's asset. +pub const ASSET_SIZE: u64 = 100; + +/// The statements sqlx ran, read from its `sqlx::query` events. +#[derive(Clone, Default)] +pub struct Statements(Arc>>); + +impl Statements { + /// Make a recorder the process's `tracing` subscriber (once per test + /// binary) and return it. + pub fn install() -> Statements { + static INSTALLED: OnceLock = OnceLock::new(); + INSTALLED + .get_or_init(|| { + let statements = Statements::default(); + let subscriber = tracing_subscriber::registry().with( + statements.clone().with_filter( + tracing_subscriber::filter::Targets::new() + .with_target("sqlx::query", tracing::Level::DEBUG), + ), + ); + tracing::subscriber::set_global_default(subscriber).expect("the only subscriber"); + statements + }) + .clone() + } + + /// How many statements ran so far. + pub fn len(&self) -> usize { + self.0.lock().unwrap().len() + } + + /// The repository size reads among the statements that ran since + /// `mark` (an earlier [`Self::len`]): the statements that select one + /// `SUM` of sizes, as `repository_size` of packages, images and files + /// each does (evaluation sums per version, in a subquery). + pub fn size_reads_since(&self, mark: usize) -> Vec { + self.0.lock().unwrap()[mark..] + .iter() + .filter(|sql| sql.trim_start().starts_with("SELECT COALESCE(SUM(")) + .cloned() + .collect() + } +} + +impl tracing_subscriber::Layer for Statements { + fn on_event( + &self, + event: &tracing::Event<'_>, + _ctx: tracing_subscriber::layer::Context<'_, S>, + ) { + /// The statement: `db.statement` when sqlx shortened it into + /// `summary`, `summary` otherwise. + #[derive(Default)] + struct Sql { + summary: String, + statement: String, + } + impl tracing::field::Visit for Sql { + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + match field.name() { + "summary" => self.summary = value.to_string(), + "db.statement" => self.statement = value.to_string(), + _ => {} + } + } + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + self.record_str(field, &format!("{value:?}")); + } + } + let mut sql = Sql::default(); + event.record(&mut sql); + self.0 + .lock() + .unwrap() + .push(format!("{} {}", sql.summary, sql.statement)); + } +} + +/// A hosted NuGet repository `name` with two versions of `lib`, 60 days old, +/// under a policy that deletes versions older than 30 days but keeps the +/// newest one, with `extra` merged into the policy. Returns its id. +pub async fn seed(pool: &PgPool, name: &str, extra: Value) -> i64 { + let repo = repository_dao::insert_repository(pool, NewRepository::hosted(name.to_string())) + .await + .unwrap(); + for version in ["1.0.0", "2.0.0"] { + // A blob of its own per version, unlike any other test binary's. + let sha = format!("{:064x}", (0x549_u128 << 64) | u128::from(rand_tag())); + blob_dao::upsert_pending(pool, &sha, ASSET_SIZE as i64, &format!("k/{sha}")) + .await + .unwrap(); + let mut tx = pool.begin().await.unwrap(); + blob_dao::commit_and_incref(&mut tx, &sha).await.unwrap(); + tx.commit().await.unwrap(); + let component = component_dao::insert_component( + pool, + NewComponent { + repository_id: repo.id, + name: "lib".into(), + original_id: "lib".into(), + version: version.into(), + version_original: version.into(), + listed: true, + }, + ) + .await + .unwrap(); + asset_dao::insert_asset( + pool, + NewAsset { + component_id: component.id, + path: format!("lib.{version}.nupkg"), + content_type: "application/octet-stream".into(), + blob_sha256: sha, + size: ASSET_SIZE as i64, + sha512_base64: None, + }, + ) + .await + .unwrap(); + sqlx::query("UPDATE component SET created_at = now() - interval '60 days' WHERE id = $1") + .bind(component.id) + .execute(pool) + .await + .unwrap(); + } + let mut policy = json!({ + "version": 1, + "rules": [{ + "id": "old", + "action": "hard_delete", + "criteria": { "keep_last_n": 1, "older_than_days": 30 } + }] + }); + policy + .as_object_mut() + .unwrap() + .extend(extra.as_object().cloned().unwrap_or_default()); + retention_dao::write_policy(pool, repo.id, Some(&policy)) + .await + .unwrap(); + repo.id +} + +/// A blob tag no other process of this test run uses. +fn rand_tag() -> u64 { + (u64::from(std::process::id()) << 32) | unique() +} + +/// Run the repository's retention once, as the worker's tick does. +pub async fn run(pool: &PgPool, repository_id: i64) -> RunSummary { + let format = + forge_core::format::SpecFormat::new(Arc::new(forge_format_nuget::Nuget::default())); + let worker = RetentionWorker::new( + pool.clone(), + Arc::new(format) as Arc, + Arc::new(NoopCacheInvalidator), + RetentionConfig { + poll_interval: StdDuration::from_millis(10), + default_run_interval: Duration::seconds(1), + lease: Duration::seconds(60), + delete_grace: Duration::seconds(60), + }, + ); + worker + .tick_repo(repository_id) + .await + .expect("the run") + .expect("a claimed repository") +} diff --git a/crates/db/src/pool.rs b/crates/db/src/pool.rs index 0c419a69..98934a5a 100644 --- a/crates/db/src/pool.rs +++ b/crates/db/src/pool.rs @@ -11,9 +11,11 @@ pub async fn connect(url: &str) -> sqlx::Result { /// cannot get a connection within `acquire_timeout` fails instead of queueing /// indefinitely behind a saturated pool. /// -/// Every acquire emits a `TRACE` event (target `sqlx::pool::acquire`, field -/// `acquired_after_secs`), which the server's metrics read; a slow one (2 s) -/// is a `WARN`, as by default. +/// With the metrics on ([`forge_telemetry::enabled`], so connect after the +/// server installed them), every acquire emits a `TRACE` event (target +/// `sqlx::pool::acquire`, field `acquired_after_secs`), which the server's +/// metrics read. Without them no acquire emits one. A slow acquire (2 s) is +/// a `WARN` either way, as by default. pub async fn connect_with( url: &str, max_connections: u32, @@ -22,11 +24,21 @@ pub async fn connect_with( sqlx::postgres::PgPoolOptions::new() .max_connections(max_connections) .acquire_timeout(acquire_timeout) - .acquire_time_level(log::LevelFilter::Trace) + .acquire_time_level(acquire_time_level()) .connect(url) .await } +/// The level of the event an acquire emits: `TRACE` for the metrics to read, +/// none (sqlx's default) when nothing records them. +fn acquire_time_level() -> log::LevelFilter { + if forge_telemetry::enabled() { + log::LevelFilter::Trace + } else { + log::LevelFilter::Off + } +} + /// Begin a transaction on a connection of `pool`. Use this, never /// `Pool::begin`: it is safe against cancellation. /// diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index f58b138a..9496edb9 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -47,25 +47,23 @@ async fn main() -> anyhow::Result<()> { telemetry.install(); forge_telemetry::set_per_repository(config.telemetry.per_repository); - // The console log, filtered by RUST_LOG (default `info`); the connection - // pool's acquire events, which the metrics read whatever RUST_LOG says; - // and the trace and log export. + // The console log, filtered by RUST_LOG (default `info`); with the + // metrics on, the connection pool's acquire events, which the metrics + // read whatever RUST_LOG says; and the trace and log export. let console_filter = telemetry::console_log_filter(); let log_level = console_filter.max_level_hint(); - let mut layers: Vec = vec![ - Box::new(tracing_subscriber::fmt::layer().with_filter(console_filter)), - Box::new( - telemetry::AcquireTimes.with_filter( - tracing_subscriber::filter::Targets::new() - .with_target(telemetry::ACQUIRE_TARGET, tracing::Level::TRACE), - ), - ), - ]; + let mut layers: Vec = vec![Box::new( + tracing_subscriber::fmt::layer().with_filter(console_filter), + )]; + if let Some(acquire_times) = telemetry::acquire_times() { + layers.push(Box::new(acquire_times)); + } layers.extend(telemetry.layers()); tracing_subscriber::registry().with(layers).init(); - // Those TRACE events raise the subscriber's level, and `init` gives it to - // the `log` bridge too: keep dependencies' `log` records at the level - // the log prints, so they are skipped as cheaply as before. + // The acquire events are TRACE and raise the subscriber's level, and + // `init` gives it to the `log` bridge too: keep dependencies' `log` + // records at the level the log prints, so they are skipped as cheaply + // as without the metrics. if let Some(level) = log_level { log::set_max_level(match level.into_level() { None => log::LevelFilter::Off, diff --git a/crates/server/src/telemetry.rs b/crates/server/src/telemetry.rs index 5ba11b67..ae7c7060 100644 --- a/crates/server/src/telemetry.rs +++ b/crates/server/src/telemetry.rs @@ -25,7 +25,8 @@ //! backlog, counted every `telemetry.inventory_interval_seconds`, never //! per collection. //! * [`AcquireTimes`], the tracing layer that turns the connection pool's -//! acquire events into `db.client.connection.wait_time`. +//! acquire events into `db.client.connection.wait_time`, installed only +//! with the metrics on ([`acquire_times`]). use std::borrow::Cow; use std::pin::Pin; @@ -842,6 +843,26 @@ pub struct AcquireTimes; /// The target of sqlx's acquire events. pub const ACQUIRE_TARGET: &str = "sqlx::pool::acquire"; +/// [`AcquireTimes`] on the pool's acquire events alone, for the process's +/// `tracing` subscriber; `None` without the metrics +/// ([`forge_telemetry::enabled`]: call it after [`Telemetry::install`]). +/// The pool emits those events only then ([`forge_db::connect_with`]), and +/// with no meter to record into the layer would only raise the +/// subscriber's level to `TRACE`. +pub fn acquire_times() -> Option< + tracing_subscriber::filter::Filtered, +> +where + S: tracing::Subscriber + for<'span> tracing_subscriber::registry::LookupSpan<'span>, +{ + forge_telemetry::enabled().then(|| { + AcquireTimes.with_filter( + tracing_subscriber::filter::Targets::new() + .with_target(ACQUIRE_TARGET, tracing::Level::TRACE), + ) + }) +} + impl Layer for AcquireTimes { fn on_event( &self, diff --git a/crates/server/tests/telemetry.rs b/crates/server/tests/telemetry.rs index a64cfca5..064a12b6 100644 --- a/crates/server/tests/telemetry.rs +++ b/crates/server/tests/telemetry.rs @@ -27,7 +27,7 @@ use axum::response::{IntoResponse, Response}; use axum::routing::get; use axum::{Json, Router}; use common::{unique, TestServer}; -use forge_server::telemetry::{AcquireTimes, Telemetry, ACQUIRE_TARGET}; +use forge_server::telemetry::Telemetry; use opentelemetry::trace::{SpanId, SpanKind, TraceId}; use opentelemetry::KeyValue; use opentelemetry_sdk::logs::{InMemoryLogExporter, SdkLoggerProvider}; @@ -38,7 +38,6 @@ use reqwest::header::{AUTHORIZATION, CONTENT_TYPE}; use serde_json::{json, Value}; use sha2::Digest as _; use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::Layer; const OCI_MANIFEST: &str = "application/vnd.oci.image.manifest.v1+json"; @@ -86,12 +85,7 @@ fn export() -> &'static Export { telemetry.install(); let subscriber = tracing_subscriber::registry() .with(telemetry.layers()) - .with( - AcquireTimes.with_filter( - tracing_subscriber::filter::Targets::new() - .with_target(ACQUIRE_TARGET, tracing::Level::TRACE), - ), - ); + .with(forge_server::telemetry::acquire_times()); tracing::subscriber::set_global_default(subscriber).expect("the only subscriber"); Export { meter, @@ -694,6 +688,25 @@ async fn metrics_move_with_pushes_pulls_and_proxy_lookups() { /// or tried with a marker no repository name contains; no attribute may /// show one, and every attribute must be one /// [`forge_telemetry::ATTRIBUTE_KEYS`] lists. +/// The server's pool emits an event per acquire with the metrics on, and +/// `db.client.connection.wait_time` counts each (#549: neither happens with +/// them off, `telemetry_off.rs`). +#[tokio::test] +async fn pool_acquires_feed_the_wait_time_histogram() { + export(); + // Built as `main` builds the server's, after the metrics were installed. + let pool = common::sized_pool(2, std::time::Duration::from_secs(10)).await; + let before = collect().count("db.client.connection.wait_time", &[]); + for _ in 0..5 { + pool.acquire().await.expect("acquire"); + } + let after = collect().count("db.client.connection.wait_time", &[]); + assert!( + after >= before + 5, + "five acquires were recorded: {before} before, {after} after" + ); +} + #[tokio::test] async fn no_metric_attribute_carries_a_package_image_user_or_path() { let srv = start().await; diff --git a/crates/server/tests/telemetry_off.rs b/crates/server/tests/telemetry_off.rs new file mode 100644 index 00000000..5c463970 --- /dev/null +++ b/crates/server/tests/telemetry_off.rs @@ -0,0 +1,81 @@ +//! With the metrics off (the default), the connection pool emits no event +//! per acquire and no layer waits for one (#549): `telemetry.rs` has the +//! acquire times with the metrics on. +//! +//! A test binary of its own: the metrics are a process's, and this one never +//! turns them on. Requires Docker (Postgres via the shared harness), or +//! `FORGE_TEST_DATABASE_URL`. + +mod common; + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use forge_server::telemetry::ACQUIRE_TARGET; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::{Layer, Registry}; + +/// Counts the events of [`ACQUIRE_TARGET`]. +#[derive(Clone, Default)] +struct AcquireEvents(Arc); + +impl Layer for AcquireEvents { + fn on_event( + &self, + event: &tracing::Event<'_>, + _ctx: tracing_subscriber::layer::Context<'_, S>, + ) { + if event.metadata().target() == ACQUIRE_TARGET { + self.0.fetch_add(1, Ordering::SeqCst); + } + } +} + +#[tokio::test] +async fn the_pool_emits_no_acquire_event_without_the_metrics() { + assert!(!forge_telemetry::enabled(), "this binary has no metrics"); + // What `main` would add to the subscriber: nothing. + assert!(forge_server::telemetry::acquire_times::().is_none()); + + // A subscriber that wants every acquire event, down to TRACE: a pool + // that emitted one would reach it. + let events = AcquireEvents::default(); + let subscriber = tracing_subscriber::registry().with( + events.clone().with_filter( + tracing_subscriber::filter::Targets::new() + .with_target(ACQUIRE_TARGET, tracing::Level::TRACE), + ), + ); + tracing::subscriber::set_global_default(subscriber).expect("the only subscriber"); + + // The server's pool, as `main` builds it. + let pool = common::sized_pool(2, Duration::from_secs(10)).await; + for _ in 0..5 { + let mut conn = pool.acquire().await.expect("acquire"); + sqlx::query("SELECT 1") + .execute(&mut *conn) + .await + .expect("query"); + } + assert_eq!( + events.0.load(Ordering::SeqCst), + 0, + "no acquire event with the metrics off" + ); + + // The counter does see such events: a pool told to emit them, as the + // server's is with the metrics on, emits one per acquire. + let options: sqlx::postgres::PgConnectOptions = pool.connect_options().as_ref().clone(); + let loud = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .acquire_time_level(log::LevelFilter::Trace) + .connect_with(options) + .await + .expect("connect"); + for _ in 0..3 { + loud.acquire().await.expect("acquire"); + } + // The connect acquires too. + assert!(events.0.load(Ordering::SeqCst) >= 3); +} diff --git a/docs/changes/2.2.0/549.fixed.md b/docs/changes/2.2.0/549.fixed.md new file mode 100644 index 00000000..04d3fa1e --- /dev/null +++ b/docs/changes/2.2.0/549.fixed.md @@ -0,0 +1,6 @@ +With telemetry off, two of its measurements still cost work: retention read +each repository's size (a `SUM` over its assets, images or files) before +every run and again after one that deleted something, and the connection +pool emitted an event on every acquire for a metrics layer that recorded +nothing. Both now happen only with `telemetry.metrics` on; a dry run and a +disabled policy, which free nothing, no longer read the size either. From 8bb56a17932b13368a061df49e017948da631da6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 09:06:49 +0000 Subject: [PATCH 2/2] Retention size tests: record statements without a new dev-dependency The statement recorder is a bare `tracing::Subscriber` instead of a `tracing-subscriber` layer, so forge-core gains no dev-dependency and the lockfile stays as on main. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_012e6DJYg356rtc9YTDE6RrF --- Cargo.lock | 1 - crates/core/Cargo.toml | 2 -- crates/core/tests/retention_size/mod.rs | 37 +++++++++++++++---------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40519e74..aef12e58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2206,7 +2206,6 @@ dependencies = [ "tokio", "tokio-util", "tracing", - "tracing-subscriber", "url", "uuid", "wiremock", diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 0397d92a..711cc0c2 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -85,8 +85,6 @@ axum = { workspace = true } # tests/retention_metrics.rs). opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true, features = ["testing"] } -# Record the statements sqlx logs (tests/retention_size). -tracing-subscriber = { workspace = true } # Mock HTTP server for discovery/JWKS unit tests (no real network). wiremock = "0.6" # Generate a throwaway RSA keypair in-test: sign a self-signed ID token and diff --git a/crates/core/tests/retention_size/mod.rs b/crates/core/tests/retention_size/mod.rs index 01bbb7e8..da701047 100644 --- a/crates/core/tests/retention_size/mod.rs +++ b/crates/core/tests/retention_size/mod.rs @@ -16,8 +16,6 @@ use forge_db::repository_dao::{self, NewRepository}; use forge_db::retention_dao::{self, RunSummary}; use serde_json::{json, Value}; use sqlx::PgPool; -use tracing_subscriber::layer::SubscriberExt as _; -use tracing_subscriber::Layer as _; use crate::common::unique; @@ -36,13 +34,8 @@ impl Statements { INSTALLED .get_or_init(|| { let statements = Statements::default(); - let subscriber = tracing_subscriber::registry().with( - statements.clone().with_filter( - tracing_subscriber::filter::Targets::new() - .with_target("sqlx::query", tracing::Level::DEBUG), - ), - ); - tracing::subscriber::set_global_default(subscriber).expect("the only subscriber"); + tracing::subscriber::set_global_default(statements.clone()) + .expect("the only subscriber"); statements }) .clone() @@ -66,12 +59,26 @@ impl Statements { } } -impl tracing_subscriber::Layer for Statements { - fn on_event( - &self, - event: &tracing::Event<'_>, - _ctx: tracing_subscriber::layer::Context<'_, S>, - ) { +/// A subscriber of sqlx's statement events (`sqlx::query`) alone; spans +/// are none of its business. +impl tracing::Subscriber for Statements { + fn enabled(&self, metadata: &tracing::Metadata<'_>) -> bool { + metadata.target() == "sqlx::query" + } + + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + + fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {} + + fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {} + + fn enter(&self, _: &tracing::span::Id) {} + + fn exit(&self, _: &tracing::span::Id) {} + + fn event(&self, event: &tracing::Event<'_>) { /// The statement: `db.statement` when sqlx shortened it into /// `summary`, `summary` otherwise. #[derive(Default)]