diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index 503e7669..209553b1 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -1422,8 +1422,9 @@ pub struct TelemetryConfig { pub logs: bool, /// Give the metrics the repository's name (`forge.repository`): /// administrators create repositories, so their number stays small. - /// Turn it off for an install with very many; the metrics then add up - /// all repositories. Default `true`. + /// Turn it off for an install with more than the per-repository metrics + /// have room for ([`crate::telemetry::REPOSITORY_BUDGET`]); the metrics + /// then add up all repositories. Default `true`. #[serde(default = "default_true")] pub per_repository: bool, /// Seconds between two counts of what the instance holds (repositories, diff --git a/crates/server/src/telemetry.rs b/crates/server/src/telemetry.rs index 41f32c57..9d6662eb 100644 --- a/crates/server/src/telemetry.rs +++ b/crates/server/src/telemetry.rs @@ -29,8 +29,9 @@ //! with the metrics on ([`acquire_times`]). use std::borrow::Cow; +use std::collections::HashSet; use std::pin::Pin; -use std::sync::Arc; +use std::sync::{Arc, Mutex, PoisonError}; use std::task::{Context, Poll}; use std::time::{Duration, Instant}; @@ -50,8 +51,11 @@ use opentelemetry::propagation::Extractor; use opentelemetry::trace::TracerProvider as _; use opentelemetry::KeyValue; use opentelemetry_otlp::WithHttpConfig as _; +use opentelemetry_sdk::error::OTelSdkResult; use opentelemetry_sdk::logs::SdkLoggerProvider; -use opentelemetry_sdk::metrics::SdkMeterProvider; +use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData, ResourceMetrics}; +use opentelemetry_sdk::metrics::exporter::PushMetricExporter; +use opentelemetry_sdk::metrics::{self as sdk_metrics, SdkMeterProvider, Stream, Temporality}; use opentelemetry_sdk::propagation::TraceContextPropagator; use opentelemetry_sdk::resource::{EnvResourceDetector, TelemetryResourceDetector}; use opentelemetry_sdk::trace::SdkTracerProvider; @@ -97,12 +101,11 @@ impl Telemetry { )?) .build() .context("build the OTLP metric exporter")?; - Some( - SdkMeterProvider::builder() - .with_resource(resource.clone()) - .with_periodic_exporter(exporter) - .build(), - ) + Some(meter_provider( + config.per_repository, + resource.clone(), + exporter, + )) } else { None }; @@ -267,6 +270,162 @@ fn direct_client(signal_timeout_var: &str) -> anyhow::Result usize { + forge_telemetry::PER_REPOSITORY_INSTRUMENTS + .iter() + .find(|(instrument, _)| per_repository && *instrument == name) + .map_or(DEFAULT_CARDINALITY_LIMIT, |(_, sets)| { + (sets * REPOSITORY_BUDGET).max(DEFAULT_CARDINALITY_LIMIT) + }) +} + +/// The meter provider `main` runs with, exporting to `exporter` every +/// `OTEL_METRIC_EXPORT_INTERVAL`: with `per_repository` +/// (`telemetry.per_repository`), a view gives each instrument that carries +/// the repository's name its [`cardinality_limit`], and an attribute set +/// that overflows a limit all the same is warned about ([`WarnOnOverflow`]). +pub fn meter_provider( + per_repository: bool, + resource: Resource, + exporter: impl PushMetricExporter, +) -> SdkMeterProvider { + let mut builder = SdkMeterProvider::builder() + .with_resource(resource) + .with_periodic_exporter(WarnOnOverflow { + inner: exporter, + per_repository, + warned: Mutex::default(), + }); + if per_repository { + builder = builder.with_view(|instrument: &sdk_metrics::Instrument| { + let limit = cardinality_limit(instrument.name(), true); + // The other instruments keep the SDK's stream. + (limit != DEFAULT_CARDINALITY_LIMIT) + .then(|| { + // A stream of the instrument's own name, unit, description + // and buckets: only the limit differs. + Stream::builder().with_cardinality_limit(limit).build().ok() + }) + .flatten() + }); + } + builder.build() +} + +/// The attribute the SDK gives the one series that holds what was recorded +/// with an attribute set past the instrument's cardinality limit. +const OVERFLOW_ATTRIBUTE: &str = "otel.metric.overflow"; + +/// A metric exporter that warns, once per instrument, when an export holds +/// the SDK's overflow series ([`OVERFLOW_ATTRIBUTE`]): the SDK says nothing, +/// and what lands there is counted without its repository, status or +/// outcome, so the alerts and panels by those attributes miss it. +struct WarnOnOverflow { + inner: E, + /// Whether the per-repository limits apply (`telemetry.per_repository`). + per_repository: bool, + /// The instruments warned about. + warned: Mutex>, +} + +impl WarnOnOverflow { + /// Warn about the instruments of `metrics` that overflowed since the + /// last warning. + fn warn(&self, metrics: &ResourceMetrics) { + for name in overflowed(metrics) { + let mut warned = self.warned.lock().unwrap_or_else(PoisonError::into_inner); + if warned.contains(name) { + continue; + } + warned.insert(name.to_string()); + drop(warned); + let limit = cardinality_limit(name, self.per_repository); + if limit == DEFAULT_CARDINALITY_LIMIT { + tracing::warn!( + instrument = name, + limit, + "a metric has more attribute sets than its limit: further ones are counted in one series, otel.metric.overflow=true, without their attributes" + ); + } else { + tracing::warn!( + instrument = name, + limit, + repositories = REPOSITORY_BUDGET, + "a per-repository metric has more attribute sets than its limit, which has room for this many repositories: further ones are counted in one series, otel.metric.overflow=true, without their repository, so alerts and panels by repository miss them. Restart Forge to drop the series of deleted repositories, or turn telemetry.per_repository off" + ); + } + } + } +} + +impl PushMetricExporter for WarnOnOverflow { + fn export( + &self, + metrics: &ResourceMetrics, + ) -> impl std::future::Future + Send { + self.warn(metrics); + self.inner.export(metrics) + } + + fn force_flush(&self) -> OTelSdkResult { + self.inner.force_flush() + } + + fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult { + self.inner.shutdown_with_timeout(timeout) + } + + fn temporality(&self) -> Temporality { + self.inner.temporality() + } +} + +/// The names of the metrics of `metrics` that hold the SDK's overflow +/// series. +fn overflowed(metrics: &ResourceMetrics) -> Vec<&str> { + fn is_overflow<'a>(mut attributes: impl Iterator) -> bool { + attributes.any(|kv| kv.key.as_str() == OVERFLOW_ATTRIBUTE) + } + fn any_overflow(data: &MetricData) -> bool { + match data { + MetricData::Gauge(g) => g.data_points().any(|p| is_overflow(p.attributes())), + MetricData::Sum(s) => s.data_points().any(|p| is_overflow(p.attributes())), + MetricData::Histogram(h) => h.data_points().any(|p| is_overflow(p.attributes())), + MetricData::ExponentialHistogram(h) => { + h.data_points().any(|p| is_overflow(p.attributes())) + } + } + } + metrics + .scope_metrics() + .flat_map(|scope| scope.metrics()) + .filter(|metric| match metric.data() { + AggregatedMetrics::F64(data) => any_overflow(data), + AggregatedMetrics::U64(data) => any_overflow(data), + AggregatedMetrics::I64(data) => any_overflow(data), + }) + .map(|metric| metric.name()) + .collect() +} + /// The filter of the console log: `RUST_LOG` (default `info`), without the /// spans meant for export. pub fn console_log_filter() -> EnvFilter { diff --git a/crates/server/tests/telemetry_cardinality.rs b/crates/server/tests/telemetry_cardinality.rs new file mode 100644 index 00000000..d58c2813 --- /dev/null +++ b/crates/server/tests/telemetry_cardinality.rs @@ -0,0 +1,254 @@ +//! The cardinality limits of the per-repository metrics, on the meter +//! provider `main` builds ([`meter_provider`]) and through Forge's own +//! instruments: more attribute sets than the SDK's default of 2,000 per +//! instrument are exported under their own attributes, and a set past the +//! raised limit, which the SDK folds into its `otel.metric.overflow` series +//! without a word, is warned about once. Needs neither Docker nor a +//! database. +//! +//! The tests of this binary share one provider, as a process does: each +//! records into, and asserts on, instruments of its own. + +use std::collections::BTreeSet; +use std::sync::{Mutex, OnceLock}; +use std::time::Duration; + +use forge_server::telemetry::{cardinality_limit, meter_provider, REPOSITORY_BUDGET}; +use forge_telemetry::proxy; +use opentelemetry::metrics::MeterProvider as _; +use opentelemetry::KeyValue; +use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData}; +use opentelemetry_sdk::metrics::{InMemoryMetricExporter, SdkMeterProvider}; +use opentelemetry_sdk::Resource; +use tracing_subscriber::layer::SubscriberExt; + +/// The SDK's limit without a view. +const SDK_DEFAULT: usize = 2000; + +/// The process's meter provider, exporting to memory, and the warnings +/// logged. +struct Export { + meter: SdkMeterProvider, + metrics: InMemoryMetricExporter, +} + +/// The warnings logged, each as its message and fields. +static WARNINGS: Mutex> = Mutex::new(Vec::new()); + +/// Keeps the warnings: the export logs from the reader's thread, so this is +/// the process's subscriber. +struct Warnings; + +impl tracing_subscriber::Layer for Warnings { + fn on_event(&self, event: &tracing::Event<'_>, _: tracing_subscriber::layer::Context<'_, S>) { + struct Text(String); + impl tracing::field::Visit for Text { + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + self.0.push_str(&format!(" {}={value:?}", field.name())); + } + } + if *event.metadata().level() == tracing::Level::WARN { + let mut text = Text(String::new()); + event.record(&mut text); + WARNINGS.lock().expect("warnings").push(text.0); + } + } +} + +/// Set up the provider as `main` does, with Forge's instruments on it, once +/// per process. +fn export() -> &'static Export { + static EXPORT: OnceLock = OnceLock::new(); + EXPORT.get_or_init(|| { + tracing::subscriber::set_global_default(tracing_subscriber::registry().with(Warnings)) + .expect("the process's subscriber"); + let metrics = InMemoryMetricExporter::default(); + let meter = meter_provider(true, Resource::builder_empty().build(), metrics.clone()); + forge_telemetry::init(&meter.meter("forge")); + Export { meter, metrics } + }) +} + +/// A data point of a metric: its attributes and, of a histogram, its +/// bucket bounds. +struct Point { + attributes: Vec, + bounds: Vec, +} + +impl Point { + fn value(&self, key: &str) -> Option { + self.attributes + .iter() + .find(|kv| kv.key.as_str() == key) + .map(|kv| kv.value.to_string()) + } +} + +/// Collect, and return the data points of metric `name` in that export. +fn points(name: &str) -> Vec { + let export = export(); + export.meter.force_flush().expect("flush"); + let exports = export.metrics.get_finished_metrics().expect("metrics"); + // Cumulative: the last export holds every series. + let last = exports.last().expect("an export"); + let mut out = Vec::new(); + for metric in last + .scope_metrics() + .flat_map(|scope| scope.metrics()) + .filter(|metric| metric.name() == name) + { + fn of(data: &MetricData, out: &mut Vec) { + match data { + MetricData::Sum(sum) => out.extend(sum.data_points().map(|p| Point { + attributes: p.attributes().cloned().collect(), + bounds: Vec::new(), + })), + MetricData::Histogram(histogram) => { + out.extend(histogram.data_points().map(|p| Point { + attributes: p.attributes().cloned().collect(), + bounds: p.bounds().collect(), + })) + } + MetricData::Gauge(_) | MetricData::ExponentialHistogram(_) => { + panic!("neither a sum nor a histogram") + } + } + } + match metric.data() { + AggregatedMetrics::F64(data) => of(data, &mut out), + AggregatedMetrics::U64(data) => of(data, &mut out), + AggregatedMetrics::I64(data) => of(data, &mut out), + } + } + out +} + +/// The warnings logged about instrument `name`. +fn warnings_about(name: &str) -> Vec { + WARNINGS + .lock() + .expect("warnings") + .iter() + .filter(|warning| warning.contains(&format!("instrument={name:?}"))) + .cloned() + .collect() +} + +/// Every point carries its own attributes: none is the overflow series. +fn assert_none_overflowed(name: &str, points: &[Point]) { + for point in points { + assert_eq!( + point.value("otel.metric.overflow"), + None, + "{name} overflowed" + ); + assert!( + point.value("forge.repository").is_some(), + "{name}: a point without its repository: {:?}", + point.attributes + ); + } +} + +#[test] +fn more_sets_than_the_sdks_default_keep_their_attributes() { + export(); + // What the issue describes: a hundred proxies, each answered with thirty + // statuses by its upstream and looking up two kinds with every outcome. + let repositories: Vec = (0..100).map(|n| format!("proxy-{n}")).collect(); + let statuses: Vec = (0..29).map(|n| 200 + n * 7).collect(); + for repository in &repositories { + for &status in &statuses { + proxy::upstream_request(repository, "GET", Some(status), Duration::from_millis(20)); + } + proxy::upstream_request(repository, "GET", None, Duration::from_millis(20)); + for kind in [proxy::Kind::Metadata, proxy::Kind::Package] { + for &outcome in proxy::Outcome::ALL { + proxy::Lookup::start(repository, kind).finish(outcome); + } + } + } + + let requests = points("http.client.request.duration"); + assert_eq!(requests.len(), 100 * 30); + assert!(requests.len() > SDK_DEFAULT); + assert_none_overflowed("http.client.request.duration", &requests); + // The combination the alerts are about: each proxy's own 429. + let rate_limited: BTreeSet = requests + .iter() + .filter(|p| p.value("http.response.status_code").as_deref() == Some("249")) + .filter_map(|p| p.value("forge.repository")) + .collect(); + assert_eq!(rate_limited.len(), 100); + // The view changes the limit alone: the buckets are the instrument's. + let bounds = &requests[0].bounds; + assert_eq!(bounds.len(), 17, "{bounds:?}"); + assert_eq!( + (bounds.first(), bounds.last()), + (Some(&0.005), Some(&120.0)) + ); + + // 1,800 sets: under the default, and still all there. + let lookups = points("forge.proxy.lookups"); + assert_eq!(lookups.len(), 100 * 2 * proxy::Outcome::ALL.len()); + assert_none_overflowed("forge.proxy.lookups", &lookups); + + for name in ["http.client.request.duration", "forge.proxy.lookups"] { + assert_eq!(warnings_about(name), Vec::::new()); + } +} + +#[test] +fn a_set_past_the_raised_limit_is_warned_about_once() { + export(); + let name = "forge.proxy.upstream.documents"; + let limit = cardinality_limit(name, true); + assert_eq!(limit, REPOSITORY_BUDGET * proxy::DocumentSource::ALL.len()); + assert!(limit > SDK_DEFAULT); + // Without the repository's name the SDK's limit stands. + assert_eq!(cardinality_limit(name, false), SDK_DEFAULT); + assert_eq!( + cardinality_limit("http.server.request.duration", true), + SDK_DEFAULT + ); + + // Exactly the limit: every repository of the budget, nothing to warn of. + for n in 0..REPOSITORY_BUDGET { + for &source in proxy::DocumentSource::ALL { + proxy::upstream_document(&format!("budgeted-{n}"), source); + } + } + let documents = points(name); + assert_eq!(documents.len(), limit); + assert_none_overflowed(name, &documents); + assert_eq!(warnings_about(name), Vec::::new()); + + // One repository more: its sets land in the overflow series. + for &source in proxy::DocumentSource::ALL { + proxy::upstream_document("one-too-many", source); + } + let documents = points(name); + assert_eq!(documents.len(), limit + 1); + let overflow: Vec<&Point> = documents + .iter() + .filter(|p| p.value("otel.metric.overflow").is_some()) + .collect(); + assert_eq!(overflow.len(), 1); + assert_eq!(overflow[0].value("forge.repository"), None); + let warnings = warnings_about(name); + assert_eq!(warnings.len(), 1, "{warnings:?}"); + for part in [ + format!("limit={limit}"), + format!("repositories={REPOSITORY_BUDGET}"), + "otel.metric.overflow=true".to_string(), + "telemetry.per_repository".to_string(), + ] { + assert!(warnings[0].contains(&part), "{part}: {warnings:?}"); + } + + // The series stays in every export; the warning is not repeated. + proxy::upstream_document("another-too-many", proxy::DocumentSource::Stored); + points(name); + assert_eq!(warnings_about(name).len(), 1); +} diff --git a/crates/telemetry/src/lib.rs b/crates/telemetry/src/lib.rs index 5f0ee58c..579398f8 100644 --- a/crates/telemetry/src/lib.rs +++ b/crates/telemetry/src/lib.rs @@ -88,6 +88,32 @@ fn with_repo(name: &str, attributes: [KeyValue; N]) -> Vec $text:literal,)* }) => { @@ -1677,6 +1703,25 @@ mod tests { "attribute {key:?} is not in ATTRIBUTE_KEYS" ); } + // The instruments with the repository attribute are the ones whose + // cardinality limit follows the repositories. + let per_repository: BTreeSet<&str> = metrics + .iter() + .filter(|(_, points)| { + points + .iter() + .flatten() + .any(|kv| kv.key.as_str() == "forge.repository") + }) + .map(|(name, _)| name.as_str()) + .collect(); + assert_eq!( + per_repository, + PER_REPOSITORY_INSTRUMENTS + .iter() + .map(|(name, _)| *name) + .collect::>() + ); for name in [ "http.server.request.duration", "http.server.active_requests", diff --git a/docs/changes/2.2.0/580.fixed.md b/docs/changes/2.2.0/580.fixed.md new file mode 100644 index 00000000..4a9b3e62 --- /dev/null +++ b/docs/changes/2.2.0/580.fixed.md @@ -0,0 +1,10 @@ +With a few dozen to about a hundred proxy repositories, the per-repository +metrics (`http.client.request.duration`, `forge.proxy.lookups`) passed the +OpenTelemetry SDK's limit of 2,000 attribute sets per instrument: what was +recorded with a further one, a proxy's first 429, landed in a single +`otel.metric.overflow=true` series without repository or status, with +nothing logged, so alerts and dashboard panels by `forge_repository` missed +it. Every instrument that carries `forge.repository` now has room for 1,000 +repositories, Forge logs a warning once per instrument when an attribute set +overflows all the same, and the deployment guide (Telemetry) names the +threshold past which to turn `telemetry.per_repository` off. diff --git a/docs/deployment.md b/docs/deployment.md index 4c108f89..9a103659 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1484,9 +1484,32 @@ counted under the *surface* of the route it matched (`forge.surface`: `nuget`, `oci`, `plugin_npm`, `plugin_raw`, `plugin_other`, `admin`, `browse`, `auth`, `health`, `ui`; `repository` for a `/repository/…` request that named no repository), never its URL. The repository's name -(`forge.repository`) is the one attribute an administrator chooses; with -very many repositories turn `telemetry.per_repository` off and the -repositories add up. +(`forge.repository`) is the one attribute an administrator chooses. + +**Up to 1,000 repositories keep their own series.** The OpenTelemetry SDK +keeps at most 2,000 attribute sets per instrument and counts what is +recorded with a further one in a single series, `otel.metric.overflow=true`, +without repository, status or outcome: alerts and dashboard panels by +`forge_repository` then miss it. A proxy gives `http.client.request.duration` +a set per method and upstream status and `forge.proxy.lookups` one per kind +and outcome, some twenty to thirty each, so Forge raises the limit of every +instrument that carries `forge.repository` to what 1,000 repositories need +(32,000 sets for `http.client.request.duration`, 18,000 for +`forge.proxy.lookups`), at a few megabytes of memory. The series are +cumulative and never dropped while the instance runs, so a deleted or +renamed repository counts until the restart. Past a limit, Forge logs a +warning once per instrument and instance: + +```text +WARN a per-repository metric has more attribute sets than its limit, which has room for this many repositories: … instrument="http.client.request.duration" limit=32000 repositories=1000 +``` + +Restart the instance when deleted repositories filled the limit. With more +than 1,000 repositories, or proxies whose upstreams answer with more than +some thirty different statuses each, turn `telemetry.per_repository` off: +the repositories then add up, and each instrument is back under the SDK's +2,000 sets. An `otel_metric_overflow="true"` series in the backend says the +same as the warning. | Area | Instruments (attributes) | | --- | --- | diff --git a/forge.toml.example b/forge.toml.example index 03f1f41b..75ba3868 100644 --- a/forge.toml.example +++ b/forge.toml.example @@ -295,7 +295,8 @@ processed_retention_seconds = 604800 # (default: 604800 = 7 days) keep processe # traces = true # (default: false) # logs = true # (default: false) what RUST_LOG prints, with its trace # The repository's name on the metrics (forge.repository). Off, the -# repositories add up: for an install with very many. +# repositories add up: for an install with more than 1,000, which is what +# the per-repository metrics have room for (docs/deployment.md → Telemetry). # per_repository = true # (default: true) # How often an instance counts its repositories, their versions and stored # bytes, and the outbox backlog for the metrics (database queries).