feat(stats): query time histogram in OpenMetrics and OTLP

.schema/pgdog.schema.json

@@ -92,6 +92,20 @@
         "query_parser_engine": "pg_query_protobuf",
         "query_size_limit": null,
         "query_size_limit_action": "warn",
+        "query_time_buckets": [
+          0.1,
+          0.3,
+          1.0,
+          3.0,
+          10.0,
+          30.0,
+          100.0,
+          300.0,
+          1000.0,
+          3000.0,
+          10000.0,
+          30000.0
+        ],
         "query_timeout": 9223372036854775807,
         "read_write_split": "include_primary",
         "read_write_strategy": "conservative",
@@ -175,7 +189,8 @@
         "endpoint": null,
         "headers": {},
         "namespace": null,
-        "push_interval": 0
+        "push_interval": 10000,
+        "temporality_preference": "Cumulative"
       }
     },
     "plugins": {
@@ -1085,6 +1100,28 @@
           "$ref": "#/$defs/QuerySizeLimitAction",
           "default": "warn"
         },
+        "query_time_buckets": {
+          "description": "Upper bounds, in milliseconds, of the `query_time_seconds` histogram buckets.\n\nEach bound emits one time series per pool, so prefer a short ladder that\nbrackets the latencies worth alerting on. Values are sorted and\ndeduplicated, and an implicit `+Inf` bucket is always appended. At most\n20 bounds are accepted and every one must be finite and greater than\nzero; a ladder PgDog cannot use is refused at startup rather than\nrepaired, so the exported buckets always match what was configured.\n\n**Note:** This setting cannot be changed at runtime. Restart PgDog after changing it.\n\n**Note:** A malformed `PGDOG_QUERY_TIME_BUCKETS` falls back to the default\nladder instead of being refused, matching every other environment variable.\n\n_Default:_ `[0.1, 0.3, 1, 3, 10, 30, 100, 300, 1000, 3000, 10000, 30000]`\n\nEnv: `PGDOG_QUERY_TIME_BUCKETS` (comma-separated milliseconds)\n\n<https://docs.pgdog.dev/configuration/pgdog.toml/general/#query_time_buckets>",
+          "type": "array",
+          "default": [
+            0.1,
+            0.3,
+            1.0,
+            3.0,
+            10.0,
+            30.0,
+            100.0,
+            300.0,
+            1000.0,
+            3000.0,
+            10000.0,
+            30000.0
+          ],
+          "items": {
+            "type": "number",
+            "format": "double"
+          }
+        },
         "query_timeout": {
           "description": "Maximum amount of time to wait for a Postgres query to finish executing.\n\n<https://docs.pgdog.dev/configuration/pgdog.toml/general/#query_timeout>",
           "type": "integer",
@@ -1598,10 +1635,42 @@
           "format": "uint64",
           "default": 10000,
           "minimum": 0
+        },
+        "temporality_preference": {
+          "description": "Describes how the exported metric points should be described.\n\nSee https://opentelemetry.io/docs/specs/otel/metrics/data-model/#metric-points\n\n_Default:_ `Cumulative`, or `Delta` when `datadog_api_key` is set.\n\nEnv: `OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE`",
+          "anyOf": [
+            {
+              "$ref": "#/$defs/OtelTemporalityPreference"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": "Cumulative"
         }
       },
       "additionalProperties": false
     },
+    "OtelTemporalityPreference": {
+      "description": "Aggregation temporality used when exporting OTLP metric points.",
+      "oneOf": [
+        {
+          "description": "Points report the value accumulated since the exporter started.",
+          "type": "string",
+          "const": "Cumulative"
+        },
+        {
+          "description": "Points report the change since the last export.",
+          "type": "string",
+          "const": "Delta"
+        },
+        {
+          "description": "Delta for sums, cumulative for histograms; minimizes exporter memory.",
+          "type": "string",
+          "const": "LowMemory"
+        }
+      ]
+    },
     "PassthroughAuth": {
       "description": "toggle automatic creation of connection pools given the user name, database and password.\n\nSee [passthrough authentication](https://docs.pgdog.dev/features/authentication/#passthrough-authentication).\n\n<https://docs.pgdog.dev/configuration/pgdog.toml/general/#passthrough_auth>",
       "oneOf": [

CONTRIBUTING.md

@@ -24,3 +24,4 @@ Contributions are welcome. If you see a bug, feel free to submit a PR with a fix
 1. Please format your code with `cargo fmt`.
 2. If you're feeling generous, `cargo clippy` as well.
 3. Please write and include tests. This is production software used in one of the most important areas of the stack.
+4. If changes have been made to configuration schemas, run `cargo run -p pgdog-jsonschema`

example.pgdog.toml

@@ -152,6 +152,18 @@ openmetrics_port = 9090
 #
 # Default: none
 openmetrics_namespace = "pgdog_"
+# Upper bounds, in milliseconds, of the "query_time_seconds" histogram buckets.
+#
+# Each bound emits one time series per pool, so prefer a short ladder that
+# brackets the latencies worth alerting on. Read once at startup: changing
+# this requires a restart.
+#
+# At most 20 bounds, each finite and greater than zero. A ladder PgDog cannot
+# use is refused at startup rather than repaired, so the buckets you scrape are
+# always the ones you configured here.
+#
+# Default: [0.1, 0.3, 1, 3, 10, 30, 100, 300, 1000, 3000, 10000, 30000]
+query_time_buckets = [0.1, 0.3, 1.0, 3.0, 10.0, 30.0, 100.0, 300.0, 1000.0, 3000.0, 10000.0, 30000.0]
 # Log output format.
 #
 # Default: text

pgdog-config/src/core.rs

@@ -6,6 +6,7 @@ use std::fs::read_to_string;
 use std::path::{Path, PathBuf};
 use tracing::{error, info, warn};
 
+use crate::otel_temporality::OtelTemporalityPreference;
 use crate::sharding::ShardedSchema;
 use crate::util::random_string;
 use crate::{
@@ -120,6 +121,7 @@ impl ConfigAndUsers {
         self.config.check();
         self.users.check(&self.config);
         self.validate_server_auth()?;
+        self.warn_if_data_dog_cumulative();
         Ok(())
     }
 
@@ -149,6 +151,33 @@ impl ConfigAndUsers {
         Ok(())
     }
 
+    fn warn_if_data_dog_cumulative(&self) {
+        match (
+            &self.config.otel.datadog_api_key,
+            &self.config.otel.temporality_preference,
+        ) {
+            (Some(_datadog_present), Some(OtelTemporalityPreference::Cumulative))
+                if std::env::var("IGNORE_DATADOG_CUMULATIVE_WARNING")
+                    .ok()
+                    .as_deref()
+                    != Some("1") =>
+            {
+                warn!(
+                    "Sending Cumulative OTLP sums/histograms to Datadog is stateful and lossy: \
+                        all points on a timeseries must reach the same Agent/exporter (constraining \
+                        how you scale collectors), the first point of a new series may be dropped \
+                        (causing gaps on restart), and histogram min/max may be missing or \
+                        approximated. See \
+                        https://docs.datadoghq.com/opentelemetry/guide/otlp_delta_temporality/?tab=python#implications-of-using-cumulative-aggregation-temporality. \
+                        Set IGNORE_DATADOG_CUMULATIVE_WARNING=1 to silence."
+                );
+            }
+            _ => {
+                // valid
+            }
+        }
+    }
+
     /// Prepared statements are enabled.
     pub fn prepared_statements(&self) -> PreparedStatements {
         // Disable prepared statements automatically in session mode
@@ -284,6 +313,7 @@ pub struct Config {
     ///
     /// <https://docs.pgdog.dev/configuration/pgdog.toml/otel/>
     #[serde(default)]
+    #[schemars(default = "Otel::schema_default")]
     pub otel: Otel,
 
     /// HashiCorp Vault settings, required for users configured with `server_auth = "vault"`.

pgdog-config/src/general.rs

@@ -344,6 +344,29 @@ pub struct General {
     /// <https://docs.pgdog.dev/configuration/pgdog.toml/general/#openmetrics_namespace>
     pub openmetrics_namespace: Option<String>,
 
+    /// Upper bounds, in milliseconds, of the `query_time_seconds` histogram buckets.
+    ///
+    /// Each bound emits one time series per pool, so prefer a short ladder that
+    /// brackets the latencies worth alerting on. Values are sorted and
+    /// deduplicated, and an implicit `+Inf` bucket is always appended. At most
+    /// 20 bounds are accepted and every one must be finite and greater than
+    /// zero; a ladder PgDog cannot use is refused at startup rather than
+    /// repaired, so the exported buckets always match what was configured.
+    ///
+    /// **Note:** This setting cannot be changed at runtime. Restart PgDog after changing it.
+    ///
+    /// **Note:** A malformed `PGDOG_QUERY_TIME_BUCKETS` falls back to the default
+    /// ladder instead of being refused, matching every other environment variable.
+    ///
+    /// _Default:_ `[0.1, 0.3, 1, 3, 10, 30, 100, 300, 1000, 3000, 10000, 30000]`
+    ///
+    /// Env: `PGDOG_QUERY_TIME_BUCKETS` (comma-separated milliseconds)
+    ///
+    /// <https://docs.pgdog.dev/configuration/pgdog.toml/general/#query_time_buckets>
+    #[serde(default = "General::query_time_buckets")]
+    #[schemars(default = "General::schema_default_query_time_buckets")]
+    pub query_time_buckets: Vec<f64>,
+
     /// Enables support for prepared statements.
     ///
     /// _Default:_ `extended`
@@ -881,6 +904,7 @@ impl Default for General {
             query_size_limit_action: Self::query_size_limit_action(),
             openmetrics_port: Self::openmetrics_port(),
             openmetrics_namespace: Self::openmetrics_namespace(),
+            query_time_buckets: Self::query_time_buckets(),
             prepared_statements: Self::prepared_statements(),
             query_parser_enabled: Self::query_parser_enabled(),
             query_parser: QueryParserLevel::default(),
@@ -954,6 +978,11 @@ impl Default for General {
 }
 
 impl General {
+    /// Default upper bounds of the `query_time_seconds` histogram, in milliseconds.
+    pub const DEFAULT_QUERY_TIME_BUCKETS: [f64; 12] = [
+        0.1, 0.3, 1.0, 3.0, 10.0, 30.0, 100.0, 300.0, 1_000.0, 3_000.0, 10_000.0, 30_000.0,
+    ];
+
     fn env_or_default<T: std::str::FromStr>(env_var: &str, default: T) -> T {
         env::var(env_var)
             .ok()
@@ -1312,6 +1341,31 @@ impl General {
         Self::env_option_string("PGDOG_OPENMETRICS_NAMESPACE")
     }
 
+    /// Default `query_time_seconds` bucket bounds, in milliseconds.
+    ///
+    /// Exponential from 100µs to 30s. A malformed env var degrades to the
+    /// defaults instead of failing startup; it degrades as a whole, so an
+    /// operator never gets a ladder made of the values that happened to parse.
+    pub fn query_time_buckets() -> Vec<f64> {
+        let Some(raw) = Self::env_option_string("PGDOG_QUERY_TIME_BUCKETS") else {
+            return Self::DEFAULT_QUERY_TIME_BUCKETS.to_vec();
+        };
+
+        raw.split(',')
+            .map(|value| value.trim().parse::<f64>())
+            .collect::<Result<Vec<_>, _>>()
+            .ok()
+            .filter(|buckets| !buckets.is_empty())
+            .unwrap_or_else(|| Self::DEFAULT_QUERY_TIME_BUCKETS.to_vec())
+    }
+
+    /// Schema-only default, so the generated schema documents the shipped
+    /// bounds rather than whatever `PGDOG_QUERY_TIME_BUCKETS` happened to be
+    /// set to when the schema was generated.
+    fn schema_default_query_time_buckets() -> Vec<f64> {
+        Self::DEFAULT_QUERY_TIME_BUCKETS.to_vec()
+    }
+
     fn default_dns_ttl() -> Option<u64> {
         Self::env_option("PGDOG_DNS_TTL")
     }
@@ -1928,6 +1982,55 @@ mod tests {
         assert_eq!(General::openmetrics_namespace(), None);
     }
 
+    #[test]
+    fn test_query_time_buckets() {
+        let _guard = remove_env_var("PGDOG_QUERY_TIME_BUCKETS");
+
+        let general = General::default();
+        assert_eq!(
+            general.query_time_buckets,
+            General::DEFAULT_QUERY_TIME_BUCKETS.to_vec()
+        );
+
+        let general: General = toml::from_str("query_time_buckets = [1.0, 10.0, 100.0]").unwrap();
+        assert_eq!(general.query_time_buckets, vec![1.0, 10.0, 100.0]);
+
+        // Omitting the setting keeps the defaults.
+        let general: General = toml::from_str("").unwrap();
+        assert_eq!(
+            general.query_time_buckets,
+            General::DEFAULT_QUERY_TIME_BUCKETS.to_vec()
+        );
+    }
+
+    #[test]
+    fn test_env_query_time_buckets() {
+        let _guard = set_env_var("PGDOG_QUERY_TIME_BUCKETS", "1, 5,25");
+        assert_eq!(General::query_time_buckets(), vec![1.0, 5.0, 25.0]);
+
+        // Nothing parseable falls back to the defaults rather than
+        // disabling bucketing.
+        let _guard = set_env_var("PGDOG_QUERY_TIME_BUCKETS", "nonsense");
+        assert_eq!(
+            General::query_time_buckets(),
+            General::DEFAULT_QUERY_TIME_BUCKETS.to_vec()
+        );
+
+        // One bad value discards the whole ladder: keeping the rest would build
+        // a histogram the operator never asked for.
+        let _guard = set_env_var("PGDOG_QUERY_TIME_BUCKETS", "1,abc,3");
+        assert_eq!(
+            General::query_time_buckets(),
+            General::DEFAULT_QUERY_TIME_BUCKETS.to_vec()
+        );
+
+        let _guard = remove_env_var("PGDOG_QUERY_TIME_BUCKETS");
+        assert_eq!(
+            General::query_time_buckets(),
+            General::DEFAULT_QUERY_TIME_BUCKETS.to_vec()
+        );
+    }
+
     #[test]
     fn test_env_invalid_enum_values() {
         let _guard = set_env_var("PGDOG_POOLER_MODE", "invalid_mode");

pgdog-config/src/lib.rs

@@ -8,6 +8,7 @@ pub mod general;
 pub mod memory;
 pub mod networking;
 pub mod otel;
+pub mod otel_temporality;
 pub mod overrides;
 pub mod pooling;
 pub mod replication;