Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion crates/core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,8 @@ zip = { version = "9.0", default-features = false, features = ["deflate"] }
testcontainers = "0.27"
testcontainers-modules = { version = "0.15", features = ["postgres"] }
axum = { workspace = true }
# Read back the metrics recorded (tests/auth_pool_timeout.rs).
# Read back the metrics recorded (tests/auth_pool_timeout.rs,
# tests/retention_metrics.rs).
opentelemetry = { workspace = true }
opentelemetry_sdk = { workspace = true, features = ["testing"] }
# Mock HTTP server for discovery/JWKS unit tests (no real network).
Expand Down
39 changes: 29 additions & 10 deletions crates/core/src/retention/worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -194,9 +194,15 @@ impl RetentionWorker {
run_id,
self.config.lease,
));
// The size before the run, for the bytes it frees (metrics only: a
// failed read records none).
let size_before = self.repository_size(&repo).await.ok();
// The size before the run, for the bytes it frees. Only the metrics
// use it: without them nothing reads the size, and with them neither
// does a dry run or a disabled policy, which frees nothing. A failed
// read records no freed bytes.
let size_before = if forge_telemetry::enabled() && frees_bytes(&repo.policy) {
self.repository_size(&repo).await.ok()
} else {
None
};
// The policy's `max_run_seconds`, counted from the claim.
let deadline =
schedule::max_run_duration(&repo.policy).map(|limit| std::time::Instant::now() + limit);
Expand Down Expand Up @@ -241,7 +247,8 @@ impl RetentionWorker {
}
}

/// The logical size of a claimed repository, as its format stores it.
/// The logical size of a claimed repository, as its format stores it: a
/// `SUM` over the repository's rows, read for the metrics only.
async fn repository_size(&self, repo: &retention_dao::ClaimedRepo) -> sqlx::Result<u64> {
let format = self.formats.resolve(&repo.format);
if format.as_ref().is_some_and(|f| f.stores_images()) {
Expand All @@ -262,15 +269,11 @@ impl RetentionWorker {
summary: &RunSummary,
size_before: Option<u64>,
) {
let dry_run = repo
.policy
.get("dry_run_only")
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
if dry_run {
if flag(&repo.policy, "dry_run_only", false) {
forge_telemetry::retention::run(&repo.name, true, 0, 0, 0);
return;
}
// `size_before` is `None` without the metrics: no second read then.
let freed = match (size_before, summary.deleted > 0) {
(Some(before), true) => self
.repository_size(repo)
Expand Down Expand Up @@ -741,6 +744,22 @@ fn panic_message(payload: &(dyn std::any::Any + Send)) -> String {
.unwrap_or_else(|| "non-string panic payload".to_string())
}

/// A boolean of a stored policy (`dry_run_only`, `enabled`), `default`
/// when the policy does not set it: the three policy shapes (packages,
/// images, files) share these.
fn flag(policy: &serde_json::Value, name: &str, default: bool) -> bool {
policy
.get(name)
.and_then(serde_json::Value::as_bool)
.unwrap_or(default)
}

/// Whether a run of `policy` can delete anything: not a dry run, and not a
/// disabled policy.
fn frees_bytes(policy: &serde_json::Value) -> bool {
flag(policy, "enabled", true) && !flag(policy, "dry_run_only", false)
}

/// Renew the run's lease every third of its length until aborted; stops on
/// its own once the lease is lost.
fn spawn_lease_heartbeat(
Expand Down
130 changes: 130 additions & 0 deletions crates/core/tests/retention_metrics.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
//! With the metrics on, a retention run reads the repository's size before
//! and after it deleted something, and `forge.retention.freed` counts the
//! difference; a dry run and a disabled policy, which free nothing, read no
//! size (#549). `retention_metrics_off.rs` is the run with the metrics off.
//!
//! The tests of this binary share the metrics and the statement recorder,
//! as a process does, so they run one at a time. Postgres only.

mod common;
mod retention_size;

use common::{pool, unique};
use opentelemetry::metrics::MeterProvider as _;
use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader, SdkMeterProvider};
use retention_size::{Statements, ASSET_SIZE};
use serde_json::json;
use std::sync::OnceLock;

/// The metrics, recorded to memory.
struct Metrics {
provider: SdkMeterProvider,
exporter: InMemoryMetricExporter,
}

/// Turn the metrics on, once per process.
fn metrics() -> &'static Metrics {
static METRICS: OnceLock<Metrics> = OnceLock::new();
METRICS.get_or_init(|| {
let exporter = InMemoryMetricExporter::default();
let provider = SdkMeterProvider::builder()
.with_reader(PeriodicReader::builder(exporter.clone()).build())
.build();
forge_telemetry::init(&provider.meter("forge"));
Metrics { provider, exporter }
})
}

impl Metrics {
/// The cumulative value of counter `name` for `repository`, `None`
/// when it has no such series.
fn counter(&self, name: &str, repository: &str) -> Option<u64> {
self.exporter.reset();
self.provider.force_flush().expect("flush");
let finished = self.exporter.get_finished_metrics().expect("metrics");
let latest = finished.last()?;
let mut total = None;
for scope in latest.scope_metrics() {
for metric in scope.metrics().filter(|m| m.name() == name) {
let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else {
panic!("{name} is a u64 counter");
};
for point in sum.data_points() {
if point.attributes().any(|kv| kv.value.as_str() == repository) {
*total.get_or_insert(0) += point.value();
}
}
}
}
total
}
}

/// One test at a time: they read the shared statement recorder.
static ONE_AT_A_TIME: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());

#[tokio::test]
async fn a_run_that_deletes_reads_the_size_twice_and_records_the_freed_bytes() {
let _one = ONE_AT_A_TIME.lock().await;
let statements = Statements::install();
let metrics = metrics();
assert!(forge_telemetry::enabled());
let pool = pool().await;
let name = format!("rsize-on-{}", unique());
let repo = retention_size::seed(&pool, &name, json!({})).await;

let mark = statements.len();
let summary = retention_size::run(&pool, repo).await;

assert_eq!(summary.deleted, 1);
assert_eq!(
statements.size_reads_since(mark).len(),
2,
"the size before and after the run: {:?}",
statements.size_reads_since(mark)
);
assert_eq!(
metrics.counter("forge.retention.freed", &name),
Some(ASSET_SIZE),
"the deleted version's bytes"
);
assert_eq!(metrics.counter("forge.retention.runs", &name), Some(1));
}

#[tokio::test]
async fn a_dry_run_and_a_disabled_policy_read_no_size() {
let _one = ONE_AT_A_TIME.lock().await;
let statements = Statements::install();
let metrics = metrics();
let pool = pool().await;

for (what, extra, would_delete) in [
("dry", json!({ "dry_run_only": true }), 1),
("disabled", json!({ "enabled": false }), 0),
] {
let name = format!("rsize-{what}-{}", unique());
let repo = retention_size::seed(&pool, &name, extra).await;

let mark = statements.len();
let summary = retention_size::run(&pool, repo).await;

assert_eq!(summary.deleted, would_delete, "{what}");
assert_eq!(
statements.size_reads_since(mark),
Vec::<String>::new(),
"{what}: a run that frees nothing reads no size"
);
// The run still counts, with nothing freed.
assert_eq!(
metrics.counter("forge.retention.runs", &name),
Some(1),
"{what}"
);
assert_eq!(
metrics.counter("forge.retention.freed", &name),
Some(0),
"{what}"
);
}
}
36 changes: 36 additions & 0 deletions crates/core/tests/retention_metrics_off.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
//! With the metrics off (the default), a retention run never reads the
//! repository's size: the read only feeds `forge.retention.freed` (#549).
//!
//! A test binary of its own: the metrics are a process's, and this one never
//! turns them on. `retention_metrics.rs` is the same run with them on.
//! Postgres only.

mod common;
mod retention_size;

use common::{pool, unique};
use retention_size::Statements;
use serde_json::json;

#[tokio::test]
async fn a_run_reads_no_repository_size_without_the_metrics() {
let statements = Statements::install();
assert!(!forge_telemetry::enabled(), "this binary has no metrics");
let pool = pool().await;
let repo = retention_size::seed(&pool, &format!("rsize-off-{}", unique()), json!({})).await;

let mark = statements.len();
let summary = retention_size::run(&pool, repo).await;

// The run did its work, and the recorder saw it.
assert_eq!(summary.deleted, 1);
assert!(
statements.len() > mark,
"the run's statements were recorded"
);
assert_eq!(
statements.size_reads_since(mark),
Vec::<String>::new(),
"no size read around a run without the metrics"
);
}
Loading
Loading