From ea626e6828a923fd5653cef52429ca988ff88df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Migone?= Date: Tue, 15 Sep 2026 15:26:22 -0300 Subject: [PATCH 1/2] feat: relax debt to escrow ratio requirements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás Migone --- crates/bin/escrow_manager/README.md | 28 +- crates/bin/escrow_manager/src/config.rs | 10 + crates/bin/escrow_manager/src/main.rs | 72 +++- crates/bin/escrow_manager/src/metrics.rs | 13 + grafana/escrow_manager.json | 410 +++++++++++++++++++++-- 5 files changed, 502 insertions(+), 31 deletions(-) diff --git a/crates/bin/escrow_manager/README.md b/crates/bin/escrow_manager/README.md index 1831e66..9018aa5 100644 --- a/crates/bin/escrow_manager/README.md +++ b/crates/bin/escrow_manager/README.md @@ -19,6 +19,30 @@ Configuration options are set via a single JSON file. The structure of the file | `dry_run` | If `true`, skip contract calls (useful for testing) | | `port_metrics` | Port for Prometheus metrics server (default: 9090) | | `update_interval_seconds` | Polling interval for the main loop | +| `balance_fill_factor` | Funding margin, in the range (0, 1] (default: 0.8) | + +## Funding Margin + +Each cycle, every receiver's debt is taken as the maximum of: receipts over the last 28 days, RAVs +on active allocations, and the manual `debts` floor. The escrow balance is then stepped up until debt sits below +`balance_fill_factor` of the balance, so the target settles at roughly `debt / balance_fill_factor`: + +| `balance_fill_factor` | Target balance | +|---|---| +| `0.6` | ~1.67x debt | +| `0.8` (default) | ~1.25x debt | +| `0.95` | ~1.05x debt | + +Steps double from 2 GRT and then grow linearly in 10,000 GRT increments, so the target only +approximates that ratio for small balances — below ~16,000 GRT the step granularity dominates and +the effective margin is wider. + +Lowering the margin frees capital but leaves less headroom to absorb query volume between cycles. +Note that total deposits are capped at 10,000 GRT per cycle (`MAX_ADJUSTMENT`), which bounds how +fast a thin margin can be refilled. **The manager never withdraws**, so raising +`balance_fill_factor` only affects receivers still being topped up — balances already above their +target drain only as receivers collect. Validate changes with `dry_run: true` first, and watch +`escrow_target_grt` against `escrow_balance_grt`. ## Sender and Signers @@ -106,12 +130,14 @@ curl http://localhost:9090/metrics |--------|------|-------------| | `escrow_total_debt_grt` | Gauge | Total outstanding debt across all receivers | | `escrow_total_balance_grt` | Gauge | Total escrow balance across all receivers | +| `escrow_total_target_grt` | Gauge | Total target escrow balance across all receivers | | `escrow_total_adjustment_grt` | Gauge | Total GRT deposited in the last cycle | | `escrow_receiver_count` | Gauge | Number of receivers being tracked | | `escrow_loop_duration_seconds` | Histogram | Duration of each polling cycle | | `escrow_debt_grt{receiver}` | Gauge | Outstanding debt per receiver | | `escrow_balance_grt{receiver}` | Gauge | Escrow balance per receiver | -| `escrow_adjustment_grt{receiver}` | Gauge | Last adjustment per receiver | +| `escrow_target_grt{receiver}` | Gauge | Target escrow balance per receiver | +| `escrow_adjustment_grt{receiver}` | Gauge | Last adjustment per receiver (0 if at or above target) | | `escrow_deposit_ok` | Counter | Successful deposit transactions | | `escrow_deposit_err` | Counter | Failed deposit transactions | | `escrow_deposit_duration` | Histogram | Deposit transaction duration | diff --git a/crates/bin/escrow_manager/src/config.rs b/crates/bin/escrow_manager/src/config.rs index 181adac..4d8a6e4 100644 --- a/crates/bin/escrow_manager/src/config.rs +++ b/crates/bin/escrow_manager/src/config.rs @@ -43,12 +43,22 @@ pub struct Config { /// Port for metrics server #[serde(default = "default_port_metrics")] pub port_metrics: u16, + /// Fraction of a receiver's target escrow balance that its debt is allowed to reach before the + /// balance is raised to the next step. This sets the funding margin: the target balance settles + /// at roughly `debt / balance_fill_factor`, so 0.6 funds ~1.67x debt and 0.8 funds ~1.25x. + /// Must be in the range (0, 1]. + #[serde(default = "default_balance_fill_factor")] + pub balance_fill_factor: f64, } fn default_port_metrics() -> u16 { 9090 } +fn default_balance_fill_factor() -> f64 { + 0.8 +} + #[derive(Debug, Deserialize)] pub struct Kafka { pub config: BTreeMap, diff --git a/crates/bin/escrow_manager/src/main.rs b/crates/bin/escrow_manager/src/main.rs index f2babbc..bb3d439 100644 --- a/crates/bin/escrow_manager/src/main.rs +++ b/crates/bin/escrow_manager/src/main.rs @@ -44,6 +44,13 @@ async fn main() -> anyhow::Result<()> { .and_then(|s| serde_json::from_str(&s).map_err(anyhow::Error::from)) .context("failed to load config")?; + anyhow::ensure!( + (config.balance_fill_factor > 0.0) && (config.balance_fill_factor <= 1.0), + "balance_fill_factor must be in the range (0, 1], got {}", + config.balance_fill_factor, + ); + tracing::info!(balance_fill_factor = config.balance_fill_factor); + if config.dry_run { tracing::info!("dry run mode enabled, contract calls will be skipped"); } @@ -215,6 +222,7 @@ async fn main() -> anyhow::Result<()> { .total_debt_grt .set(debts.values().sum::() as f64 / GRT as f64); + let mut total_target: u128 = 0; let adjustments: Vec<(Address, u128)> = receivers .into_iter() .filter_map(|receiver| { @@ -223,8 +231,21 @@ async fn main() -> anyhow::Result<()> { debts.get(&receiver).copied().unwrap_or(0), config.debts.get(&receiver).copied().unwrap_or(0) as u128 * GRT, ); - let next_balance = next_balance(debt); + let next_balance = next_balance(debt, config.balance_fill_factor); + total_target += next_balance; let adjustment = next_balance.saturating_sub(balance); + // Record the target and adjustment for every receiver, including those already at + // or above their target. Skipping them would leave the gauges holding the last + // value they were set to, indefinitely. + let receiver_str = format!("{receiver:?}"); + metrics::METRICS + .target_grt + .with_label_values(&[&receiver_str]) + .set(next_balance as f64 / GRT as f64); + metrics::METRICS + .adjustment_grt + .with_label_values(&[&receiver_str]) + .set(adjustment as f64 / GRT as f64); if adjustment == 0 { return None; } @@ -232,16 +253,15 @@ async fn main() -> anyhow::Result<()> { ?receiver, balance_grt = (balance as f64) / (GRT as f64), debt_grt = (debt as f64) / (GRT as f64), + target_grt = (next_balance as f64) / (GRT as f64), adjustment_grt = (adjustment as f64) / (GRT as f64), ); - let receiver_str = format!("{receiver:?}"); - metrics::METRICS - .adjustment_grt - .with_label_values(&[&receiver_str]) - .set(adjustment as f64 / GRT as f64); Some((receiver, adjustment)) }) .collect(); + metrics::METRICS + .total_target_grt + .set(total_target as f64 / GRT as f64); let total_adjustment: u128 = adjustments.iter().map(|(_, a)| a).sum(); tracing::info!(total_adjustment_grt = ((total_adjustment as f64) * 1e-18).ceil() as u64); @@ -298,9 +318,12 @@ async fn main() -> anyhow::Result<()> { } } -fn next_balance(debt: u128) -> u128 { +/// Target escrow balance for a receiver with the given debt. The balance steps up while debt +/// reaches `fill_factor` of the current step, so the target settles at roughly `debt / fill_factor` +/// once the steps are fine-grained (above `MAX_ADJUSTMENT`, where they stop doubling). +fn next_balance(debt: u128, fill_factor: f64) -> u128 { let mut next_round = (MIN_DEPOSIT / GRT) as u32; - while (debt as f64) >= ((next_round as u128 * GRT) as f64 * 0.6) { + while (debt as f64) >= ((next_round as u128 * GRT) as f64 * fill_factor) { next_round = next_round .saturating_mul(2) .min(next_round + (MAX_ADJUSTMENT / GRT) as u32); @@ -356,7 +379,38 @@ mod tests { (100 * GRT, 256 * GRT), ]; for (debt, expected) in tests { - assert_eq!(super::next_balance(debt), expected); + assert_eq!(super::next_balance(debt, 0.6), expected); + } + } + + #[test] + fn next_balance_fill_factor() { + // A higher fill factor packs debt closer to the target balance, funding a thinner margin. + let tests = [ + (0, MIN_DEPOSIT), + (MIN_DEPOSIT, MIN_DEPOSIT * 2), + (30 * GRT, 64 * GRT), + (70 * GRT, 128 * GRT), + // 100 GRT of debt is funded to 256 GRT at 0.6, but only 128 GRT at 0.8. + (100 * GRT, 128 * GRT), + ]; + for (debt, expected) in tests { + assert_eq!(super::next_balance(debt, 0.8), expected); + } + } + + #[test] + fn next_balance_margin_converges_above_step_cap() { + // Once the steps stop doubling, the target tracks debt / fill_factor closely. + for fill_factor in [0.6, 0.8, 0.95] { + let debt = 580_000 * GRT; + let target = super::next_balance(debt, fill_factor); + let ratio = (target as f64) / (debt as f64); + let expected = 1.0 / fill_factor; + assert!( + (ratio >= expected) && (ratio < (expected + 0.05)), + "fill_factor {fill_factor}: ratio {ratio} not just above {expected}", + ); } } } diff --git a/crates/bin/escrow_manager/src/metrics.rs b/crates/bin/escrow_manager/src/metrics.rs index cc68e32..8591548 100644 --- a/crates/bin/escrow_manager/src/metrics.rs +++ b/crates/bin/escrow_manager/src/metrics.rs @@ -11,6 +11,7 @@ lazy_static! { pub struct Metrics { pub total_debt_grt: Gauge, pub total_balance_grt: Gauge, + pub total_target_grt: Gauge, pub total_adjustment_grt: Gauge, pub receiver_count: IntGauge, pub loop_duration: Histogram, @@ -18,6 +19,7 @@ pub struct Metrics { // Per-receiver metrics pub debt_grt: GaugeVec, pub balance_grt: GaugeVec, + pub target_grt: GaugeVec, pub adjustment_grt: GaugeVec, } @@ -34,6 +36,11 @@ impl Metrics { "total escrow balance across all receivers in GRT" ) .unwrap(), + total_target_grt: register_gauge!( + "escrow_total_target_grt", + "total target escrow balance across all receivers in GRT" + ) + .unwrap(), total_adjustment_grt: register_gauge!( "escrow_total_adjustment_grt", "total GRT deposited in the last cycle" @@ -62,6 +69,12 @@ impl Metrics { &["receiver"] ) .unwrap(), + target_grt: register_gauge_vec!( + "escrow_target_grt", + "target escrow balance per receiver in GRT", + &["receiver"] + ) + .unwrap(), adjustment_grt: register_gauge_vec!( "escrow_adjustment_grt", "last adjustment per receiver in GRT", diff --git a/grafana/escrow_manager.json b/grafana/escrow_manager.json index 11d9d8b..f2afb50 100644 --- a/grafana/escrow_manager.json +++ b/grafana/escrow_manager.json @@ -60,7 +60,7 @@ }, "gridPos": { "h": 4, - "w": 6, + "w": 4, "x": 0, "y": 1 }, @@ -119,8 +119,8 @@ }, "gridPos": { "h": 4, - "w": 6, - "x": 6, + "w": 4, + "x": 4, "y": 1 }, "id": 2, @@ -156,6 +156,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Sum of the target escrow balance the funding algorithm is currently aiming for, across all receivers. Driven by debt and `balance_fill_factor`.", "fieldConfig": { "defaults": { "color": { @@ -167,7 +168,7 @@ "mode": "absolute", "steps": [ { - "color": "yellow", + "color": "blue", "value": 0 } ] @@ -178,10 +179,137 @@ }, "gridPos": { "h": 4, - "w": 6, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.0-22326976726", + "targets": [ + { + "expr": "escrow_total_target_grt", + "refId": "A" + } + ], + "title": "Total Target (GRT)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Escrow balance above what the algorithm is asking for. The manager never withdraws, so this only drains as receivers collect. A large value means drift from past debt, not the current funding margin.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 100000 + }, + { + "color": "orange", + "value": 500000 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, "x": 12, "y": 1 }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.0-22326976726", + "targets": [ + { + "expr": "escrow_total_balance_grt - escrow_total_target_grt", + "refId": "A" + } + ], + "title": "Idle Capital (GRT)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, "id": 3, "options": { "colorMode": "value", @@ -237,8 +365,8 @@ }, "gridPos": { "h": 4, - "w": 6, - "x": 18, + "w": 4, + "x": 20, "y": 1 }, "id": 4, @@ -355,6 +483,35 @@ } } ] + }, + { + "matcher": { + "id": "byName", + "options": "Target" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] } ] }, @@ -388,13 +545,18 @@ "legendFormat": "Debt", "refId": "A" }, + { + "expr": "escrow_total_target_grt", + "legendFormat": "Target", + "refId": "C" + }, { "expr": "escrow_total_balance_grt", "legendFormat": "Balance", "refId": "B" } ], - "title": "Debt vs Balance", + "title": "Debt vs Target vs Balance", "type": "timeseries" }, { @@ -402,6 +564,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Total balance as a percentage of total debt. The algorithm targets ~1/balance_fill_factor (125% at 0.8, 167% at 0.6). Sustained readings well above target indicate idle capital.", "fieldConfig": { "defaults": { "color": { @@ -413,16 +576,24 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "red", "value": 0 }, { "color": "yellow", - "value": 150 + "value": 110 }, { - "color": "red", - "value": 200 + "color": "green", + "value": 125 + }, + { + "color": "yellow", + "value": 175 + }, + { + "color": "orange", + "value": 225 } ] }, @@ -1271,6 +1442,99 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Balance above target, per receiver. These accounts receive no deposits until their debt grows back into their balance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "GRT", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.0.0-22326976726", + "targets": [ + { + "expr": "topk(10, escrow_balance_grt - escrow_target_grt)", + "legendFormat": "{{receiver}}", + "refId": "A" + } + ], + "title": "Idle Capital by Receiver (Top 10)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Debt, target, balance and idle capital per receiver. Idle = balance - target; receivers with positive idle are not being funded this cycle.", "fieldConfig": { "defaults": { "color": { @@ -1366,11 +1630,11 @@ }, { "color": "yellow", - "value": 150 + "value": 110 }, { "color": "green", - "value": 167 + "value": 125 } ] } @@ -1383,14 +1647,77 @@ } } ] + }, + { + "matcher": { + "id": "byName", + "options": "Target" + }, + "properties": [ + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 10000 + }, + { + "color": "orange", + "value": 50000 + } + ] + } + } + ] } ] }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 31 + "w": 24, + "x": 0, + "y": 39 }, "id": 16, "options": { @@ -1399,7 +1726,7 @@ "sortBy": [ { "desc": true, - "displayName": "Ratio %" + "displayName": "Idle" } ] }, @@ -1422,6 +1749,18 @@ "format": "table", "instant": true, "refId": "C" + }, + { + "expr": "escrow_target_grt", + "format": "table", + "instant": true, + "refId": "D" + }, + { + "expr": "escrow_balance_grt - escrow_target_grt", + "format": "table", + "instant": true, + "refId": "E" } ], "title": "All Receivers", @@ -1440,43 +1779,72 @@ "Time 1": true, "Time 2": true, "Time 3": true, + "Time 4": true, + "Time 5": true, "__name__": true, "__name__ 1": true, "__name__ 2": true, "__name__ 3": true, + "__name__ 4": true, + "__name__ 5": true, "cluster 1": true, "cluster 2": true, "cluster 3": true, + "cluster 4": true, + "cluster 5": true, "container 1": true, "container 2": true, "container 3": true, + "container 4": true, + "container 5": true, "endpoint 1": true, "endpoint 2": true, "endpoint 3": true, + "endpoint 4": true, + "endpoint 5": true, "instance": true, "instance 1": true, "instance 2": true, "instance 3": true, + "instance 4": true, + "instance 5": true, "job": true, "job 1": true, "job 2": true, "job 3": true, + "job 4": true, + "job 5": true, "namespace 1": true, "namespace 2": true, "namespace 3": true, + "namespace 4": true, + "namespace 5": true, "pod 1": true, "pod 2": true, "pod 3": true, + "pod 4": true, + "pod 5": true, "prometheus 1": true, "prometheus 2": true, - "prometheus 3": true + "prometheus 3": true, + "prometheus 4": true, + "prometheus 5": true }, "includeByName": {}, - "indexByName": {}, + "indexByName": { + "Value #A": 1, + "Value #B": 3, + "Value #C": 5, + "Value #D": 2, + "Value #E": 4, + "receiver": 0 + }, "renameByName": { "Value #A": "Debt", "Value #B": "Balance", "Value #C": "Ratio %", + "Value #D": "Target", + "Value #E": "Idle", "receiver": "Receiver" } } @@ -1515,6 +1883,6 @@ "timezone": "browser", "title": "Graph Tally Escrow Manager", "uid": "graph-tally-escrow-manager", - "version": 2, + "version": 3, "weekStart": "" } \ No newline at end of file From 4baac902ded440ae8e4753a55f4ccf81133bc27f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Migone?= Date: Tue, 15 Sep 2026 16:22:39 -0300 Subject: [PATCH 2/2] fix: clippy compilation error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás Migone --- crates/bin/escrow_manager/src/contracts.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/bin/escrow_manager/src/contracts.rs b/crates/bin/escrow_manager/src/contracts.rs index 52506e4..72d533f 100644 --- a/crates/bin/escrow_manager/src/contracts.rs +++ b/crates/bin/escrow_manager/src/contracts.rs @@ -19,7 +19,9 @@ sol!( ); use ERC20::ERC20Instance; sol!( - #[allow(missing_docs)] + // `sol!` generates a constructor per event, and some events (e.g. GraphDirectoryInitialized) + // have more parameters than clippy's threshold. Nothing we can restructure. + #[allow(missing_docs, clippy::too_many_arguments)] #[sol(rpc)] #[derive(Debug)] PaymentsEscrow, @@ -27,7 +29,7 @@ sol!( ); use PaymentsEscrow::{PaymentsEscrowErrors, PaymentsEscrowInstance}; sol!( - #[allow(missing_docs)] + #[allow(missing_docs, clippy::too_many_arguments)] #[sol(rpc)] #[derive(Debug)] GraphTallyCollector,