From 875bd813a7ab2b452939796e8ad19a8f2412abfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Migone?= Date: Fri, 18 Sep 2026 09:45:05 -0300 Subject: [PATCH 01/12] feat(escrow_manager): add withdraw functionality MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás Migone --- crates/bin/escrow_manager/README.md | 67 +- .../src/abi/PaymentsEscrow.abi.json | 34 + crates/bin/escrow_manager/src/config.rs | 25 + crates/bin/escrow_manager/src/contracts.rs | 90 ++ crates/bin/escrow_manager/src/main.rs | 400 +++++- crates/bin/escrow_manager/src/metrics.rs | 26 + crates/bin/escrow_manager/src/subgraphs.rs | 35 +- grafana/escrow_manager.json | 1230 ++++++++++++++++- 8 files changed, 1838 insertions(+), 69 deletions(-) diff --git a/crates/bin/escrow_manager/README.md b/crates/bin/escrow_manager/README.md index 9018aa5..2e30a1d 100644 --- a/crates/bin/escrow_manager/README.md +++ b/crates/bin/escrow_manager/README.md @@ -39,10 +39,60 @@ the effective margin is wider. Lowering the margin frees capital but leaves less headroom to absorb query volume between cycles. Note that total deposits are capped at 10,000 GRT per cycle (`MAX_ADJUSTMENT`), which bounds how -fast a thin margin can be refilled. **The manager never withdraws**, so raising -`balance_fill_factor` only affects receivers still being topped up — balances already above their -target drain only as receivers collect. Validate changes with `dry_run: true` first, and watch -`escrow_target_grt` against `escrow_balance_grt`. +fast a thin margin can be refilled. Unless reclamation is enabled (see below) the manager only ever +deposits, so raising `balance_fill_factor` only affects receivers still being topped up — balances +already above their target drain only as receivers collect. Validate changes with `dry_run: true` +first, and watch `escrow_target_grt` against `escrow_balance_grt`. + +## Reclaiming Escrow + +Deposits are one-directional, so a receiver's balance is a high-water mark of its past debt. When +query volume drops the target falls but the balance does not follow, leaving escrow idle. Setting +`withdraw_enabled: true` lets the manager reclaim that idle escrow back to the payer wallet. + +| Field | Default | Meaning | +|---|---|---| +| `withdraw_enabled` | `false` | Reclaim escrow above the target balance | +| `withdraw_margin` | `0.25` | Headroom kept above target before reclaiming | +| `min_withdraw_grt` | `500` | Minimum excess required to start reclaiming | + +`dry_run: true` covers reclamation along with everything else: the manager logs the full plan each +cycle — grep for `dry run: skipping` — without sending any transaction, deposits included. + +Escrow is only thawed above `target * (1 + withdraw_margin)`, so at the defaults the manager holds +roughly 1.7x debt and reclaims the rest. There is no separate handling for abandoned accounts: a +receiver with no debt has a target of 2 GRT, so it falls out of the same rule as the large-excess +end of the range. + +Reclaiming is not instant. It goes through the escrow contract's `thaw` and `withdraw`, so funds +only return after the contract's thawing period — **28 days on mainnet**. The manager uses +`adjustThaw` rather than `thaw`, which means a pending amount is tracked downward as debt grows +while keeping its original maturity date, instead of being cancelled and restarted. In practice the +amount withdrawn at maturity has been continuously re-validated against current debt for the whole +period. + +Two consequences worth knowing: + +- Because the thawing period matches the 28-day receipt window the debt estimate is built from, the + debt picture rolls over completely before a thaw matures. The margin is what absorbs debt growth + over that period, so size it against how much a single receiver's debt can grow in a month — not + just against how much idle capital you want to reclaim. +- A receiver collecting against its escrow shrinks a pending thaw, and a collection that drains the + balance cancels it outright. This is normal and needs no intervention, but it means a reclamation + in flight is not guaranteed to complete. + +Funding decisions use the balance net of any pending withdrawal, matching the escrow contract's own +`getBalance`, so coverage still holds once a withdrawal lands. A receiver is therefore never funded +and reclaimed in the same cycle. + +Setting `withdraw_enabled` back to `false` is inert: anything already thawing is left exactly as it +is and logged as a warning each cycle, rather than being cancelled or withdrawn. To stop reclaiming +without abandoning work in flight, leave it enabled and raise `withdraw_margin` instead — pending +amounts then shrink or cancel through the normal path. + +Validate with `dry_run: true` first, and watch `escrow_total_thawing_grt` and +`escrow_thawing_count`. Given the 28-day round trip, that observation period is the only practical +way to tune these values. ## Sender and Signers @@ -132,12 +182,21 @@ curl http://localhost:9090/metrics | `escrow_total_balance_grt` | Gauge | Total escrow balance across all receivers | | `escrow_total_target_grt` | Gauge | Total target escrow balance across all receivers | | `escrow_total_adjustment_grt` | Gauge | Total GRT deposited in the last cycle | +| `escrow_total_thawing_grt` | Gauge | Total escrow pending withdrawal across all receivers | | `escrow_receiver_count` | Gauge | Number of receivers being tracked | +| `escrow_thawing_count` | Gauge | Number of receivers with escrow pending withdrawal | | `escrow_loop_duration_seconds` | Histogram | Duration of each polling cycle | | `escrow_debt_grt{receiver}` | Gauge | Outstanding debt per receiver | | `escrow_balance_grt{receiver}` | Gauge | Escrow balance per receiver | | `escrow_target_grt{receiver}` | Gauge | Target escrow balance per receiver | | `escrow_adjustment_grt{receiver}` | Gauge | Last adjustment per receiver (0 if at or above target) | +| `escrow_thawing_grt{receiver}` | Gauge | Escrow pending withdrawal per receiver | | `escrow_deposit_ok` | Counter | Successful deposit transactions | | `escrow_deposit_err` | Counter | Failed deposit transactions | | `escrow_deposit_duration` | Histogram | Deposit transaction duration | +| `escrow_adjust_thaw_ok` | Counter | Successful thaw adjustment transactions | +| `escrow_adjust_thaw_err` | Counter | Failed thaw adjustment transactions | +| `escrow_adjust_thaw_duration` | Histogram | Thaw adjustment transaction duration | +| `escrow_withdraw_ok` | Counter | Successful withdrawal transactions | +| `escrow_withdraw_err` | Counter | Failed withdrawal transactions | +| `escrow_withdraw_duration` | Histogram | Withdrawal transaction duration | diff --git a/crates/bin/escrow_manager/src/abi/PaymentsEscrow.abi.json b/crates/bin/escrow_manager/src/abi/PaymentsEscrow.abi.json index 62002fe..c8c97d9 100644 --- a/crates/bin/escrow_manager/src/abi/PaymentsEscrow.abi.json +++ b/crates/bin/escrow_manager/src/abi/PaymentsEscrow.abi.json @@ -378,6 +378,40 @@ "stateMutability": "view", "type": "function" }, + { + "inputs": [ + { + "internalType": "address", + "name": "collector", + "type": "address" + }, + { + "internalType": "address", + "name": "receiver", + "type": "address" + }, + { + "internalType": "uint256", + "name": "tokensToThaw", + "type": "uint256" + }, + { + "internalType": "bool", + "name": "evenIfTimerReset", + "type": "bool" + } + ], + "name": "adjustThaw", + "outputs": [ + { + "internalType": "uint256", + "name": "tokensThawing", + "type": "uint256" + } + ], + "stateMutability": "nonpayable", + "type": "function" + }, { "inputs": [ { diff --git a/crates/bin/escrow_manager/src/config.rs b/crates/bin/escrow_manager/src/config.rs index 4d8a6e4..d6158dd 100644 --- a/crates/bin/escrow_manager/src/config.rs +++ b/crates/bin/escrow_manager/src/config.rs @@ -49,6 +49,23 @@ pub struct Config { /// Must be in the range (0, 1]. #[serde(default = "default_balance_fill_factor")] pub balance_fill_factor: f64, + /// Reclaim escrow that sits above a receiver's target balance. When disabled the manager only + /// ever deposits, and a receiver's balance is a high-water mark of its past debt. Reclaiming + /// takes effect via `thaw` and `withdraw`, so funds only return after the escrow contract's + /// thawing period (28 days on mainnet) has elapsed. + #[serde(default)] + pub withdraw_enabled: bool, + /// Headroom kept above the target balance before any escrow is reclaimed: escrow is only + /// thawed above `target * (1 + withdraw_margin)`. This absorbs debt growth over the thawing + /// period and provides hysteresis against the step ladder, so it should comfortably exceed the + /// debt growth expected for a single receiver over that period. Must be non-negative. + #[serde(default = "default_withdraw_margin")] + pub withdraw_margin: f64, + /// Minimum excess, in whole GRT, required to start thawing a receiver's escrow. Excess below + /// this is left alone. Once a receiver is already thawing, the amount tracks the excess down + /// past this threshold rather than being cancelled, so a shrinking excess keeps its timer. + #[serde(default = "default_min_withdraw_grt")] + pub min_withdraw_grt: u64, } fn default_port_metrics() -> u16 { @@ -59,6 +76,14 @@ fn default_balance_fill_factor() -> f64 { 0.8 } +fn default_withdraw_margin() -> f64 { + 0.25 +} + +fn default_min_withdraw_grt() -> u64 { + 500 +} + #[derive(Debug, Deserialize)] pub struct Kafka { pub config: BTreeMap, diff --git a/crates/bin/escrow_manager/src/contracts.rs b/crates/bin/escrow_manager/src/contracts.rs index 72d533f..7970e37 100644 --- a/crates/bin/escrow_manager/src/contracts.rs +++ b/crates/bin/escrow_manager/src/contracts.rs @@ -133,6 +133,96 @@ impl Contracts { Ok(block_number) } + /// Thawing period enforced by the escrow contract before withdrawals can be executed. Set at + /// deployment and immutable, so it is read once at startup rather than assumed. + pub async fn withdraw_escrow_thawing_period(&self) -> anyhow::Result { + self.payments_escrow + .WITHDRAW_ESCROW_THAWING_PERIOD() + .call() + .await + .context("get withdraw escrow thawing period")? + .try_into() + .context("result out of bounds") + } + + /// Set each receiver's thawing amount to `tokens`, batched into one transaction. + /// + /// Uses `adjustThaw` rather than `thaw`/`cancelThaw` because it is the only variant that is + /// safe to call every cycle: decreasing an amount preserves the existing maturity timestamp, + /// increasing one is refused outright when it would reset the timer (`evenIfTimerReset` is + /// false), and a no-op change writes nothing. A plain `thaw` would restart the thawing period + /// on every call. + pub async fn adjust_thaw_many( + &self, + adjustments: impl IntoIterator, + ) -> anyhow::Result { + let calls: Vec = adjustments + .into_iter() + .map(|(receiver, tokens)| { + self.payments_escrow + .adjustThaw( + *self.graph_tally_collector.address(), + receiver, + U256::from(tokens), + false, + ) + .calldata() + .clone() + }) + .collect(); + + let receipt = self + .payments_escrow + .multicall(calls) + .send() + .await + .map_err(decoded_err::)? + .with_timeout(Some(Duration::from_secs(30))) + .with_required_confirmations(1) + .get_receipt() + .await?; + + receipt + .block_number + .ok_or_else(|| anyhow!("invalid adjust thaw receipt")) + } + + /// Withdraw each receiver's matured thawing amount back to the payer, batched into one + /// transaction. + /// + /// The contract requires the maturity timestamp to be strictly in the past and reverts with + /// `PaymentsEscrowStillThawing` otherwise, which in a multicall fails the whole batch. Callers + /// must only include receivers whose thaw has definitely matured. + pub async fn withdraw_many( + &self, + receivers: impl IntoIterator, + ) -> anyhow::Result { + let calls: Vec = receivers + .into_iter() + .map(|receiver| { + self.payments_escrow + .withdraw(*self.graph_tally_collector.address(), receiver) + .calldata() + .clone() + }) + .collect(); + + let receipt = self + .payments_escrow + .multicall(calls) + .send() + .await + .map_err(decoded_err::)? + .with_timeout(Some(Duration::from_secs(30))) + .with_required_confirmations(1) + .get_receipt() + .await?; + + receipt + .block_number + .ok_or_else(|| anyhow!("invalid withdraw receipt")) + } + pub async fn authorize_signer(&self, signer: &PrivateKeySigner) -> anyhow::Result<()> { let chain_id = self .graph_tally_collector diff --git a/crates/bin/escrow_manager/src/main.rs b/crates/bin/escrow_manager/src/main.rs index bb3d439..6e63fd8 100644 --- a/crates/bin/escrow_manager/src/main.rs +++ b/crates/bin/escrow_manager/src/main.rs @@ -8,16 +8,19 @@ use std::{ collections::{BTreeMap, BTreeSet}, io::Write as _, net::{IpAddr, Ipv4Addr, SocketAddr}, - time::{Duration, Instant}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; -use alloy::{primitives::Address, signers::local::PrivateKeySigner}; +use alloy::{ + primitives::{Address, BlockNumber}, + signers::local::PrivateKeySigner, +}; use anyhow::{anyhow, Context as _}; use axum::{http::StatusCode, routing, Router}; use config::Config; use contracts::Contracts; use prometheus::Encoder as _; -use subgraphs::{active_allocations, authorized_signers, escrow_accounts}; +use subgraphs::{active_allocations, authorized_signers, escrow_accounts, EscrowAccount}; use thegraph_client_subgraphs::Client as SubgraphClient; use tokio::{ net::TcpListener, @@ -31,6 +34,10 @@ static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc; const GRT: u128 = 1_000_000_000_000_000_000; const MIN_DEPOSIT: u128 = 2 * GRT; const MAX_ADJUSTMENT: u128 = 10_000 * GRT; +/// Grace period added to a thaw's maturity before a withdrawal is attempted. The escrow contract +/// requires the maturity timestamp to be strictly in the past, and reverts the whole multicall +/// otherwise, so this absorbs subgraph lag and block timestamp drift. +const WITHDRAW_MATURITY_SLACK_SECONDS: u64 = 300; #[tokio::main] async fn main() -> anyhow::Result<()> { @@ -51,6 +58,16 @@ async fn main() -> anyhow::Result<()> { ); tracing::info!(balance_fill_factor = config.balance_fill_factor); + anyhow::ensure!( + config.withdraw_margin.is_finite() && (config.withdraw_margin >= 0.0), + "withdraw_margin must be non-negative, got {}", + config.withdraw_margin, + ); + // Converted to basis points so every subsequent calculation on token amounts stays in integer + // arithmetic. `u128` GRT amounts exceed f64's exact range, and these numbers decide transactions. + let withdraw_margin_bps = (config.withdraw_margin * 10_000.0).round() as u128; + let min_withdraw = config.min_withdraw_grt as u128 * GRT; + if config.dry_run { tracing::info!("dry run mode enabled, contract calls will be skipped"); } @@ -101,6 +118,23 @@ async fn main() -> anyhow::Result<()> { } } + if config.withdraw_enabled { + // Read rather than assume: the period is set at deployment and capped at 90 days by the + // escrow contract, so the schedule this manager plans against has to come from the chain. + let thawing_period = contracts + .withdraw_escrow_thawing_period() + .await + .context("get withdraw escrow thawing period")?; + tracing::info!( + withdraw_margin = config.withdraw_margin, + min_withdraw_grt = config.min_withdraw_grt, + thawing_period_days = thawing_period as f64 / 86400.0, + "escrow reclamation enabled" + ); + } else { + tracing::info!("escrow reclamation disabled, deposits only"); + } + let mut allowance = contracts.allowance().await?; let expected_allowance = config.grt_allowance as u128 * GRT; tracing::info!(allowance = allowance as f64 * 1e-18); @@ -181,7 +215,13 @@ async fn main() -> anyhow::Result<()> { metrics::METRICS.receiver_count.set(receivers.len() as i64); metrics::METRICS .total_balance_grt - .set(escrow_accounts.values().sum::() as f64 / GRT as f64); + .set(escrow_accounts.values().map(|a| a.balance).sum::() as f64 / GRT as f64); + metrics::METRICS + .total_thawing_grt + .set(escrow_accounts.values().map(|a| a.thawing).sum::() as f64 / GRT as f64); + metrics::METRICS + .thawing_count + .set(escrow_accounts.values().filter(|a| a.thawing > 0).count() as i64); let mut indexer_ravs: BTreeMap = Default::default(); { @@ -207,11 +247,15 @@ async fn main() -> anyhow::Result<()> { ravs = %format!("{:.6}", ravs as f64 * 1e-18), ); let receiver_str = format!("{receiver:?}"); - let balance = escrow_accounts.get(receiver).copied().unwrap_or(0); + let account = escrow_accounts.get(receiver).copied().unwrap_or_default(); metrics::METRICS .balance_grt .with_label_values(&[&receiver_str]) - .set(balance as f64 / GRT as f64); + .set(account.balance as f64 / GRT as f64); + metrics::METRICS + .thawing_grt + .with_label_values(&[&receiver_str]) + .set(account.thawing as f64 / GRT as f64); metrics::METRICS .debt_grt .with_label_values(&[&receiver_str]) @@ -222,43 +266,95 @@ async fn main() -> anyhow::Result<()> { .total_debt_grt .set(debts.values().sum::() as f64 / GRT as f64); + let now_unix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let mut total_target: u128 = 0; - let adjustments: Vec<(Address, u128)> = receivers - .into_iter() - .filter_map(|receiver| { - let balance = escrow_accounts.get(&receiver).cloned().unwrap_or(0); - let debt = u128::max( - debts.get(&receiver).copied().unwrap_or(0), - config.debts.get(&receiver).copied().unwrap_or(0) as u128 * GRT, - ); - let next_balance = next_balance(debt, config.balance_fill_factor); - total_target += next_balance; - let adjustment = next_balance.saturating_sub(balance); - // Record the target and adjustment for every receiver, including those already at - // or above their target. Skipping them would leave the gauges holding the last - // value they were set to, indefinitely. - let receiver_str = format!("{receiver:?}"); - metrics::METRICS - .target_grt - .with_label_values(&[&receiver_str]) - .set(next_balance as f64 / GRT as f64); - metrics::METRICS - .adjustment_grt - .with_label_values(&[&receiver_str]) - .set(adjustment as f64 / GRT as f64); - if adjustment == 0 { - return None; - } + let mut adjustments: Vec<(Address, u128)> = Default::default(); + let mut thaw_adjustments: Vec<(Address, u128)> = Default::default(); + let mut withdrawals: Vec
= Default::default(); + for receiver in receivers { + let account = escrow_accounts.get(&receiver).copied().unwrap_or_default(); + let debt = u128::max( + debts.get(&receiver).copied().unwrap_or(0), + config.debts.get(&receiver).copied().unwrap_or(0) as u128 * GRT, + ); + let target = next_balance(debt, config.balance_fill_factor); + total_target += target; + + // Amount we want thawing once this cycle's calls land. With reclamation disabled the + // existing amount is left exactly as it is, so turning the feature off is inert. + let desired_thaw = if config.withdraw_enabled { + desired_thawing(&account, target, withdraw_margin_bps, min_withdraw) + } else { + account.thawing + }; + + // Fund against the balance that survives a pending withdrawal, so coverage still holds + // once it lands. This is also how the escrow contract's own `getBalance` is defined. + let effective_balance = account.balance.saturating_sub(desired_thaw); + let adjustment = target.saturating_sub(effective_balance); + + // Record for every receiver, including those needing no action. Skipping them would + // leave the gauges holding the last value they were set to, indefinitely. + let receiver_str = format!("{receiver:?}"); + metrics::METRICS + .target_grt + .with_label_values(&[&receiver_str]) + .set(target as f64 / GRT as f64); + metrics::METRICS + .adjustment_grt + .with_label_values(&[&receiver_str]) + .set(adjustment as f64 / GRT as f64); + + if desired_thaw != account.thawing { tracing::info!( ?receiver, - balance_grt = (balance as f64) / (GRT as f64), + balance_grt = (account.balance as f64) / (GRT as f64), debt_grt = (debt as f64) / (GRT as f64), - target_grt = (next_balance as f64) / (GRT as f64), - adjustment_grt = (adjustment as f64) / (GRT as f64), + target_grt = (target as f64) / (GRT as f64), + thawing_grt = (account.thawing as f64) / (GRT as f64), + desired_thawing_grt = (desired_thaw as f64) / (GRT as f64), + "thaw adjustment", + ); + thaw_adjustments.push((receiver, desired_thaw)); + } else if !config.withdraw_enabled && (account.thawing > 0) { + tracing::warn!( + ?receiver, + thawing_grt = (account.thawing as f64) / (GRT as f64), + "escrow is thawing but reclamation is disabled, leaving it untouched", ); - Some((receiver, adjustment)) - }) - .collect(); + } + + // Only withdraw a thaw that has definitely matured and still has a justified amount. + // Cancelling clears the timestamp, so a receiver being cancelled this cycle must be + // left out or the withdrawal reverts and takes the whole batch with it. + let matured = (account.thaw_end_timestamp != 0) + && (now_unix > (account.thaw_end_timestamp + WITHDRAW_MATURITY_SLACK_SECONDS)); + if config.withdraw_enabled && matured && (desired_thaw > 0) { + tracing::info!( + ?receiver, + tokens_grt = (desired_thaw as f64) / (GRT as f64), + "withdrawal matured", + ); + withdrawals.push(receiver); + } + + if adjustment == 0 { + continue; + } + tracing::info!( + ?receiver, + balance_grt = (account.balance as f64) / (GRT as f64), + effective_balance_grt = (effective_balance as f64) / (GRT as f64), + debt_grt = (debt as f64) / (GRT as f64), + target_grt = (target as f64) / (GRT as f64), + adjustment_grt = (adjustment as f64) / (GRT as f64), + ); + adjustments.push((receiver, adjustment)); + } metrics::METRICS .total_target_grt .set(total_target as f64 / GRT as f64); @@ -268,6 +364,42 @@ async fn main() -> anyhow::Result<()> { metrics::METRICS .total_adjustment_grt .set(total_adjustment as f64 / GRT as f64); + // Tracked across all three phases so a failure in one does not skip the others, and the + // subgraph client is only pinned forward once. + let mut latest_tx_block: Option = None; + + // Thaw adjustments run first: shrinking or cancelling a thaw is the protective action, and + // the deposit amounts computed above already assume it has happened. + if !thaw_adjustments.is_empty() { + if config.dry_run { + for (receiver, tokens) in &thaw_adjustments { + tracing::info!( + ?receiver, + tokens_grt = (*tokens as f64) / (GRT as f64), + "dry run: skipping adjust thaw" + ); + } + } else { + let start = Instant::now(); + let result = contracts.adjust_thaw_many(thaw_adjustments).await; + metrics::METRICS + .adjust_thaw + .duration + .observe(start.elapsed().as_secs_f64()); + match result { + Ok(block) => { + metrics::METRICS.adjust_thaw.ok.inc(); + latest_tx_block = latest_tx_block.max(Some(block)); + tracing::info!("thaw adjustments complete"); + } + Err(adjust_thaw_err) => { + metrics::METRICS.adjust_thaw.err.inc(); + tracing::error!("{:#}", adjust_thaw_err.context("adjust thaw")); + } + } + } + } + if total_adjustment > 0 { let adjustments = if total_adjustment <= MAX_ADJUSTMENT { adjustments @@ -282,25 +414,54 @@ async fn main() -> anyhow::Result<()> { "dry run: skipping deposit" ); } - continue; + } else { + let deposit_start = Instant::now(); + let deposit_result = contracts.deposit_many(adjustments).await; + metrics::METRICS + .deposit + .duration + .observe(deposit_start.elapsed().as_secs_f64()); + match deposit_result { + Ok(block) => { + metrics::METRICS.deposit.ok.inc(); + latest_tx_block = latest_tx_block.max(Some(block)); + tracing::info!("adjustments complete"); + } + Err(deposit_err) => { + metrics::METRICS.deposit.err.inc(); + tracing::error!("{:#}", deposit_err.context("deposit")); + } + } } - let deposit_start = Instant::now(); - let deposit_result = contracts.deposit_many(adjustments).await; - metrics::METRICS - .deposit - .duration - .observe(deposit_start.elapsed().as_secs_f64()); - let tx_block = match deposit_result { - Ok(block) => { - metrics::METRICS.deposit.ok.inc(); - block + } + + if !withdrawals.is_empty() { + if config.dry_run { + for receiver in &withdrawals { + tracing::info!(?receiver, "dry run: skipping withdraw"); } - Err(deposit_err) => { - metrics::METRICS.deposit.err.inc(); - tracing::error!("{:#}", deposit_err.context("deposit")); - continue; + } else { + let start = Instant::now(); + let result = contracts.withdraw_many(withdrawals).await; + metrics::METRICS + .withdraw + .duration + .observe(start.elapsed().as_secs_f64()); + match result { + Ok(block) => { + metrics::METRICS.withdraw.ok.inc(); + latest_tx_block = latest_tx_block.max(Some(block)); + tracing::info!("withdrawals complete"); + } + Err(withdraw_err) => { + metrics::METRICS.withdraw.err.inc(); + tracing::error!("{:#}", withdraw_err.context("withdraw")); + } } - }; + } + } + + if let Some(tx_block) = latest_tx_block { network_subgraph = SubgraphClient::builder( network_subgraph.http_client, network_subgraph.subgraph_url, @@ -308,8 +469,6 @@ async fn main() -> anyhow::Result<()> { .with_auth_token(Some(config.query_auth.clone())) .with_subgraph_latest_block(tx_block) .build(); - - tracing::info!("adjustments complete"); } metrics::METRICS @@ -331,6 +490,34 @@ fn next_balance(debt: u128, fill_factor: f64) -> u128 { next_round as u128 * GRT } +/// Escrow that should be thawing for a receiver once this cycle's calls land. +/// +/// Escrow is only reclaimed above `target * (1 + margin)`. That headroom absorbs debt growth over +/// the thawing period, and provides hysteresis against the target's step ladder. +/// +/// `adjustThaw` refuses an increase that would reset a running timer, so a thaw already in flight +/// can only be tracked downward. This mirrors that, both so the amount withdrawn at maturity has +/// been continuously re-validated against current debt, and so funding decisions agree with what +/// the contract will actually hold. The minimum only gates starting a thaw: once one is running the +/// amount follows the excess down past the minimum, because shrinking keeps the maturity timestamp +/// whereas cancelling would forfeit the whole thawing period. +fn desired_thawing( + account: &EscrowAccount, + target: u128, + margin_bps: u128, + min_withdraw: u128, +) -> u128 { + let floor = target + (target / 10_000) * margin_bps; + let excess = account.balance.saturating_sub(floor); + if account.thawing > 0 { + excess.min(account.thawing) + } else if excess >= min_withdraw { + excess + } else { + 0 + } +} + fn reduce_adjustments(adjustments: Vec<(Address, u128)>) -> Vec<(Address, u128)> { let desired: BTreeMap = adjustments.into_iter().collect(); assert!(desired.values().sum::() > MAX_ADJUSTMENT); @@ -364,7 +551,106 @@ async fn handle_metrics() -> impl axum::response::IntoResponse { #[cfg(test)] mod tests { - use super::{GRT, MIN_DEPOSIT}; + use super::{EscrowAccount, GRT, MIN_DEPOSIT}; + + const MARGIN_BPS: u128 = 2_500; + const MIN_WITHDRAW: u128 = 500 * GRT; + + fn account(balance: u128, thawing: u128) -> EscrowAccount { + EscrowAccount { + balance, + thawing, + thaw_end_timestamp: if thawing > 0 { 1 } else { 0 }, + } + } + + fn desired(balance: u128, thawing: u128, target: u128) -> u128 { + super::desired_thawing(&account(balance, thawing), target, MARGIN_BPS, MIN_WITHDRAW) + } + + #[test] + fn desired_thawing_applies_margin_above_target() { + let target = 10_000 * GRT; + // Nothing is reclaimed until the balance clears target * 1.25. + assert_eq!(desired(12_500 * GRT, 0, target), 0); + assert_eq!(desired(12_999 * GRT, 0, target), 0); + // Above the floor, only the excess over it is reclaimed. + assert_eq!(desired(20_000 * GRT, 0, target), 7_500 * GRT); + } + + #[test] + fn desired_thawing_respects_minimum_when_starting() { + let target = 10_000 * GRT; + // An excess under the minimum is not worth a 28 day round trip. + assert_eq!(desired(12_999 * GRT, 0, target), 0); + assert_eq!(desired(13_000 * GRT, 0, target), 500 * GRT); + } + + #[test] + fn desired_thawing_tracks_a_running_thaw_downward() { + // Target climbed a rung, so the justified excess shrank. Following it down preserves the + // maturity timestamp; cancelling and re-thawing would forfeit the thawing period. + assert_eq!( + desired(20_000 * GRT, 7_500 * GRT, 14_000 * GRT), + 2_500 * GRT + ); + // Below the minimum too: the threshold only gates starting a thaw. + assert_eq!(desired(20_000 * GRT, 7_500 * GRT, 15_900 * GRT), 125 * GRT); + // Excess gone entirely: cancel. + assert_eq!(desired(20_000 * GRT, 7_500 * GRT, 16_000 * GRT), 0); + assert_eq!(desired(20_000 * GRT, 7_500 * GRT, 30_000 * GRT), 0); + } + + #[test] + fn desired_thawing_never_grows_a_running_thaw() { + // The contract refuses an increase that would reset the timer, so asking for one would + // silently do nothing. Report what the contract will actually hold instead. + let target = 10_000 * GRT; + assert_eq!(desired(40_000 * GRT, 7_500 * GRT, target), 7_500 * GRT); + } + + #[test] + fn deposits_and_reclaims_are_mutually_exclusive() { + // A receiver can never be funded and reclaimed in the same cycle: reclaiming leaves the + // effective balance at the floor, which is above target by construction. + for debt_grt in [0, 1, 500, 12_345, 100_000, 580_000] { + for balance_grt in [0, 2, 1_000, 40_000, 96_384, 1_000_000] { + for thawing_grt in [0, 100, 20_000] { + let balance = balance_grt * GRT; + let thawing = u128::min(thawing_grt * GRT, balance); + let target = super::next_balance(debt_grt * GRT, 0.8); + let thaw = super::desired_thawing( + &account(balance, thawing), + target, + MARGIN_BPS, + MIN_WITHDRAW, + ); + let adjustment = target.saturating_sub(balance.saturating_sub(thaw)); + assert!( + (thaw == 0) || (adjustment == 0), + "debt {debt_grt} balance {balance_grt} thawing {thawing_grt}: \ + thaw {thaw} and adjustment {adjustment}", + ); + } + } + } + } + + #[test] + fn desired_thawing_never_exceeds_balance() { + for balance_grt in [0, 2, 1_000, 96_384] { + for target_grt in [2, 1_000, 96_384] { + let balance = balance_grt * GRT; + let thaw = super::desired_thawing( + &account(balance, 0), + target_grt * GRT, + MARGIN_BPS, + MIN_WITHDRAW, + ); + assert!(thaw <= balance, "balance {balance_grt} target {target_grt}"); + } + } + } #[test] fn next_balance() { diff --git a/crates/bin/escrow_manager/src/metrics.rs b/crates/bin/escrow_manager/src/metrics.rs index 8591548..6d58d97 100644 --- a/crates/bin/escrow_manager/src/metrics.rs +++ b/crates/bin/escrow_manager/src/metrics.rs @@ -13,14 +13,19 @@ pub struct Metrics { pub total_balance_grt: Gauge, pub total_target_grt: Gauge, pub total_adjustment_grt: Gauge, + pub total_thawing_grt: Gauge, pub receiver_count: IntGauge, + pub thawing_count: IntGauge, pub loop_duration: Histogram, pub deposit: ResponseMetrics, + pub adjust_thaw: ResponseMetrics, + pub withdraw: ResponseMetrics, // Per-receiver metrics pub debt_grt: GaugeVec, pub balance_grt: GaugeVec, pub target_grt: GaugeVec, pub adjustment_grt: GaugeVec, + pub thawing_grt: GaugeVec, } impl Metrics { @@ -46,17 +51,32 @@ impl Metrics { "total GRT deposited in the last cycle" ) .unwrap(), + total_thawing_grt: register_gauge!( + "escrow_total_thawing_grt", + "total escrow pending withdrawal across all receivers in GRT" + ) + .unwrap(), receiver_count: register_int_gauge!( "escrow_receiver_count", "number of receivers being tracked" ) .unwrap(), + thawing_count: register_int_gauge!( + "escrow_thawing_count", + "number of receivers with escrow pending withdrawal" + ) + .unwrap(), loop_duration: register_histogram!( "escrow_loop_duration_seconds", "duration of each polling cycle in seconds" ) .unwrap(), deposit: ResponseMetrics::new("escrow_deposit", "escrow deposit transaction"), + adjust_thaw: ResponseMetrics::new( + "escrow_adjust_thaw", + "escrow thaw adjustment transaction", + ), + withdraw: ResponseMetrics::new("escrow_withdraw", "escrow withdrawal transaction"), debt_grt: register_gauge_vec!( "escrow_debt_grt", "outstanding debt per receiver in GRT", @@ -81,6 +101,12 @@ impl Metrics { &["receiver"] ) .unwrap(), + thawing_grt: register_gauge_vec!( + "escrow_thawing_grt", + "escrow pending withdrawal per receiver in GRT", + &["receiver"] + ) + .unwrap(), } } } diff --git a/crates/bin/escrow_manager/src/subgraphs.rs b/crates/bin/escrow_manager/src/subgraphs.rs index 7abd15a..cc24410 100644 --- a/crates/bin/escrow_manager/src/subgraphs.rs +++ b/crates/bin/escrow_manager/src/subgraphs.rs @@ -36,10 +36,21 @@ pub async fn authorized_signers( Ok(signers) } +/// Escrow account state for a single receiver. +#[derive(Clone, Copy, Debug, Default)] +pub struct EscrowAccount { + /// Total escrow balance. Thawing does not reduce this; only withdrawing and collecting do. + pub balance: u128, + /// Amount currently thawing. Still collectable by the receiver, but committed to leaving. + pub thawing: u128, + /// Unix timestamp at which the thawing amount becomes withdrawable, or 0 if not thawing. + pub thaw_end_timestamp: u64, +} + pub async fn escrow_accounts( network_subgraph: &mut SubgraphClient, payer: &Address, -) -> anyhow::Result> { +) -> anyhow::Result> { let query = format!( r#" paymentsEscrowAccounts( @@ -54,6 +65,8 @@ pub async fn escrow_accounts( ) {{ id balance + totalAmountThawing + thawEndTimestamp receiver {{ id }} @@ -62,9 +75,14 @@ pub async fn escrow_accounts( ); #[serde_as] #[derive(serde::Deserialize)] - struct EscrowAccount { + #[serde(rename_all = "camelCase")] + struct EscrowAccount_ { #[serde_as(as = "serde_with::DisplayFromStr")] balance: u128, + #[serde_as(as = "serde_with::DisplayFromStr")] + total_amount_thawing: u128, + #[serde_as(as = "serde_with::DisplayFromStr")] + thaw_end_timestamp: u64, receiver: Receiver, } #[derive(serde::Deserialize)] @@ -72,12 +90,21 @@ pub async fn escrow_accounts( id: Address, } let response = network_subgraph - .paginated_query::(query, 500) + .paginated_query::(query, 500) .await; match response { Ok(accounts) => Ok(accounts .into_iter() - .map(|a| (a.receiver.id, a.balance)) + .map(|a| { + ( + a.receiver.id, + EscrowAccount { + balance: a.balance, + thawing: a.total_amount_thawing, + thaw_end_timestamp: a.thaw_end_timestamp, + }, + ) + }) .collect()), Err(PaginatedQueryError::EmptyResponse) => Ok(Default::default()), Err(err) => Err(anyhow!(err)), diff --git a/grafana/escrow_manager.json b/grafana/escrow_manager.json index fc93932..918b5b2 100644 --- a/grafana/escrow_manager.json +++ b/grafana/escrow_manager.json @@ -3031,6 +3031,1110 @@ } } } + }, + "panel-24": { + "kind": "Panel", + "spec": { + "id": 24, + "title": "Thawing (GRT)", + "description": "Escrow currently pending withdrawal back to the payer. Thawing does not reduce the balance and does not stop a receiver collecting; the funds leave only when the 28 day thawing period matures and the withdrawal executes.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "max(escrow_total_thawing_grt{job=~\"$job\"})", + "instant": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "blue" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-25": { + "kind": "Panel", + "spec": { + "id": 25, + "title": "Accounts Thawing", + "description": "Receivers with escrow pending withdrawal. Compare against Receivers to see how much of the set is being reclaimed from.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "max(escrow_thawing_count{job=~\"$job\"})", + "instant": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "purple" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-26": { + "kind": "Panel", + "spec": { + "id": 26, + "title": "Thaw Adjustments Failed", + "description": "Failed `adjustThaw` transactions in the selected range. These start, shrink and cancel pending withdrawals; a sustained failure means reclamation amounts are no longer tracking debt.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "sum(increase(escrow_adjust_thaw_err{job=~\"$job\"}[$__range]))", + "instant": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 1, + "color": "red" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-27": { + "kind": "Panel", + "spec": { + "id": 27, + "title": "Withdrawals Failed", + "description": "Failed `withdraw` transactions in the selected range. The escrow contract reverts the whole batch if any thaw has not matured, so repeated failures point at clock or subgraph lag beyond the maturity slack.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "sum(increase(escrow_withdraw_err{job=~\"$job\"}[$__range]))", + "instant": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 1, + "color": "red" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-28": { + "kind": "Panel", + "spec": { + "id": 28, + "title": "Thawing Over Time", + "description": "Total escrow pending withdrawal. Steps up when reclamation starts on a receiver, down as amounts are tracked against growing debt, and to zero as withdrawals mature.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "max(escrow_total_thawing_grt{job=~\"$job\"})", + "legendFormat": "Thawing" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "GRT", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-29": { + "kind": "Panel", + "spec": { + "id": 29, + "title": "Reclamation In Flight", + "description": "Receivers with escrow pending withdrawal, one row each. Thawing is the amount due back to the payer at maturity; it is re-validated against debt every cycle, so it shrinks if debt grows and disappears if the excess does. Balance minus Thawing is what funding decisions use.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "max by (receiver) (escrow_thawing_grt{job=~\"$job\"}) > 0", + "format": "table", + "instant": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "max by (receiver) (escrow_balance_grt{job=~\"$job\"}) and max by (receiver) (escrow_thawing_grt{job=~\"$job\"}) > 0", + "format": "table", + "instant": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "max by (receiver) (escrow_debt_grt{job=~\"$job\"}) and max by (receiver) (escrow_thawing_grt{job=~\"$job\"}) > 0", + "format": "table", + "instant": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "C", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "max by (receiver) (escrow_target_grt{job=~\"$job\"}) and max by (receiver) (escrow_thawing_grt{job=~\"$job\"}) > 0", + "format": "table", + "instant": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "D", + "hidden": false + } + } + ], + "transformations": [ + { + "kind": "Transformation", + "group": "seriesToColumns", + "spec": { + "options": { + "byField": "receiver" + } + } + }, + { + "kind": "Transformation", + "group": "organize", + "spec": { + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "Time 1": true, + "__name__ 1": true, + "Time 2": true, + "__name__ 2": true, + "Time 3": true, + "__name__ 3": true, + "Time 4": true, + "__name__ 4": true + }, + "includeByName": {}, + "indexByName": { + "receiver": 0, + "Value #C": 1, + "Value #D": 2, + "Value #B": 3, + "Value #A": 4 + }, + "renameByName": { + "receiver": "Receiver", + "Value #A": "Thawing", + "Value #B": "Balance", + "Value #C": "Debt", + "Value #D": "Target" + } + } + } + } + ], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "table", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Thawing" + } + ] + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Thawing" + }, + "properties": [ + { + "id": "decimals", + "value": 2 + }, + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-BlPu" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Balance" + }, + "properties": [ + { + "id": "decimals", + "value": 2 + }, + { + "id": "unit", + "value": "none" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Debt" + }, + "properties": [ + { + "id": "decimals", + "value": 2 + }, + { + "id": "unit", + "value": "none" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Target" + }, + "properties": [ + { + "id": "decimals", + "value": 2 + }, + { + "id": "unit", + "value": "none" + } + ] + } + ] + } + } + } + } + }, + "panel-30": { + "kind": "Panel", + "spec": { + "id": 30, + "title": "Reclamation Transactions", + "description": "Thaw adjustments start, shrink and cancel pending withdrawals and run whenever the justified amount changes; withdrawals only fire once a thaw matures, so they are expected to be rare.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "sum(increase(escrow_adjust_thaw_ok{job=~\"$job\"}[1h]))", + "legendFormat": "Thaw Adjust OK" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "sum(increase(escrow_adjust_thaw_err{job=~\"$job\"}[1h]))", + "legendFormat": "Thaw Adjust Errors" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "sum(increase(escrow_withdraw_ok{job=~\"$job\"}[1h]))", + "legendFormat": "Withdraw OK" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "C", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "sum(increase(escrow_withdraw_err{job=~\"$job\"}[1h]))", + "legendFormat": "Withdraw Errors" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "D", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + } + } + } + } + }, + "panel-31": { + "kind": "Panel", + "spec": { + "id": 31, + "title": "Reclamation Duration", + "description": "Average transaction duration for thaw adjustments and withdrawals.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "rate(escrow_adjust_thaw_duration_sum{job=~\"$job\"}[5m]) / rate(escrow_adjust_thaw_duration_count{job=~\"$job\"}[5m])", + "legendFormat": "Thaw Adjust ({{pod}})" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "expr": "rate(escrow_withdraw_duration_sum{job=~\"$job\"}[5m]) / rate(escrow_withdraw_duration_count{job=~\"$job\"}[5m])", + "legendFormat": "Withdraw ({{pod}})" + }, + "labels": { + "grafana.app/export-label": "prometheus-1", + "grafana.app/export-datasource-name": "VM graph-mainnet" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.3.0-34942711622", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } } }, "layout": { @@ -3133,7 +4237,7 @@ "height": 4, "element": { "kind": "ElementReference", - "name": "panel-3" + "name": "panel-24" } } }, @@ -3146,7 +4250,7 @@ "height": 4, "element": { "kind": "ElementReference", - "name": "panel-4" + "name": "panel-25" } } }, @@ -3155,6 +4259,58 @@ "spec": { "x": 0, "y": 8, + "width": 6, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 6, + "y": 8, + "width": 6, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 8, + "width": 6, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-26" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 18, + "y": 8, + "width": 6, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-27" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 12, "width": 12, "height": 8, "element": { @@ -3167,7 +4323,7 @@ "kind": "GridLayoutItem", "spec": { "x": 12, - "y": 8, + "y": 12, "width": 6, "height": 8, "element": { @@ -3180,7 +4336,7 @@ "kind": "GridLayoutItem", "spec": { "x": 18, - "y": 8, + "y": 12, "width": 6, "height": 8, "element": { @@ -3352,6 +4508,72 @@ } } }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Reclamation", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-28" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-29" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 8, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-30" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 8, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-31" + } + } + } + ] + } + } + } + }, { "kind": "RowsLayoutRow", "spec": { From d88fda76966f04e860672534bd628714d1b76c94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Migone?= Date: Fri, 18 Sep 2026 17:38:21 -0300 Subject: [PATCH 02/12] fix: easy fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás Migone --- crates/bin/escrow_manager/src/config.rs | 2 +- crates/bin/escrow_manager/src/contracts.rs | 16 +++++++++ crates/bin/escrow_manager/src/main.rs | 41 +++++++++++++++------- grafana/escrow_manager.json | 2 +- 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/crates/bin/escrow_manager/src/config.rs b/crates/bin/escrow_manager/src/config.rs index d6158dd..e976273 100644 --- a/crates/bin/escrow_manager/src/config.rs +++ b/crates/bin/escrow_manager/src/config.rs @@ -58,7 +58,7 @@ pub struct Config { /// Headroom kept above the target balance before any escrow is reclaimed: escrow is only /// thawed above `target * (1 + withdraw_margin)`. This absorbs debt growth over the thawing /// period and provides hysteresis against the step ladder, so it should comfortably exceed the - /// debt growth expected for a single receiver over that period. Must be non-negative. + /// debt growth expected for a single receiver over that period. Must be in the range [0, 1]. #[serde(default = "default_withdraw_margin")] pub withdraw_margin: f64, /// Minimum excess, in whole GRT, required to start thawing a receiver's escrow. Excess below diff --git a/crates/bin/escrow_manager/src/contracts.rs b/crates/bin/escrow_manager/src/contracts.rs index 7970e37..261456c 100644 --- a/crates/bin/escrow_manager/src/contracts.rs +++ b/crates/bin/escrow_manager/src/contracts.rs @@ -1,6 +1,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use alloy::{ + eips::BlockId, network::EthereumWallet, primitives::{keccak256, Address, BlockNumber, Bytes, U256}, providers::{DynProvider, Provider as _, ProviderBuilder, WalletProvider}, @@ -145,6 +146,21 @@ impl Contracts { .context("result out of bounds") } + /// Timestamp of the latest block, for planning against contract-side time checks. + /// + /// Block timestamps are non-decreasing, so this is a lower bound on the timestamp of whatever + /// block a transaction sent now lands in. Planning against it can only be conservative. + pub async fn latest_block_timestamp(&self) -> anyhow::Result { + let block = self + .payments_escrow + .provider() + .get_block(BlockId::latest()) + .await + .context("get latest block")? + .ok_or_else(|| anyhow!("no latest block"))?; + Ok(block.header.timestamp) + } + /// Set each receiver's thawing amount to `tokens`, batched into one transaction. /// /// Uses `adjustThaw` rather than `thaw`/`cancelThaw` because it is the only variant that is diff --git a/crates/bin/escrow_manager/src/main.rs b/crates/bin/escrow_manager/src/main.rs index 6e63fd8..31a4d8e 100644 --- a/crates/bin/escrow_manager/src/main.rs +++ b/crates/bin/escrow_manager/src/main.rs @@ -8,7 +8,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, io::Write as _, net::{IpAddr, Ipv4Addr, SocketAddr}, - time::{Duration, Instant, SystemTime, UNIX_EPOCH}, + time::{Duration, Instant}, }; use alloy::{ @@ -34,10 +34,6 @@ static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc; const GRT: u128 = 1_000_000_000_000_000_000; const MIN_DEPOSIT: u128 = 2 * GRT; const MAX_ADJUSTMENT: u128 = 10_000 * GRT; -/// Grace period added to a thaw's maturity before a withdrawal is attempted. The escrow contract -/// requires the maturity timestamp to be strictly in the past, and reverts the whole multicall -/// otherwise, so this absorbs subgraph lag and block timestamp drift. -const WITHDRAW_MATURITY_SLACK_SECONDS: u64 = 300; #[tokio::main] async fn main() -> anyhow::Result<()> { @@ -63,6 +59,13 @@ async fn main() -> anyhow::Result<()> { "withdraw_margin must be non-negative, got {}", config.withdraw_margin, ); + // Upper bound catches a fraction written as a percentage. A margin of 25 rather than 0.25 puts + // the floor at 26x target, which no balance ever clears, so reclamation silently never runs. + anyhow::ensure!( + config.withdraw_margin <= 1.0, + "withdraw_margin must be at most 1.0 (a fraction, not a percentage), got {}", + config.withdraw_margin, + ); // Converted to basis points so every subsequent calculation on token amounts stays in integer // arithmetic. `u128` GRT amounts exceed f64's exact range, and these numbers decide transactions. let withdraw_margin_bps = (config.withdraw_margin * 10_000.0).round() as u128; @@ -266,10 +269,21 @@ async fn main() -> anyhow::Result<()> { .total_debt_grt .set(debts.values().sum::() as f64 / GRT as f64); - let now_unix = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); + // Maturity is a contract-side comparison against the block timestamp, so the reference + // time has to come from the chain; the local clock has no defined relationship to it. A + // failure here skips withdrawals for the cycle rather than falling back to wall clock, + // since being early reverts the whole batch while being late costs nothing against a + // 28 day horizon. + let chain_now = match config.withdraw_enabled { + false => None, + true => match contracts.latest_block_timestamp().await { + Ok(timestamp) => Some(timestamp), + Err(err) => { + tracing::error!("{:#}", err.context("get latest block timestamp")); + None + } + }, + }; let mut total_target: u128 = 0; let mut adjustments: Vec<(Address, u128)> = Default::default(); @@ -328,11 +342,12 @@ async fn main() -> anyhow::Result<()> { ); } - // Only withdraw a thaw that has definitely matured and still has a justified amount. - // Cancelling clears the timestamp, so a receiver being cancelled this cycle must be - // left out or the withdrawal reverts and takes the whole batch with it. + // Only withdraw a thaw that has matured and still has a justified amount. Cancelling + // clears the timestamp, so a receiver being cancelled this cycle must be left out or + // the withdrawal reverts and takes the whole batch with it. The comparison mirrors the + // contract's, which is strict. let matured = (account.thaw_end_timestamp != 0) - && (now_unix > (account.thaw_end_timestamp + WITHDRAW_MATURITY_SLACK_SECONDS)); + && chain_now.is_some_and(|now| now > account.thaw_end_timestamp); if config.withdraw_enabled && matured && (desired_thaw > 0) { tracing::info!( ?receiver, diff --git a/grafana/escrow_manager.json b/grafana/escrow_manager.json index 918b5b2..9168f0e 100644 --- a/grafana/escrow_manager.json +++ b/grafana/escrow_manager.json @@ -3293,7 +3293,7 @@ "spec": { "id": 27, "title": "Withdrawals Failed", - "description": "Failed `withdraw` transactions in the selected range. The escrow contract reverts the whole batch if any thaw has not matured, so repeated failures point at clock or subgraph lag beyond the maturity slack.", + "description": "Failed `withdraw` transactions in the selected range. The escrow contract reverts the whole batch if any thaw has not matured. Maturity is planned against the latest block timestamp, which cannot run ahead of the executing block, so repeated failures point at a receiver collecting mid-cycle rather than at clock drift.", "links": [], "data": { "kind": "QueryGroup", From 40f45f86871bd6962fdbe5be1a6f7d59905c742a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Migone?= Date: Mon, 21 Sep 2026 12:01:50 -0300 Subject: [PATCH 03/12] refactor: simplify algorithm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás Migone --- crates/bin/escrow_manager/README.md | 45 ++- .../src/abi/PaymentsEscrow.abi.json | 34 -- crates/bin/escrow_manager/src/contracts.rs | 21 +- crates/bin/escrow_manager/src/main.rs | 379 +++++++++++------- crates/bin/escrow_manager/src/metrics.rs | 7 +- grafana/escrow_manager.json | 26 +- 6 files changed, 292 insertions(+), 220 deletions(-) diff --git a/crates/bin/escrow_manager/README.md b/crates/bin/escrow_manager/README.md index 2e30a1d..fbdc90a 100644 --- a/crates/bin/escrow_manager/README.md +++ b/crates/bin/escrow_manager/README.md @@ -65,11 +65,25 @@ receiver with no debt has a target of 2 GRT, so it falls out of the same rule as end of the range. Reclaiming is not instant. It goes through the escrow contract's `thaw` and `withdraw`, so funds -only return after the contract's thawing period — **28 days on mainnet**. The manager uses -`adjustThaw` rather than `thaw`, which means a pending amount is tracked downward as debt grows -while keeping its original maturity date, instead of being cancelled and restarted. In practice the -amount withdrawn at maturity has been continuously re-validated against current debt for the whole -period. +only return after the contract's thawing period — **28 days on mainnet**. + +Each cycle a receiver gets exactly one action, decided in this order: + +1. **Withdraw** — a thaw has matured, so the escrow returns to the payer. +2. **Deposit** — the balance net of anything thawing is short of the target, so top it up. +3. **Thaw** — nothing is thawing and the balance clears `target * (1 + withdraw_margin)` by at + least `min_withdraw_grt`, so start reclaiming the excess. +4. Otherwise nothing. + +Because the branches are exclusive, the three transaction batches share no receivers and none +depends on another having landed first. + +A thaw is never resized once running: the contract cannot grow one without resetting its 28-day +timer, so escrow that builds up meanwhile waits for the next round. Debt that grows during a thaw +is answered by the deposit branch instead, which is why the thawing amount does not need to track +it. The trade is that money occasionally round-trips through the payer wallet — the full thawed +amount leaves at maturity and a deposit covers the difference — rather than being kept in place. +That costs one deposit against the alternative of rewriting the thaw every cycle for 28 days. Two consequences worth knowing: @@ -81,14 +95,15 @@ Two consequences worth knowing: balance cancels it outright. This is normal and needs no intervention, but it means a reclamation in flight is not guaranteed to complete. -Funding decisions use the balance net of any pending withdrawal, matching the escrow contract's own -`getBalance`, so coverage still holds once a withdrawal lands. A receiver is therefore never funded -and reclaimed in the same cycle. +Funding always uses the balance net of anything thawing, matching the escrow contract's own +`getBalance`, so coverage holds throughout a reclamation and across the withdrawal that ends it. -Setting `withdraw_enabled` back to `false` is inert: anything already thawing is left exactly as it -is and logged as a warning each cycle, rather than being cancelled or withdrawn. To stop reclaiming -without abandoning work in flight, leave it enabled and raise `withdraw_margin` instead — pending -amounts then shrink or cancel through the normal path. +Setting `withdraw_enabled` back to `false` is inert with respect to the thaw itself: anything +already thawing is left exactly as it is and logged as a warning each cycle, rather than being +cancelled or withdrawn. It is not free, though — that escrow stays permanently excluded from the +effective balance and is never reclaimed, so the manager tops the account up around it. To stop +reclaiming without stranding work in flight, leave it enabled and raise `withdraw_margin` instead, +which prevents new thaws while letting pending ones complete. Validate with `dry_run: true` first, and watch `escrow_total_thawing_grt` and `escrow_thawing_count`. Given the 28-day round trip, that observation period is the only practical @@ -194,9 +209,9 @@ curl http://localhost:9090/metrics | `escrow_deposit_ok` | Counter | Successful deposit transactions | | `escrow_deposit_err` | Counter | Failed deposit transactions | | `escrow_deposit_duration` | Histogram | Deposit transaction duration | -| `escrow_adjust_thaw_ok` | Counter | Successful thaw adjustment transactions | -| `escrow_adjust_thaw_err` | Counter | Failed thaw adjustment transactions | -| `escrow_adjust_thaw_duration` | Histogram | Thaw adjustment transaction duration | +| `escrow_thaw_ok` | Counter | Successful thaw transactions | +| `escrow_thaw_err` | Counter | Failed thaw transactions | +| `escrow_thaw_duration` | Histogram | Thaw transaction duration | | `escrow_withdraw_ok` | Counter | Successful withdrawal transactions | | `escrow_withdraw_err` | Counter | Failed withdrawal transactions | | `escrow_withdraw_duration` | Histogram | Withdrawal transaction duration | diff --git a/crates/bin/escrow_manager/src/abi/PaymentsEscrow.abi.json b/crates/bin/escrow_manager/src/abi/PaymentsEscrow.abi.json index c8c97d9..62002fe 100644 --- a/crates/bin/escrow_manager/src/abi/PaymentsEscrow.abi.json +++ b/crates/bin/escrow_manager/src/abi/PaymentsEscrow.abi.json @@ -378,40 +378,6 @@ "stateMutability": "view", "type": "function" }, - { - "inputs": [ - { - "internalType": "address", - "name": "collector", - "type": "address" - }, - { - "internalType": "address", - "name": "receiver", - "type": "address" - }, - { - "internalType": "uint256", - "name": "tokensToThaw", - "type": "uint256" - }, - { - "internalType": "bool", - "name": "evenIfTimerReset", - "type": "bool" - } - ], - "name": "adjustThaw", - "outputs": [ - { - "internalType": "uint256", - "name": "tokensThawing", - "type": "uint256" - } - ], - "stateMutability": "nonpayable", - "type": "function" - }, { "inputs": [ { diff --git a/crates/bin/escrow_manager/src/contracts.rs b/crates/bin/escrow_manager/src/contracts.rs index 261456c..5dfa0d8 100644 --- a/crates/bin/escrow_manager/src/contracts.rs +++ b/crates/bin/escrow_manager/src/contracts.rs @@ -161,26 +161,23 @@ impl Contracts { Ok(block.header.timestamp) } - /// Set each receiver's thawing amount to `tokens`, batched into one transaction. + /// Start thawing `tokens` for each receiver, batched into one transaction. /// - /// Uses `adjustThaw` rather than `thaw`/`cancelThaw` because it is the only variant that is - /// safe to call every cycle: decreasing an amount preserves the existing maturity timestamp, - /// increasing one is refused outright when it would reset the timer (`evenIfTimerReset` is - /// false), and a no-op change writes nothing. A plain `thaw` would restart the thawing period - /// on every call. - pub async fn adjust_thaw_many( + /// Only ever called for receivers with nothing currently thawing. `thaw` restarts the thawing + /// period on every call, and the contract will not grow a running thaw without resetting its + /// timer either, so escrow that builds up while one is in flight waits for the next. + pub async fn thaw_many( &self, - adjustments: impl IntoIterator, + thaws: impl IntoIterator, ) -> anyhow::Result { - let calls: Vec = adjustments + let calls: Vec = thaws .into_iter() .map(|(receiver, tokens)| { self.payments_escrow - .adjustThaw( + .thaw( *self.graph_tally_collector.address(), receiver, U256::from(tokens), - false, ) .calldata() .clone() @@ -200,7 +197,7 @@ impl Contracts { receipt .block_number - .ok_or_else(|| anyhow!("invalid adjust thaw receipt")) + .ok_or_else(|| anyhow!("invalid thaw receipt")) } /// Withdraw each receiver's matured thawing amount back to the payer, batched into one diff --git a/crates/bin/escrow_manager/src/main.rs b/crates/bin/escrow_manager/src/main.rs index 31a4d8e..f0df287 100644 --- a/crates/bin/escrow_manager/src/main.rs +++ b/crates/bin/escrow_manager/src/main.rs @@ -285,9 +285,15 @@ async fn main() -> anyhow::Result<()> { }, }; + let reclaim = config.withdraw_enabled.then_some(Reclaim { + chain_now, + margin_bps: withdraw_margin_bps, + min_withdraw, + }); + let mut total_target: u128 = 0; let mut adjustments: Vec<(Address, u128)> = Default::default(); - let mut thaw_adjustments: Vec<(Address, u128)> = Default::default(); + let mut thaws: Vec<(Address, u128)> = Default::default(); let mut withdrawals: Vec
= Default::default(); for receiver in receivers { let account = escrow_accounts.get(&receiver).copied().unwrap_or_default(); @@ -298,18 +304,7 @@ async fn main() -> anyhow::Result<()> { let target = next_balance(debt, config.balance_fill_factor); total_target += target; - // Amount we want thawing once this cycle's calls land. With reclamation disabled the - // existing amount is left exactly as it is, so turning the feature off is inert. - let desired_thaw = if config.withdraw_enabled { - desired_thawing(&account, target, withdraw_margin_bps, min_withdraw) - } else { - account.thawing - }; - - // Fund against the balance that survives a pending withdrawal, so coverage still holds - // once it lands. This is also how the escrow contract's own `getBalance` is defined. - let effective_balance = account.balance.saturating_sub(desired_thaw); - let adjustment = target.saturating_sub(effective_balance); + let action = decide(&account, target, reclaim); // Record for every receiver, including those needing no action. Skipping them would // leave the gauges holding the last value they were set to, indefinitely. @@ -321,20 +316,12 @@ async fn main() -> anyhow::Result<()> { metrics::METRICS .adjustment_grt .with_label_values(&[&receiver_str]) - .set(adjustment as f64 / GRT as f64); + .set(match action { + Action::Deposit(amount) => amount as f64 / GRT as f64, + _ => 0.0, + }); - if desired_thaw != account.thawing { - tracing::info!( - ?receiver, - balance_grt = (account.balance as f64) / (GRT as f64), - debt_grt = (debt as f64) / (GRT as f64), - target_grt = (target as f64) / (GRT as f64), - thawing_grt = (account.thawing as f64) / (GRT as f64), - desired_thawing_grt = (desired_thaw as f64) / (GRT as f64), - "thaw adjustment", - ); - thaw_adjustments.push((receiver, desired_thaw)); - } else if !config.withdraw_enabled && (account.thawing > 0) { + if !config.withdraw_enabled && (account.thawing > 0) { tracing::warn!( ?receiver, thawing_grt = (account.thawing as f64) / (GRT as f64), @@ -342,33 +329,39 @@ async fn main() -> anyhow::Result<()> { ); } - // Only withdraw a thaw that has matured and still has a justified amount. Cancelling - // clears the timestamp, so a receiver being cancelled this cycle must be left out or - // the withdrawal reverts and takes the whole batch with it. The comparison mirrors the - // contract's, which is strict. - let matured = (account.thaw_end_timestamp != 0) - && chain_now.is_some_and(|now| now > account.thaw_end_timestamp); - if config.withdraw_enabled && matured && (desired_thaw > 0) { - tracing::info!( - ?receiver, - tokens_grt = (desired_thaw as f64) / (GRT as f64), - "withdrawal matured", - ); - withdrawals.push(receiver); - } - - if adjustment == 0 { - continue; + match action { + Action::Nothing => (), + Action::Withdraw => { + tracing::info!( + ?receiver, + thawing_grt = (account.thawing as f64) / (GRT as f64), + "withdrawal matured", + ); + withdrawals.push(receiver); + } + Action::Deposit(amount) => { + tracing::info!( + ?receiver, + balance_grt = (account.balance as f64) / (GRT as f64), + thawing_grt = (account.thawing as f64) / (GRT as f64), + debt_grt = (debt as f64) / (GRT as f64), + target_grt = (target as f64) / (GRT as f64), + adjustment_grt = (amount as f64) / (GRT as f64), + ); + adjustments.push((receiver, amount)); + } + Action::Thaw(amount) => { + tracing::info!( + ?receiver, + balance_grt = (account.balance as f64) / (GRT as f64), + debt_grt = (debt as f64) / (GRT as f64), + target_grt = (target as f64) / (GRT as f64), + thaw_grt = (amount as f64) / (GRT as f64), + "thawing idle escrow", + ); + thaws.push((receiver, amount)); + } } - tracing::info!( - ?receiver, - balance_grt = (account.balance as f64) / (GRT as f64), - effective_balance_grt = (effective_balance as f64) / (GRT as f64), - debt_grt = (debt as f64) / (GRT as f64), - target_grt = (target as f64) / (GRT as f64), - adjustment_grt = (adjustment as f64) / (GRT as f64), - ); - adjustments.push((receiver, adjustment)); } metrics::METRICS .total_target_grt @@ -380,36 +373,35 @@ async fn main() -> anyhow::Result<()> { .total_adjustment_grt .set(total_adjustment as f64 / GRT as f64); // Tracked across all three phases so a failure in one does not skip the others, and the - // subgraph client is only pinned forward once. + // subgraph client is only pinned forward once. The phases share no receivers, so their + // order carries no meaning and a failure in one cannot corrupt another. let mut latest_tx_block: Option = None; - // Thaw adjustments run first: shrinking or cancelling a thaw is the protective action, and - // the deposit amounts computed above already assume it has happened. - if !thaw_adjustments.is_empty() { + if !thaws.is_empty() { if config.dry_run { - for (receiver, tokens) in &thaw_adjustments { + for (receiver, tokens) in &thaws { tracing::info!( ?receiver, tokens_grt = (*tokens as f64) / (GRT as f64), - "dry run: skipping adjust thaw" + "dry run: skipping thaw" ); } } else { let start = Instant::now(); - let result = contracts.adjust_thaw_many(thaw_adjustments).await; + let result = contracts.thaw_many(thaws).await; metrics::METRICS - .adjust_thaw + .thaw .duration .observe(start.elapsed().as_secs_f64()); match result { Ok(block) => { - metrics::METRICS.adjust_thaw.ok.inc(); + metrics::METRICS.thaw.ok.inc(); latest_tx_block = latest_tx_block.max(Some(block)); - tracing::info!("thaw adjustments complete"); + tracing::info!("thaws complete"); } - Err(adjust_thaw_err) => { - metrics::METRICS.adjust_thaw.err.inc(); - tracing::error!("{:#}", adjust_thaw_err.context("adjust thaw")); + Err(thaw_err) => { + metrics::METRICS.thaw.err.inc(); + tracing::error!("{:#}", thaw_err.context("thaw")); } } } @@ -505,31 +497,80 @@ fn next_balance(debt: u128, fill_factor: f64) -> u128 { next_round as u128 * GRT } -/// Escrow that should be thawing for a receiver once this cycle's calls land. -/// -/// Escrow is only reclaimed above `target * (1 + margin)`. That headroom absorbs debt growth over -/// the thawing period, and provides hysteresis against the target's step ladder. -/// -/// `adjustThaw` refuses an increase that would reset a running timer, so a thaw already in flight -/// can only be tracked downward. This mirrors that, both so the amount withdrawn at maturity has -/// been continuously re-validated against current debt, and so funding decisions agree with what -/// the contract will actually hold. The minimum only gates starting a thaw: once one is running the -/// amount follows the excess down past the minimum, because shrinking keeps the maturity timestamp -/// whereas cancelling would forfeit the whole thawing period. -fn desired_thawing( - account: &EscrowAccount, - target: u128, +/// Reclamation policy for a cycle, present only when `withdraw_enabled`. +#[derive(Clone, Copy)] +struct Reclaim { + /// Latest block timestamp, or `None` when it could not be read this cycle. Withdrawals are + /// then skipped rather than planned against the local clock, which has no defined relationship + /// to the block timestamp the contract compares against. + chain_now: Option, margin_bps: u128, min_withdraw: u128, -) -> u128 { - let floor = target + (target / 10_000) * margin_bps; - let excess = account.balance.saturating_sub(floor); +} + +/// The one action taken for a receiver in a cycle. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Action { + Nothing, + /// A matured thaw is ready to come back to the payer. The contract withdraws whatever is + /// thawing at execution time, so no amount is carried here. + Withdraw, + /// Effective balance is short of the target; top it up by this much. + Deposit(u128), + /// Idle escrow above the margin is worth reclaiming; start thawing this much. + Thaw(u128), +} + +/// Decide the single action to take for a receiver this cycle. +/// +/// Exactly one action applies, which is what keeps the three transaction batches independent: no +/// receiver ever appears in more than one, so no batch depends on another having landed first. +/// +/// The branches are ordered so each is decided against state the earlier ones cannot invalidate: +/// +/// - Withdrawal comes first because it leaves `balance - thawing` untouched — the contract zeroes +/// both together — so taking it can never leave an account short. It only defers a deposit by a +/// cycle. +/// - Funding is decided against `balance - thawing`, matching the escrow contract's own +/// `getBalance`. Escrow already committed to leaving cannot count as coverage. +/// - A thaw only ever starts from nothing. The contract cannot grow a running thaw without +/// resetting its timer, so excess accumulating after one starts waits for the next rather than +/// being chased every cycle. Debt that grows meanwhile is covered by the deposit branch, which +/// is why the reclaimed amount need not be re-validated against it. +fn decide(account: &EscrowAccount, target: u128, reclaim: Option) -> Action { + // `tokensThawing == 0` implies `thawEndTimestamp == 0` on chain, but the timestamp is what the + // contract checks, and a withdrawal that reverts takes the whole batch with it. The comparison + // mirrors the contract's, which is strict. + let matured = (account.thaw_end_timestamp != 0) + && (account.thawing > 0) + && reclaim + .and_then(|reclaim| reclaim.chain_now) + .is_some_and(|now| now > account.thaw_end_timestamp); + if matured { + return Action::Withdraw; + } + + let effective_balance = account.balance.saturating_sub(account.thawing); + let deposit = target.saturating_sub(effective_balance); + if deposit > 0 { + return Action::Deposit(deposit); + } + + let Some(reclaim) = reclaim else { + return Action::Nothing; + }; if account.thawing > 0 { - excess.min(account.thawing) - } else if excess >= min_withdraw { - excess - } else { - 0 + return Action::Nothing; + } + // Escrow is only reclaimed above `target * (1 + margin)`. That headroom absorbs debt growth + // over the thawing period, and keeps ordinary fluctuation from bouncing between depositing and + // thawing — without it the deposit and thaw branches would partition the whole range and one + // of them would fire every cycle. + let floor = target + (target / 10_000) * reclaim.margin_bps; + let excess = account.balance.saturating_sub(floor); + match (excess > 0) && (excess >= reclaim.min_withdraw) { + true => Action::Thaw(excess), + false => Action::Nothing, } } @@ -566,107 +607,163 @@ async fn handle_metrics() -> impl axum::response::IntoResponse { #[cfg(test)] mod tests { - use super::{EscrowAccount, GRT, MIN_DEPOSIT}; + use super::{Action, EscrowAccount, Reclaim, GRT, MIN_DEPOSIT}; const MARGIN_BPS: u128 = 2_500; const MIN_WITHDRAW: u128 = 500 * GRT; + const NOW: u64 = 1_000_000; + const MATURED: u64 = NOW - 1; + const PENDING: u64 = NOW + 1; - fn account(balance: u128, thawing: u128) -> EscrowAccount { + fn account(balance: u128, thawing: u128, thaw_end_timestamp: u64) -> EscrowAccount { EscrowAccount { balance, thawing, - thaw_end_timestamp: if thawing > 0 { 1 } else { 0 }, + thaw_end_timestamp, } } - fn desired(balance: u128, thawing: u128, target: u128) -> u128 { - super::desired_thawing(&account(balance, thawing), target, MARGIN_BPS, MIN_WITHDRAW) + fn reclaim(chain_now: Option) -> Option { + Some(Reclaim { + chain_now, + margin_bps: MARGIN_BPS, + min_withdraw: MIN_WITHDRAW, + }) + } + + fn decide(balance: u128, thawing: u128, thaw_end_timestamp: u64, target: u128) -> Action { + super::decide( + &account(balance, thawing, thaw_end_timestamp), + target, + reclaim(Some(NOW)), + ) } #[test] - fn desired_thawing_applies_margin_above_target() { + fn withdraws_a_matured_thaw() { + // Maturity mirrors the contract's strict comparison against the block timestamp. + assert_eq!( + decide(20_000 * GRT, 7_500 * GRT, MATURED, 10_000 * GRT), + Action::Withdraw + ); + assert_eq!( + decide(20_000 * GRT, 7_500 * GRT, NOW, 10_000 * GRT), + Action::Nothing + ); + } + + #[test] + fn withdrawal_comes_before_funding() { + // A withdrawal zeroes balance and thawing together, leaving `balance - thawing` untouched, + // so taking it first cannot leave the account short. The deposit follows next cycle. + assert_eq!( + decide(20_000 * GRT, 7_500 * GRT, MATURED, 18_000 * GRT), + Action::Withdraw + ); + } + + #[test] + fn funds_against_the_balance_net_of_thawing() { + // Escrow committed to leaving cannot count as coverage, so debt growth during a thaw is + // answered with a deposit. This is what makes re-validating the thawing amount unnecessary. + assert_eq!( + decide(20_000 * GRT, 7_500 * GRT, PENDING, 18_000 * GRT), + Action::Deposit(5_500 * GRT) + ); + } + + #[test] + fn leaves_a_running_thaw_alone() { + // Covered and already thawing: no attempt to chase the excess, which the contract would + // refuse anyway without resetting the 28 day timer. + assert_eq!( + decide(20_000 * GRT, 7_500 * GRT, PENDING, 10_000 * GRT), + Action::Nothing + ); + // Even with far more idle escrow than the running thaw covers. + assert_eq!( + decide(100_000 * GRT, 7_500 * GRT, PENDING, 10_000 * GRT), + Action::Nothing + ); + } + + #[test] + fn thaws_idle_escrow_above_the_margin() { let target = 10_000 * GRT; // Nothing is reclaimed until the balance clears target * 1.25. - assert_eq!(desired(12_500 * GRT, 0, target), 0); - assert_eq!(desired(12_999 * GRT, 0, target), 0); + assert_eq!(decide(12_500 * GRT, 0, 0, target), Action::Nothing); + assert_eq!(decide(12_999 * GRT, 0, 0, target), Action::Nothing); // Above the floor, only the excess over it is reclaimed. - assert_eq!(desired(20_000 * GRT, 0, target), 7_500 * GRT); + assert_eq!( + decide(20_000 * GRT, 0, 0, target), + Action::Thaw(7_500 * GRT) + ); } #[test] - fn desired_thawing_respects_minimum_when_starting() { + fn respects_the_withdrawal_minimum() { let target = 10_000 * GRT; // An excess under the minimum is not worth a 28 day round trip. - assert_eq!(desired(12_999 * GRT, 0, target), 0); - assert_eq!(desired(13_000 * GRT, 0, target), 500 * GRT); + assert_eq!(decide(12_999 * GRT, 0, 0, target), Action::Nothing); + assert_eq!(decide(13_000 * GRT, 0, 0, target), Action::Thaw(500 * GRT)); } #[test] - fn desired_thawing_tracks_a_running_thaw_downward() { - // Target climbed a rung, so the justified excess shrank. Following it down preserves the - // maturity timestamp; cancelling and re-thawing would forfeit the thawing period. + fn reclamation_disabled_only_ever_deposits() { + let idle = account(20_000 * GRT, 0, 0); + let thawing = account(20_000 * GRT, 7_500 * GRT, MATURED); + // No thaws started, and a matured thaw is left exactly where it is. + assert_eq!(super::decide(&idle, 10_000 * GRT, None), Action::Nothing); + assert_eq!(super::decide(&thawing, 10_000 * GRT, None), Action::Nothing); + // Funding still nets out the thawing amount, matching the contract's `getBalance`. assert_eq!( - desired(20_000 * GRT, 7_500 * GRT, 14_000 * GRT), - 2_500 * GRT + super::decide(&thawing, 18_000 * GRT, None), + Action::Deposit(5_500 * GRT) ); - // Below the minimum too: the threshold only gates starting a thaw. - assert_eq!(desired(20_000 * GRT, 7_500 * GRT, 15_900 * GRT), 125 * GRT); - // Excess gone entirely: cancel. - assert_eq!(desired(20_000 * GRT, 7_500 * GRT, 16_000 * GRT), 0); - assert_eq!(desired(20_000 * GRT, 7_500 * GRT, 30_000 * GRT), 0); } #[test] - fn desired_thawing_never_grows_a_running_thaw() { - // The contract refuses an increase that would reset the timer, so asking for one would - // silently do nothing. Report what the contract will actually hold instead. - let target = 10_000 * GRT; - assert_eq!(desired(40_000 * GRT, 7_500 * GRT, target), 7_500 * GRT); + fn skips_withdrawals_without_a_chain_timestamp() { + // Planning maturity against the local clock would risk reverting the whole batch, so a + // failed timestamp read holds the withdrawal rather than guessing. + let matured = account(20_000 * GRT, 7_500 * GRT, MATURED); + assert_eq!( + super::decide(&matured, 10_000 * GRT, reclaim(None)), + Action::Nothing + ); } #[test] - fn deposits_and_reclaims_are_mutually_exclusive() { - // A receiver can never be funded and reclaimed in the same cycle: reclaiming leaves the - // effective balance at the floor, which is above target by construction. + fn thawing_never_drops_the_balance_below_target() { + // The floor sits above target by construction, so what remains after a thaw still covers + // debt. This is what lets the thaw and deposit branches stay mutually exclusive. for debt_grt in [0, 1, 500, 12_345, 100_000, 580_000] { for balance_grt in [0, 2, 1_000, 40_000, 96_384, 1_000_000] { for thawing_grt in [0, 100, 20_000] { let balance = balance_grt * GRT; let thawing = u128::min(thawing_grt * GRT, balance); + let thaw_end = if thawing > 0 { PENDING } else { 0 }; let target = super::next_balance(debt_grt * GRT, 0.8); - let thaw = super::desired_thawing( - &account(balance, thawing), + let action = super::decide( + &account(balance, thawing, thaw_end), target, - MARGIN_BPS, - MIN_WITHDRAW, + reclaim(Some(NOW)), ); - let adjustment = target.saturating_sub(balance.saturating_sub(thaw)); + let Action::Thaw(amount) = action else { + continue; + }; + // `thaw(0)` reverts, so a thaw is never queued for nothing. + assert!(amount > 0, "debt {debt_grt} balance {balance_grt}"); assert!( - (thaw == 0) || (adjustment == 0), - "debt {debt_grt} balance {balance_grt} thawing {thawing_grt}: \ - thaw {thaw} and adjustment {adjustment}", + balance.saturating_sub(amount) >= target, + "debt {debt_grt} balance {balance_grt}: \ + thaw {amount} leaves less than target {target}", ); } } } } - #[test] - fn desired_thawing_never_exceeds_balance() { - for balance_grt in [0, 2, 1_000, 96_384] { - for target_grt in [2, 1_000, 96_384] { - let balance = balance_grt * GRT; - let thaw = super::desired_thawing( - &account(balance, 0), - target_grt * GRT, - MARGIN_BPS, - MIN_WITHDRAW, - ); - assert!(thaw <= balance, "balance {balance_grt} target {target_grt}"); - } - } - } - #[test] fn next_balance() { let tests = [ diff --git a/crates/bin/escrow_manager/src/metrics.rs b/crates/bin/escrow_manager/src/metrics.rs index 6d58d97..d1fab32 100644 --- a/crates/bin/escrow_manager/src/metrics.rs +++ b/crates/bin/escrow_manager/src/metrics.rs @@ -18,7 +18,7 @@ pub struct Metrics { pub thawing_count: IntGauge, pub loop_duration: Histogram, pub deposit: ResponseMetrics, - pub adjust_thaw: ResponseMetrics, + pub thaw: ResponseMetrics, pub withdraw: ResponseMetrics, // Per-receiver metrics pub debt_grt: GaugeVec, @@ -72,10 +72,7 @@ impl Metrics { ) .unwrap(), deposit: ResponseMetrics::new("escrow_deposit", "escrow deposit transaction"), - adjust_thaw: ResponseMetrics::new( - "escrow_adjust_thaw", - "escrow thaw adjustment transaction", - ), + thaw: ResponseMetrics::new("escrow_thaw", "escrow thaw transaction"), withdraw: ResponseMetrics::new("escrow_withdraw", "escrow withdrawal transaction"), debt_grt: register_gauge_vec!( "escrow_debt_grt", diff --git a/grafana/escrow_manager.json b/grafana/escrow_manager.json index 9168f0e..c541653 100644 --- a/grafana/escrow_manager.json +++ b/grafana/escrow_manager.json @@ -3204,8 +3204,8 @@ "kind": "Panel", "spec": { "id": 26, - "title": "Thaw Adjustments Failed", - "description": "Failed `adjustThaw` transactions in the selected range. These start, shrink and cancel pending withdrawals; a sustained failure means reclamation amounts are no longer tracking debt.", + "title": "Thaws Failed", + "description": "Failed `thaw` transactions in the selected range. Thaws start a reclamation; a sustained failure means idle escrow is no longer being reclaimed, which is visible as a flat Thawing line.", "links": [], "data": { "kind": "QueryGroup", @@ -3222,7 +3222,7 @@ "name": "${datasource}" }, "spec": { - "expr": "sum(increase(escrow_adjust_thaw_err{job=~\"$job\"}[$__range]))", + "expr": "sum(increase(escrow_thaw_err{job=~\"$job\"}[$__range]))", "instant": true }, "labels": { @@ -3381,7 +3381,7 @@ "spec": { "id": 28, "title": "Thawing Over Time", - "description": "Total escrow pending withdrawal. Steps up when reclamation starts on a receiver, down as amounts are tracked against growing debt, and to zero as withdrawals mature.", + "description": "Total escrow pending withdrawal. Steps up when reclamation starts on a receiver, down as receivers collect against a thawing balance, and to zero as withdrawals mature.", "links": [], "data": { "kind": "QueryGroup", @@ -3500,7 +3500,7 @@ "spec": { "id": 29, "title": "Reclamation In Flight", - "description": "Receivers with escrow pending withdrawal, one row each. Thawing is the amount due back to the payer at maturity; it is re-validated against debt every cycle, so it shrinks if debt grows and disappears if the excess does. Balance minus Thawing is what funding decisions use.", + "description": "Receivers with escrow pending withdrawal, one row each. Thawing is the amount due back to the payer at maturity; it is fixed when the thaw starts and is not resized, so debt growing meanwhile is answered by a deposit instead. Balance minus Thawing is what funding decisions use.", "links": [], "data": { "kind": "QueryGroup", @@ -3781,7 +3781,7 @@ "spec": { "id": 30, "title": "Reclamation Transactions", - "description": "Thaw adjustments start, shrink and cancel pending withdrawals and run whenever the justified amount changes; withdrawals only fire once a thaw matures, so they are expected to be rare.", + "description": "Thaws start a reclamation and only fire for receivers with nothing already thawing; withdrawals only fire once a thaw matures. Both are expected to be rare -- a receiver can start at most one thaw per 28 day period.", "links": [], "data": { "kind": "QueryGroup", @@ -3798,8 +3798,8 @@ "name": "${datasource}" }, "spec": { - "expr": "sum(increase(escrow_adjust_thaw_ok{job=~\"$job\"}[1h]))", - "legendFormat": "Thaw Adjust OK" + "expr": "sum(increase(escrow_thaw_ok{job=~\"$job\"}[1h]))", + "legendFormat": "Thaw OK" }, "labels": { "grafana.app/export-label": "prometheus-1", @@ -3821,8 +3821,8 @@ "name": "${datasource}" }, "spec": { - "expr": "sum(increase(escrow_adjust_thaw_err{job=~\"$job\"}[1h]))", - "legendFormat": "Thaw Adjust Errors" + "expr": "sum(increase(escrow_thaw_err{job=~\"$job\"}[1h]))", + "legendFormat": "Thaw Errors" }, "labels": { "grafana.app/export-label": "prometheus-1", @@ -3998,7 +3998,7 @@ "spec": { "id": 31, "title": "Reclamation Duration", - "description": "Average transaction duration for thaw adjustments and withdrawals.", + "description": "Average transaction duration for thaws and withdrawals.", "links": [], "data": { "kind": "QueryGroup", @@ -4015,8 +4015,8 @@ "name": "${datasource}" }, "spec": { - "expr": "rate(escrow_adjust_thaw_duration_sum{job=~\"$job\"}[5m]) / rate(escrow_adjust_thaw_duration_count{job=~\"$job\"}[5m])", - "legendFormat": "Thaw Adjust ({{pod}})" + "expr": "rate(escrow_thaw_duration_sum{job=~\"$job\"}[5m]) / rate(escrow_thaw_duration_count{job=~\"$job\"}[5m])", + "legendFormat": "Thaw ({{pod}})" }, "labels": { "grafana.app/export-label": "prometheus-1", From 8cb81012163c3551a49331a1db410018cf21ed24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Migone?= Date: Mon, 21 Sep 2026 14:49:23 -0300 Subject: [PATCH 04/12] refactor: keep provider instance in Contracts struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás Migone --- .claude/settings.local.json | 17 +++++++++++++++++ crates/bin/escrow_manager/src/contracts.rs | 13 +++++-------- 2 files changed, 22 insertions(+), 8 deletions(-) create mode 100644 .claude/settings.local.json diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..b275659 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,17 @@ +{ + "permissions": { + "allow": [ + "Bash(gh pr view:*)", + "Bash(gh pr diff:*)", + "Bash(grep:*)", + "Bash(python3 -c ' *)", + "Bash(python3 *)", + "Bash(cargo test *)", + "Bash(cargo clippy *)", + "Bash(cargo fmt *)", + "Bash(ocr delegate *)", + "Read(//Users/tomi/git/thegraph/skills/skills/pr-description/**)", + "Bash(gh pr *)" + ] + } +} diff --git a/crates/bin/escrow_manager/src/contracts.rs b/crates/bin/escrow_manager/src/contracts.rs index 5dfa0d8..51773b0 100644 --- a/crates/bin/escrow_manager/src/contracts.rs +++ b/crates/bin/escrow_manager/src/contracts.rs @@ -39,6 +39,8 @@ sol!( use GraphTallyCollector::{GraphTallyCollectorErrors, GraphTallyCollectorInstance}; pub struct Contracts { + /// Held directly for chain-level calls that belong to no particular contract. + provider: DynProvider, payments_escrow: PaymentsEscrowInstance, graph_tally_collector: GraphTallyCollectorInstance, token: ERC20Instance, @@ -64,6 +66,7 @@ impl Contracts { GraphTallyCollectorInstance::new(graph_tally_collector, provider.clone()); let token = ERC20Instance::new(token, provider.clone()); Self { + provider, payments_escrow, graph_tally_collector, token, @@ -152,8 +155,7 @@ impl Contracts { /// block a transaction sent now lands in. Planning against it can only be conservative. pub async fn latest_block_timestamp(&self) -> anyhow::Result { let block = self - .payments_escrow - .provider() + .provider .get_block(BlockId::latest()) .await .context("get latest block")? @@ -237,12 +239,7 @@ impl Contracts { } pub async fn authorize_signer(&self, signer: &PrivateKeySigner) -> anyhow::Result<()> { - let chain_id = self - .graph_tally_collector - .provider() - .get_chain_id() - .await - .context("get chain ID")?; + let chain_id = self.provider.get_chain_id().await.context("get chain ID")?; let deadline_offset_s = 60; let deadline = U256::from( SystemTime::now() From a946ce08b3b311536b937fb50a234708965840ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Migone?= Date: Mon, 21 Sep 2026 14:42:57 -0300 Subject: [PATCH 05/12] chore: apply suggestion from @tmigone --- crates/bin/escrow_manager/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bin/escrow_manager/src/config.rs b/crates/bin/escrow_manager/src/config.rs index e976273..d382e2b 100644 --- a/crates/bin/escrow_manager/src/config.rs +++ b/crates/bin/escrow_manager/src/config.rs @@ -57,7 +57,7 @@ pub struct Config { pub withdraw_enabled: bool, /// Headroom kept above the target balance before any escrow is reclaimed: escrow is only /// thawed above `target * (1 + withdraw_margin)`. This absorbs debt growth over the thawing - /// period and provides hysteresis against the step ladder, so it should comfortably exceed the + /// period and provides hysteresis against the deposit step ladder, so it should comfortably exceed the /// debt growth expected for a single receiver over that period. Must be in the range [0, 1]. #[serde(default = "default_withdraw_margin")] pub withdraw_margin: f64, From f99e83dfea85538602fe46fbe9efda971665f7c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Migone?= Date: Mon, 21 Sep 2026 14:50:57 -0300 Subject: [PATCH 06/12] chore: remove claude local cofig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás Migone --- .claude/settings.local.json | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 .claude/settings.local.json diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index b275659..0000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(gh pr view:*)", - "Bash(gh pr diff:*)", - "Bash(grep:*)", - "Bash(python3 -c ' *)", - "Bash(python3 *)", - "Bash(cargo test *)", - "Bash(cargo clippy *)", - "Bash(cargo fmt *)", - "Bash(ocr delegate *)", - "Read(//Users/tomi/git/thegraph/skills/skills/pr-description/**)", - "Bash(gh pr *)" - ] - } -} From 642f1b0874bb0e2b3fad85720d38724d7849df12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Migone?= Date: Mon, 21 Sep 2026 14:51:38 -0300 Subject: [PATCH 07/12] chore: apply suggestion from @tmigone --- crates/bin/escrow_manager/src/contracts.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/bin/escrow_manager/src/contracts.rs b/crates/bin/escrow_manager/src/contracts.rs index 51773b0..108deae 100644 --- a/crates/bin/escrow_manager/src/contracts.rs +++ b/crates/bin/escrow_manager/src/contracts.rs @@ -39,7 +39,6 @@ sol!( use GraphTallyCollector::{GraphTallyCollectorErrors, GraphTallyCollectorInstance}; pub struct Contracts { - /// Held directly for chain-level calls that belong to no particular contract. provider: DynProvider, payments_escrow: PaymentsEscrowInstance, graph_tally_collector: GraphTallyCollectorInstance, From 964c64d678b371831ad2e7882d577bede9d77cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Migone?= Date: Mon, 21 Sep 2026 15:01:03 -0300 Subject: [PATCH 08/12] chore: apply suggestion from @tmigone --- crates/bin/escrow_manager/src/main.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/bin/escrow_manager/src/main.rs b/crates/bin/escrow_manager/src/main.rs index f0df287..4bf493d 100644 --- a/crates/bin/escrow_manager/src/main.rs +++ b/crates/bin/escrow_manager/src/main.rs @@ -122,8 +122,6 @@ async fn main() -> anyhow::Result<()> { } if config.withdraw_enabled { - // Read rather than assume: the period is set at deployment and capped at 90 days by the - // escrow contract, so the schedule this manager plans against has to come from the chain. let thawing_period = contracts .withdraw_escrow_thawing_period() .await From 71c72d11f519c170509c575ed758d78cd3611842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Migone?= Date: Mon, 21 Sep 2026 15:32:56 -0300 Subject: [PATCH 09/12] chore: small algo tweaks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás Migone --- crates/bin/escrow_manager/src/config.rs | 9 +++--- crates/bin/escrow_manager/src/contracts.rs | 12 -------- crates/bin/escrow_manager/src/main.rs | 33 +++++++++------------- crates/bin/escrow_manager/src/subgraphs.rs | 4 +-- 4 files changed, 21 insertions(+), 37 deletions(-) diff --git a/crates/bin/escrow_manager/src/config.rs b/crates/bin/escrow_manager/src/config.rs index d382e2b..ef25d67 100644 --- a/crates/bin/escrow_manager/src/config.rs +++ b/crates/bin/escrow_manager/src/config.rs @@ -57,13 +57,14 @@ pub struct Config { pub withdraw_enabled: bool, /// Headroom kept above the target balance before any escrow is reclaimed: escrow is only /// thawed above `target * (1 + withdraw_margin)`. This absorbs debt growth over the thawing - /// period and provides hysteresis against the deposit step ladder, so it should comfortably exceed the - /// debt growth expected for a single receiver over that period. Must be in the range [0, 1]. + /// period and provides hysteresis against the deposit step ladder, so it should comfortably + /// exceed the debt growth expected for a single receiver over that period. Must be in the + /// range [0, 1]. #[serde(default = "default_withdraw_margin")] pub withdraw_margin: f64, /// Minimum excess, in whole GRT, required to start thawing a receiver's escrow. Excess below - /// this is left alone. Once a receiver is already thawing, the amount tracks the excess down - /// past this threshold rather than being cancelled, so a shrinking excess keeps its timer. + /// this is left alone, since reclaiming it is not worth the thawing period. It gates starting a + /// thaw only: a thaw already running is never resized, so the threshold is not reapplied to it. #[serde(default = "default_min_withdraw_grt")] pub min_withdraw_grt: u64, } diff --git a/crates/bin/escrow_manager/src/contracts.rs b/crates/bin/escrow_manager/src/contracts.rs index 108deae..0ffe9ac 100644 --- a/crates/bin/escrow_manager/src/contracts.rs +++ b/crates/bin/escrow_manager/src/contracts.rs @@ -136,18 +136,6 @@ impl Contracts { Ok(block_number) } - /// Thawing period enforced by the escrow contract before withdrawals can be executed. Set at - /// deployment and immutable, so it is read once at startup rather than assumed. - pub async fn withdraw_escrow_thawing_period(&self) -> anyhow::Result { - self.payments_escrow - .WITHDRAW_ESCROW_THAWING_PERIOD() - .call() - .await - .context("get withdraw escrow thawing period")? - .try_into() - .context("result out of bounds") - } - /// Timestamp of the latest block, for planning against contract-side time checks. /// /// Block timestamps are non-decreasing, so this is a lower bound on the timestamp of whatever diff --git a/crates/bin/escrow_manager/src/main.rs b/crates/bin/escrow_manager/src/main.rs index 4bf493d..70be0f7 100644 --- a/crates/bin/escrow_manager/src/main.rs +++ b/crates/bin/escrow_manager/src/main.rs @@ -67,7 +67,8 @@ async fn main() -> anyhow::Result<()> { config.withdraw_margin, ); // Converted to basis points so every subsequent calculation on token amounts stays in integer - // arithmetic. `u128` GRT amounts exceed f64's exact range, and these numbers decide transactions. + // arithmetic. `u128` GRT amounts exceed f64's exact range, and these numbers decide + // transactions. let withdraw_margin_bps = (config.withdraw_margin * 10_000.0).round() as u128; let min_withdraw = config.min_withdraw_grt as u128 * GRT; @@ -122,14 +123,9 @@ async fn main() -> anyhow::Result<()> { } if config.withdraw_enabled { - let thawing_period = contracts - .withdraw_escrow_thawing_period() - .await - .context("get withdraw escrow thawing period")?; tracing::info!( withdraw_margin = config.withdraw_margin, min_withdraw_grt = config.min_withdraw_grt, - thawing_period_days = thawing_period as f64 / 86400.0, "escrow reclamation enabled" ); } else { @@ -267,11 +263,6 @@ async fn main() -> anyhow::Result<()> { .total_debt_grt .set(debts.values().sum::() as f64 / GRT as f64); - // Maturity is a contract-side comparison against the block timestamp, so the reference - // time has to come from the chain; the local clock has no defined relationship to it. A - // failure here skips withdrawals for the cycle rather than falling back to wall clock, - // since being early reverts the whole batch while being late costs nothing against a - // 28 day horizon. let chain_now = match config.withdraw_enabled { false => None, true => match contracts.latest_block_timestamp().await { @@ -366,13 +357,20 @@ async fn main() -> anyhow::Result<()> { .set(total_target as f64 / GRT as f64); let total_adjustment: u128 = adjustments.iter().map(|(_, a)| a).sum(); - tracing::info!(total_adjustment_grt = ((total_adjustment as f64) * 1e-18).ceil() as u64); + let total_thaw: u128 = thaws.iter().map(|(_, t)| t).sum(); + // Withdrawals can only be guesstimated so we log the count + tracing::info!( + total_adjustment_grt = ((total_adjustment as f64) * 1e-18).ceil() as u64, + total_thaw_grt = ((total_thaw as f64) * 1e-18).ceil() as u64, + withdrawals = withdrawals.len(), + "cycle plan", + ); metrics::METRICS .total_adjustment_grt .set(total_adjustment as f64 / GRT as f64); - // Tracked across all three phases so a failure in one does not skip the others, and the - // subgraph client is only pinned forward once. The phases share no receivers, so their - // order carries no meaning and a failure in one cannot corrupt another. + + // Whenever a transaction lands, track the block number. We use this to pin the network + // subgraph snapshot so the decision algorithm does not operate on stale data. let mut latest_tx_block: Option = None; if !thaws.is_empty() { @@ -405,7 +403,7 @@ async fn main() -> anyhow::Result<()> { } } - if total_adjustment > 0 { + if !adjustments.is_empty() { let adjustments = if total_adjustment <= MAX_ADJUSTMENT { adjustments } else { @@ -536,9 +534,6 @@ enum Action { /// being chased every cycle. Debt that grows meanwhile is covered by the deposit branch, which /// is why the reclaimed amount need not be re-validated against it. fn decide(account: &EscrowAccount, target: u128, reclaim: Option) -> Action { - // `tokensThawing == 0` implies `thawEndTimestamp == 0` on chain, but the timestamp is what the - // contract checks, and a withdrawal that reverts takes the whole batch with it. The comparison - // mirrors the contract's, which is strict. let matured = (account.thaw_end_timestamp != 0) && (account.thawing > 0) && reclaim diff --git a/crates/bin/escrow_manager/src/subgraphs.rs b/crates/bin/escrow_manager/src/subgraphs.rs index cc24410..32ec60c 100644 --- a/crates/bin/escrow_manager/src/subgraphs.rs +++ b/crates/bin/escrow_manager/src/subgraphs.rs @@ -76,7 +76,7 @@ pub async fn escrow_accounts( #[serde_as] #[derive(serde::Deserialize)] #[serde(rename_all = "camelCase")] - struct EscrowAccount_ { + struct EscrowAccountRow { #[serde_as(as = "serde_with::DisplayFromStr")] balance: u128, #[serde_as(as = "serde_with::DisplayFromStr")] @@ -90,7 +90,7 @@ pub async fn escrow_accounts( id: Address, } let response = network_subgraph - .paginated_query::(query, 500) + .paginated_query::(query, 500) .await; match response { Ok(accounts) => Ok(accounts From 2384d90a24a869ce04e28833c4c10eb48475c0d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Migone?= Date: Tue, 22 Sep 2026 10:11:12 -0300 Subject: [PATCH 10/12] fix: network subgraph query not keying by collector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás Migone --- crates/bin/escrow_manager/src/contracts.rs | 7 +++++++ crates/bin/escrow_manager/src/main.rs | 7 ++++++- crates/bin/escrow_manager/src/subgraphs.rs | 8 ++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/bin/escrow_manager/src/contracts.rs b/crates/bin/escrow_manager/src/contracts.rs index 0ffe9ac..d05e363 100644 --- a/crates/bin/escrow_manager/src/contracts.rs +++ b/crates/bin/escrow_manager/src/contracts.rs @@ -77,6 +77,13 @@ impl Contracts { self.payer } + /// Collector every escrow call in this module targets. Escrow accounts are scoped by + /// `(payer, collector, receiver)`, so anything reading account state has to filter on this + /// exact address or it will plan against a different collector's balances. + pub fn collector(&self) -> Address { + *self.graph_tally_collector.address() + } + pub async fn allowance(&self) -> anyhow::Result { self.token .allowance(self.payer(), *self.payments_escrow.address()) diff --git a/crates/bin/escrow_manager/src/main.rs b/crates/bin/escrow_manager/src/main.rs index 70be0f7..c8bd73b 100644 --- a/crates/bin/escrow_manager/src/main.rs +++ b/crates/bin/escrow_manager/src/main.rs @@ -194,7 +194,12 @@ async fn main() -> anyhow::Result<()> { } }; let mut receivers: BTreeSet
= allocations.iter().map(|a| a.indexer).collect(); - let escrow_accounts = match escrow_accounts(&mut network_subgraph, &contracts.payer()).await + let escrow_accounts = match escrow_accounts( + &mut network_subgraph, + &contracts.payer(), + &contracts.collector(), + ) + .await { Ok(escrow_accounts) => escrow_accounts, Err(escrow_accounts_err) => { diff --git a/crates/bin/escrow_manager/src/subgraphs.rs b/crates/bin/escrow_manager/src/subgraphs.rs index 32ec60c..3a22cfd 100644 --- a/crates/bin/escrow_manager/src/subgraphs.rs +++ b/crates/bin/escrow_manager/src/subgraphs.rs @@ -47,9 +47,16 @@ pub struct EscrowAccount { pub thaw_end_timestamp: u64, } +/// Escrow accounts held by `payer` under `collector`, keyed by receiver. +/// +/// Accounts are scoped by `(payer, collector, receiver)` on both the contract and the subgraph, so +/// the collector filter is load-bearing rather than an optimisation: without it the payer's +/// accounts under every other collector come back too, and collapsing to a receiver-keyed map +/// would let one of them silently displace the account the transactions actually target. pub async fn escrow_accounts( network_subgraph: &mut SubgraphClient, payer: &Address, + collector: &Address, ) -> anyhow::Result> { let query = format!( r#" @@ -61,6 +68,7 @@ pub async fn escrow_accounts( where: {{ id_gt: $last payer: "{payer:?}" + collector: "{collector:?}" }} ) {{ id From 3bfe0471d32e63c273bc48f8ebba7768ffd5c99e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Migone?= Date: Tue, 22 Sep 2026 10:13:20 -0300 Subject: [PATCH 11/12] chore: comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás Migone --- crates/bin/escrow_manager/src/subgraphs.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crates/bin/escrow_manager/src/subgraphs.rs b/crates/bin/escrow_manager/src/subgraphs.rs index 3a22cfd..1747e4d 100644 --- a/crates/bin/escrow_manager/src/subgraphs.rs +++ b/crates/bin/escrow_manager/src/subgraphs.rs @@ -48,11 +48,6 @@ pub struct EscrowAccount { } /// Escrow accounts held by `payer` under `collector`, keyed by receiver. -/// -/// Accounts are scoped by `(payer, collector, receiver)` on both the contract and the subgraph, so -/// the collector filter is load-bearing rather than an optimisation: without it the payer's -/// accounts under every other collector come back too, and collapsing to a receiver-keyed map -/// would let one of them silently displace the account the transactions actually target. pub async fn escrow_accounts( network_subgraph: &mut SubgraphClient, payer: &Address, From 83cfae1604b92b51a63f5b554b80730435ddf722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Migone?= Date: Tue, 22 Sep 2026 12:46:47 -0300 Subject: [PATCH 12/12] fix: ensure debt snapshot can be trusted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás Migone --- crates/bin/escrow_manager/src/main.rs | 43 +++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/crates/bin/escrow_manager/src/main.rs b/crates/bin/escrow_manager/src/main.rs index c8bd73b..8e2ca1a 100644 --- a/crates/bin/escrow_manager/src/main.rs +++ b/crates/bin/escrow_manager/src/main.rs @@ -268,7 +268,15 @@ async fn main() -> anyhow::Result<()> { .total_debt_grt .set(debts.values().sum::() as f64 / GRT as f64); - let chain_now = match config.withdraw_enabled { + // Ensure we can trust the debt snapshot + let reclaim_enabled = config.withdraw_enabled && debt_ready(&debts); + if config.withdraw_enabled && !reclaim_enabled { + tracing::warn!( + "no debt recorded for any receiver, skipping reclamation this cycle for safety" + ); + } + + let chain_now = match reclaim_enabled { false => None, true => match contracts.latest_block_timestamp().await { Ok(timestamp) => Some(timestamp), @@ -279,7 +287,7 @@ async fn main() -> anyhow::Result<()> { }, }; - let reclaim = config.withdraw_enabled.then_some(Reclaim { + let reclaim = reclaim_enabled.then_some(Reclaim { chain_now, margin_bps: withdraw_margin_bps, min_withdraw, @@ -522,6 +530,15 @@ enum Action { Thaw(u128), } +/// Whether this cycle's debt picture can be trusted for reclamation decisions. +/// +/// Debt comes from Kafka consumers that publish an eventually consistent snapshot over a watch +/// channel, with no readiness signal: an empty table means "nothing loaded yet" and "nothing is +/// owed" alike, and the reader cannot tell them apart. +fn debt_ready(debts: &BTreeMap) -> bool { + debts.values().any(|debt| *debt > 0) +} + /// Decide the single action to take for a receiver this cycle. /// /// Exactly one action applies, which is what keeps the three transaction batches independent: no @@ -605,6 +622,10 @@ async fn handle_metrics() -> impl axum::response::IntoResponse { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + + use alloy::primitives::Address; + use super::{Action, EscrowAccount, Reclaim, GRT, MIN_DEPOSIT}; const MARGIN_BPS: u128 = 2_500; @@ -706,6 +727,24 @@ mod tests { assert_eq!(decide(13_000 * GRT, 0, 0, target), Action::Thaw(500 * GRT)); } + #[test] + fn debt_ready_needs_one_receiver_with_debt() { + let debts = |values: &[u128]| -> BTreeMap { + values + .iter() + .enumerate() + .map(|(i, debt)| (Address::repeat_byte(i as u8), *debt)) + .collect() + }; + // Cold start: the consumers have published nothing, or nothing but zeroes. + assert!(!super::debt_ready(&debts(&[]))); + assert!(!super::debt_ready(&debts(&[0, 0, 0]))); + // A single receiver with debt is enough to show the consumers have caught up. It does not + // prove the picture is complete, only that it is no longer empty. + assert!(super::debt_ready(&debts(&[0, 0, 1]))); + assert!(super::debt_ready(&debts(&[5_000 * GRT]))); + } + #[test] fn reclamation_disabled_only_ever_deposits() { let idle = account(20_000 * GRT, 0, 0);