diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b8d1c8d83..9a9f73c6e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * [#4032](https://github.com/sei-protocol/sei-chain/pull/4032) fix(config): the default `telemetry.prometheus-retention-time` drops from `7200` to `0`, so neither app.toml-generation pipeline (`seid init`, or the file a node writes for itself on any other subcommand) starts the Prometheus metrics sink unless an operator sets a positive retention. Freshly generated nodes keep the bounded in-memory telemetry sink used by SIGUSR1 dumps. Existing `app.toml` files are unchanged. * [#4021](https://github.com/sei-protocol/sei-chain/pull/4021) feat(grpc): per-IP rate-limit admission for the gRPC plane, off by default behind `[grpc] rate-limiting-enabled` (new `ip-rate-limit-rps` / `ip-rate-limit-burst` / `trusted-proxy-cidrs`, defaults 10 rps / 20 burst / trust no proxy). Native gRPC (:9090) is admitted by a tap handler and gRPC-Web (:9091) by HTTP middleware, both before the request is protobuf-decoded, so a throttled caller cannot spend the decoder; streams pay one token to establish and one per inbound message. Both planes draw from the same per-IP buckets. Over-budget callers get `ResourceExhausted` on :9090 and HTTP 429 on :9091, counted by `rpc_rate_limit_rejected_total{plane="grpc", method_namespace}`. * [#4078](https://github.com/sei-protocol/sei-chain/pull/4078) feat(grpc): bound concurrent in-flight RPCs and open connections per IP on the gRPC query plane. New `[grpc] max-connections-per-ip` and `[grpc-web] max-connections-per-ip` (default 0, unlimited) optionally cap one address's share of the global connection budget on :9090 and :9091, regardless of `rate-limiting-enabled`. New `[grpc] max-in-flight-per-ip` (default 100) caps concurrent RPCs per address when `rate-limiting-enabled = true`: the slot is taken at the HTTP/2 HEADERS frame and returned when the RPC ends. Both planes draw from the same per-IP pool. Concurrency rejections return `ResourceExhausted` on :9090 and HTTP 429 on :9091, counted by `rpc_inflight_rejected_total{plane, method_namespace}`; refused connections are counted by `rpc_connection_rejected_total{plane}`. +* [#4145](https://github.com/sei-protocol/sei-chain/pull/4145) fix(flatkv): FlatKV keeps 72 old PebbleDB checkpoints instead of the 1 it inherited from memIAVL's `state-commit.sc-keep-recent`. Its snapshot *interval* is still taken from `sc-snapshot-interval`; only the retention count is now independent. This raises the guaranteed reach of `migrate-evm-status`, `dump-flatkv`, a cross-backend digest and a FlatKV rollback from 10,000 blocks (about 74 minutes) to 720,000 blocks (about 89 hours), and costs about 20 GiB of extra disk on a mainnet-sized node, because checkpoints hardlink their SSTs and so only pin what compaction has since obsoleted. No `app.toml` change is needed, and an explicit `state-commit.flatkv.snapshot-keep-recent` left over from an old template still has no effect. * [#4117](https://github.com/sei-protocol/sei-chain/pull/4117) chore(giga): remove the unused evmone/evmc execution path from the Giga executor. The Giga executor's production path already ran on go-ethereum's native interpreter; evmone was only reachable through a best-effort VM init that nothing consumed. Release images no longer ship `libevmone.*.so`/`.dylib` under `/usr/lib`, and the `SEI_EVMONE_LIB_DIR` operator override is removed. ### Upgrade guide diff --git a/sei-cosmos/server/config/config.go b/sei-cosmos/server/config/config.go index 70db7578e3..5f1fb382a6 100644 --- a/sei-cosmos/server/config/config.go +++ b/sei-cosmos/server/config/config.go @@ -557,9 +557,10 @@ func GetConfig(v *viper.Viper) (Config, error) { // FlatKV knobs are not rendered in the default app.toml template. GetConfig // is a faithful parse of app.toml/flags: it only reads the explicit // state-commit.flatkv.* keys (if an operator adds them by hand) on top of the - // in-code defaults. The FlatKV-follows-memIAVL mirror (and snapshot cadence - // normalization) is applied later by composite.alignFlatKVSnapshotWithMemIAVL - // at store construction, so we deliberately do not mirror the sc-* keys here. + // in-code defaults. The FlatKV-follows-memIAVL interval mirror (and its + // cadence normalization) is applied later by + // composite.alignFlatKVSnapshotIntervalWithMemIAVL at store construction, so + // we deliberately do not mirror the sc-* keys here. flatKVConfig := config.DefaultStateCommitConfig().FlatKVConfig if v.IsSet("state-commit.flatkv.fsync") { flatKVConfig.Fsync = v.GetBool("state-commit.flatkv.fsync") diff --git a/sei-cosmos/server/config/config_test.go b/sei-cosmos/server/config/config_test.go index d74196247b..df6c13bd3c 100644 --- a/sei-cosmos/server/config/config_test.go +++ b/sei-cosmos/server/config/config_test.go @@ -566,8 +566,9 @@ func TestGetConfigParsesRawSnapshotKeepRecent(t *testing.T) { require.NoError(t, err) // GetConfig is a faithful parse of app.toml/flags: the raw 0 is preserved for // memIAVL here and only floored later at store construction. FlatKV does not - // mirror the sc-* keys in GetConfig (that is composite.alignFlatKVSnapshotWithMemIAVL's - // job), so it keeps its in-code default. + // mirror the sc-* keys in GetConfig (the interval mirror is + // composite.alignFlatKVSnapshotIntervalWithMemIAVL's job, and the retention + // count is never mirrored), so it keeps its in-code default. require.Equal(t, uint32(0), cfg.StateCommit.MemIAVLConfig.SnapshotKeepRecent) require.Equal(t, seidbconfig.DefaultStateCommitConfig().FlatKVConfig.SnapshotKeepRecent, cfg.StateCommit.FlatKVConfig.SnapshotKeepRecent) } @@ -594,9 +595,10 @@ func TestGetConfigHonorsExplicitFlatKVOverrides(t *testing.T) { } // TestGetConfigFlatKVDefaultsWhenSCSnapshotAbsent locks in the regression fix: -// GetConfig does not mirror the sc-* keys onto FlatKV (that is -// composite.alignFlatKVSnapshotWithMemIAVL's job at store construction), and an -// absent sc-snapshot-interval / sc-keep-recent must preserve the in-code FlatKV +// GetConfig does not mirror the sc-* keys onto FlatKV (the interval mirror is +// composite.alignFlatKVSnapshotIntervalWithMemIAVL's job at store construction, +// and the retention count is never mirrored), and an absent +// sc-snapshot-interval / sc-keep-recent must preserve the in-code FlatKV // defaults rather than reading back 0 (which disables FlatKV snapshots and drops // all old snapshots). func TestGetConfigFlatKVDefaultsWhenSCSnapshotAbsent(t *testing.T) { diff --git a/sei-db/config/ss_config_test.go b/sei-db/config/ss_config_test.go index 543f50ba0c..81251ab10a 100644 --- a/sei-db/config/ss_config_test.go +++ b/sei-db/config/ss_config_test.go @@ -66,8 +66,9 @@ func TestAlignSSSnapshotWithSCZeroesCadenceWhenDisabled(t *testing.T) { require.Zero(t, ssConfig.SnapshotMinTimeInterval) } -// FlatKV and SS both mirror memIAVL's cadence, and they must resolve it -// identically or the two backends drift onto different snapshot heights. +// SS mirrors memIAVL's whole cadence and FlatKV mirrors its interval, and every +// mirror must resolve that interval identically or the backends drift onto +// different snapshot heights. func TestAlignSSSnapshotMatchesEffectiveMemIAVLCadence(t *testing.T) { for _, tc := range []struct { name string diff --git a/sei-db/state_db/sc/composite/store.go b/sei-db/state_db/sc/composite/store.go index fc94ec0050..c5cf1e5193 100644 --- a/sei-db/state_db/sc/composite/store.go +++ b/sei-db/state_db/sc/composite/store.go @@ -163,7 +163,7 @@ func NewCompositeCommitStore( if err := cfg.Validate(); err != nil { return nil, fmt.Errorf("invalid state commit config: %w", err) } - alignFlatKVSnapshotWithMemIAVL(&cfg) + alignFlatKVSnapshotIntervalWithMemIAVL(&cfg) var memIAVL *memiavl.CommitStore if cfg.WriteMode != types.FlatKVOnly { @@ -233,38 +233,38 @@ func (cs *CompositeCommitStore) recordFlatKVHash(_ context.Context, _ int64, has return nil } -// alignFlatKVSnapshotWithMemIAVL keeps the two backends' snapshot cadence in -// sync. FlatKV has no independently-exposed snapshot knobs in app.toml, so it -// derives its snapshot-interval / keep-recent from memIAVL's sc-* keys. This is -// the single place both backends are constructed from the same config, so it is -// where the alignment is enforced. +// alignFlatKVSnapshotIntervalWithMemIAVL makes FlatKV take its snapshot interval +// from memIAVL's sc-snapshot-interval. This is the single place both backends are +// constructed from the same config, so it is where the alignment is enforced. // -// This derivation is intentionally unconditional across write modes, including -// FlatKVOnly — where NewCompositeCommitStore never constructs a memIAVL store. -// The sc-* keys are the only operator-visible snapshot-cadence knobs now that -// the flatkv.* keys are hidden from the app.toml template, so they must govern -// FlatKV's cadence in every mode; otherwise FlatKVOnly would have no -// template-visible way to tune it. It is harmless when memIAVL is absent: the -// sc-* defaults match FlatKV's own in-code defaults, and only cfg.FlatKVConfig -// is read when building the FlatKVOnly store. +// The interval must match because a composite operation needs a version *both* +// backends hold a snapshot for: a rollback rewinds memIAVL and then FlatKV, and a +// cross-backend digest has to open each at the same height. Two backends +// checkpointing on different heights have no such version in common. // -// FlatKV mirrors memIAVL's *effective* cadence: a zero memIAVL value is first -// resolved to the same default Options.FillDefaults would apply at OpenDB -// (interval 0 -> DefaultSnapshotInterval, keep-recent 0 -> DefaultSnapshotKeepRecent), -// then assigned to FlatKV unconditionally. Resolving-then-assigning (rather than -// skipping on a zero and letting FlatKV keep its own in-code default) keeps the -// two backends in true lockstep without relying on FlatKV's default happening to -// equal memIAVL's healed default. That reliance is fragile — the defaults are -// only kept equal by hand — and it breaks for an upgrading node whose old -// app.toml still carries an explicit state-commit.flatkv.snapshot-keep-recent -// (rendered by the old template) alongside sc-keep-recent = 0: skipping would -// leave FlatKV pinned to the stale explicit value while memIAVL healed to a -// different default. Note that mirroring a raw 0 is never correct here (0 means -// "disable auto-snapshots" for FlatKV), which is why the zero is resolved first. -func alignFlatKVSnapshotWithMemIAVL(cfg *config.StateCommitConfig) { - interval, keepRecent := config.EffectiveMemIAVLSnapshotCadence(cfg.MemIAVLConfig) +// Retention count is deliberately not mirrored. It is a per-backend disk decision +// rather than a cadence, and the two backends' costs differ by a factor of +// roughly 200: measured at mainnet state size, one further retained snapshot +// costs 56,782 MiB on memIAVL, whose snapshots are independent full copies, and +// about 286 MiB on FlatKV, whose checkpoints hardlink their SSTs. A single shared +// count cannot serve both — the depth FlatKV wants for forensic reach into the +// migration window would ask memIAVL for more than the volume holds. FlatKV +// therefore keeps config.DefaultSnapshotKeepRecent, which is sized for that reach. +// +// The mirror is unconditional across write modes, including FlatKVOnly, where +// NewCompositeCommitStore never constructs a memIAVL store. sc-snapshot-interval +// is the only operator-visible cadence knob, since the flatkv.* keys are hidden +// from the app.toml template and the production reader does not consult them, so +// it has to govern FlatKV's interval in every mode. It is harmless when memIAVL is +// absent, because only cfg.FlatKVConfig is read when building that store. +// +// A zero is resolved before it is assigned, to the same default +// Options.FillDefaults would apply at OpenDB. Mirroring a raw 0 is never correct: +// 0 disables auto-snapshots for FlatKV, which lets the WAL grow without bound and +// makes every restart replay from snapshot-0. +func alignFlatKVSnapshotIntervalWithMemIAVL(cfg *config.StateCommitConfig) { + interval, _ := config.EffectiveMemIAVLSnapshotCadence(cfg.MemIAVLConfig) cfg.FlatKVConfig.SnapshotInterval = interval - cfg.FlatKVConfig.SnapshotKeepRecent = keepRecent } // Initialize records the set of child store names that should exist on diff --git a/sei-db/state_db/sc/composite/store_test.go b/sei-db/state_db/sc/composite/store_test.go index c8aefc2578..e2a5a9dd70 100644 --- a/sei-db/state_db/sc/composite/store_test.go +++ b/sei-db/state_db/sc/composite/store_test.go @@ -17,6 +17,7 @@ import ( gigatypes "github.com/sei-protocol/sei-chain/sei-db/state_db/giga/types" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" + flatkvconfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" @@ -1058,12 +1059,13 @@ func evmMigratedConfig() config.StateCommitConfig { cfg.MemIAVLConfig.SnapshotInterval = 1 cfg.MemIAVLConfig.SnapshotMinTimeInterval = 0 cfg.MemIAVLConfig.AsyncCommitBuffer = 0 - // With SnapshotInterval=1 every commit produces a snapshot, and FlatKV - // mirrors this cadence via alignFlatKVSnapshotWithMemIAVL. The default - // keep-recent of 1 would prune all but the two newest snapshots, so a + // With SnapshotInterval=1 every commit produces a snapshot, and memIAVL's + // default keep-recent of 1 would prune all but the two newest, so a // rollback/reconcile to an older version (e.g. v3 after committing v5) // could no longer find a base snapshot at-or-below the target. Retain all // snapshots for the short duration of a test so those paths stay valid. + // FlatKV needs no equivalent here: it mirrors the interval but keeps its + // own retention count, which is deep enough already. cfg.MemIAVLConfig.SnapshotKeepRecent = 100 return cfg } @@ -2595,31 +2597,16 @@ func TestLoadVersionReadOnlyDuringMigrateEVMTransition(t *testing.T) { require.Equal(t, []byte(evmVal), got) } -func TestAlignFlatKVSnapshotWithMemIAVL(t *testing.T) { - t.Run("FlatKV derives interval and keep-recent from a non-zero memIAVL", func(t *testing.T) { +func TestAlignFlatKVSnapshotIntervalWithMemIAVL(t *testing.T) { + t.Run("FlatKV derives its interval from a non-zero memIAVL", func(t *testing.T) { cfg := config.DefaultStateCommitConfig() cfg.MemIAVLConfig.SnapshotInterval = 5000 - cfg.MemIAVLConfig.SnapshotKeepRecent = 3 - // Start FlatKV from divergent values to prove they get overwritten. + // Start FlatKV from a divergent value to prove it gets overwritten. cfg.FlatKVConfig.SnapshotInterval = 111 - cfg.FlatKVConfig.SnapshotKeepRecent = 222 - alignFlatKVSnapshotWithMemIAVL(&cfg) + alignFlatKVSnapshotIntervalWithMemIAVL(&cfg) require.Equal(t, uint32(5000), cfg.FlatKVConfig.SnapshotInterval) - require.Equal(t, uint32(3), cfg.FlatKVConfig.SnapshotKeepRecent) - }) - - t.Run("a zero memIAVL keep-recent resolves to the healed default", func(t *testing.T) { - cfg := config.DefaultStateCommitConfig() - cfg.MemIAVLConfig.SnapshotKeepRecent = 0 - // FlatKV must not mirror the raw 0 (which would prune everything but the - // latest). Instead it mirrors the value FillDefaults will heal memIAVL to, - // keeping the two in lockstep. memIAVL's own 0 is left for FillDefaults. - alignFlatKVSnapshotWithMemIAVL(&cfg) - - require.Equal(t, uint32(0), cfg.MemIAVLConfig.SnapshotKeepRecent) - require.Equal(t, uint32(memiavl.DefaultSnapshotKeepRecent), cfg.FlatKVConfig.SnapshotKeepRecent) }) t.Run("a zero memIAVL interval resolves to the healed default", func(t *testing.T) { @@ -2627,28 +2614,51 @@ func TestAlignFlatKVSnapshotWithMemIAVL(t *testing.T) { cfg.MemIAVLConfig.SnapshotInterval = 0 // A raw 0 would disable FlatKV auto-snapshots; instead FlatKV mirrors the // value FillDefaults will heal memIAVL's interval to. - alignFlatKVSnapshotWithMemIAVL(&cfg) + alignFlatKVSnapshotIntervalWithMemIAVL(&cfg) require.Equal(t, uint32(memiavl.DefaultSnapshotInterval), cfg.FlatKVConfig.SnapshotInterval) require.NotZero(t, cfg.FlatKVConfig.SnapshotInterval) }) - t.Run("an explicit FlatKV override loses to memIAVL's healed default", func(t *testing.T) { - // Upgrade scenario: an old app.toml still pins an explicit FlatKV - // keep-recent/interval (the previous template rendered flatkv.* keys) - // while sc-* is 0. FlatKV must follow memIAVL's effective (healed) cadence - // rather than staying pinned to the stale explicit value, otherwise the - // two backends diverge (memIAVL heals 0 -> default, FlatKV keeps the old - // explicit value). + t.Run("retention count is not mirrored", func(t *testing.T) { + // The two backends share a cadence and not a disk budget. A retained + // memIAVL snapshot is an independent full copy where a FlatKV checkpoint + // hardlinks its SSTs, so one shared count cannot serve both. + cfg := config.DefaultStateCommitConfig() + cfg.MemIAVLConfig.SnapshotKeepRecent = 3 + cfg.FlatKVConfig.SnapshotKeepRecent = 222 + + alignFlatKVSnapshotIntervalWithMemIAVL(&cfg) + + require.Equal(t, uint32(222), cfg.FlatKVConfig.SnapshotKeepRecent) + require.Equal(t, uint32(3), cfg.MemIAVLConfig.SnapshotKeepRecent) + }) + + t.Run("a zero memIAVL keep-recent does not reach FlatKV", func(t *testing.T) { + // A zero memIAVL keep-recent is healed to memiavl.DefaultSnapshotKeepRecent + // for memIAVL's own use. That healed value must not reach FlatKV, whose + // own default stands. cfg := config.DefaultStateCommitConfig() cfg.MemIAVLConfig.SnapshotKeepRecent = 0 - cfg.MemIAVLConfig.SnapshotInterval = 0 - cfg.FlatKVConfig.SnapshotKeepRecent = 2 - cfg.FlatKVConfig.SnapshotInterval = 7777 - alignFlatKVSnapshotWithMemIAVL(&cfg) + alignFlatKVSnapshotIntervalWithMemIAVL(&cfg) - require.Equal(t, uint32(memiavl.DefaultSnapshotKeepRecent), cfg.FlatKVConfig.SnapshotKeepRecent) - require.Equal(t, uint32(memiavl.DefaultSnapshotInterval), cfg.FlatKVConfig.SnapshotInterval) + require.Equal(t, uint32(0), cfg.MemIAVLConfig.SnapshotKeepRecent) + require.Equal(t, flatkvconfig.DefaultSnapshotKeepRecent, cfg.FlatKVConfig.SnapshotKeepRecent) + require.NotEqual(t, uint32(memiavl.DefaultSnapshotKeepRecent), cfg.FlatKVConfig.SnapshotKeepRecent, + "FlatKV must not inherit memIAVL's retention count") }) } + +// The default exists to bound what can be asked about a past height, so a reach +// shorter than the migration window it was sized for is the regression to catch. +// 72 checkpoints at a 10000-block interval is 720,000 blocks, about 89 hours at +// mainnet's measured 2.247 blocks/s, against an 85-hour drain at K=1024. +func TestFlatKVDefaultRetentionSpansTheMigrationWindow(t *testing.T) { + cfg := config.DefaultStateCommitConfig() + alignFlatKVSnapshotIntervalWithMemIAVL(&cfg) + + reach := uint64(cfg.FlatKVConfig.SnapshotKeepRecent) * uint64(cfg.FlatKVConfig.SnapshotInterval) + require.GreaterOrEqual(t, reach, uint64(690_000), + "default FlatKV retention must reach back across an 85-hour drain at 2.247 blocks/s") +} diff --git a/sei-db/state_db/sc/flatkv/config/config.go b/sei-db/state_db/sc/flatkv/config/config.go index 67e4e19c4b..3171ff8068 100644 --- a/sei-db/state_db/sc/flatkv/config/config.go +++ b/sei-db/state_db/sc/flatkv/config/config.go @@ -10,6 +10,23 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash" ) +// DefaultSnapshotKeepRecent is how many old checkpoints (besides the latest) to +// keep, which at the default snapshot interval of 10000 is a guaranteed reach of +// 720,000 blocks — about 89 hours at mainnet's block rate, so it spans the EVM +// migration window at the rate that window is planned for. +// +// It is this deep because a FlatKV checkpoint is nearly free. Checkpoints +// hardlink their SSTs, so one only costs the bytes compaction has since made +// obsolete: measured at mainnet state size, 261 MiB of pinned SSTs plus about +// 25 MiB of retained state WAL. 72 of them is roughly 20 GiB. The cost is linear +// in depth, because each older checkpoint pins exactly the files obsoleted during +// its own interval and those sets are disjoint. +// +// Reach matters because it bounds what can be answered about a past height at +// all. Below it, migrate-evm-status, dump-flatkv and a cross-backend digest +// cannot open a version, and a rollback has no base snapshot to rewind to. +const DefaultSnapshotKeepRecent uint32 = 72 + // Config defines configuration for the FlatKV (EVM) commit store. type Config struct { // DataDir is the root directory for the FlatKV data files. @@ -34,6 +51,11 @@ type Config struct { // SnapshotKeepRecent defines how many old snapshots to keep besides the // latest one. 0 means keep only the current snapshot (no old snapshots). // Ignored entirely when ExternalPruning is set. + // + // It is not mirrored from memIAVL's sc-keep-recent, and the production store + // reads no app.toml key for it, so a node runs the DefaultConfig value. See + // composite.alignFlatKVSnapshotIntervalWithMemIAVL for why the two backends + // share an interval but not a retention count. SnapshotKeepRecent uint32 `mapstructure:"snapshot-keep-recent"` // MaxSnapshotLagBlocks is how many committed blocks may queue up behind a snapshot that is still @@ -136,7 +158,7 @@ func DefaultConfig() *Config { Fsync: false, AsyncWriteBuffer: 0, SnapshotInterval: 10000, - SnapshotKeepRecent: 1, + SnapshotKeepRecent: DefaultSnapshotKeepRecent, MaxSnapshotLagBlocks: 64, EnablePebbleMetrics: true, AccountDBConfig: pebbledb.DefaultConfig(),