fix(flatkv): retain 72 checkpoints instead of mirroring memIAVL's count

CHANGELOG.md

@@ -35,6 +35,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * [#4032](https://github.com/sei-protocol/sei-chain/pull/4032) fix(config): the default `telemetry.prometheus-retention-time` drops from `7200` to `0`, so neither app.toml-generation pipeline (`seid init`, or the file a node writes for itself on any other subcommand) starts the Prometheus metrics sink unless an operator sets a positive retention. Freshly generated nodes keep the bounded in-memory telemetry sink used by SIGUSR1 dumps. Existing `app.toml` files are unchanged.
 * [#4021](https://github.com/sei-protocol/sei-chain/pull/4021) feat(grpc): per-IP rate-limit admission for the gRPC plane, off by default behind `[grpc] rate-limiting-enabled` (new `ip-rate-limit-rps` / `ip-rate-limit-burst` / `trusted-proxy-cidrs`, defaults 10 rps / 20 burst / trust no proxy). Native gRPC (:9090) is admitted by a tap handler and gRPC-Web (:9091) by HTTP middleware, both before the request is protobuf-decoded, so a throttled caller cannot spend the decoder; streams pay one token to establish and one per inbound message. Both planes draw from the same per-IP buckets. Over-budget callers get `ResourceExhausted` on :9090 and HTTP 429 on :9091, counted by `rpc_rate_limit_rejected_total{plane="grpc", method_namespace}`.
 * [#4078](https://github.com/sei-protocol/sei-chain/pull/4078) feat(grpc): bound concurrent in-flight RPCs and open connections per IP on the gRPC query plane. New `[grpc] max-connections-per-ip` and `[grpc-web] max-connections-per-ip` (default 0, unlimited) optionally cap one address's share of the global connection budget on :9090 and :9091, regardless of `rate-limiting-enabled`. New `[grpc] max-in-flight-per-ip` (default 100) caps concurrent RPCs per address when `rate-limiting-enabled = true`: the slot is taken at the HTTP/2 HEADERS frame and returned when the RPC ends. Both planes draw from the same per-IP pool. Concurrency rejections return `ResourceExhausted` on :9090 and HTTP 429 on :9091, counted by `rpc_inflight_rejected_total{plane, method_namespace}`; refused connections are counted by `rpc_connection_rejected_total{plane}`.
+* [#4145](https://github.com/sei-protocol/sei-chain/pull/4145) fix(flatkv): FlatKV keeps 72 old PebbleDB checkpoints instead of the 1 it inherited from memIAVL's `state-commit.sc-keep-recent`. Its snapshot *interval* is still taken from `sc-snapshot-interval`; only the retention count is now independent. This raises the guaranteed reach of `migrate-evm-status`, `dump-flatkv`, a cross-backend digest and a FlatKV rollback from 10,000 blocks (about 74 minutes) to 720,000 blocks (about 89 hours), and costs about 20 GiB of extra disk on a mainnet-sized node, because checkpoints hardlink their SSTs and so only pin what compaction has since obsoleted. No `app.toml` change is needed, and an explicit `state-commit.flatkv.snapshot-keep-recent` left over from an old template still has no effect.
 * [#4117](https://github.com/sei-protocol/sei-chain/pull/4117) chore(giga): remove the unused evmone/evmc execution path from the Giga executor. The Giga executor's production path already ran on go-ethereum's native interpreter; evmone was only reachable through a best-effort VM init that nothing consumed. Release images no longer ship `libevmone.*.so`/`.dylib` under `/usr/lib`, and the `SEI_EVMONE_LIB_DIR` operator override is removed.
 
 ### Upgrade guide

sei-cosmos/server/config/config.go

@@ -557,9 +557,10 @@ func GetConfig(v *viper.Viper) (Config, error) {
 	// FlatKV knobs are not rendered in the default app.toml template. GetConfig
 	// is a faithful parse of app.toml/flags: it only reads the explicit
 	// state-commit.flatkv.* keys (if an operator adds them by hand) on top of the
-	// in-code defaults. The FlatKV-follows-memIAVL mirror (and snapshot cadence
-	// normalization) is applied later by composite.alignFlatKVSnapshotWithMemIAVL
-	// at store construction, so we deliberately do not mirror the sc-* keys here.
+	// in-code defaults. The FlatKV-follows-memIAVL interval mirror (and its
+	// cadence normalization) is applied later by
+	// composite.alignFlatKVSnapshotIntervalWithMemIAVL at store construction, so
+	// we deliberately do not mirror the sc-* keys here.
 	flatKVConfig := config.DefaultStateCommitConfig().FlatKVConfig
 	if v.IsSet("state-commit.flatkv.fsync") {
 		flatKVConfig.Fsync = v.GetBool("state-commit.flatkv.fsync")

sei-cosmos/server/config/config_test.go

@@ -566,8 +566,9 @@ func TestGetConfigParsesRawSnapshotKeepRecent(t *testing.T) {
 	require.NoError(t, err)
 	// GetConfig is a faithful parse of app.toml/flags: the raw 0 is preserved for
 	// memIAVL here and only floored later at store construction. FlatKV does not
-	// mirror the sc-* keys in GetConfig (that is composite.alignFlatKVSnapshotWithMemIAVL's
-	// job), so it keeps its in-code default.
+	// mirror the sc-* keys in GetConfig (the interval mirror is
+	// composite.alignFlatKVSnapshotIntervalWithMemIAVL's job, and the retention
+	// count is never mirrored), so it keeps its in-code default.
 	require.Equal(t, uint32(0), cfg.StateCommit.MemIAVLConfig.SnapshotKeepRecent)
 	require.Equal(t, seidbconfig.DefaultStateCommitConfig().FlatKVConfig.SnapshotKeepRecent, cfg.StateCommit.FlatKVConfig.SnapshotKeepRecent)
 }
@@ -594,9 +595,10 @@ func TestGetConfigHonorsExplicitFlatKVOverrides(t *testing.T) {
 }
 
 // TestGetConfigFlatKVDefaultsWhenSCSnapshotAbsent locks in the regression fix:
-// GetConfig does not mirror the sc-* keys onto FlatKV (that is
-// composite.alignFlatKVSnapshotWithMemIAVL's job at store construction), and an
-// absent sc-snapshot-interval / sc-keep-recent must preserve the in-code FlatKV
+// GetConfig does not mirror the sc-* keys onto FlatKV (the interval mirror is
+// composite.alignFlatKVSnapshotIntervalWithMemIAVL's job at store construction,
+// and the retention count is never mirrored), and an absent
+// sc-snapshot-interval / sc-keep-recent must preserve the in-code FlatKV
 // defaults rather than reading back 0 (which disables FlatKV snapshots and drops
 // all old snapshots).
 func TestGetConfigFlatKVDefaultsWhenSCSnapshotAbsent(t *testing.T) {

sei-db/config/ss_config_test.go

@@ -66,8 +66,9 @@ func TestAlignSSSnapshotWithSCZeroesCadenceWhenDisabled(t *testing.T) {
 	require.Zero(t, ssConfig.SnapshotMinTimeInterval)
 }
 
-// FlatKV and SS both mirror memIAVL's cadence, and they must resolve it
-// identically or the two backends drift onto different snapshot heights.
+// SS mirrors memIAVL's whole cadence and FlatKV mirrors its interval, and every
+// mirror must resolve that interval identically or the backends drift onto
+// different snapshot heights.
 func TestAlignSSSnapshotMatchesEffectiveMemIAVLCadence(t *testing.T) {
 	for _, tc := range []struct {
 		name                 string

sei-db/state_db/sc/composite/store.go

@@ -163,7 +163,7 @@ func NewCompositeCommitStore(
 	if err := cfg.Validate(); err != nil {
 		return nil, fmt.Errorf("invalid state commit config: %w", err)
 	}
-	alignFlatKVSnapshotWithMemIAVL(&cfg)
+	alignFlatKVSnapshotIntervalWithMemIAVL(&cfg)
 
 	var memIAVL *memiavl.CommitStore
 	if cfg.WriteMode != types.FlatKVOnly {
@@ -233,38 +233,38 @@ func (cs *CompositeCommitStore) recordFlatKVHash(_ context.Context, _ int64, has
 	return nil
 }
 
-// alignFlatKVSnapshotWithMemIAVL keeps the two backends' snapshot cadence in
-// sync. FlatKV has no independently-exposed snapshot knobs in app.toml, so it
-// derives its snapshot-interval / keep-recent from memIAVL's sc-* keys. This is
-// the single place both backends are constructed from the same config, so it is
-// where the alignment is enforced.
+// alignFlatKVSnapshotIntervalWithMemIAVL makes FlatKV take its snapshot interval
+// from memIAVL's sc-snapshot-interval. This is the single place both backends are
+// constructed from the same config, so it is where the alignment is enforced.
 //
-// This derivation is intentionally unconditional across write modes, including
-// FlatKVOnly — where NewCompositeCommitStore never constructs a memIAVL store.
-// The sc-* keys are the only operator-visible snapshot-cadence knobs now that
-// the flatkv.* keys are hidden from the app.toml template, so they must govern
-// FlatKV's cadence in every mode; otherwise FlatKVOnly would have no
-// template-visible way to tune it. It is harmless when memIAVL is absent: the
-// sc-* defaults match FlatKV's own in-code defaults, and only cfg.FlatKVConfig
-// is read when building the FlatKVOnly store.
+// The interval must match because a composite operation needs a version *both*
+// backends hold a snapshot for: a rollback rewinds memIAVL and then FlatKV, and a
+// cross-backend digest has to open each at the same height. Two backends
+// checkpointing on different heights have no such version in common.
 //
-// FlatKV mirrors memIAVL's *effective* cadence: a zero memIAVL value is first
-// resolved to the same default Options.FillDefaults would apply at OpenDB
-// (interval 0 -> DefaultSnapshotInterval, keep-recent 0 -> DefaultSnapshotKeepRecent),
-// then assigned to FlatKV unconditionally. Resolving-then-assigning (rather than
-// skipping on a zero and letting FlatKV keep its own in-code default) keeps the
-// two backends in true lockstep without relying on FlatKV's default happening to
-// equal memIAVL's healed default. That reliance is fragile — the defaults are
-// only kept equal by hand — and it breaks for an upgrading node whose old
-// app.toml still carries an explicit state-commit.flatkv.snapshot-keep-recent
-// (rendered by the old template) alongside sc-keep-recent = 0: skipping would
-// leave FlatKV pinned to the stale explicit value while memIAVL healed to a
-// different default. Note that mirroring a raw 0 is never correct here (0 means
-// "disable auto-snapshots" for FlatKV), which is why the zero is resolved first.
-func alignFlatKVSnapshotWithMemIAVL(cfg *config.StateCommitConfig) {
-	interval, keepRecent := config.EffectiveMemIAVLSnapshotCadence(cfg.MemIAVLConfig)
+// Retention count is deliberately not mirrored. It is a per-backend disk decision
+// rather than a cadence, and the two backends' costs differ by a factor of
+// roughly 200: measured at mainnet state size, one further retained snapshot
+// costs 56,782 MiB on memIAVL, whose snapshots are independent full copies, and
+// about 286 MiB on FlatKV, whose checkpoints hardlink their SSTs. A single shared
+// count cannot serve both — the depth FlatKV wants for forensic reach into the
+// migration window would ask memIAVL for more than the volume holds. FlatKV
+// therefore keeps config.DefaultSnapshotKeepRecent, which is sized for that reach.
+//
+// The mirror is unconditional across write modes, including FlatKVOnly, where
+// NewCompositeCommitStore never constructs a memIAVL store. sc-snapshot-interval
+// is the only operator-visible cadence knob, since the flatkv.* keys are hidden
+// from the app.toml template and the production reader does not consult them, so
+// it has to govern FlatKV's interval in every mode. It is harmless when memIAVL is
+// absent, because only cfg.FlatKVConfig is read when building that store.
+//
+// A zero is resolved before it is assigned, to the same default
+// Options.FillDefaults would apply at OpenDB. Mirroring a raw 0 is never correct:
+// 0 disables auto-snapshots for FlatKV, which lets the WAL grow without bound and
+// makes every restart replay from snapshot-0.
+func alignFlatKVSnapshotIntervalWithMemIAVL(cfg *config.StateCommitConfig) {
+	interval, _ := config.EffectiveMemIAVLSnapshotCadence(cfg.MemIAVLConfig)
 	cfg.FlatKVConfig.SnapshotInterval = interval
-	cfg.FlatKVConfig.SnapshotKeepRecent = keepRecent
 }
 
 // Initialize records the set of child store names that should exist on

sei-db/state_db/sc/composite/store_test.go

@@ -17,6 +17,7 @@ import (
 
 	gigatypes "github.com/sei-protocol/sei-chain/sei-db/state_db/giga/types"
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv"
+	flatkvconfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config"
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype"
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash"
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl"
@@ -1058,12 +1059,13 @@ func evmMigratedConfig() config.StateCommitConfig {
 	cfg.MemIAVLConfig.SnapshotInterval = 1
 	cfg.MemIAVLConfig.SnapshotMinTimeInterval = 0
 	cfg.MemIAVLConfig.AsyncCommitBuffer = 0
-	// With SnapshotInterval=1 every commit produces a snapshot, and FlatKV
-	// mirrors this cadence via alignFlatKVSnapshotWithMemIAVL. The default
-	// keep-recent of 1 would prune all but the two newest snapshots, so a
+	// With SnapshotInterval=1 every commit produces a snapshot, and memIAVL's
+	// default keep-recent of 1 would prune all but the two newest, so a
 	// rollback/reconcile to an older version (e.g. v3 after committing v5)
 	// could no longer find a base snapshot at-or-below the target. Retain all
 	// snapshots for the short duration of a test so those paths stay valid.
+	// FlatKV needs no equivalent here: it mirrors the interval but keeps its
+	// own retention count, which is deep enough already.
 	cfg.MemIAVLConfig.SnapshotKeepRecent = 100
 	return cfg
 }
@@ -2595,60 +2597,68 @@ func TestLoadVersionReadOnlyDuringMigrateEVMTransition(t *testing.T) {
 	require.Equal(t, []byte(evmVal), got)
 }
 
-func TestAlignFlatKVSnapshotWithMemIAVL(t *testing.T) {
-	t.Run("FlatKV derives interval and keep-recent from a non-zero memIAVL", func(t *testing.T) {
+func TestAlignFlatKVSnapshotIntervalWithMemIAVL(t *testing.T) {
+	t.Run("FlatKV derives its interval from a non-zero memIAVL", func(t *testing.T) {
 		cfg := config.DefaultStateCommitConfig()
 		cfg.MemIAVLConfig.SnapshotInterval = 5000
-		cfg.MemIAVLConfig.SnapshotKeepRecent = 3
-		// Start FlatKV from divergent values to prove they get overwritten.
+		// Start FlatKV from a divergent value to prove it gets overwritten.
 		cfg.FlatKVConfig.SnapshotInterval = 111
-		cfg.FlatKVConfig.SnapshotKeepRecent = 222
 
-		alignFlatKVSnapshotWithMemIAVL(&cfg)
+		alignFlatKVSnapshotIntervalWithMemIAVL(&cfg)
 
 		require.Equal(t, uint32(5000), cfg.FlatKVConfig.SnapshotInterval)
-		require.Equal(t, uint32(3), cfg.FlatKVConfig.SnapshotKeepRecent)
-	})
-
-	t.Run("a zero memIAVL keep-recent resolves to the healed default", func(t *testing.T) {
-		cfg := config.DefaultStateCommitConfig()
-		cfg.MemIAVLConfig.SnapshotKeepRecent = 0
-		// FlatKV must not mirror the raw 0 (which would prune everything but the
-		// latest). Instead it mirrors the value FillDefaults will heal memIAVL to,
-		// keeping the two in lockstep. memIAVL's own 0 is left for FillDefaults.
-		alignFlatKVSnapshotWithMemIAVL(&cfg)
-
-		require.Equal(t, uint32(0), cfg.MemIAVLConfig.SnapshotKeepRecent)
-		require.Equal(t, uint32(memiavl.DefaultSnapshotKeepRecent), cfg.FlatKVConfig.SnapshotKeepRecent)
 	})
 
 	t.Run("a zero memIAVL interval resolves to the healed default", func(t *testing.T) {
 		cfg := config.DefaultStateCommitConfig()
 		cfg.MemIAVLConfig.SnapshotInterval = 0
 		// A raw 0 would disable FlatKV auto-snapshots; instead FlatKV mirrors the
 		// value FillDefaults will heal memIAVL's interval to.
-		alignFlatKVSnapshotWithMemIAVL(&cfg)
+		alignFlatKVSnapshotIntervalWithMemIAVL(&cfg)
 
 		require.Equal(t, uint32(memiavl.DefaultSnapshotInterval), cfg.FlatKVConfig.SnapshotInterval)
 		require.NotZero(t, cfg.FlatKVConfig.SnapshotInterval)
 	})
 
-	t.Run("an explicit FlatKV override loses to memIAVL's healed default", func(t *testing.T) {
-		// Upgrade scenario: an old app.toml still pins an explicit FlatKV
-		// keep-recent/interval (the previous template rendered flatkv.* keys)
-		// while sc-* is 0. FlatKV must follow memIAVL's effective (healed) cadence
-		// rather than staying pinned to the stale explicit value, otherwise the
-		// two backends diverge (memIAVL heals 0 -> default, FlatKV keeps the old
-		// explicit value).
+	t.Run("retention count is not mirrored", func(t *testing.T) {
+		// The two backends share a cadence and not a disk budget. A retained
+		// memIAVL snapshot is an independent full copy where a FlatKV checkpoint
+		// hardlinks its SSTs, so one shared count cannot serve both.
+		cfg := config.DefaultStateCommitConfig()
+		cfg.MemIAVLConfig.SnapshotKeepRecent = 3
+		cfg.FlatKVConfig.SnapshotKeepRecent = 222
+
+		alignFlatKVSnapshotIntervalWithMemIAVL(&cfg)
+
+		require.Equal(t, uint32(222), cfg.FlatKVConfig.SnapshotKeepRecent)
+		require.Equal(t, uint32(3), cfg.MemIAVLConfig.SnapshotKeepRecent)
+	})
+
+	t.Run("a zero memIAVL keep-recent does not reach FlatKV", func(t *testing.T) {
+		// A zero memIAVL keep-recent is healed to memiavl.DefaultSnapshotKeepRecent
+		// for memIAVL's own use. That healed value must not reach FlatKV, whose
+		// own default stands.
 		cfg := config.DefaultStateCommitConfig()
 		cfg.MemIAVLConfig.SnapshotKeepRecent = 0
-		cfg.MemIAVLConfig.SnapshotInterval = 0
-		cfg.FlatKVConfig.SnapshotKeepRecent = 2
-		cfg.FlatKVConfig.SnapshotInterval = 7777
 
-		alignFlatKVSnapshotWithMemIAVL(&cfg)
+		alignFlatKVSnapshotIntervalWithMemIAVL(&cfg)
 
-		require.Equal(t, uint32(memiavl.DefaultSnapshotKeepRecent), cfg.FlatKVConfig.SnapshotKeepRecent)
-		require.Equal(t, uint32(memiavl.DefaultSnapshotInterval), cfg.FlatKVConfig.SnapshotInterval)
+		require.Equal(t, uint32(0), cfg.MemIAVLConfig.SnapshotKeepRecent)
+		require.Equal(t, flatkvconfig.DefaultSnapshotKeepRecent, cfg.FlatKVConfig.SnapshotKeepRecent)
+		require.NotEqual(t, uint32(memiavl.DefaultSnapshotKeepRecent), cfg.FlatKVConfig.SnapshotKeepRecent,
+			"FlatKV must not inherit memIAVL's retention count")
 	})
 }
+
+// The default exists to bound what can be asked about a past height, so a reach
+// shorter than the migration window it was sized for is the regression to catch.
+// 72 checkpoints at a 10000-block interval is 720,000 blocks, about 89 hours at
+// mainnet's measured 2.247 blocks/s, against an 85-hour drain at K=1024.
+func TestFlatKVDefaultRetentionSpansTheMigrationWindow(t *testing.T) {
+	cfg := config.DefaultStateCommitConfig()
+	alignFlatKVSnapshotIntervalWithMemIAVL(&cfg)
+
+	reach := uint64(cfg.FlatKVConfig.SnapshotKeepRecent) * uint64(cfg.FlatKVConfig.SnapshotInterval)
+	require.GreaterOrEqual(t, reach, uint64(690_000),
+		"default FlatKV retention must reach back across an 85-hour drain at 2.247 blocks/s")
+}