diff --git a/test/integration/giga_mixed_release_test.go b/test/integration/giga_mixed_release_test.go index c472d57..92801e4 100644 --- a/test/integration/giga_mixed_release_test.go +++ b/test/integration/giga_mixed_release_test.go @@ -8,41 +8,40 @@ import ( "net/http" "os/signal" "strconv" - "strings" "syscall" "testing" "time" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "github.com/sei-protocol/sei-k8s-controller/internal/keygen" "github.com/sei-protocol/sei-k8s-controller/sdk/sei" ) -// TestNightlyGigaMixedRelease proves giga_store_migration.md's central safety claim — -// "Giga SS Store is a per-node SS change that is invisible to the network" — -// under real conformance load, not just a healthy-boot check. It provisions one -// 4-validator chain with two plain RPC followers, migrates ONE follower to giga -// (evm-ss-split=true) via the same StateSync recipe TestNightlyGigaStoreMigration -// drives, leaves the other at the shipped default (every node ships with -// ss-enable=true already; evm-ss-split=false is the only thing distinguishing a -// "giga" node from a plain one), then runs the external release-test conformance -// harness against BOTH followers concurrently, each signing with its own funded -// admin account so the two runs never race a shared account's tx sequence. -// Parity is: both runs pass identically — the same 749-case suite, the same exit -// code — regardless of which storage layout answered the queries. +// TestNightlyGigaMixedRelease proves giga_store_migration.md's central safety +// claim — "Giga SS Store is a per-node SS change that is invisible to the +// network" — under real conformance load rather than a healthy-boot check. It +// provisions one 4-validator, snapshot-producing chain with two plain RPC +// followers, migrates ONE follower to giga (evm-ss-split=true, pebbledb) via the +// same StateSync recipe TestNightlyGigaStoreMigration drives, and leaves the other +// at the shipped default (the v2 control). It then runs the external release-test +// conformance harness against the chain and asserts the invariant: +// +// a mixed giga/v2 follower set is invisible to consensus — through real +// transaction load the chain keeps producing blocks and BOTH followers stay at +// the validator head (a follower that halted or diverged under load would fall +// behind, so head-parity is the SDK-observable proof of no divergence), and the +// migrated giga node comes up and serves EVM under evm-ss-split. // -// Honest ceiling: the release-test image is external and opaque to this suite -// (no source here); if any of its cases assert something about GLOBAL chain -// state (e.g. total supply) rather than state scoped to their own admin account, -// running two instances concurrently against the same chain could cross-talk in -// a way neither run's own log would explain. Reviewed and accepted for the -// nightly run; if cross-talk is ever observed, the fallback is running the two -// harness Jobs sequentially instead of concurrently (accept the added wall-clock -// for a cleaner isolation guarantee). +// Why ONE conformance run, not two: the harness runs against the v2 follower with +// exclusive chain access. Its stateful sequences (EVM filters, send-then-wait) +// assume a single consistent mempool + filter-store view — the same reason +// TestNightlyRelease runs exactly one node. Two concurrent runs (one per follower) +// would cross-talk on GLOBAL block state (the EIP-1559 base fee is per-block, +// driven by every tx in the block, not per-account) and on inclusion timing, which +// no separate-admin isolation prevents; per-node SS read+serve parity is already +// covered by TestNightlyGigaStoreMigration. This suite's distinct job is the +// mixed-fleet-under-load consensus invariant, which needs only one traffic source. +// The harness targets the v2 control follower — the full-history, shipped-default +// node — while the migrated giga follower is exercised by the head-parity check. // // Inputs (env): SEI_CHAIN_ID, SEID_IMAGE, RELEASE_TEST_IMAGE [required]; // SEI_NAMESPACE [optional]. Run with -test.timeout 0. @@ -54,9 +53,10 @@ func TestNightlyGigaMixedRelease(t *testing.T) { ns := envOr("SEI_NAMESPACE", "") runLabels := map[string]string{runLabelKey: chainID} - // The giga migration workflow and two release-test harness runs (each up to - // 60m per releaseJob's own deadline) can overlap; size well above both. - ctx, cancel := context.WithTimeout(context.Background(), 120*time.Minute) + // Sequential critical path: provision + giga migration + one conformance run. + // TestNightlyRelease budgets 80m for provision + a single 60m release Job; this + // suite adds a ~15m giga migration ahead of the same run, so size ~30m above it. + ctx, cancel := context.WithTimeout(context.Background(), 110*time.Minute) defer cancel() ctx, stop := signal.NotifyContext(ctx, syscall.SIGTERM, syscall.SIGINT) defer stop() @@ -65,22 +65,17 @@ func TestNightlyGigaMixedRelease(t *testing.T) { cs := clientset(t) hc := &http.Client{Timeout: 10 * time.Second} - // Two independent admin accounts, one per target node, so the two concurrent - // harness runs never race a shared account's tx sequence. - adminV2, err := keygen.Derive() + // One funded admin: the single conformance run (against the v2 follower) signs + // with it. The giga follower carries no load of its own here. + admin, err := keygen.Derive() if err != nil { - t.Fatalf("derive v2 admin key: %v", err) - } - adminGiga, err := keygen.Derive() - if err != nil { - t.Fatalf("derive giga admin key: %v", err) + t.Fatalf("derive admin key: %v", err) } - // Provision: 4 validators, snapshot-producing (the migration below - // state-syncs rpcNodes[1] from a network snapshot); 2 plain RPC followers, - // non-producing (rpcConfig zeroes the interval — a follower is a witness, - // not a snapshot source). rpcNodes[0] stays at the shipped default (the v2 - // control); rpcNodes[1] is migrated to giga below. + // 1. Provision one chain with two plain RPC followers. rpcNodes[0] stays at the + // shipped default (the v2 control); rpcNodes[1] is migrated to giga below. + // Validators are snapshot-producing so the migration can state-sync from a + // snapshot; the followers are non-producing witnesses (interval 0). ch, err := provision(ctx, t, c, spec{ chainID: chainID, runID: chainID, @@ -90,43 +85,105 @@ func TestNightlyGigaMixedRelease(t *testing.T) { rpcNodes: 2, storageConfig: mergeConfig(releaseBaseConfig, snapshotProductionConfig), rpcConfig: mergeConfig(releaseRPCConfig, map[string]string{"storage.snapshot_interval": "0"}), - accounts: []sei.GenesisAccount{ - {Address: adminV2.Address, Balance: releaseAdminBalance}, - {Address: adminGiga.Address, Balance: releaseAdminBalance}, - }, + accounts: []sei.GenesisAccount{{Address: admin.Address, Balance: releaseAdminBalance}}, }) cleanupChain(t, ch) if err != nil { t.Fatalf("provision: %v", err) } - net := ch.network v2Node := ch.rpcNodes[0] gigaNode := ch.rpcNodes[1] - t.Logf("network %s: ready (4 validators, 2 rpc followers, 2 admins funded)", chainID) + t.Logf("network %s: ready (4 validators, 2 rpc followers, admin %s funded)", chainID, admin.Address) - // A follower can only state-sync from a snapshot that already exists. - // Advance the network one interval + margin so the migration below has one - // to fetch (the same wait bringUpStateSyncFollower uses). + // The migration state-syncs from a snapshot; step 5 reuses the snapshot interval + // as the follower head-lag tolerance (the same freshness yardstick assertWitnessFresh uses). interval, err := strconv.Atoi(snapshotProductionConfig["storage.snapshot_interval"]) if err != nil { t.Fatalf("snapshot_interval not an int: %v", err) } + + // 2. Migrate rpcNodes[1] onto the giga SS store in place, witnessed by genesis + // validator-0 and the untouched v2 follower. + migrateFollowerToGiga(ctx, t, c, hc, ch.network, gigaNode, v2Node, interval, runLabels) + + // 3. Confirm the migrated giga follower is live under evm-ss-split, and the v2 + // control is still healthy, before handing the chain any load. + assertGigaFollowerServing(ctx, t, hc, gigaNode) + if err := sei.WaitCaughtUp(ctx, hc, v2Node.TendermintRPC()); err != nil { + t.Fatalf("v2 control follower %s not caught up before load: %v", v2Node.Name(), err) + } + t.Logf("v2 control follower %s: caught up, untouched by the migration", v2Node.Name()) + + // Baseline the validator head so step 5 can prove the chain advanced under load + // (catching_up is a one-way latch and cannot certify liveness — see assertWitnessFresh). + preHead, ok := sei.LatestHeight(ctx, hc, ch.network.TendermintRPC()) + if !ok { + t.Fatalf("read validator head before load") + } + + // 4. Run the conformance harness once, against the v2 follower, with exclusive + // chain access. Its exit code is the functional verdict. + runReleaseTest(ctx, t, cs, releaseRunParams{ + hc: hc, + net: ch.network, + node: v2Node, + admin: admin, + image: releaseImage, + runLabels: runLabels, + label: "v2", + }) + t.Logf("release-test PASSED against the v2 follower") + + // 5. The invariant: through the conformance load the chain kept producing blocks + // and BOTH followers stayed at the validator head — the giga migration was + // invisible to consensus. Head-parity, not catching_up: the latch already + // flipped in steps 1-3, so it would false-green a post-load halt or a giga + // follower silently falling behind (the one failure mode this suite exists to catch). + postHead, ok := sei.LatestHeight(ctx, hc, ch.network.TendermintRPC()) + if !ok { + t.Fatalf("read validator head after load") + } + if postHead <= preHead { + t.Fatalf("validator head did not advance through the conformance load: %d -> %d (the chain stalled under real traffic)", preHead, postHead) + } + assertFollowerAtHead(ctx, t, hc, gigaNode, "giga", postHead, interval) + assertFollowerAtHead(ctx, t, hc, v2Node, "v2 control", postHead, interval) + t.Logf("chain advanced %d -> %d and both followers tracked head through conformance load — mixed storage invisible to the network, TestNightlyGigaMixedRelease OK", preHead, postHead) +} + +// migrateFollowerToGiga re-bootstraps gigaNode in place onto the giga SS store +// (evm-ss-split=true, pebbledb) via the StateSync + ConfigMigration recipe, +// witnessed by genesis validator-0 and the untouched v2 follower. In order it: +// advances the chain past one snapshot interval (a follower can only state-sync +// from a snapshot that already exists), asserts the witnesses are fresh enough to +// serve that snapshot, drives the workflow to Complete, and anchors on the compiled +// plan carrying exactly the giga app.toml flags. It returns once the migration has +// materialized; the caller gates the node's runtime health separately +// (assertGigaFollowerServing). +func migrateFollowerToGiga( + ctx context.Context, t *testing.T, c *sei.Client, hc *http.Client, + net *sei.Network, gigaNode, v2Witness *sei.Node, interval int, runLabels map[string]string, +) { + t.Helper() + chainID := net.Name() + ns := net.Namespace() + if err := sei.WaitHeightAdvances(ctx, hc, net.TendermintRPC(), int64(interval)+10); err != nil { t.Fatalf("network did not advance past the snapshot interval: %v", err) } - // Migrate rpcNodes[1] to giga. Witnesses are explicit (validator-0 + the v2 - // control node): a plain follower's ResolvedStateSyncers is populated only - // for nodes created with an explicit StateSync spec, which neither follower - // has here. A witness serves RPC trust points only; its own storage layout - // is not consensus-relevant to serving them, so the v2 node is a valid + // Witnesses are explicit (validator-0 + the v2 control): a plain follower's + // ResolvedStateSyncers is empty — neither follower was created with a StateSync + // spec — and a witness serves RPC trust points only, so its own storage layout + // is not consensus-relevant to serving them. That makes the v2 node a valid // witness for the giga target's migration. witnessNS := witnessNamespace(t, net) witnesses := []string{ nodeRPC(fmt.Sprintf("%s-0", chainID), witnessNS), // genesis validator-0 - nodeRPC(v2Node.Name(), witnessNS), // the v2 control node + nodeRPC(v2Witness.Name(), witnessNS), // the v2 control follower } assertWitnessFresh(ctx, t, hc, witnesses[0], witnesses[1], interval) + wf, err := c.CreateWorkflow(ctx, sei.WorkflowSpec{ Name: "giga-" + chainID, Namespace: ns, @@ -149,147 +206,45 @@ func TestNightlyGigaMixedRelease(t *testing.T) { awaitWorkflowComplete(ctx, t, wf, workflowWaitTimeout) t.Logf("workflow %s: complete", wf.Name()) + + // Anchor: the compiled plan carries exactly the giga app.toml flags — proof the + // ConfigMigration materialized end to end (inspects the persisted status.plan, + // not a log line, so it cannot false-green). assertGigaConfigPatch(t, wf, "pebbledb") +} +// assertGigaFollowerServing gates the migrated follower's runtime health: Ready +// (the resync cycled seid), caught up, then EVM serving under evm-ss-split. Per the +// giga migration doc's startup safety checks, a node with evm-ss-split=true and an +// unpopulated EVM-SS layout refuses to boot, so a served EVM endpoint IS the +// SDK-observable evidence that giga went live. +func assertGigaFollowerServing(ctx context.Context, t *testing.T, hc *http.Client, gigaNode *sei.Node) { + t.Helper() if err := gigaNode.WaitReady(ctx); err != nil { - t.Fatalf("giga node %s not Running after migration: %v", gigaNode.Name(), err) + t.Fatalf("giga follower %s not Running after migration: %v", gigaNode.Name(), err) } if err := sei.WaitCaughtUp(ctx, hc, gigaNode.TendermintRPC()); err != nil { - t.Fatalf("giga node %s not caught up after migration: %v", gigaNode.Name(), err) + t.Fatalf("giga follower %s not caught up after migration: %v", gigaNode.Name(), err) } if err := sei.WaitEVMServing(ctx, hc, gigaNode.EVMRPC()); err != nil { - t.Fatalf("giga node %s EVM not serving under evm-ss-split=true after migration: %v (a node with an unpopulated EVM-SS layout refuses to boot, so a non-serving EVM here means giga did not go live)", gigaNode.Name(), err) - } - t.Logf("giga node %s: caught up + EVM serving under evm-ss-split=true", gigaNode.Name()) - - // The v2 control node was never touched by the migration; confirm it is - // still healthy before handing both nodes to the conformance harness. - if err := sei.WaitCaughtUp(ctx, hc, v2Node.TendermintRPC()); err != nil { - t.Fatalf("v2 control node %s not caught up: %v", v2Node.Name(), err) + t.Fatalf("giga follower %s EVM not serving under evm-ss-split=true after migration: %v (a node with an unpopulated EVM-SS layout refuses to boot, so a non-serving EVM here means giga did not go live)", gigaNode.Name(), err) } - t.Logf("v2 control node %s: still caught up, untouched by the migration", v2Node.Name()) - - // Launch one release-test Job per node. Job/Secret creation and t.Cleanup - // registration happen here, on the test's own goroutine — only the - // wait-and-log step below runs concurrently, and it never touches t.Fatal/ - // t.Cleanup (unsafe outside the test's own goroutine), only t.Logf (documented - // safe for concurrent use) and plain error returns. - targets := []struct { - label string - node *sei.Node - admin keygen.Identity - }{ - {"v2", v2Node, adminV2}, - {"giga", gigaNode, adminGiga}, - } - jobNames := make([]string, len(targets)) - for i, tgt := range targets { - rest := tgt.node.REST() - if rest == "" { - t.Fatalf("%s node %q exposes no REST endpoint (release-test needs SEI_REST_ENDPOINT)", tgt.label, tgt.node.Name()) - } - if err := sei.WaitRESTServing(ctx, hc, rest); err != nil { - t.Fatalf("%s node %q REST serving: %v", tgt.label, tgt.node.Name(), err) - } - t.Logf("%s node %s: REST serving at %s", tgt.label, tgt.node.Name(), rest) - - secretName := "admin-" + tgt.label + "-" + chainID - createMnemonicSecret(ctx, t, cs, net.Namespace(), secretName, runLabels, tgt.admin.Mnemonic) - - job := releaseJob(releaseParams{ - name: "release-test-" + tgt.label + "-" + chainID, - namespace: net.Namespace(), - image: releaseImage, - runID: chainID, - chainID: chainID, - adminAddr: tgt.admin.Address, - secretName: secretName, - tmRPC: tgt.node.TendermintRPC(), - evmRPC: tgt.node.EVMRPC(), - rest: rest, - }) - if _, err := cs.BatchV1().Jobs(net.Namespace()).Create(ctx, job, metav1.CreateOptions{}); err != nil { - t.Fatalf("create %s release-test job: %v", tgt.label, err) - } - jobNames[i] = job.Name - t.Cleanup(func(name string) func() { - return func() { - delCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - bg := metav1.DeletePropagationBackground - _ = cs.BatchV1().Jobs(net.Namespace()).Delete(delCtx, name, metav1.DeleteOptions{PropagationPolicy: &bg}) - } - }(job.Name)) - t.Logf("%s release-test job launched (%s)", tgt.label, releaseImage) - } - - type result struct { - label string - err error - } - results := make(chan result, len(targets)) - for i, tgt := range targets { - go func(label, ns, jobName string) { - results <- result{label, waitJobErr(ctx, cs, ns, jobName)} - }(tgt.label, net.Namespace(), jobNames[i]) - } - - var failed []string - for range targets { - r := <-results - if r.err != nil { - failed = append(failed, fmt.Sprintf("%s: %v", r.label, r.err)) - continue - } - t.Logf("%s release-test job completed", r.label) - } - if len(failed) > 0 { - t.Fatalf("release-test failed: %s", strings.Join(failed, "; ")) - } - - // Both nodes stayed live through the concurrent harness runs. - if err := sei.WaitCaughtUp(ctx, hc, v2Node.TendermintRPC()); err != nil { - t.Errorf("post-release v2 node %s not caught up: %v", v2Node.Name(), err) - } - if err := sei.WaitCaughtUp(ctx, hc, gigaNode.TendermintRPC()); err != nil { - t.Errorf("post-release giga node %s not caught up: %v", gigaNode.Name(), err) - } - t.Logf("release-test PASSED against both v2 and giga followers — parity confirmed, TestNightlyGigaMixedRelease OK") + t.Logf("giga follower %s: caught up + EVM serving under evm-ss-split=true", gigaNode.Name()) } -// waitJobErr mirrors waitJob's polling logic but returns an error instead of -// calling t.Fatalf, so it is safe to run from a goroutine other than the one -// running the test (T.FailNow — which Fatalf calls — must only be called from -// the test's own goroutine; log methods and plain returns carry no such -// restriction). -func waitJobErr(ctx context.Context, cs *kubernetes.Clientset, ns, name string) error { - tick := time.NewTicker(10 * time.Second) - defer tick.Stop() - for { - job, err := cs.BatchV1().Jobs(ns).Get(ctx, name, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("get job %q: %w", name, err) - } - for _, cond := range job.Status.Conditions { - if cond.Type == batchv1.JobComplete && cond.Status == corev1.ConditionTrue { - return nil - } - if cond.Type == batchv1.JobFailed && cond.Status == corev1.ConditionTrue { - // Fresh, short-lived context for the log read: ctx may be near its - // own deadline here, which would truncate the read that explains why. - logCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - tail := podLogTail(logCtx, cs, ns, name) - cancel() - return fmt.Errorf("job %q failed: %s\n--- pod log (tail) ---\n%s", name, cond.Message, tail) - } - } - select { - case <-ctx.Done(): - logCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - tail := podLogTail(logCtx, cs, ns, name) - cancel() - return fmt.Errorf("job %q did not finish before deadline: %w\n--- pod log (tail) ---\n%s", name, ctx.Err(), tail) - case <-tick.C: - } - } +// assertFollowerAtHead fails the suite unless the follower is within maxLag blocks +// of the given validator head — proof it tracked consensus rather than halting or +// diverging under load. catching_up is a one-way latch (see assertWitnessFresh) and +// cannot certify this, so freshness is asserted directly on heights, the same way +// assertWitnessFresh gates a state-sync witness. role tags the message. +func assertFollowerAtHead(ctx context.Context, t *testing.T, hc *http.Client, node *sei.Node, role string, head int64, maxLag int) { + t.Helper() + h, ok := sei.LatestHeight(ctx, hc, node.TendermintRPC()) + if !ok { + t.Fatalf("%s follower %s: read height after load", role, node.Name()) + } + if gap := head - h; gap > int64(maxLag) { + t.Fatalf("%s follower %s at height %d lags validator head %d by %d blocks (> %d) after conformance load: a follower that fell behind or diverged under real traffic is not invisible to the network", role, node.Name(), h, head, gap, maxLag) + } + t.Logf("%s follower %s: at height %d, validator head %d after load", role, node.Name(), h, head) } diff --git a/test/integration/release_test.go b/test/integration/release_test.go index 21ec28c..f98efcf 100644 --- a/test/integration/release_test.go +++ b/test/integration/release_test.go @@ -106,59 +106,107 @@ func TestNightlyRelease(t *testing.T) { t.Logf("network %s: ready (4 validators, admin %s funded)", chainID, admin.Address) hc := &http.Client{Timeout: 10 * time.Second} - rest := node.REST() + + // Run the external release-test image once against the RPC node; its exit code + // is the functional verdict. + runReleaseTest(ctx, t, cs, releaseRunParams{ + hc: hc, + net: net, + node: node, + admin: admin, + image: releaseImage, + runLabels: runLabels, + label: "", + }) + + // The chain stayed live through the release suite: the follower is still + // caught up (it can't catch up to a halted chain, so this covers quorum). + if err := sei.WaitCaughtUp(ctx, hc, node.TendermintRPC()); err != nil { + t.Errorf("post-release %s not caught up: %v", rpcName, err) + } + t.Logf("chain live post-release — TestNightlyRelease OK") +} + +// releaseRunParams selects a single conformance run: the RPC node the external +// harness drives, the funded admin identity it signs with, and a DNS-safe label +// distinguishing this run's resources ("" for TestNightlyRelease's lone run, "v2" +// for the mixed-release suite's run against its v2 follower). The chain id and +// namespace are read from net, which is authoritative (provision names the network +// after the chain id; env may leave the namespace "" for the SDK to resolve). +type releaseRunParams struct { + hc *http.Client + net *sei.Network + node *sei.Node + admin keygen.Identity + image string + runLabels map[string]string + label string +} + +// runReleaseTest runs the external release-test conformance harness once against a +// single RPC node and gates on its exit code. In order it: probes REST actually +// serves (the status advertises REST as soon as the endpoint is composed, but the +// LCD listener binds later than the EVM one, so a cold REST surfaces here, not +// mid-test), hands the admin mnemonic to the pod via a labeled Secret, launches the +// harness as a one-shot Job wired to the node's endpoints, waits for the Job's +// verdict, and archives the harness log tail — even on success, since exit 0 alone +// doesn't show which sub-cases ran, so a skip-but-pass is otherwise invisible. The +// caller owns all pre/post chain-health assertions. +// +// Exactly one node with exclusive chain access is deliberate: the harness runs +// stateful EVM-filter and send-then-wait sequences that need one consistent mempool +// + filter-store view. +func runReleaseTest(ctx context.Context, t *testing.T, cs *kubernetes.Clientset, p releaseRunParams) { + t.Helper() + chainID := p.net.Name() + ns := p.net.Namespace() + + rest := p.node.REST() if rest == "" { - t.Fatalf("rpc node %q exposes no REST endpoint (release-test needs SEI_REST_ENDPOINT)", rpcName) + t.Fatalf("rpc node %q exposes no REST endpoint (release-test needs SEI_REST_ENDPOINT)", p.node.Name()) } - // The status advertises REST as soon as the endpoint is composed, but the LCD - // listener binds later than the EVM one — probe it actually serves before - // handing the URL to the harness, so a cold REST surfaces here, not mid-test. - if err := sei.WaitRESTServing(ctx, hc, rest); err != nil { - t.Fatalf("rpc node %q REST serving: %v", rpcName, err) + if err := sei.WaitRESTServing(ctx, p.hc, rest); err != nil { + t.Fatalf("rpc node %q REST serving: %v", p.node.Name(), err) } - t.Logf("rpc node %s: REST serving at %s", rpcName, rest) + t.Logf("rpc node %s: REST serving at %s", p.node.Name(), rest) + + // DNS-safe per-run resource names: "admin-" / "release-test-" for + // the lone run, "…-