diff --git a/img_hashes.txt b/img_hashes.txt new file mode 100644 index 0000000000..40effed125 --- /dev/null +++ b/img_hashes.txt @@ -0,0 +1,172 @@ +d41d8cd98f00b204e9800998ecf8427e simplyblock_core/__init__.py +921aad174688b985519e0c72d1d6d0e7 simplyblock_core/cluster_ops.py +eb941f1f1c0fbc49bf6c569fef7d31f8 simplyblock_core/constants.py +d41d8cd98f00b204e9800998ecf8427e simplyblock_core/controllers/__init__.py +a440b45b21c4f14f2e40240dbf2d9bc5 simplyblock_core/controllers/backup_controller.py +eb0275929e5d1dd522763c44b3b40fda simplyblock_core/controllers/backup_events.py +43f023313267a2434a2029deda471f61 simplyblock_core/controllers/cluster_events.py +d41d8cd98f00b204e9800998ecf8427e simplyblock_core/controllers/cluster_expansion/__init__.py +38cba0e6dd4e9374b37c3fe2cd0c6b8f simplyblock_core/controllers/cluster_expansion/executor.py +7e3ed535a62cdcd5e087044502437269 simplyblock_core/controllers/cluster_expansion/orchestrator.py +5388e317669ef6ed861b882f75522e27 simplyblock_core/controllers/cluster_expansion/planner.py +cb317fde8a270cda8bba528d0e4bf286 simplyblock_core/controllers/cluster_expansion/preconditions.py +1dd883da6a0e99bad07ea12cb6ed2727 simplyblock_core/controllers/device_controller.py +49318a96a77b96cd2438b1389a927600 simplyblock_core/controllers/device_events.py +9563d5d608189dc3ee1cc7dc6575d8c8 simplyblock_core/controllers/events_controller.py +96466463b95d74b79cce2a398949956a simplyblock_core/controllers/fdb_backup_controller.py +ef01b95df3f3c501bbd0df35753eeea0 simplyblock_core/controllers/health_controller.py +583833a31731ee88d6c5e4ed9ed8b2f9 simplyblock_core/controllers/host_auth.py +122238a7f90215505a85c9fb4e1d06d7 simplyblock_core/controllers/lvol_controller.py +8ed20e8ada22c39ad2b40dc1aeb91436 simplyblock_core/controllers/lvol_events.py +d7261581a5308b13e1fc830910b4202d simplyblock_core/controllers/mgmt_events.py +4495ba0571fed37306e9d6684e993e87 simplyblock_core/controllers/migration_bdev_ops.py +7a7a80de2a192802cfab2a4d52a619c4 simplyblock_core/controllers/migration_controller.py +faf2454ca99acce7a0b8b12f2a0a1c2e simplyblock_core/controllers/migration_events.py +3329fdc76d009ab22043a4adfdd31162 simplyblock_core/controllers/ops_gate.py +16b973f2a2a44a20ae1d76329572ed33 simplyblock_core/controllers/pool_controller.py +1c79832885997693b642ea125465c224 simplyblock_core/controllers/pool_events.py +10fbc91df11af77ad23a83c9d0dbe66b simplyblock_core/controllers/qos_controller.py +8ef27ba9a004a7f365f7fdc810796cfa simplyblock_core/controllers/replication_policy_controller.py +a4a06dc366b026908f02fc21502792d2 simplyblock_core/controllers/snapshot_controller.py +99f7f70f3a3c0e909aac03802f5006b3 simplyblock_core/controllers/snapshot_events.py +9c08682f202955ed566ed96d9535c829 simplyblock_core/controllers/storage_events.py +441da86657062e418f150da66efa5390 simplyblock_core/controllers/tasks_controller.py +c5cc66cb98bde4768092cdb1a0131ae4 simplyblock_core/controllers/tasks_events.py +b8d69479e568944c6d26da7e9b61bb56 simplyblock_core/controllers/tcp_ports_events.py +91613b33cc8eecfd8e36488c42b1efcb simplyblock_core/db_controller.py +84a808ea97b86654e5547b13472faf48 simplyblock_core/distr_controller.py +212bb9c4ba9198b35be212d5dfd89601 simplyblock_core/exceptions.py +c577aec17bc3aa0428af060e5cbbd8f0 simplyblock_core/fw_api_client.py +3fc3f4fc5f60993d7b138fa9b4adbcad simplyblock_core/jm_raid.py +74e7b278ad56e4adbfc6f5105f23fa70 simplyblock_core/kms/__init__.py +b21dfac624800ae4695edb6dead53a7c simplyblock_core/kms/_base.py +eaa1c679331e65af0a8533856173c486 simplyblock_core/kms/_exceptions.py +707d37b161f048797f4be19654345709 simplyblock_core/kms/_fdb.py +7f917ea95665ee16c0ffd1c08ddbd9c2 simplyblock_core/kms/_hcp.py +fcd90d3a9b7af72c76369eb6fd67e955 simplyblock_core/mgmt_node_ops.py +d41d8cd98f00b204e9800998ecf8427e simplyblock_core/models/__init__.py +f4a6dff300d400be9d9983ea6bcd02b0 simplyblock_core/models/backup.py +1991ef4a4e6f043d90f08232ccd6998a simplyblock_core/models/base_model.py +b7a0b089043fcd1bbc4114cf866f2a98 simplyblock_core/models/cluster.py +41c4c62321b77e7381709d39345be74c simplyblock_core/models/events.py +c1d95ed3eb8531a3736a74ba9d522a16 simplyblock_core/models/hub_cooldown.py +84a5e67573439d80fc8217bf6956e86d simplyblock_core/models/hublvol.py +aaa0f08f3724359abfccac3a6f912651 simplyblock_core/models/iface.py +bad6cffda3d269eac4528a9fbe3bceca simplyblock_core/models/job_schedule.py +fbf8c48e05b78ad0cfadd0d0b6952e3e simplyblock_core/models/lvol_migration.py +93e027ba3737c5d3a66a76c4bef1eb99 simplyblock_core/models/lvol_migration_group.py +7233db81712cd3c02277cf88b2c5509d simplyblock_core/models/lvol_model.py +79a31818d478ecce83c63050ece62065 simplyblock_core/models/lvstore_lock.py +5e8aeda97b87058699292ed0fd3603b9 simplyblock_core/models/mgmt_node.py +b1076bf14319f93df9906a1ff77e9920 simplyblock_core/models/nvme_device.py +16e6498b741406a6ca52e531a2a34611 simplyblock_core/models/pool.py +d350fce28254cf53da607facfdab1cce simplyblock_core/models/port_stat.py +76b86dcb5850954d0b78928a772e4616 simplyblock_core/models/qos.py +82eb6d6570452eaa7052d84dd5a2b47e simplyblock_core/models/replication.py +7e34fc1f2577c045667418687e82796d simplyblock_core/models/restart_lock.py +b17735bfa9fb6099d948ca1ab954a907 simplyblock_core/models/snapshot.py +930d082cbb3d3f2f9076837e38a5283d simplyblock_core/models/stats.py +afadba47b1b755674ecfb2edbae2dafd simplyblock_core/models/storage_node.py +f820ca63541b36fc34dc938a1419da53 simplyblock_core/prom_client.py +9ef4b502175a7d5cc32f6ea82d110dfa simplyblock_core/release_upgrades/__init__.py +cba45f129303eca7295214719b56df4b simplyblock_core/release_upgrades/jc_compression_upgrade.py +ae7b5ff638b16236f39f38ecd52563e5 simplyblock_core/rpc_client.py +fb8278ed7174bd9ca12b44a545b108e6 simplyblock_core/scripts/__init__.py +649b3072b7625d6bbb732a9e66566dc2 simplyblock_core/scripts/collect_logs.py +d41d8cd98f00b204e9800998ecf8427e simplyblock_core/scripts/helpers/__init__.py +455c553a6598cd0178ecb36433f07d10 simplyblock_core/scripts/helpers/nvme_disconnect_by_ip.py +a371a8ebc56236f463fa49c6674faa51 simplyblock_core/scripts/restore_fdb_from_kvfiledump.py +4c20bb18b5f6e04234455dfe8112b3d8 simplyblock_core/services/__init__.py +89ea5203a611d9eb7ba04d1879aa840c simplyblock_core/services/cap_monitor.py +c74e062d9cd028e98503b07fd3a7ee04 simplyblock_core/services/capacity_and_stats_collector.py +9310b22ccdc7d6f5d850e932a6d6098d simplyblock_core/services/device_monitor.py +3b982e4e58d489fc2cc861ee9f5d9e88 simplyblock_core/services/health_check_service.py +8773b45be91a706f4dd50bf6ba14ff87 simplyblock_core/services/hub_controller_manager.py +f22a5aa9946183ec33061a253166b9f0 simplyblock_core/services/lvol_monitor.py +5af944ba3a09848e24984cc560012c00 simplyblock_core/services/lvol_stat_collector.py +e286e508f7f34ea28b6894df29b6819e simplyblock_core/services/main_distr_event_collector.py +db8fd72f53396244adcd18f1e0348655 simplyblock_core/services/mgmt_node_monitor.py +66a30a4a6386a08eb720530341b3ee0e simplyblock_core/services/new_device_discovery.py +1da9113a14a2e5c72438f8174d4d1301 simplyblock_core/services/replication_final_step.py +f49cf6719f484f43c6f543f09a59356d simplyblock_core/services/snapshot_monitor.py +0103fd4d054878e22b450350e72cfd04 simplyblock_core/services/snapshot_replication.py +a8e8f2ac6baaec3a2781cbc774886036 simplyblock_core/services/spdk_http_proxy_server.py +d21aa89347671199e4c4f93db11067e6 simplyblock_core/services/storage_node_monitor.py +f5a0397d343cfc2148eb86192c9e8ebd simplyblock_core/services/tasks_cluster_status.py +659ee98597f9f0f8e9607209a3abebd1 simplyblock_core/services/tasks_runner_backup.py +10a97c1171915bbaa645c649cff88da2 simplyblock_core/services/tasks_runner_backup_merge.py +ed954f1ff465008efb0222c09bb68deb simplyblock_core/services/tasks_runner_batch_migration.py +b4048ca32a99789e6ed76ae16c834985 simplyblock_core/services/tasks_runner_cluster_expand.py +13faa65654c71c55bd8f3a19e34cba15 simplyblock_core/services/tasks_runner_failed_migration.py +4894a2715a481f116ee2f092fe963c8b simplyblock_core/services/tasks_runner_fdb_backup.py +d7f6eb76fb513c62e8a3d5abd5192fa1 simplyblock_core/services/tasks_runner_jc_comp.py +1c1ac93f66e096d52addb6373db50de2 simplyblock_core/services/tasks_runner_lvol_migration.py +9763fc859165deda77a7433f48dc3825 simplyblock_core/services/tasks_runner_migration.py +a45d0d3c7e952a7598ff15c91e90c6af simplyblock_core/services/tasks_runner_new_dev_migration.py +2cd5d1df8cfda73e0eaf1b9e1d96b8b9 simplyblock_core/services/tasks_runner_node_add.py +2b57c422d936551428c28546ce9d7a02 simplyblock_core/services/tasks_runner_node_removal.py +5015f0ed7eb5d465a2e61f0d414a8f19 simplyblock_core/services/tasks_runner_port_allow.py +75434f8a8951e5dabee51f20bd8d8c3a simplyblock_core/services/tasks_runner_replication_final.py +dfafaedfafff3b69802d3f2c9f680390 simplyblock_core/services/tasks_runner_restart.py +6f7c8d54b32d8a3cb60f18eff49a1217 simplyblock_core/services/tasks_runner_sync_lvol_del.py +a3c56c87ab44a01ba71bcb12d778d23e simplyblock_core/settings.py +c0fb4c93ff5b5d32005d63bcdb813d0c simplyblock_core/shell_utils.py +03cda4cb8275cd62cb084da29d8a09af simplyblock_core/snapshot_retention.py +f1bc395dbd5223ae85a7efe1fee9b9cb simplyblock_core/snode_client.py +b12f37e2116565a4801be071c6a2fc10 simplyblock_core/storage_node_ops.py +cc67f35ed45f89fa61d44fc74069898b simplyblock_core/test/conftest.py +4e36ab773d8359225f213cecaf429aab simplyblock_core/test/test_ana_namespace_scope.py +a77e777d1924e7964788ab3e1624b9b3 simplyblock_core/test/test_async_delete_poll.py +93ee14f68659902bb5eb33efabdb7039 simplyblock_core/test/test_backpressure_safety_valve.py +a99d0eb13088fa96da90263687413d29 simplyblock_core/test/test_chain_lock_scope.py +4ebd88bc0d2f073317bebe6398a92023 simplyblock_core/test/test_connect_path_resolution.py +0fdd9f09d496833cbf4450237868243f simplyblock_core/test/test_del_sync_gate_release.py +5b6b39afa4d99b0f3185f7d071178f28 simplyblock_core/test/test_delete_absent_is_done.py +1739de1f19e94c6af8368e0d169be4bf simplyblock_core/test/test_failback_chain_parent.py +b2f945c83409c635074eb2179036e043 simplyblock_core/test/test_failover_clone_race.py +6cd40edbbf823f2d5e8482c3cd39f720 simplyblock_core/test/test_failover_snapshot_selection.py +f202c98493ee5e1e182dd902b40eccb2 simplyblock_core/test/test_failover_target.py +cef1f86b3ef92e3de9a181555edbd587 simplyblock_core/test/test_internal_snapshot_backpressure.py +d394ebf8b8c7e5451817fa4ec07d000d simplyblock_core/test/test_internal_snapshot_scheduler.py +51ab8b315c56ddebb6524922d1f05c03 simplyblock_core/test/test_internal_subsystem_exemption.py +cbc0dd39dd808c076c3b6a0de6e97eea simplyblock_core/test/test_jc_dual_node.py +98a25a8b439113f8730b070e32a68151 simplyblock_core/test/test_lvstore_mutation_lock.py +494ffa54376289a7a3746d003291a785 simplyblock_core/test/test_models.py +468f1f2ee94262aea2a960e1ba79a519 simplyblock_core/test/test_pool_id_or_name.py +8eee2578bf5bffcd5b682212499e8b61 simplyblock_core/test/test_replication_backlog_chain.py +4df442ad73560df92de19c3f02a8499a simplyblock_core/test/test_replication_backlog_destination.py +858ad1c5544632989bf9b3a71ced0019 simplyblock_core/test/test_replication_backlog_reporting.py +87fc1665b2d89826c316e5b9adc6dc06 simplyblock_core/test/test_replication_chain_completeness.py +d6b2259d2fd45d8b672c58dfdcf333d4 simplyblock_core/test/test_replication_chain_predecessor.py +142c27321f4107cb8dcd6f80cebc6d71 simplyblock_core/test/test_replication_commit.py +27c3901c3b17efe712cfeb0441b750ed simplyblock_core/test/test_replication_destination_resolution.py +2aef47c5d33aa862031042362abb4552 simplyblock_core/test/test_replication_failback.py +cbcedc215d05ce9d13e5aa1297da65c4 simplyblock_core/test/test_replication_failback_pool.py +efdcf72be12304a4d6ea97570052b564 simplyblock_core/test/test_replication_final_step.py +ec169938b233d87ed30f5356867b1a02 simplyblock_core/test/test_replication_health.py +428cce0e1f68056e361ec7e7affa4861 simplyblock_core/test/test_replication_leader_follow.py +7fd80e9a3b36f0cbe9f8204d67fccbb7 simplyblock_core/test/test_replication_models.py +674100457e09a93981f8d1bbeca581b4 simplyblock_core/test/test_replication_policies.py +142f851c9f1a973a3b69ad6587ab084f simplyblock_core/test/test_replication_progress.py +c5d56733f8e64b4934a83dfa773dd9e8 simplyblock_core/test/test_replication_relationship_and_delete_source.py +9deb75aea6214d0dc6ad81527332acb2 simplyblock_core/test/test_replication_runner_resilience.py +722b01fc4a5a39f1165d01d365fdccfa simplyblock_core/test/test_replication_source_leader.py +d060d8d46b38d9fca593db7206f0d33a simplyblock_core/test/test_shutdown_task_guard.py +78de0df795baaf32863f4e6916a34967 simplyblock_core/test/test_snapshot_chain_target.py +d6e30188e4eb37f5f54e72c031ca1ccd simplyblock_core/test/test_snapshot_instance_handoff.py +98c1f729b8a91aadcfe5bd15ac5466b3 simplyblock_core/test/test_snapshot_monitor_live_clone.py +7088fd00c8673421cb9d100ec7612b16 simplyblock_core/test/test_snapshot_retention.py +9a95119047fe465944e67c499cd925b3 simplyblock_core/test/test_sync_delete_peer.py +33b8ddf70012155e20620d95b495dfcd simplyblock_core/test/test_tasks_runner_replication_final.py +bc53f292fae277b435d6b1952cdc6daf simplyblock_core/test/test_transfer_hub_heal.py +0c9a3e9dc14315ea372a1f631802c354 simplyblock_core/test/test_utils.py +9e4fa1baf48be1c833f34da643b1a81f simplyblock_core/utils/__init__.py +fc87695c06fb33ca74293dae12f6bb15 simplyblock_core/utils/dial_backoff.py +4d255423494ff9d96f38b2cae3175195 simplyblock_core/utils/helpers.py +70c1bf1532768cc2ff86d061b4b8d17b simplyblock_core/utils/hublvol_reconnect.py +7dcbc009ce235b4df483d38d5b34c13b simplyblock_core/utils/nvme.py +b0b3ac01438e9bbea6b7eeb678f4f52f simplyblock_core/utils/pci.py +9d2effb926da5d0e36be1e86778bbdde simplyblock_core/utils/port_block.py +b45bc54582bfbe1bea24580cf20e8010 simplyblock_core/utils/secrets.py +57af18bc39674a7a16cbb7bc9a8684b5 simplyblock_core/utils/ttl_cache.py +dbf1184940d69c0d5bf7a1f22f000b63 simplyblock_core/workers/cleanup_foundationdb.py diff --git a/scripts/aws_dual_node_outage_soak_multipath.py b/scripts/aws_dual_node_outage_soak_multipath.py index f2b1794df2..6b816bc7f7 100755 --- a/scripts/aws_dual_node_outage_soak_multipath.py +++ b/scripts/aws_dual_node_outage_soak_multipath.py @@ -130,6 +130,18 @@ "fio: pid=", "Killed", "Terminated", + # fio prints verify failures WITHOUT the "fio: " prefix -- the line reads + # "verify: bad magic header a8a4, wanted acca at file ...". Neither + # "fio: verify" nor "verify failed" matches that, so run 20260825_155730 + # corrupted vol6 at 18:53 and vol2 at 19:43 and the soak kept applying + # outages for another two hours, reporting PASS each time, until vol4's + # fio *process* died at 20:49 and the rc-file branch finally caught it. + # A data verification failure is the single most important thing this + # harness can find; it must never again depend on fio also crashing. + "verify: bad", + "bad magic header", + "bad header offset", + "verify: got", ) #: fio stderr markers for a --max_latency violation. Fatal in phase 1 (a #: single-NIC outage must be transparent), counted in phase 2 (promotion @@ -206,6 +218,14 @@ def parse_args(): help="Seconds the NIC stays down on all nodes (default 30).") nic.add_argument("--nic-phase-settle", type=int, default=30, help="Seconds to wait after NIC restore before verifying (default 30).") + nic.add_argument("--no-nic-phase", action="store_true", + help="Disable phase 1 (the all-nodes single-NIC outage) " + "entirely. REQUIRED on a single-data-NIC, non-multipath " + "cluster: with one data NIC per node, phase 1 takes that " + "NIC down on every node at once and isolates the whole " + "cluster instead of exercising path redundancy. Note that " + "--nic-phase-every 0 does NOT disable phase 1 -- it means " + "'once, on iteration 1'.") nic.add_argument("--nic-phase-every", type=int, default=1, help="Run the NIC phase every N iterations. 0 = once, before the " "first pair only (default 1).") @@ -1220,8 +1240,15 @@ def start_fio(self, volumes): if args.fio_max_latency > 0: fio_cmd += f"--max_latency={args.fio_max_latency}s " if args.fio_verify: + # verify_dump writes ..received / .expected on a + # mismatch. Without it a verify failure gives only fio's one + # line, and every corruption so far has died with the returned + # bytes unidentified -- we could not tell stale data from + # parity noise from a neighbouring block, and the volumes are + # on instance store so they vanish when the fleet is stopped. + # The dumps are 4 KiB each and only appear on failure. fio_cmd += (f"--verify={args.fio_verify} --verify_fatal=1 " - f"--verify_backlog=1024 ") + f"--verify_backlog=1024 --verify_dump=1 ") fio_cmd += f"--output={shlex.quote(volume['fio_log'])}" start_script = ( @@ -1602,6 +1629,12 @@ def _rpc_json(subcmd, what): # node passive. active_passive means one NIC carries all hub IO. to_check = [(b, "hublvol") for b in hublvol_bdevs] to_check += [(b, "remote") for b in remote_bdevs[:max(0, self.args.policy_sample)]] + # Only meaningful with more than one path: on a single-data-NIC + # (non-multipath) cluster a bdev legitimately reports active_passive, + # and asserting active_active there would make every bdev a "problem", + # so the heal gate could never converge. + if len(self.args.data_nics) < 2: + to_check = [] for bdev_name, kind in to_check: try: bdevs = _rpc_json(f"bdev_get_bdevs -b {shlex.quote(bdev_name)}", @@ -2152,8 +2185,13 @@ def run(self): + ", ".join(f"{n['uuid'][:12]}:{n['status']}" for n in current)) uuids = [n["uuid"] for n in current] - nic_due = (iteration == 1 if args.nic_phase_every == 0 - else (iteration - 1) % args.nic_phase_every == 0) + # --nic-phase-every 0 means "once, on iteration 1"; only + # --no-nic-phase disables phase 1 outright, which is required on a + # single-data-NIC cluster where taking "one" NIC down on every node + # isolates the whole cluster instead of testing path redundancy. + nic_due = ((not args.no_nic_phase) + and (iteration == 1 if args.nic_phase_every == 0 + else (iteration - 1) % args.nic_phase_every == 0)) if nic_due: nic = args.data_nics[(iteration - 1) % len(args.data_nics)] if self.run_nic_phase(iteration, nic, uuids): diff --git a/scripts/deploy_gate_and_soak.py b/scripts/deploy_gate_and_soak.py index c41a32dd06..24dd2943ae 100644 --- a/scripts/deploy_gate_and_soak.py +++ b/scripts/deploy_gate_and_soak.py @@ -32,7 +32,16 @@ #: mtime: the ultra Dockerfile bakes `git log` of the spdk repo into #: /root/spdk/git_log.txt, which pins the image to a commit. See the SPDK_IMAGE #: pin in setup_perf_test_multipath.py for why this is checked, not assumed. -EXPECT_SPDK_COMMIT = "a311a6852" +EXPECT_SPDK_COMMIT = "554c80f11" +#: The ultra commit that must be running. b44de698 defers the JC leadership +#: signal until the parity-desynchronisation check completes. Before it, a +#: reactively promoted distrib announced leadership while parity was still +#: desynchronised, so reads served in that window came off desynchronised +#: parity and returned arbitrary bytes -- the 2026-08-24 iteration-12 "bad +#: magic" failures, whose received magics were random rather than fio's +#: 0xacca, ruling out stale-but-valid data. Without this pin the run would +#: re-test the build that already failed. +EXPECT_ULTRA_COMMIT = "b44de698" #: Lines probe_bin.sh must print, with what their absence would mean. These #: distinguish upstream d528e1a67 (zeroes retry state at the submission entry #: point) from the superseded local attempt that zeroed it at completion and so @@ -123,6 +132,11 @@ def gate(mgmt, sn): f"— /root/spdk/git_log.txt names a different commit, so the node is " f"on a stale image. Check the SPDK_IMAGE pin and that the " f"spdk-core R26.3 tags finished rebuilding (amd64 included).") + if EXPECT_ULTRA_COMMIT not in out: + raise RuntimeError( + f"GATE FAILED: image was not built from ultra " + f"{EXPECT_ULTRA_COMMIT} — the parity-desync leadership fix under " + f"test is absent, so the run would prove nothing.") for fragment, why in REQUIRED_FIX_LINES: if fragment not in out: raise RuntimeError(f"GATE FAILED: {why} ({fragment!r} absent)") diff --git a/scripts/hotfix_repl_lab.py b/scripts/hotfix_repl_lab.py index c8d3f75f7f..bc00e73589 100644 --- a/scripts/hotfix_repl_lab.py +++ b/scripts/hotfix_repl_lab.py @@ -21,6 +21,7 @@ Usage: python hotfix_repl_lab.py [--verify-only] """ import json +import shlex import subprocess import sys import time @@ -61,6 +62,12 @@ # dependency alongside the file that needs it. "simplyblock_core/controllers/ops_gate.py": "controllers/ops_gate.py", "simplyblock_core/controllers/tasks_controller.py": "controllers/tasks_controller.py", + # The JC dual-node flag lives here: apply_jc_dual_node() is called on + # node add AND on every bring-up, and case 6 restarts a node mid-run -- + # a restarted node comes back with the flag off unless the CP re-applies + # it, which is exactly when the survivor would abort. + "simplyblock_core/storage_node_ops.py": "storage_node_ops.py", + "simplyblock_core/rpc_client.py": "rpc_client.py", } #: service -> its own module (mounted on top of the shared set) SERVICES = { @@ -71,7 +78,8 @@ "app_TasksRunnerReplicationFinal": { "simplyblock_core/services/tasks_runner_replication_final.py": "services/tasks_runner_replication_final.py"}, - "app_LVolMonitor": {}, + "app_LVolMonitor": { + "simplyblock_core/services/lvol_monitor.py": "services/lvol_monitor.py"}, } #: also refreshed on the mgmt HOST: the harness runs `sudo python3 -c #: "...lvol_controller.get_replication_info..."` there, and sbctl imports it. @@ -79,6 +87,56 @@ HOST_FILES.update({p: v for s in SERVICES.values() for p, v in s.items()}) HOST_CLI = "simplyblock_cli/clibase.py" +CRLF, LF = bytes([13, 10]), bytes([10]) + +#: filled in by discover_drift() at run time: repo path -> path under +#: simplyblock_core, for every module that differs from the deployed image +DRIFT = {} + + +def discover_drift(mgmt): + """Every simplyblock_core module that differs from the one in the IMAGE. + + A hand-maintained mount list is wrong by construction: it must be extended + for every new module a fix happens to touch. Forgetting one fails loudly + (ImportError: cannot import name 'ConsistencyGroup' -- the service + crash-looping, lab 2026-08-26) or, worse, quietly: the fix mounted while a + module it depends on stays at the image's version. Ask the image what it + has and mount everything that does not match. + """ + import hashlib + ref = ssh(mgmt, "sudo docker ps --format '{{.Names}}' | grep -m1 " + "TasksRunnerReplicationFinal", check=False).strip() + if not ref: + log("no reference container; falling back to the static mount list") + return {} + probe = ("cd /usr/local/lib/python3.12/site-packages && " + 'find simplyblock_core -name "*.py" | sort | xargs md5sum') + listing = ssh(mgmt, f"sudo docker exec {ref} sh -c {shlex.quote(probe)}", + check=False) + image = {} + for line in listing.splitlines(): + parts = line.split() + if len(parts) == 2 and parts[1].endswith(".py"): + image[parts[1]] = parts[0] + if not image: + log("could not read the image's module hashes; using the static list") + return {} + + drift, core = {}, REPO / "simplyblock_core" + for path in core.rglob("*.py"): + rel = path.relative_to(REPO).as_posix() + if "__pycache__" in rel or rel.startswith("simplyblock_core/test"): + continue + raw = path.read_bytes().replace(CRLF, LF) # image files are LF + digest = hashlib.md5(raw).hexdigest() + if image.get(rel) != digest: + drift[rel] = path.relative_to(core).as_posix() + log(f"image drift: {len(drift)} module(s) differ from the deployed image") + for rel in sorted(drift): + log(f" {rel}{'' if rel in image else ' (new)'}") + return drift + def log(msg): print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) @@ -98,6 +156,11 @@ def ssh(host, cmd, timeout=600, check=True): def stage(mgmt): log(f"staging {len(HOST_FILES) + 1} files in {HOTFIX_DIR}") + names = [Path(x).name for x in HOST_FILES] + dupes = sorted({n for n in names if names.count(n) > 1}) + if dupes: + raise RuntimeError(f"modules share a basename and would collide in the " + f"flat staging dir: {dupes}") ssh(mgmt, f"sudo mkdir -p {HOTFIX_DIR}/backup && sudo chown ec2-user {HOTFIX_DIR}") locals_ = [str(REPO / p) for p in HOST_FILES] + [str(REPO / HOST_CLI)] run(["scp", *SSH_OPTS, *locals_, f"ec2-user@{mgmt}:{HOTFIX_DIR}/"], timeout=900) @@ -168,10 +231,17 @@ def verify(mgmt): def main(): + global DRIFT, HOST_FILES meta = json.loads((HERE / "cluster_metadata_repl.json").read_text()) mgmt = meta["mgmt"]["public_ip"] log(f"mgmt={mgmt}") if "--verify-only" not in sys.argv: + # Mount everything that differs from the image, not a hand-kept list. + DRIFT = discover_drift(mgmt) + SHARED.update(DRIFT) + HOST_FILES = dict(SHARED) + HOST_FILES.update({p: v for sv in SERVICES.values() + for p, v in sv.items()}) stage(mgmt) mount_services(mgmt) patch_host(mgmt) diff --git a/scripts/repl_soak.py b/scripts/repl_soak.py new file mode 100755 index 0000000000..b5286ec82f --- /dev/null +++ b/scripts/repl_soak.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +"""One-liner for the async-replication soak: deploy a lab from a branch, run cases, report. + + python scripts/repl_soak.py --cases case7,case9 --env CHAOS_EVENTS=100 + python scripts/repl_soak.py --cases case9 --env CHAOS_EVENTS=100 --teardown on-pass + python scripts/repl_soak.py --cases case10,case11 --branch main --spdk-image public.ecr.aws/simply-block/ultra:main-latest-amd64 + python scripts/repl_soak.py --cases case3 --reuse-lab # existing lab, no redeploy + +What --branch means: the sbcli checkout that is (1) pip-installed on every +lab node by the deployer, (2) the source of the control-plane hotfix mounted +over the running services, and (3) the copy of the test driver that is staged +to the management node. So a case runs against exactly that branch's code, +whatever the pre-built CP image lags behind. The SPDK image is independent +(--spdk-image, default: the digest pinned in setup_repl_test_2clusters.py); +--cp-image overrides the control-plane image (SIMPLYBLOCK_DOCKER_IMAGE). + +Steps: clone/refresh the branch -> deploy (unless --reuse-lab) -> hotfix +(unless --no-hotfix) -> stage + run the cases -> wait for the driver -> +print the SUMMARY -> optional teardown. Exit code 0 only when every case +passed. +""" +import argparse +import json +import os +import re +import shlex +import subprocess +import sys +import time +from pathlib import Path + +KEY = os.environ.get("REPL_KEY", "C:/Users/Michael/.ssh/mtes01.pem") +REPO_URL = "github.com/simplyblock/sbcli.git" +WORK_ROOT = Path(os.environ.get("REPL_SOAK_ROOT", + Path(__file__).resolve().parent.parent.parent / "soak-labs")) +SSH_OPTS = ["-o", "StrictHostKeyChecking=no", "-o", "LogLevel=ERROR", + "-o", "ConnectTimeout=30", "-i", KEY] + + +# Any credential that reaches a log line is leaked: these logs are pasted into +# tickets and chat, and the lab copies keep them on disk for the life of the +# instance. The fetch URL carries a GitHub token, so redact before printing +# rather than trusting each call site to remember. +_SECRET_RE = re.compile(r"(x-access-token:)[^@\s]+(@)") + + +def redact(msg): + return _SECRET_RE.sub(r"\1***\2", str(msg)) + + +def log(msg): + print(f"[{time.strftime('%H:%M:%S')}] {redact(msg)}", flush=True) + + +def sh(cmd, cwd=None, env=None, check=True, capture=False): + log("$ " + (cmd if isinstance(cmd, str) else " ".join(shlex.quote(c) for c in cmd))) + r = subprocess.run(cmd, cwd=cwd, env=env, shell=isinstance(cmd, str), + text=True, capture_output=capture) + if check and r.returncode != 0: + if capture: + print(redact(r.stdout[-2000:]), redact(r.stderr[-2000:])) + raise SystemExit(f"step failed (rc={r.returncode}): {redact(cmd)}") + return r + + +def gh_token(): + r = subprocess.run(["C:/Users/Michael/.local/bin/gh.exe", "auth", "token"], + text=True, capture_output=True, check=True) + return r.stdout.strip() + + +def ssh(mgmt, remote_cmd, check=True): + r = subprocess.run(["ssh", *SSH_OPTS, f"ec2-user@{mgmt}", remote_cmd], + text=True, capture_output=True) + if check and r.returncode != 0: + raise SystemExit(f"ssh failed on {mgmt}: {r.stderr.strip()[-500:]}") + return r.stdout + + +def checkout(branch): + """A dedicated clone per branch under WORK_ROOT (never the developer's tree).""" + WORK_ROOT.mkdir(parents=True, exist_ok=True) + dest = WORK_ROOT / f"sbcli-{re.sub(r'[^A-Za-z0-9._-]', '_', branch)}" + url = f"https://x-access-token:{gh_token()}@{REPO_URL}" + if not (dest / ".git").exists(): + sh(["git", "clone", "-q", "--branch", branch, url, str(dest)]) + else: + sh(["git", "-C", str(dest), "fetch", "-q", url, branch]) + sh(["git", "-C", str(dest), "checkout", "-q", "-B", branch, "FETCH_HEAD"]) + head = sh(["git", "-C", str(dest), "log", "--oneline", "-1"], capture=True).stdout.strip() + log(f"branch {branch} @ {head}") + return dest + + +def load_meta(scripts_dir): + p = scripts_dir / "cluster_metadata_repl.json" + if not p.exists(): + raise SystemExit(f"no lab metadata at {p} (deploy first, or drop --reuse-lab)") + return json.loads(p.read_text()) + + +def lab_is_reachable(mgmt, timeout=20): + r = subprocess.run(["ssh", "-o", "StrictHostKeyChecking=no", "-o", "LogLevel=ERROR", + "-o", f"ConnectTimeout={timeout}", "-i", KEY, + f"ec2-user@{mgmt}", "true"], capture_output=True, text=True) + return r.returncode == 0 + + +def require_reachable(mgmt, reused): + """A stale metadata file is the normal case, not the exception: the repo + TRACKS scripts/cluster_metadata_repl.json, so every fresh clone arrives + carrying whichever lab was alive when it was last committed. Probe before + doing anything that would otherwise fail deep inside the hotfix.""" + if lab_is_reachable(mgmt): + return + hint = ("that metadata is the copy committed in the repo, pointing at a lab " + "that no longer exists -- drop --reuse-lab to deploy a fresh one" + if reused else + "the deploy reported success but the management node is unreachable") + raise SystemExit(f"management node {mgmt} is not reachable: {hint}") + + +def deploy(scripts_dir, branch, spdk_image, cp_image): + env = dict(os.environ, SBCLI_BRANCH=branch) + if spdk_image: + env["SPDK_IMAGE"] = spdk_image + if cp_image: + env["SIMPLYBLOCK_DOCKER_IMAGE"] = cp_image + sh([sys.executable, "-u", "setup_repl_test_2clusters.py", "-d"], cwd=scripts_dir, env=env) + + +def hotfix(scripts_dir): + sh([sys.executable, "hotfix_repl_lab.py"], cwd=scripts_dir) + + +def run_cases(scripts_dir, cases, env_kv): + sh([sys.executable, "stage_and_run_repl_cases.py", cases, *env_kv], cwd=scripts_dir) + + +def wait_for_driver(mgmt, poll=120): + """Follow the remote driver to === DONE ===, then return (summary, passed).""" + logfile = "" + for _ in range(20): + logfile = ssh(mgmt, "cat ~/repl_log 2>/dev/null", check=False).strip() + if logfile: + break + time.sleep(15) + if not logfile: + raise SystemExit("driver never registered a log file on the management node") + log(f"following {logfile}") + while True: + out = ssh(mgmt, + f"if grep -q '=== DONE ===' {logfile}; then echo FINISHED; " + f"grep -A12 '=== SUMMARY ===' {logfile}; " + f"elif ! pgrep -f '[t]est_async_replication.py' >/dev/null; then echo DIED; " + f"tail -15 {logfile}; else echo RUNNING; " + f"grep -E '^==========' {logfile} | tail -1; fi", check=False) + if out.startswith("FINISHED"): + summary = out.split("\n", 1)[1] + passed = ("FAIL" not in summary) and ("PASS" in summary) + return summary, passed + if out.startswith("DIED"): + return out, False + log("running: " + (out.split("\n", 1)[1].strip() if "\n" in out else "...")) + time.sleep(poll) + + +def teardown(meta): + ips = [meta["mgmt"]["private_ip"]] + [c["private_ip"] for c in meta.get("clients", [])] + for cl in meta["clusters"].values(): + ips += cl["storage_private_ips"] + r = sh(["aws", "ec2", "describe-instances", "--filters", + f"Name=private-ip-address,Values={','.join(ips)}", + "Name=instance-state-name,Values=running,stopped,pending", + "--query", "Reservations[].Instances[].InstanceId", "--output", "text"], + capture=True) + ids = r.stdout.split() + if ids: + sh(["aws", "ec2", "terminate-instances", "--instance-ids", *ids], capture=True) + log(f"terminated {len(ids)} lab instances") + else: + log("no lab instances found to terminate") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--cases", required=True, nargs="+", metavar="CASE", + help="cases or a group name: 'case6,case7', 'case6 case7', " + "'case6, case7' and 'all9' all work") + ap.add_argument("--branch", default="replication-features", + help="sbcli branch to deploy/hotfix/stage (default replication-features)") + ap.add_argument("--spdk-image", default="", help="ultra image ref; default = digest pinned in the deployer") + ap.add_argument("--cp-image", default="", help="control-plane docker image (SIMPLYBLOCK_DOCKER_IMAGE)") + ap.add_argument("--env", action="append", default=[], metavar="KEY=VAL", + help="driver knobs, e.g. CHAOS_EVENTS=100, CASE11_RUNTIME_MIN=30 (repeatable)") + ap.add_argument("--reuse-lab", action="store_true", help="skip deploy; use the branch clone's existing metadata") + ap.add_argument("--no-hotfix", action="store_true", help="skip mounting the branch's python over the CP services") + ap.add_argument("--teardown", choices=["never", "on-pass", "always"], default="never") + args = ap.parse_args() + # Accept every shape a shell hands us: "a,b", "a, b" (the space makes the + # shell pass two argv entries) and "a b" all mean the same list. + args.cases = ",".join(c for tok in args.cases for c in tok.split(",") if c) + if not args.cases: + ap.error("--cases got no case names") + for kv in args.env: + if "=" not in kv: + ap.error(f"--env expects KEY=VAL, got {kv!r}") + + clone = checkout(args.branch) + scripts_dir = clone / "scripts" + + if not args.reuse_lab: + deploy(scripts_dir, args.branch, args.spdk_image, args.cp_image) + meta = load_meta(scripts_dir) + mgmt = meta["mgmt"]["public_ip"] + shape = ", ".join("{}={}({}n)".format(k, v["cluster_uuid"][:8], v["nodes"]) + for k, v in meta["clusters"].items()) + log(f"lab mgmt {mgmt}: {shape}") + require_reachable(mgmt, args.reuse_lab) + + if not args.no_hotfix: + hotfix(scripts_dir) + + run_cases(scripts_dir, args.cases, args.env) + summary, passed = wait_for_driver(mgmt) + print("\n" + summary.strip() + "\n") + log("RESULT: " + ("ALL PASSED" if passed else "FAILED")) + + if args.teardown == "always" or (args.teardown == "on-pass" and passed): + teardown(meta) + elif not passed: + log(f"lab kept for diagnosis: ssh -i {KEY} ec2-user@{mgmt} (log: $(cat ~/repl_log))") + raise SystemExit(0 if passed else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/setup_lab_perf_test1.py b/scripts/setup_lab_perf_test1.py index 305d836fd9..1fd47806af 100644 --- a/scripts/setup_lab_perf_test1.py +++ b/scripts/setup_lab_perf_test1.py @@ -42,7 +42,7 @@ IFACE = "eth0" DATA_IFACE = "eth1" BRANCH = "inline-checksum-validation" -MAX_LVOL = "100" +MAX_LVOL = "75" # capped by constants.MAX_SUBSYSTEMS_PER_NODE (75): above it, values were silently clamped at placement time, so a node reserved huge pages for subsystems it could never serve. cluster create rejects it since 8bcedce79. # Same volume plan layout as the AWS variant; consumed by downstream perf tooling. VOLUME_PLAN = [ diff --git a/scripts/setup_perf_test1.py b/scripts/setup_perf_test1.py index 37cd1f03fe..3aea191e65 100644 --- a/scripts/setup_perf_test1.py +++ b/scripts/setup_perf_test1.py @@ -5,6 +5,18 @@ import select import time from concurrent.futures import ThreadPoolExecutor +import sys + +# Streamed remote stderr is arbitrary UTF-8 -- systemctl alone prints +# "Created symlink ... -> ..." with U+2192. On Windows a redirected stdout +# defaults to cp1252, so the first such byte killed a 6-node deployment at +# phase 2a with UnicodeEncodeError (2026-08-26). Never let log formatting +# abort a deployment. +for _stream in (sys.stdout, sys.stderr): + try: + _stream.reconfigure(encoding="utf-8", errors="replace") + except Exception: + pass import boto3 # noqa: E402 (must follow the stdout reconfigure above) import paramiko # noqa: E402 (must follow the stdout reconfigure above) @@ -20,12 +32,18 @@ AZ = "us-east-1a" SG_NAME = "default" BRANCH = "main" -MAX_LVOL = "100" +MAX_LVOL = "75" # capped by constants.MAX_SUBSYSTEMS_PER_NODE (75): above it, values were silently clamped at placement time, so a node reserved huge pages for subsystems it could never serve. cluster create rejects it since 8bcedce79. # --- Manual Network Config --- # Replace this with your actual Subnet ID (e.g., "subnet-0593459d6b931ee4c") SUBNET_ID = "subnet-0593459d6b931ee4c" STORAGE_SG_ID = "sg-02e89a1372e9f39e9" SN_TYPE = "i3en.2xlarge" +#: Pin the SPDK/ultra image so a run is reproducible and comparable against +#: another run. Without it, sn add-node takes the control-plane default, which +#: drifts as main is rebuilt -- useless when the whole point of a run is to +#: hold the build constant. Empty string = control-plane default. +SPDK_IMAGE = "public.ecr.aws/simply-block/ultra:main-d91ff03a-amd64" + SN_COUNT = 6 MGMT_TYPE = "m6i.2xlarge" # --- Selectable Client Specification --- @@ -37,7 +55,7 @@ USER = "ec2-user" AZ = "us-east-1a" IFACE = "eth0" -MAX_LVOL = "100" +MAX_LVOL = "75" VOLUME_PLAN = [ {"idx": 0, "node_idx": 0, "qty": 5, "size": "100G", "client": "client1", "io_queues": 12}, @@ -372,6 +390,63 @@ def close(self): +#: Soak wiring. The deployment is only useful if the soak actually starts, and +#: getting the mgmt node ready by hand is where runs get lost: the soak runs ON +#: mgmt (--run-on-mgmt) and SSHes to the nodes and client, so it needs the key, +#: the metadata this script writes, and its own source there. +SOAK_SCRIPT = "aws_dual_node_outage_soak_multipath.py" +SOAK_LAUNCHER = "start_soak_base.sh" +SOAK_EXTRA_FILES = ("collect_logs.py",) +METADATA_FILE = "cluster_metadata_base.json" + + +def stage_soak(mgmt_ip): + """Put the soak, its launcher, the ssh key and the metadata on mgmt.""" + here = os.path.dirname(os.path.abspath(__file__)) + payload = [os.path.join(here, f) for f in + (SOAK_SCRIPT, SOAK_LAUNCHER) + SOAK_EXTRA_FILES] + payload.append(os.path.abspath(METADATA_FILE)) + missing = [f for f in payload if not os.path.isfile(f)] + if missing: + raise RuntimeError(f"cannot stage soak, missing files: {missing}") + + print(f"\n--- Phase 7: Stage soak on mgmt {mgmt_ip} ---") + ssh_exec(mgmt_ip, ["mkdir -p ~/.ssh"], check=True) + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(mgmt_ip, username="ec2-user", key_filename=KEY_PATH, + allow_agent=False, look_for_keys=False) + try: + sftp = ssh.open_sftp() + try: + for src in payload: + dst = "/home/ec2-user/" + os.path.basename(src) + sftp.put(src, dst) + print(f" staged {os.path.basename(src)}") + # The soak reaches the storage nodes and the client from mgmt, so + # the private key has to travel with it. + sftp.put(KEY_PATH, "/home/ec2-user/.ssh/mtes01.pem") + print(" staged ssh key") + finally: + sftp.close() + finally: + ssh.close() + # The launcher is written with LF endings on purpose; a CRLF shell script + # fails on the node with a bare-CR "set" option error. + ssh_exec(mgmt_ip, [ + "chmod 600 ~/.ssh/mtes01.pem", + f"chmod +x ~/{SOAK_LAUNCHER}", + f"sed -i 's/\r$//' ~/{SOAK_LAUNCHER}", + ], check=True) + + +def start_soak(mgmt_ip, env_prefix=""): + print(f"\n--- Phase 8: Start soak on mgmt {mgmt_ip} ---") + out = ssh_exec(mgmt_ip, [f"{env_prefix}bash ~/{SOAK_LAUNCHER}"], + get_output=True, check=True)[0] + print(out) + return out + def main(): @@ -486,7 +561,8 @@ def add_one_node(priv_ip): for attempt in range(5): try: ssh_exec(mgmt_ip, [ - f"sudo /usr/local/bin/sbctl -d sn add-node {cluster_uuid} {priv_ip}:5000 {IFACE} --ha-jm-count 4" + f"sudo /usr/local/bin/sbctl -d --dev sn add-node {cluster_uuid} {priv_ip}:5000 {IFACE} --ha-jm-count 4" + + (f" --spdk-image {SPDK_IMAGE}" if SPDK_IMAGE else "") ], check=True) return except RuntimeError: @@ -592,6 +668,34 @@ def add_one_node(priv_ip): print("\n--- Setup Complete ---") print(f"Cluster {cluster_uuid} is active. Metadata saved.") + # A deployment that stops here needs a second, hand-assembled step before + # it produces any data, and that is where runs get lost. Stage the soak and + # start it, unless explicitly told not to. + if "--no-soak" in sys.argv: + stage_soak(mgmt_ip) + print("--no-soak: files staged, soak NOT started. Start it with:") + print(f" ssh -i {KEY_PATH} ec2-user@{mgmt_ip} " + f"\"PLACEMENT_DUMPS=1 bash ~/{SOAK_LAUNCHER}\"") + return + + stage_soak(mgmt_ip) + env = "PLACEMENT_DUMPS=1 " + for flag, var in (("--iterations", "ITERATIONS"), + ("--runtime", "RUNTIME"), + ("--restart-timeout", "RESTART_TIMEOUT"), + ("--start-iteration", "START_ITERATION")): + if flag in sys.argv: + env += f"{var}={sys.argv[sys.argv.index(flag) + 1]} " + start_soak(mgmt_ip, env_prefix=env) + print() + print("Soak running on mgmt. Follow it with:") + print(f" ssh -i {KEY_PATH} ec2-user@{mgmt_ip} " + "\"tail -f ~/soak_base_$(cat ~/soak_ts).out\"") + print("Check verdicts / faults with:") + print(f" ssh -i {KEY_PATH} ec2-user@{mgmt_ip} " + "\"grep -E '####|PASS|FAIL|ERROR|verify:' " + "~/soak_base_$(cat ~/soak_ts).out | tail -20\"") + diff --git a/scripts/setup_perf_test_3node.py b/scripts/setup_perf_test_3node.py index 20eddb4621..b80f1550ba 100644 --- a/scripts/setup_perf_test_3node.py +++ b/scripts/setup_perf_test_3node.py @@ -22,7 +22,7 @@ AZ = "us-east-1a" SG_NAME = "default" BRANCH = "main" -MAX_LVOL = "100" +MAX_LVOL = "75" # capped by constants.MAX_SUBSYSTEMS_PER_NODE (75): above it, values were silently clamped at placement time, so a node reserved huge pages for subsystems it could never serve. cluster create rejects it since 8bcedce79. # --- Manual Network Config --- SUBNET_ID = "subnet-0593459d6b931ee4c" STORAGE_SG_ID = "sg-02e89a1372e9f39e9" @@ -45,7 +45,7 @@ USER = "ec2-user" AZ = "us-east-1a" IFACE = "eth0" -MAX_LVOL = "100" +MAX_LVOL = "75" # --- Helper: Management Node with 30GB Root --- diff --git a/scripts/setup_perf_test_failure_domain.py b/scripts/setup_perf_test_failure_domain.py index 5827e270dd..136b73aa0e 100644 --- a/scripts/setup_perf_test_failure_domain.py +++ b/scripts/setup_perf_test_failure_domain.py @@ -22,7 +22,7 @@ # --failure-domain) lives on this branch. Installing from it also pulls the # matching SPDK ultra image pinned in simplyblock_core/env_var. BRANCH = "main" -MAX_LVOL = "100" +MAX_LVOL = "75" # capped by constants.MAX_SUBSYSTEMS_PER_NODE (75): above it, values were silently clamped at placement time, so a node reserved huge pages for subsystems it could never serve. cluster create rejects it since 8bcedce79. # --- Manual Network Config --- # Replace this with your actual Subnet ID (e.g., "subnet-0593459d6b931ee4c") SUBNET_ID = "subnet-0593459d6b931ee4c" @@ -82,7 +82,7 @@ USER = "ec2-user" AZ = "us-east-1a" IFACE = "eth0" -MAX_LVOL = "100" +MAX_LVOL = "75" VOLUME_PLAN = [ {"idx": 0, "node_idx": 0, "qty": 5, "size": "100G", "client": "client1", "io_queues": 12}, diff --git a/scripts/setup_perf_test_multipath.py b/scripts/setup_perf_test_multipath.py index 6cce555a85..1b41abc6e5 100644 --- a/scripts/setup_perf_test_multipath.py +++ b/scripts/setup_perf_test_multipath.py @@ -87,7 +87,7 @@ #: itself, so a stale pin fails loudly instead of running old code quietly. #: 8d2e5215 = ultra main "Build main on the R26.3 spdk-core base", built on #: spdk-core R26.3 a311a6852 which carries upstream d528e1a67 (spdk/spdk#3686). -SPDK_IMAGE = "public.ecr.aws/simply-block/ultra:main-8d2e5215-amd64" +SPDK_IMAGE = "public.ecr.aws/simply-block/ultra:main-b44de698-amd64" USER = "ec2-user" MGMT_IFACE = "eth0" DATA_NICS = ["eth1", "eth2"] # Names the OS assigns to ENI index 1, 2 diff --git a/scripts/setup_repl_test_2clusters.py b/scripts/setup_repl_test_2clusters.py index 91b99b26a1..5c8436cd96 100644 --- a/scripts/setup_repl_test_2clusters.py +++ b/scripts/setup_repl_test_2clusters.py @@ -19,6 +19,7 @@ """ import os import json +import sys import re import time from concurrent.futures import ThreadPoolExecutor @@ -39,12 +40,18 @@ # tasks_runner_replication_final, replication_final_step, cluster # add-replication), so main is what we test; the default # SIMPLY_BLOCK_DOCKER_IMAGE (simplyblock/simplyblock:main) matches it. -BRANCH = "main" +# Overridable for the repl_soak.py one-liner: SBCLI_BRANCH selects the sbcli +# checkout installed on every node (and the hotfix source), SPDK_IMAGE the +# pinned ultra image, SIMPLYBLOCK_DOCKER_IMAGE the control-plane image. +BRANCH = os.environ.get("SBCLI_BRANCH", "replication-features") SN_TYPE = "i3en.2xlarge" MGMT_TYPE = "m6i.2xlarge" CLIENT_TYPE = "m6in.8xlarge" -CLIENT_COUNT = 1 # client(s) used by the test process +CLIENT_COUNT = 2 # client(s) used by the test process +#: 2, not 1: case 7 spreads 20 namespaced volumes across at least two +#: clients, and growing an existing lab with `add_client` after the fact +#: is an extra manual step before every namespaced run. USER = "ec2-user" IFACE = "eth0" @@ -56,15 +63,18 @@ # The FIRST cluster (bootstrap=True) is created with `cluster create`; every # other cluster is attached to the same CP with `cluster add`. CLUSTERS = [ + # THREE nodes per cluster, everywhere. Two-node clusters are not a + # supported configuration (product minimum is 3), and the 2026-08-24/25 + # campaign showed exactly why: with one node down the survivor holds 1 of + # 2 journal members and the JC aborts it, and the restart rebalance has no + # third failure domain to place into, so its device_migration loops on + # "no allowed placement" forever and pins the cluster in REBALANCING. { - "name": "src", # 1+1 HA pair - "nodes": 2, + "name": "src", + "nodes": 3, "ndcs": 1, # data-chunks-per-stripe "npcs": 1, # parity-chunks-per-stripe (FT=1) - # None => let the CP resolve the required count (3 for FT=1). An explicit - # 2 is now rejected by resolve_ha_jm_count(); a 2-node cluster simply - # ends up with the 2 host-disjoint journals it can place. - "ha_jm_count": None, + "ha_jm_count": 3, "bootstrap": True, # `cluster create` "pool": "pool_src", }, @@ -83,10 +93,10 @@ # only run cases 1-3/5/6. { "name": "fresh", - "nodes": 2, + "nodes": 3, "ndcs": 1, "npcs": 1, - "ha_jm_count": None, + "ha_jm_count": 3, "bootstrap": False, "pool": "pool_fresh", }, @@ -100,9 +110,17 @@ # parallel reads, spdk R26.3 bdd97c1d8/ce876a169) exist only from the # 2026-08-22 build onward, and ultra:main-latest's manifest list has a live # race that leaves its amd64 entry pointing at the PREVIOUS build (observed -# 2026-08-17, -21 and -22). This digest = main-d89a595c-amd64, 2026-08-22. -SPDK_IMAGE = ("public.ecr.aws/simply-block/ultra@" - "sha256:8a007dca5bf6a1fa89cc96945f43164539cac65f0cdafa56d5ee44961e9902af") +# 2026-08-17, -21 and -22). This digest = main-d91ff03a-amd64, 2026-08-25: +# the first ultra build FROM spdk-core:master-latest (spdk master = R26.3 +# merged + the ANA-transition change reverted). NOTE the digest printed in +# the CI push log is docker.io's; ECR's differs -- resolve it against +# public.ecr.aws (docker manifest inspect -v ). Previous pin -- +# the first build carrying the promotion-window ANA-transition fix +# (spdk R26.3 554c80f11), verified built FROM spdk-core:R26.3-latest +# whose manifest was created 18:41:57, before this ultra build started. +SPDK_IMAGE = os.environ.get( + "SPDK_IMAGE", + "public.ecr.aws/simply-block/ultra@sha256:d929b4d7ececee0fa0e1ad5973f87fa4e4cf79f7079cdf454bfbb27e2df51cb6") SN_COUNT = sum(c["nodes"] for c in CLUSTERS) SBCTL = "sudo /usr/local/bin/sbctl" @@ -368,7 +386,38 @@ def add_one(priv_ip): # --------------------------------------------------------------------------- # +CLIENT_PREP_CMDS = [ + "sudo dnf install nvme-cli fio -y", + "sudo modprobe nvme-tcp", + "echo 'nvme-tcp' | sudo tee /etc/modules-load.d/nvme-tcp.conf", +] + + +def add_client(): + """Add one client instance to an EXISTING deployment (case 7 needs >= 2 + clients; redeploying a healthy two-cluster lab for that is wasteful).""" + with open("cluster_metadata_repl.json") as f: + metadata = json.load(f) + print("Launching 1 additional client...") + clients = launch_instances("SB-Repl-Client", CLIENT_TYPE, 1) + for inst in clients: + inst.wait_until_running() + inst.reload() + ip = clients[0].public_ip_address + wait_for_ssh(ip) + print(f"Prepping client {ip}...") + ssh_exec(ip, CLIENT_PREP_CMDS, check=True) + metadata.setdefault("clients", []).append( + {"public_ip": ip, "private_ip": clients[0].private_ip_address}) + with open("cluster_metadata_repl.json", "w") as f: + json.dump(metadata, f, indent=4) + print(f"Client added: {ip} ({len(metadata['clients'])} clients in metadata).") + + def main(): + if len(sys.argv) > 1 and sys.argv[1] == "add_client": + add_client() + return print(f"Launching control plane + {SN_COUNT} storage nodes + {CLIENT_COUNT} client(s)...") mgmt = launch_instances("SB-Repl-Mgmt", MGMT_TYPE, 1, with_net=False) sns = launch_instances("SB-Repl-Storage", SN_TYPE, SN_COUNT) @@ -487,11 +536,7 @@ def main(): # --- Phase 6: prep clients --- if client_pub_ips: print("Prepping clients...") - client_cmds = [ - "sudo dnf install nvme-cli fio -y", - "sudo modprobe nvme-tcp", - "echo 'nvme-tcp' | sudo tee /etc/modules-load.d/nvme-tcp.conf", - ] + client_cmds = CLIENT_PREP_CMDS for ip in client_pub_ips: wait_for_ssh(ip) with ThreadPoolExecutor(max_workers=max(1, len(client_pub_ips))) as ex: diff --git a/scripts/stage_and_run_repl_cases.py b/scripts/stage_and_run_repl_cases.py index ccbe485ae0..b0c5980065 100644 --- a/scripts/stage_and_run_repl_cases.py +++ b/scripts/stage_and_run_repl_cases.py @@ -56,9 +56,14 @@ def remote_metadata(meta): def main(): cases = sys.argv[1] if len(sys.argv) > 1 else "all_c3_last" + # Any further NAME=VALUE arguments are exported for the remote driver, so + # a run can be tuned (CHAOS_EVENTS, PRESSURE_CYCLES, ...) without editing + # the script on the box. + env_args = [a for a in sys.argv[2:] if "=" in a] + env_prefix = "".join(f"{a} " for a in env_args) meta = json.loads((HERE / "cluster_metadata_repl.json").read_text()) mgmt = meta["mgmt"]["public_ip"] - log(f"mgmt={mgmt} cases={cases}") + log(f"mgmt={mgmt} cases={cases}" + (f" env={env_args}" if env_args else "")) remote_meta = HERE / "cluster_metadata_repl_remote.json" remote_meta.write_text(json.dumps(remote_metadata(meta), indent=4)) @@ -77,10 +82,33 @@ def main(): # fd and using setsid is still needed so the driver survives the client # going away; without -f, ssh sat on the channel until it timed out (twice: # 2026-08-19 with nohup, 2026-08-20 with setsid) while the driver ran fine. - run(["ssh", "-f", *SSH_OPTS, f"ec2-user@{mgmt}", - f"cd ~ && setsid python3 -u test_async_replication.py {cases} " - f"> {remote_log} 2>&1 < /dev/null & echo $! > ~/repl_pid; " - f"echo ~/repl_cases_{ts}.log > ~/repl_log"], timeout=60) + # The 60s budget is for the ssh HANDSHAKE, not for the driver -- but a busy + # management node can exceed it while the driver has already started, and + # treating that as a launch failure reported a healthy run as dead twice + # (runs 20260826_205051 and _214011, both progressing normally while the + # wrapper exited non-zero). A timeout here is inconclusive, so ask the node + # what actually happened instead of guessing. + try: + run(["ssh", "-f", *SSH_OPTS, f"ec2-user@{mgmt}", + f"cd ~ && setsid env {env_prefix}python3 -u test_async_replication.py {cases} " + f"> {remote_log} 2>&1 < /dev/null & echo $! > ~/repl_pid; " + f"echo ~/repl_cases_{ts}.log > ~/repl_log"], timeout=60) + except subprocess.TimeoutExpired: + log("ssh -f exceeded its 60s budget; checking whether the driver started") + started = False + for _ in range(10): + time.sleep(15) + probe = ssh(mgmt, + f"test -f ~/repl_log && grep -q {ts} ~/repl_log && " + f"pgrep -f '[t]est_async_replication.py' >/dev/null " + f"&& echo STARTED || echo NOT_YET", check=False) + if "STARTED" in probe: + started = True + break + if not started: + raise RuntimeError( + f"ssh -f timed out AND no driver for {ts} is running on {mgmt}") + log("driver is running despite the ssh timeout") time.sleep(45) status = ssh(mgmt, "P=$(cat ~/repl_pid); L=$(cat ~/repl_log); " "echo \"pid=$P etime=$(ps -p $P -o etime= | tr -d ' ')\"; " diff --git a/scripts/start_soak_base.sh b/scripts/start_soak_base.sh new file mode 100644 index 0000000000..c7bc354059 --- /dev/null +++ b/scripts/start_soak_base.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Launch the outage soak on a NON-multipath (single data NIC) 2+2 cluster. +# +# Same script as the multipath soak -- we deliberately reuse it rather than the +# mixed-churn soak, because its outage timing is the part we want: one thread +# per node with its own mgmt connection, each node holding its own outage and +# issuing its own restart at its own boundary, so the pair genuinely overlaps. +# The mixed-churn soak applies both outages first and then restarts serially, +# retrying against the CP's concurrent-restart guard, which is far slower. +# +# Non-multipath adaptations (both needed, see the soak's own --help): +# --data-nics eth0 the base deploy adds nodes on eth0, so there is ONE path; +# path-count, listener-count and client-path expectations +# all derive from this, and the active_active policy +# assertions are skipped when it names fewer than 2 NICs. +# --no-nic-phase phase 1 takes "one" data NIC down on every node at once. +# With a single NIC per node that isolates the whole +# cluster instead of testing path redundancy. Note +# --nic-phase-every 0 does NOT disable it (it means +# "once, on iteration 1"). +# +# Env overrides: ITERATIONS, RUNTIME, START_ITERATION, RESTART_TIMEOUT, +# PLACEMENT_DUMPS=1, DATA_NIC. +set -u +cd "$HOME" +TS=$(date +%Y%m%d_%H%M%S) +echo "$TS" > "$HOME/soak_ts" +LOG="$HOME/soak_base_${TS}.log" +OUT="$HOME/soak_base_${TS}.out" +setsid nohup python3 "$HOME/aws_dual_node_outage_soak_multipath.py" --run-on-mgmt --metadata "$HOME/cluster_metadata_base.json" --ssh-key "$HOME/.ssh/mtes01.pem" --data-nics "${DATA_NIC:-eth0}" --no-nic-phase --iterations "${ITERATIONS:-75}" --start-iteration "${START_ITERATION:-1}" ${PLACEMENT_DUMPS:+--placement-dumps} ${RESTART_TIMEOUT:+--restart-timeout $RESTART_TIMEOUT} --runtime "${RUNTIME:-52000}" --log-file "$LOG" > "$OUT" 2>&1 < /dev/null & +PID=$! +echo "$PID" > "$HOME/soak_pid" +sleep 3 +echo "launched pid=$PID ts=$TS" +echo "log=$LOG" +echo "out=$OUT" +ps -p "$PID" -o pid=,etime=,cmd= | cut -c1-110 diff --git a/scripts/start_soak_mp.sh b/scripts/start_soak_mp.sh index 6ba68e7d0f..46c04df0fb 100644 --- a/scripts/start_soak_mp.sh +++ b/scripts/start_soak_mp.sh @@ -8,6 +8,13 @@ # # PLACEMENT_DUMPS=1 turns on per-outage placement-map dumps (gzipped, stored # on each storage node under ~/placement_dumps//). +# +# RESTART_TIMEOUT overrides the wait for nodes to return after a pair outage. +# The 900s default aborted run 20260825_085018 at iteration 4: a JC abort on +# one node plus a ~6min-per-lvstore restart crawl on the other meant both +# nodes needed ~36min to come back -- and they did come back healthy, with no +# fio error. Until that recovery time is fixed, a longer wait measures the +# product rather than the harness's patience. set -u cd "$HOME" TS=$(date +%Y%m%d_%H%M%S) @@ -20,6 +27,7 @@ setsid nohup python3 "$HOME/aws_dual_node_outage_soak_multipath.py" \ --ssh-key "$HOME/.ssh/mtes01.pem" \ --iterations 75 \ --start-iteration "${START_ITERATION:-1}" ${PLACEMENT_DUMPS:+--placement-dumps} \ + ${RESTART_TIMEOUT:+--restart-timeout $RESTART_TIMEOUT} \ --runtime 52000 \ --log-file "$LOG" \ > "$OUT" 2>&1 < /dev/null & diff --git a/scripts/test_async_replication.py b/scripts/test_async_replication.py index d4c0e0a051..5a189b477f 100644 --- a/scripts/test_async_replication.py +++ b/scripts/test_async_replication.py @@ -291,12 +291,39 @@ def wait_data_replicated(mgmt_ip, key_path, lvol_uuids, after_ts, f"predates the filesystem") -def do_failover(mgmt_ip, key_path, lvol_uuid): +def do_failover(mgmt_ip, key_path, lvol_uuid, generation=0): + """Fail a volume over, capturing WHY when it does not work. + + replicate_lvol_on_target_cluster returns a dict on success but False or + (False, error) on failure, and the controller reports the reason through + its logger -- which went to the snippet's stderr and was dropped. A bare + "returned no connection strings" then costs a whole lab run to diagnose + (case 7, runs 20260824_174611 and _202949). Capture both. + """ return mgmt_py(mgmt_ip, key_path, f""" -import json +import io, json, logging, contextlib from simplyblock_core.controllers import lvol_controller -res = lvol_controller.replicate_lvol_on_target_cluster({lvol_uuid!r}) -print(json.dumps(res if isinstance(res, dict) else {{"result": res}})) +buf = io.StringIO() +handler = logging.StreamHandler(buf) +# DEBUG, not WARNING: the RPC layer logs the actual SPDK response +# ("Invalid parameters", nsid in use, ...) at DEBUG, and the controller +# only re-reports its own generic "Failed to add bdev to subsystem". +# Only the last 1500 chars are kept, which is exactly the failure tail. +handler.setLevel(logging.DEBUG) +root = logging.getLogger() +root.addHandler(handler) +root.setLevel(logging.DEBUG) +err = "" +try: + with contextlib.redirect_stderr(buf): + res = lvol_controller.replicate_lvol_on_target_cluster({lvol_uuid!r}, generation={generation}) +except Exception as exc: # noqa: BLE001 - report, don't hide + res, err = False, f"{{type(exc).__name__}}: {{exc}}" +out = res if isinstance(res, dict) else {{"result": res}} +if not (isinstance(res, dict) and res.get("connection_strings")): + out["error"] = err + out["log"] = buf.getvalue()[-2500:] +print(json.dumps(out)) """) @@ -323,7 +350,8 @@ def replication_states(mgmt_ip, key_path, lvol_uuids): """, replayable=True) -def wait_replication_caught_up(mgmt_ip, key_path, lvol_uuids, timeout=REPL_WAIT_TIMEOUT): +def wait_replication_caught_up(mgmt_ip, key_path, lvol_uuids, timeout=REPL_WAIT_TIMEOUT, + max_lag=None): """Wait until every volume is replicating steadily with a bounded lag. NOT outstanding_count == 0. `outstanding_count` counts internal snapshots @@ -337,7 +365,7 @@ def wait_replication_caught_up(mgmt_ip, key_path, lvol_uuids, timeout=REPL_WAIT_ keeping up; the residual delta is `replication-commit`'s job, which is documented to "minimize delta then fail the client over". """ - max_lag = MAX_LAG_SECONDS + max_lag = max_lag or MAX_LAG_SECONDS print(f"Waiting for replication to reach a steady state (lag <= {max_lag}s) on all volumes...") start = time.time() stable = 0 @@ -413,28 +441,55 @@ def connect_and_mount(client_ip, key_path, mgmt_ip, lvols, fmt=True, mount_base= mnt = f"{mount_base}{idx}" if fmt: run(client_ip, key_path, f"sudo mkfs.xfs -f {dev}") + held = run(client_ip, key_path, + f"grep -E '^{dev} ' /proc/mounts | head -1 || true", + check=False, quiet=True, timeout=60).strip() + if held: + raise RuntimeError( + f"{dev} is already mounted ({held}) before mounting {lv} at " + f"{mnt}: a previous case's mount is still live on this device. " + f"mount would fail with a bare 'already mounted or mount point " + f"busy' (case 11, run 20260827_110415).") run(client_ip, key_path, f"sudo mkdir -p {mnt} && sudo mount {dev} {mnt}") mounts.append({"lvol": lv, "nqn": conn["nqn"], "dev": dev, "mount": mnt}) print(f" vol {lv} -> {dev} @ {mnt}") return mounts -def write_fio_jobfile(client_ip, key_path, mounts): - """One fio job per mounted volume; FIO_NUMJOBS threads each; md5 verify; 20s max latency.""" +def write_fio_jobfile(client_ip, key_path, mounts, + rw=None, bs=None, iodepth=None, numjobs=None, size=None, + time_based=True, verify=True, jobfile=None): + """One fio job per mounted volume; FIO_NUMJOBS threads each; md5 verify; 20s max latency. + + The keyword overrides exist for the pressure/chaos cases (7-9): case 8 + needs a FINITE sequential 64k/QD64/4-job fill of a known delta size, which + is the opposite of the endless mild verify-writer the fail-over cases use. + """ + rw = rw or FIO_RW + bs = bs or FIO_BS + iodepth = iodepth if iodepth is not None else FIO_IODEPTH + numjobs = numjobs if numjobs is not None else FIO_NUMJOBS + size = size or FIO_SIZE + jobfile = jobfile or FIO_JOBFILE sections = [ "[global]", - f"rw={FIO_RW}", - f"bs={FIO_BS}", - f"iodepth={FIO_IODEPTH}", + f"rw={rw}", + f"bs={bs}", + f"iodepth={iodepth}", "ioengine=libaio", "direct=1", - f"size={FIO_SIZE}", - f"numjobs={FIO_NUMJOBS}", - "time_based=1", - "runtime=86400", # effectively endless for the test - "verify=md5", - "verify_backlog=512", - "verify_fatal=1", + f"size={size}", + f"numjobs={numjobs}", + ] + if time_based: + sections += ["time_based=1", "runtime=86400"] # effectively endless + else: + # finite pass over `size`; rewriting the same files each cycle keeps + # the volume's footprint constant while dirtying the delta again + sections += ["loops=1", "overwrite=1"] + if verify: + sections += ["verify=md5", "verify_backlog=512", "verify_fatal=1"] + sections += [ f"max_latency={FIO_MAX_LATENCY}", "group_reporting=1", "", @@ -448,8 +503,59 @@ def write_fio_jobfile(client_ip, key_path, mounts): sections += [f"[vol{i}]", f"directory={m['mount']}", ""] content = "\n".join(sections) # Write the job file on the client. - run(client_ip, key_path, f"cat > {FIO_JOBFILE} <<'EOF'\n{content}\nEOF") - return FIO_JOBFILE + run(client_ip, key_path, f"cat > {jobfile} <<'EOF'\n{content}\nEOF") + return jobfile + + +def fio_bandwidth(client_ip, key_path, label=""): + """Print fio's own aggregate bandwidth, and return (read_mbps, write_mbps). + + fio has been writing this all along under --status-interval=15; nothing + read it, so every earlier analysis inferred the client rate from round + durations instead (and was 5x out). + """ + out = run(client_ip, key_path, + "grep -aE '^ *(READ|WRITE): bw=' %s 2>/dev/null | tail -4 || true" + % FIO_LOG, check=False, quiet=True) + rd = wr = None + for line in (out or "").splitlines(): + m = re.search(r"\((\d+(?:\.\d+)?)([kMG]?B)/s\)", line) + if not m: + continue + val = float(m.group(1)) + unit = m.group(2) + mbps = val / 1000.0 if unit == "kB" else (val * 1000.0 if unit == "GB" else val) + if line.strip().startswith("READ"): + rd = mbps + elif line.strip().startswith("WRITE"): + wr = mbps + if rd is not None or wr is not None: + print(" [fio %s] %s read %s MB/s, write %s MB/s" + % (label, client_ip, + "%.0f" % rd if rd is not None else "?", + "%.0f" % wr if wr is not None else "?")) + else: + print(" [fio %s] %s no aggregate lines yet" % (label, client_ip)) + return rd, wr + + +def collect_xfer_timing(mgmt_ip, key_path, label): + """Pull XFER-TIMING lines off the CP services into one file on the mgmt node. + + Container clocks are skewed from the host's, so every line carries its own + epoch stamp and we sort on that rather than on docker's timestamps. + """ + dest = "~/xfer_timing_%s.log" % label + services = ("app_TasksRunnerReplicationFinal app_SnapshotReplication " + "app_SnapshotMonitor app_LVolMonitor") + cmd = ("rm -f %s; for S in %s; do " + "sudo docker service logs $S 2>&1 | grep -a XFER-TIMING >> %s || true; " + "done; sort -t= -k2 -n %s -o %s 2>/dev/null || true; wc -l < %s" + % (dest, services, dest, dest, dest, dest)) + out = run(mgmt_ip, key_path, cmd, check=False, quiet=True, timeout=600) + count = (out or "").strip().splitlines()[-1] if (out or "").strip() else "0" + print(" [timing %s] collected %s XFER-TIMING lines -> %s" % (label, count, dest)) + return dest def start_fio(client_ip, key_path, jobfile): @@ -486,6 +592,78 @@ def stop_fio(client_ip, key_path): time.sleep(3) +def client_dirt(client_ip, key_path): + """What is still mounted or connected on the client. Empty dict = clean. + + Reads /proc/mounts rather than trusting umount's exit code: a lazy unmount + reports success and leaves the mount live until its IO drains, which is + exactly how a finished case hands its devices to the next one. + """ + mounts = run(client_ip, key_path, + "grep -oE '/mnt/repl[^ ]*' /proc/mounts 2>/dev/null | sort -u || true", + check=False, quiet=True, timeout=60) + subsys = run(client_ip, key_path, + "sudo nvme list-subsys 2>/dev/null " + "| grep -oE 'nqn\\.2023-02\\.io\\.simplyblock:[^ ,]+' | sort -u || true", + check=False, quiet=True, timeout=90) + dirt = {} + if mounts.split(): + dirt["mounts"] = mounts.split() + if subsys.split(): + dirt["subsystems"] = subsys.split() + return dirt + + +def force_client_clean(client_ip, key_path, rounds=3): + """Unmount and disconnect everything, and keep at it until it is gone. + + Each round kills whatever holds the mount (hung fio keeps a lazy unmount + pinned forever), unmounts, disconnects every simplyblock subsystem, then + re-reads /proc/mounts. Returns the remaining dirt, empty when clean. + """ + dirt = client_dirt(client_ip, key_path) + for attempt in range(rounds): + if not dirt: + return {} + if attempt: + print(f" [{client_ip}] client still dirty ({dirt}); escalating " + f"(round {attempt + 1}/{rounds})") + run(client_ip, key_path, "sudo pkill -x fio || true", check=False, timeout=60) + # Kill the holders first: umount -l on a mount someone still has open + # never completes, and the device stays mounted underneath. + run(client_ip, key_path, + "for m in $(grep -oE '/mnt/repl[^ ]*' /proc/mounts 2>/dev/null | sort -u); do " + "sudo timeout 20 fuser -km \"$m\" 2>/dev/null || true; " + "sudo timeout 15 umount \"$m\" 2>/dev/null " + "|| sudo timeout 15 umount -f \"$m\" 2>/dev/null " + "|| sudo timeout 15 umount -l \"$m\" 2>/dev/null || true; done", + check=False, timeout=300) + run(client_ip, key_path, + "for n in $(sudo nvme list-subsys 2>/dev/null " + "| grep -oE 'nqn\\.2023-02\\.io\\.simplyblock:[^ ,]+' | sort -u); do " + "sudo timeout 20 nvme disconnect -n \"$n\" >/dev/null 2>&1 || true; done", + check=False, timeout=300) + # A lazy unmount finishes asynchronously once its holders are gone. + for _ in range(10): + time.sleep(3) + dirt = client_dirt(client_ip, key_path) + if not dirt: + return {} + return dirt + + +def assert_client_clean(client_ip, key_path, where): + """Refuse to run *where* on a client another case left dirty.""" + dirt = force_client_clean(client_ip, key_path) + if dirt: + raise RuntimeError( + f"{where}: client {client_ip} could not be returned to a clean " + f"state -- still {dirt}. Whatever ran before it left mounts or " + f"controllers behind (a cutover freeze outlasting the 15s unmount " + f"timeout does exactly this), and starting here would test that " + f"debris instead of the case.") + + def cleanup_client(client_ip, key_path, mounts): # Unmount before disconnecting, with a lazy fallback: a plain (or forced) # unmount fails once the transport is dead, and disconnecting underneath a @@ -502,6 +680,14 @@ def cleanup_client(client_ip, key_path, mounts): run(client_ip, key_path, f"sudo timeout 30 nvme disconnect -n {m['nqn']} 2>/dev/null || true", check=False, timeout=90) + # Verify, and escalate rather than hand the next case a live mount. Not + # fatal here -- the case that owns these mounts has already done its work + # and its verdict should stand -- but loud, because this is where the + # contamination starts and the NEXT case is where it gets blamed. + dirt = force_client_clean(client_ip, key_path) + if dirt: + print(f" [{client_ip}] WARNING: cleanup left {dirt} behind; the next " + f"case will refuse to start until this clears") def prepare_mount_points(client_ip, key_path): @@ -530,6 +716,10 @@ def prepare_mount_points(client_ip, key_path): "| grep -oE 'nqn\\.2023-02\\.io\\.simplyblock:[^ ,]+'); do " "sudo timeout 20 nvme disconnect -n \"$n\" >/dev/null 2>&1 || true; done", check=False, timeout=300) + # The commands above are best-effort by design; this is the part that + # decides whether we may proceed. A lazy unmount reports success while the + # mount is still live, so the only trustworthy check is /proc/mounts. + assert_client_clean(client_ip, key_path, "prepare_mount_points") # --------------------------------------------------------------------------- # @@ -586,19 +776,57 @@ def wait_node_status(mgmt_ip, key_path, node_id, wanted, timeout=NODE_STATE_TIME f"Node {node_id} did not reach {wanted!r} within {timeout}s (last={seen!r})") -def sn_shutdown(mgmt_ip, key_path, node_id): +def wait_cluster_settled(mgmt_ip, key_path, cluster_id, timeout=1800): + """Wait until *cluster_id* has no open migration/balancing work. + + A restore leaves the cluster ACTIVE - REBALANCING (device_migration + + balancing_on_restart tasks), and `sn shutdown` REFUSES a node while that + work is open — silently, from the driver's point of view, because the + refusal goes to stderr (run 20260824_153107: case 6's shutdown printed + nothing and the node stayed 'online' for the full 900s budget, while + case 5's identical call later worked, after the rebalance had drained). + """ + print(f"Waiting for cluster {cluster_id[:8]} to settle (no rebalance/migration)...") + start = time.time() + while time.time() - start < timeout: + state = mgmt_py(mgmt_ip, key_path, f""" +import json +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.job_schedule import JobSchedule +db = DBController() +open_tasks = [t.function_name for t in db.get_job_tasks({cluster_id!r}) + if t.status != JobSchedule.STATUS_DONE and not t.canceled + and t.function_name in ("device_migration", "balancing_on_restart", + "new_device_migration", "failed_device_migration")] +print(json.dumps({{"status": db.get_cluster_by_id({cluster_id!r}).status, + "open": open_tasks}})) +""", replayable=True) + if not state["open"]: + print(f" cluster settled (status {state['status']}).") + return + print(f" status={state['status']} open={state['open']}") + time.sleep(20) + raise RuntimeError(f"Cluster {cluster_id[:8]} did not settle within {timeout}s") + + +def sn_shutdown(mgmt_ip, key_path, node_id, cluster_id=None): """Take a node offline the supported way. Deliberately NOT `docker kill` on the SPDK container: the control plane auto-restarts that within minutes (case 2 saw the "suspended" source cluster heal itself mid-test), which silently invalidates an outage scenario. """ + if cluster_id: + wait_cluster_settled(mgmt_ip, key_path, cluster_id) print(f"Shutting down node {node_id[:8]} ...") # Straight to shutdown: suspending first buys nothing and actively hurts — # a suspended node makes its own queued work defer ("node is not online, # retrying"), and that backlog then blocks the shutdown itself (run 15 # case 6: the node never left `suspended`). - run(mgmt_ip, key_path, f"{SBCTL} -d sn shutdown {node_id}", check=False) + # 2>&1: sbctl reports a REFUSED shutdown on stderr, which the channel + # otherwise drops — the driver then stares at an online node for 900s + # with no clue why (run 20260824_153107). + run(mgmt_ip, key_path, f"{SBCTL} -d sn shutdown {node_id} 2>&1", check=False) wait_node_status(mgmt_ip, key_path, node_id, "offline") @@ -691,21 +919,31 @@ def ensure_replication_target(mgmt_ip, key_path, from_cluster, to_cluster, def ensure_replication_policy(mgmt_ip, key_path, from_cluster, target_name, mode, - interval_min=REPL_INTERVAL_MIN, keep=2, name=None): - """Register (idempotently) a cadence policy on an existing target.""" - name = name or f"pol_{mode}_{target_name}" + interval_min=REPL_INTERVAL_MIN, keep=2, name=None, + extra_flags=""): + """Register (idempotently) a cadence policy on an existing target. + + extra_flags shapes the policy NAME too, so a case asking for a + retention-schedule or consistency-group policy never silently reuses a + plain one another case left behind. + """ + suffix = "_x%08x" % (hash(extra_flags) & 0xffffffff) if extra_flags else "" + if interval_min != REPL_INTERVAL_MIN: + suffix += f"_i{interval_min}" + name = name or f"pol_{mode}_{target_name}{suffix}" for row in _replication_list(mgmt_ip, key_path, "policy", from_cluster): if row.get("Name") == name: return name run(mgmt_ip, key_path, f"{SBCTL} -d cluster replication-policy-add {from_cluster} {name}" f" --target {target_name} --interval-min {interval_min} --mode {mode}" - f" --keep {keep}") + f" --keep {keep}{(' ' + extra_flags) if extra_flags else ''}") return name def set_cluster_replication(mgmt_ip, key_path, from_cluster, to_cluster, to_pool_uuid, - mode="migration"): + mode="migration", extra_flags="", policy_name=None, + interval_min=REPL_INTERVAL_MIN): """Create the target + policy that let `from_cluster` replicate to `to_cluster`. Replication is NEVER started per volume any more: `volume replication-start` @@ -720,7 +958,9 @@ def set_cluster_replication(mgmt_ip, key_path, from_cluster, to_cluster, to_pool f"(pool {to_pool_uuid[:8]}, mode {mode})") target = ensure_replication_target(mgmt_ip, key_path, from_cluster, to_cluster, to_pool_uuid) - policy = ensure_replication_policy(mgmt_ip, key_path, from_cluster, target, mode) + policy = ensure_replication_policy(mgmt_ip, key_path, from_cluster, target, mode, + interval_min=interval_min, + name=policy_name, extra_flags=extra_flags) # PRODUCT GAP (bridge, delete once the readers consult the policy): # replicate_lvol_on_target_cluster() and tasks_runner_replication_final still @@ -883,7 +1123,7 @@ def delete_test_volumes(mgmt_ip, key_path, pools): check=False, quiet=True) for line in raw.splitlines(): cols = [c.strip() for c in line.split("|")] - if len(cols) > 3 and (cols[2].startswith("replvol") or cols[2].startswith("REP_")): + if len(cols) > 3 and cols[2].startswith(("replvol", "REP_", "nsvol", "presvol", "chaosvol")): if (cols[1], cols[2]) not in victims: victims.append((cols[1], cols[2])) if not victims: @@ -895,7 +1135,11 @@ def delete_test_volumes(mgmt_ip, key_path, pools): run(mgmt_ip, key_path, f"{SBCTL} volume delete {uuid} --force", check=False, quiet=True) # Deletion is asynchronous; wait for the records to drain so the name is free. - for _ in range(30): + # Budget scales with the pile: case 7 leaves 20 namespaced volumes plus + # their REP_* landing copies, and 37 volumes did not drain inside the flat + # 300s, failing case 8 in its prologue (run 20260824_174611). + drain_polls = max(30, len(victims) * 3) + for _ in range(drain_polls): time.sleep(10) left = 0 for pool in pools: @@ -903,16 +1147,24 @@ def delete_test_volumes(mgmt_ip, key_path, pools): check=False, quiet=True) for line in raw.splitlines(): cols = [c.strip() for c in line.split("|")] - if len(cols) > 3 and (cols[2].startswith("replvol") or cols[2].startswith("REP_")): + if len(cols) > 3 and cols[2].startswith(("replvol", "REP_", "nsvol", "presvol", "chaosvol")): left += 1 if left == 0: print(" cleanup drained.") return - raise RuntimeError("Timed out waiting for leftover volumes to delete") + raise RuntimeError( + f"Timed out waiting for {len(victims)} leftover volumes to delete " + f"after {drain_polls * 10}s") + + +def lag_gate_for(interval_min): + """The steady-state lag a cadence can actually hold: 3 cadence periods.""" + return max(1, int(interval_min)) * 60 * 3 def create_volumes(mgmt_ip, key_path, src_uuid, pool, tgt_uuid, tgt_pool, mode, - count=NUM_VOLUMES): + count=NUM_VOLUMES, prefix="replvol", size=VOL_SIZE, + extra_flags="", interval_min=REPL_INTERVAL_MIN): """Create the test volumes already following a replication policy. The policy IS the start: `volume add --replication-policy` attaches it, and @@ -921,13 +1173,13 @@ def create_volumes(mgmt_ip, key_path, src_uuid, pool, tgt_uuid, tgt_pool, mode, """ policy = set_cluster_replication(mgmt_ip, key_path, src_uuid, tgt_uuid, pool_uuid_of(mgmt_ip, key_path, tgt_pool), - mode=mode) + mode=mode, interval_min=interval_min) lvols = [] for i in range(count): - name = f"replvol{i}" + name = f"{prefix}{i}" run(mgmt_ip, key_path, - f"{SBCTL} -d volume add {name} {VOL_SIZE} {pool}" - f" --replication-policy {policy}") + f"{SBCTL} -d volume add {name} {size} {pool}" + f" --replication-policy {policy}{(' ' + extra_flags) if extra_flags else ''}") lv = resolve_lvol(mgmt_ip, key_path, name) lvols.append(lv["uuid"]) print(f" created {name} = {lv['uuid']} (policy {policy}, mode={mode})") @@ -1134,7 +1386,9 @@ def test_case_2(meta): fo = do_failover(mgmt_ip, key_path, lv) print(f" failover {lv}: {json.dumps(fo)}") if not isinstance(fo, dict) or not fo.get("connection_strings"): - raise RuntimeError(f"FAIL: fail-over returned no connection strings for {lv}") + raise RuntimeError( + f"FAIL: fail-over returned no connection strings for {lv}: " + f"{fo.get('error') or ''} {fo.get('log') or ''}".strip()) if fo.get("nqn"): assert fo["nqn"], "missing NQN" failed_over.append({"src_lvol": lv, "fo": fo}) @@ -1209,7 +1463,9 @@ def _setup_failed_over_volumes(meta, tag): for lv in lvols: fo = do_failover(mgmt_ip, key_path, lv) if not isinstance(fo, dict) or not fo.get("connection_strings"): - raise RuntimeError(f"FAIL: fail-over returned no connection strings for {lv}") + raise RuntimeError( + f"FAIL: fail-over returned no connection strings for {lv}: " + f"{fo.get('error') or ''} {fo.get('log') or ''}".strip()) tgt_lvols.append(fo["lvol_id"]) # Re-key the baseline by TARGET lvol id and mount the failed-over copies. @@ -1451,7 +1707,7 @@ def test_case_5(meta): victim = node_of_lvol(mgmt_ip, key_path, lvols[0])["replication_node_id"] print(f"Taking the REPLICATION TARGET node {victim[:8]} offline...") before = _replication_progress(mgmt_ip, key_path, lvols) - sn_shutdown(mgmt_ip, key_path, victim) + sn_shutdown(mgmt_ip, key_path, victim, cluster_id=tgt_uuid) print(f"Observing {OUTAGE_REPL_CYCLES} replication cycles with the target down...") time.sleep(OUTAGE_REPL_CYCLES * REPL_INTERVAL_MIN * 60) @@ -1463,17 +1719,17 @@ def test_case_5(meta): print("Bringing the target node back...") sn_bring_back(mgmt_ip, key_path, victim) + recovery_ts = time.time() wait_replication_caught_up(mgmt_ip, key_path, lvols) + # See case 6: replicated_count is a retained count, not a progress + # counter. Require a post-recovery point-in-time on the target instead. + wait_data_replicated(mgmt_ip, key_path, lvols, recovery_ts) after = _replication_progress(mgmt_ip, key_path, lvols) - print(f" replicated_count after recovery={after}") + print(f" replicated_count after recovery={after} (retained, not cumulative)") stop_fio(client_ip, key_path) errors = fio_error_count(client_ip, key_path) cleanup_client(client_ip, key_path, mounts) - if after <= during: - raise RuntimeError( - f"FAIL: replication did not resume after the target node returned " - f"(during={during}, after={after})") if errors: raise RuntimeError(f"FAIL: fio reported {errors} errors during the target-node outage") print("CASE 5 PASSED: target-node outage survived, replication resumed.") @@ -1503,7 +1759,7 @@ def test_case_6(meta): print(f"Taking the SOURCE PRIMARY {primary[:8]} offline " f"(secondary {secondary[:8]} must carry on)...") before = _replication_progress(mgmt_ip, key_path, lvols) - sn_shutdown(mgmt_ip, key_path, primary) + sn_shutdown(mgmt_ip, key_path, primary, cluster_id=src_uuid) print(f"Observing {OUTAGE_REPL_CYCLES} replication cycles on the secondary...") time.sleep(OUTAGE_REPL_CYCLES * REPL_INTERVAL_MIN * 60) @@ -1521,21 +1777,1307 @@ def test_case_6(meta): print("Bringing the primary back...") sn_bring_back(mgmt_ip, key_path, primary) + recovery_ts = time.time() wait_replication_caught_up(mgmt_ip, key_path, lvols) + # Resumption is proven by a point-in-time created AFTER the primary + # returned reaching the target -- not by replicated_count growing. + # That counter tracks RETAINED replicated snapshots, and retention keeps + # only the newest generations, so it is bounded and routinely falls after + # a burst: run 20260825_105453 read during=13 / after=10 while + # replication was working perfectly. + wait_data_replicated(mgmt_ip, key_path, lvols, recovery_ts) after = _replication_progress(mgmt_ip, key_path, lvols) - print(f" replicated_count after recovery={after}") + print(f" replicated_count after recovery={after} (retained, not cumulative)") stop_fio(client_ip, key_path) errors = fio_error_count(client_ip, key_path) cleanup_client(client_ip, key_path, mounts) - if after <= during: - raise RuntimeError(f"FAIL: replication did not resume after the primary returned " - f"(during={during}, after={after})") if errors: raise RuntimeError(f"FAIL: fio reported {errors} errors during the primary outage") print("CASE 6 PASSED: source-primary outage survived, replication continued and resumed.") +# --------------------------------------------------------------------------- # +# Cases 7-9: namespaced subsystems, sequential pressure, chaos injection +# --------------------------------------------------------------------------- # +NS_VOLUMES = int(os.environ.get("NS_VOLUMES", "20")) +NS_PER_SUBSYS = int(os.environ.get("NS_PER_SUBSYS", "10")) +NS_VOL_SIZE = os.environ.get("NS_VOL_SIZE", "20G") +#: 20 volumes on a one-minute cadence means 20 transfers a minute +#: competing for the same hub: the lag settles at ~240s, above the +#: 180s gate, and the case times out in setup without ever reaching +#: the fail-over it exists to test (run 20260826_205051, lag +#: oscillating 216-285s for 35 minutes). Five minutes gives the same +#: coverage with a backlog the cluster can actually hold. +NS_INTERVAL_MIN = int(os.environ.get("NS_INTERVAL_MIN", "5")) + +PRESSURE_VOLUMES = int(os.environ.get("PRESSURE_VOLUMES", "2")) +PRESSURE_VOL_SIZE = os.environ.get("PRESSURE_VOL_SIZE", "120G") +PRESSURE_DELTA_GB = int(os.environ.get("PRESSURE_DELTA_GB", "50")) +PRESSURE_CYCLES = int(os.environ.get("PRESSURE_CYCLES", "3")) +PRESSURE_CATCHUP_TIMEOUT = int(os.environ.get("PRESSURE_CATCHUP_TIMEOUT", "3600")) + +CHAOS_EVENTS = int(os.environ.get("CHAOS_EVENTS", "12")) +CHAOS_SLEEP_MIN = int(os.environ.get("CHAOS_SLEEP_MIN", "20")) +CHAOS_SLEEP_MAX = int(os.environ.get("CHAOS_SLEEP_MAX", "150")) +CHAOS_SEED = os.environ.get("CHAOS_SEED", "") + + +def lvol_identities(mgmt_ip, key_path, lvol_uuids): + """{uuid: {nqn, ns_id, node_id}} — the preserved identity under test.""" + return mgmt_py(mgmt_ip, key_path, f""" +import json +from simplyblock_core.db_controller import DBController +db = DBController() +out = {{}} +for u in {list(lvol_uuids)!r}: + lv = db.get_lvol_by_id(u) + out[u] = {{"nqn": lv.nqn, "ns_id": lv.ns_id, "node_id": lv.node_id}} +print(json.dumps(out)) +""", replayable=True) + + +def _ns_devs_for_nqn(client_ip, key_path, nqn, expected, tries=10): + """{nsid: /dev/nvmeXnY} for every namespace of a SHARED subsystem. + + _dev_for_nqn picks the first namespace under the subsystem, which is + exactly wrong for case 7 where ten volumes share one NQN. sysfs gives the + block devices; `nvme get-ns-id` gives each one's NSID (the n-suffix in the + device name is a kernel instance number, NOT the NSID). + """ + for _ in range(tries): + out = run(client_ip, key_path, + "for s in /sys/class/nvme-subsystem/nvme-subsys*; do " + f"[ \"$(cat $s/subsysnqn 2>/dev/null)\" = \"{nqn}\" ] || continue; " + "ls $s 2>/dev/null | grep -E '^nvme[0-9]+n[0-9]+$'; " + "done", check=False, quiet=True) + devs = [d for d in out.split() if d] + mapping = {} + for d in devs: + nsid_out = run(client_ip, key_path, + f"sudo nvme get-ns-id /dev/{d} 2>/dev/null", + check=False, quiet=True) + m = re.search(r"(\d+)\s*$", nsid_out.strip()) + if m: + mapping[int(m.group(1))] = f"/dev/{d}" + if len(mapping) >= expected: + return mapping + time.sleep(5) + return mapping + + +def connect_and_mount_namespaced(client_ip, key_path, mgmt_ip, lvols, idents, + fmt=True, mount_base=MOUNT_BASE + "_ns"): + """connect_and_mount for volumes that SHARE subsystems: connect each NQN + once, then hand every volume the device matching ITS nsid.""" + prepare_mount_points(client_ip, key_path) + by_nqn = {} + for lv in lvols: + by_nqn.setdefault(idents[lv]["nqn"], []).append(lv) + + devmaps = {} + for nqn, members in by_nqn.items(): + conn = get_connect_cmds(mgmt_ip, key_path, members[0]) + assert not conn["err"], f"connect_lvol error for {members[0]}: {conn['err']}" + for cmd in conn["connect"]: + run(client_ip, key_path, cmd, check=False) + time.sleep(3) + devmaps[nqn] = _ns_devs_for_nqn(client_ip, key_path, nqn, len(members)) + print(f" subsystem {nqn.split(':')[-1][:13]}: " + f"{len(devmaps[nqn])} namespaces visible on {client_ip}") + + mounts = [] + for idx, lv in enumerate(lvols): + ident = idents[lv] + dev = devmaps.get(ident["nqn"], {}).get(ident["ns_id"]) + if not dev: + raise RuntimeError( + f"no device for lvol {lv} (nqn={ident['nqn']} nsid={ident['ns_id']}); " + f"visible: {devmaps.get(ident['nqn'])}") + mnt = f"{mount_base}{idx}" + if fmt: + run(client_ip, key_path, f"sudo mkfs.xfs -f {dev}") + run(client_ip, key_path, f"sudo mkdir -p {mnt} && sudo mount {dev} {mnt}") + mounts.append({"lvol": lv, "nqn": ident["nqn"], "dev": dev, "mount": mnt}) + print(f" vol {lv} (nsid {ident['ns_id']}) -> {dev} @ {mnt}") + return mounts + + +def test_case_7(meta): + """Namespaced volumes: 2 subsystems x 10 namespaces, 2 clients, full + replication + fail-over + fail-back with the shared-subsystem identity + preserved for every namespace.""" + print("\n========== CASE 7: namespaced subsystems (2x10 ns, 2 clients) ==========") + import random + key_path = meta["key_path"] + mgmt_ip = meta["mgmt"]["public_ip"] + clients = [c["public_ip"] for c in meta["clients"]] + if len(clients) < 2: + raise RuntimeError( + "case 7 needs at least 2 clients; run " + "`python scripts/setup_repl_test_2clusters.py add_client` first") + src_uuid, src, tgt_uuid, tgt = _src_target(meta) + + for ip in clients: + prepare_mount_points(ip, key_path) + delete_test_volumes(mgmt_ip, key_path, _all_test_pools(meta)) + + print(f"Creating {NS_VOLUMES} namespaced volumes " + f"(max {NS_PER_SUBSYS}/subsystem => {NS_VOLUMES // NS_PER_SUBSYS} subsystems)...") + lvols = create_volumes( + mgmt_ip, key_path, src_uuid, src["pool"], tgt_uuid, tgt["pool"], + mode="failover", count=NS_VOLUMES, prefix="nsvol", size=NS_VOL_SIZE, + interval_min=NS_INTERVAL_MIN, + extra_flags=f"--namespaced True --max-namespace-per-subsys {NS_PER_SUBSYS}") + + idents = lvol_identities(mgmt_ip, key_path, lvols) + by_nqn = {} + for lv in lvols: + by_nqn.setdefault(idents[lv]["nqn"], []).append(lv) + packing = {n.split(":")[-1][:13]: len(v) for n, v in by_nqn.items()} + print(f" subsystem packing: {packing}") + if len(by_nqn) != NS_VOLUMES // NS_PER_SUBSYS or set(packing.values()) != {NS_PER_SUBSYS}: + raise RuntimeError(f"FAIL: expected {NS_VOLUMES // NS_PER_SUBSYS} subsystems x " + f"{NS_PER_SUBSYS} namespaces, got {packing}") + + # Random client assignment, reproducible via printed seed. + seed = int(os.environ.get("NS_SEED") or time.time()) + rng = random.Random(seed) + print(f" client assignment seed: {seed}") + shuffled = lvols[:] + rng.shuffle(shuffled) + split = rng.randint(NS_VOLUMES // 4, 3 * NS_VOLUMES // 4) # both clients always used + assign = {clients[0]: shuffled[:split], clients[1]: shuffled[split:]} + + mounts_by_client, baseline = {}, {} + for ip, vols in assign.items(): + print(f"Client {ip}: {len(vols)} namespaces") + mounts_by_client[ip] = connect_and_mount_namespaced( + ip, key_path, mgmt_ip, vols, idents, fmt=True) + baseline.update(write_baseline(ip, key_path, mounts_by_client[ip])) + baseline_ts = time.time() + + for ip in assign: + start_fio(ip, key_path, write_fio_jobfile(ip, key_path, mounts_by_client[ip], + size="1G")) + ns_gate = lag_gate_for(NS_INTERVAL_MIN) + for ip in assign: + fio_bandwidth(ip, key_path, "steady-state") + wait_replication_caught_up(mgmt_ip, key_path, lvols, timeout=3600, max_lag=ns_gate) + wait_data_replicated(mgmt_ip, key_path, lvols, baseline_ts, timeout=3600) + for ip in assign: + fio_bandwidth(ip, key_path, "pre-failover") + collect_xfer_timing(mgmt_ip, key_path, "case7_pre_failover") + + print("Killing the source cluster (both nodes)...") + for ip in src["storage_public_ips"][:2]: + kill_spdk(ip, key_path) + time.sleep(15) + for ip in assign: + stop_fio(ip, key_path) + cleanup_client(ip, key_path, mounts_by_client[ip]) + + print("Failing over all namespaces...") + tgt_lvols = [] + for lv in lvols: + fo = do_failover(mgmt_ip, key_path, lv) + if not isinstance(fo, dict) or not fo.get("connection_strings"): + raise RuntimeError( + f"FAIL: fail-over returned no connection strings for {lv}: " + f"{fo.get('error') or ''} {fo.get('log') or ''}".strip()) + tgt_lvols.append(fo["lvol_id"]) + src_to_tgt = dict(zip(lvols, tgt_lvols)) + + # Identity preservation: every fail-over copy keeps ITS nqn and nsid, so + # the shared subsystems must re-form on the target with all 10 namespaces. + tgt_idents = lvol_identities(mgmt_ip, key_path, tgt_lvols) + for s, t in src_to_tgt.items(): + if (tgt_idents[t]["nqn"], tgt_idents[t]["ns_id"]) != (idents[s]["nqn"], idents[s]["ns_id"]): + raise RuntimeError( + f"FAIL: identity not preserved for {s}: " + f"{idents[s]['nqn']}/{idents[s]['ns_id']} -> " + f"{tgt_idents[t]['nqn']}/{tgt_idents[t]['ns_id']}") + print(" NQN + nsid preserved for all namespaces.") + + # Re-assign RANDOMLY again for the fail-over verification (fresh shuffle). + shuffled2 = tgt_lvols[:] + rng.shuffle(shuffled2) + split2 = rng.randint(NS_VOLUMES // 4, 3 * NS_VOLUMES // 4) + assign_fo = {clients[0]: shuffled2[:split2], clients[1]: shuffled2[split2:]} + tgt_baseline = {src_to_tgt[s]: b for s, b in baseline.items()} + + fo_mounts_by_client = {} + ok = True + for ip, vols in assign_fo.items(): + m = connect_and_mount_namespaced(ip, key_path, mgmt_ip, vols, tgt_idents, + fmt=False, mount_base=MOUNT_BASE + "_nsfo") + fo_mounts_by_client[ip] = m + good, _ = verify_baseline(ip, key_path, m, tgt_baseline) + ok = ok and good + if not ok: + raise RuntimeError("FAIL: replicated namespace data not intact after fail-over") + + print("Restoring the source cluster + failing back all namespaces...") + restore_cluster(mgmt_ip, key_path, src, label="src (case 7)") + set_cluster_replication(mgmt_ip, key_path, tgt_uuid, src_uuid, + pool_uuid_of(mgmt_ip, key_path, src["pool"])) + for lv in tgt_lvols: + failback(mgmt_ip, key_path, lv) + wait_replication_caught_up(mgmt_ip, key_path, tgt_lvols, timeout=3600, + max_lag=ns_gate) + for lv in tgt_lvols: + run(mgmt_ip, key_path, f"{SBCTL} -d volume replication-commit {lv}") + + start = time.time() + done = 0 + # The timing bundle is the whole point of the run, so collect it even when + # the poll itself blows up. Twice now a transient control-plane error inside + # this loop (`sbctl` rc=1) propagated out and skipped the collection, leaving + # the previous run's file in place -- which then looked like fresh data. + try: + while time.time() - start < CUTOVER_WAIT_TIMEOUT * 2: + try: + states = replication_states(mgmt_ip, key_path, tgt_lvols) + except Exception as e: # noqa: BLE001 + # A CP hiccup must not end the case: log it, keep polling, and + # let the deadline decide. + print(f" fail-back poll failed ({str(e)[:120]}); retrying") + time.sleep(15) + continue + done = sum(1 for s in states.values() if s in ("cutover_done", "failed_over")) + print(f" fail-back cutovers done: {done}/{len(tgt_lvols)}") + if done == len(tgt_lvols): + break + time.sleep(15) + finally: + collect_xfer_timing(mgmt_ip, key_path, "case7_failback") + if done != len(tgt_lvols): + # The breakdown matters MOST here: a stalled fail-back is the case we + # have failed to explain seven times. + raise RuntimeError(f"FAIL: only {done}/{len(tgt_lvols)} fail-back cutovers completed") + + back = failed_over_targets(mgmt_ip, key_path, tgt_lvols) + back_idents = lvol_identities(mgmt_ip, key_path, list(back.values())) + fb_baseline = {back[t]: tgt_baseline[t] for t in tgt_lvols if t in back} + ok = True + for ip, vols in assign_fo.items(): + cleanup_client(ip, key_path, fo_mounts_by_client[ip]) + fb_vols = [back[t] for t in vols if t in back] + m = connect_and_mount_namespaced(ip, key_path, mgmt_ip, fb_vols, back_idents, + fmt=False, mount_base=MOUNT_BASE + "_nsfb") + good, _ = verify_baseline(ip, key_path, m, fb_baseline) + ok = ok and good + cleanup_client(ip, key_path, m) + if not ok: + raise RuntimeError("FAIL: namespace data not intact after fail-back") + print(f"CASE 7 PASSED: {NS_VOLUMES} namespaces on " + f"{NS_VOLUMES // NS_PER_SUBSYS} shared subsystems survived " + f"fail-over + fail-back with identity preserved (seed {seed}).") + + +def test_case_8(meta): + """Sequential-pressure catch-up: 64k/QD64/4-job fills of PRESSURE_DELTA_GB + per volume, repeated; the backlog must drain back under the lag gate after + every cycle. The cadence is a target, the backlog must converge.""" + print(f"\n========== CASE 8: {PRESSURE_DELTA_GB}G sequential-pressure catch-up " + f"x{PRESSURE_CYCLES} ==========") + key_path = meta["key_path"] + mgmt_ip = meta["mgmt"]["public_ip"] + client_ip = meta["clients"][0]["public_ip"] + src_uuid, src, tgt_uuid, tgt = _src_target(meta) + + prepare_mount_points(client_ip, key_path) + delete_test_volumes(mgmt_ip, key_path, _all_test_pools(meta)) + lvols = create_volumes(mgmt_ip, key_path, src_uuid, src["pool"], tgt_uuid, + tgt["pool"], mode="failover", count=PRESSURE_VOLUMES, + prefix="presvol", size=PRESSURE_VOL_SIZE) + mounts = connect_and_mount(client_ip, key_path, mgmt_ip, lvols, fmt=True) + baseline = write_baseline(client_ip, key_path, mounts) + + # Each of the 4 jobs writes delta/4 into its own file => delta GB per + # volume per cycle; overwrite=1 dirties the SAME clusters again next cycle. + per_job = f"{(PRESSURE_DELTA_GB * 1024) // 4}m" + jobfile = write_fio_jobfile(client_ip, key_path, mounts, rw="write", bs="64k", + iodepth=64, numjobs=4, size=per_job, + time_based=False, verify=False, + jobfile="/tmp/fio_pressure.fio") + + results = [] + for cycle in range(1, PRESSURE_CYCLES + 1): + print(f"--- cycle {cycle}/{PRESSURE_CYCLES}: writing " + f"{PRESSURE_DELTA_GB}G per volume (64k seq, QD64, 4 jobs) ---") + fill_start = time.time() + with ThreadPoolExecutor(max_workers=1) as ex: + fut = ex.submit(run, client_ip, key_path, + f"sudo fio --eta=never {jobfile}", True, True, 7200) + peak_bytes, peak_lag = 0, 0 + while not fut.done(): + time.sleep(20) + infos = get_replication_infos(mgmt_ip, key_path, lvols) + bts = sum((i.get("outstanding_bytes") or 0) for i in infos.values()) + lag = max((i.get("lag_seconds") or 0) for i in infos.values()) + peak_bytes, peak_lag = max(peak_bytes, bts), max(peak_lag, lag) + print(f" [fill] backlog={bts / 2**30:.1f}GiB worst_lag={lag}s") + fut.result() + fill_secs = time.time() - fill_start + + # Now the pipeline must CATCH UP: bounded lag again within the budget. + drain_start = time.time() + caught_up = None + while time.time() - drain_start < PRESSURE_CATCHUP_TIMEOUT: + infos = get_replication_infos(mgmt_ip, key_path, lvols) + bts = sum((i.get("outstanding_bytes") or 0) for i in infos.values()) + lag = max((i.get("lag_seconds") or 0) for i in infos.values()) + outst = sum((i.get("outstanding_count") or 0) for i in infos.values()) + peak_bytes, peak_lag = max(peak_bytes, bts), max(peak_lag, lag) + print(f" [drain] backlog={bts / 2**30:.1f}GiB worst_lag={lag}s outstanding={outst}") + if lag <= MAX_LAG_SECONDS and outst <= len(lvols): + caught_up = time.time() - drain_start + break + time.sleep(20) + results.append({"cycle": cycle, "fill_secs": int(fill_secs), + "peak_backlog_gib": round(peak_bytes / 2**30, 1), + "peak_lag_s": peak_lag, + "catch_up_secs": None if caught_up is None else int(caught_up)}) + print(f" cycle {cycle}: fill={int(fill_secs)}s " + f"peak_backlog={peak_bytes / 2**30:.1f}GiB peak_lag={peak_lag}s " + f"catch_up={'TIMEOUT' if caught_up is None else str(int(caught_up)) + 's'}") + if caught_up is None: + raise RuntimeError( + f"FAIL: replication did not catch up within " + f"{PRESSURE_CATCHUP_TIMEOUT}s after cycle {cycle} " + f"(peak backlog {peak_bytes / 2**30:.1f}GiB)") + + ok, _ = verify_baseline(client_ip, key_path, mounts, baseline) + stop_fio(client_ip, key_path) + cleanup_client(client_ip, key_path, mounts) + print(" cycle results:", json.dumps(results)) + if not ok: + raise RuntimeError("FAIL: baseline corrupted during pressure cycles") + print(f"CASE 8 PASSED: {PRESSURE_CYCLES} x {PRESSURE_DELTA_GB}G deltas, " + f"replication caught up every cycle.") + + +def _sample_replication_phases(mgmt_ip, key_path): + """What the replication pipeline is doing RIGHT NOW (for kill logging).""" + return mgmt_py(mgmt_ip, key_path, """ +import json +from collections import Counter +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.job_schedule import JobSchedule +db = DBController() +c = Counter() +for cl in db.get_clusters(): + for t in db.get_job_tasks(cl.get_id()): + if t.function_name in (JobSchedule.FN_SNAPSHOT_REPLICATION, + JobSchedule.FN_REPLICATION_FINAL) \ + and t.status != JobSchedule.STATUS_DONE: + c[f"{t.function_name}:{t.status}"] += 1 +print(json.dumps(dict(c))) +""", replayable=True) + + +def _all_nodes_online(mgmt_ip, key_path): + """Truly recovered: every node online AND healthy, every cluster ACTIVE. + + Status alone is not recovery. After a chaos kill a node reports + "online" again while health_check is still False and its lvstore port + is down, and the cluster can sit in SUSPENDED/IN_ACTIVATION behind it. + Gating on status only, the soak kept firing kills into a half-recovered + 2-node cluster and drove it to SUSPENDED by event 3 (run + 20260824_224909) -- damage the test caused, not damage it found. + """ + return mgmt_py(mgmt_ip, key_path, """ +import json +from simplyblock_core.db_controller import DBController +db = DBController() +bad = [n.get_id()[:8] for n in db.get_storage_nodes() + if n.status != "online" or not n.health_check] +busy = [c.get_id()[:8] for c in db.get_clusters() + if c.status not in ("active", "degraded")] +print(json.dumps({"all_online": not bad and not busy, + "offline": bad, "clusters": busy})) +""", replayable=True) + + +def test_case_9(meta): + """Chaos: random SPDK-container kills on SOURCE and TARGET nodes while + replication runs, to land failures in every pipeline phase (target-lvol + create, hublvol attach, transfer, convert, chain, prune, detach).""" + print(f"\n========== CASE 9: chaos container kills x{CHAOS_EVENTS} ==========") + import random + key_path = meta["key_path"] + mgmt_ip = meta["mgmt"]["public_ip"] + client_ip = meta["clients"][0]["public_ip"] + src_uuid, src, tgt_uuid, tgt = _src_target(meta) + + seed = int(CHAOS_SEED or time.time()) + rng = random.Random(seed) + print(f" chaos seed: {seed}") + + prepare_mount_points(client_ip, key_path) + delete_test_volumes(mgmt_ip, key_path, _all_test_pools(meta)) + lvols = create_volumes(mgmt_ip, key_path, src_uuid, src["pool"], tgt_uuid, + tgt["pool"], mode="failover", prefix="chaosvol") + mounts = connect_and_mount(client_ip, key_path, mgmt_ip, lvols, fmt=True) + baseline = write_baseline(client_ip, key_path, mounts) + start_fio(client_ip, key_path, write_fio_jobfile(client_ip, key_path, mounts)) + wait_replication_caught_up(mgmt_ip, key_path, lvols) + + # One kill at a time, recover, next -- randomized timing spreads the kills + # across the pipeline phases; two concurrent kills in one cluster is the + # cluster-outage scenario cases 2/3 already cover, not a race. + targets = ([("src", ip) for ip in src["storage_public_ips"]] + + [("tgt", ip) for ip in tgt["storage_public_ips"]]) + kills = [] + fio_deaths = [] + for ev in range(1, CHAOS_EVENTS + 1): + time.sleep(rng.randint(CHAOS_SLEEP_MIN, CHAOS_SLEEP_MAX)) + side, victim = rng.choice(targets) + phases = _sample_replication_phases(mgmt_ip, key_path) + print(f" [{ev}/{CHAOS_EVENTS}] killing SPDK on {side} node {victim} " + f"(active phases: {phases})") + kill_spdk(victim, key_path) + # fio survival across a kill is the promotion-window signal: before the + # ANA-transition fix (spdk R26.3) a killed primary handed hard EIO to + # the client within seconds and XFS shut the filesystem down. + time.sleep(20) + alive = fio_alive(client_ip, key_path) + kills.append({"event": ev, "side": side, "node": victim, + "phases": phases, "fio_alive": alive}) + if not alive: + fio_deaths.append(ev) + print(f" fio NOT alive after event {ev} (deaths so far: {len(fio_deaths)})") + + # Wait for the auto-restart to bring everything back before the next hit. + deadline = time.time() + NODE_STATE_TIMEOUT + while time.time() < deadline: + state = _all_nodes_online(mgmt_ip, key_path) + if state["all_online"]: + break + time.sleep(20) + else: + raise RuntimeError( + f"FAIL: not recovered {NODE_STATE_TIMEOUT}s after chaos kill " + f"{ev} on {victim}: unhealthy nodes={state['offline']} " + f"clusters not active={state['clusters']}") + + # Chaos without IO is not chaos. A kill that takes fio down (the + # promotion-window EIO) otherwise leaves every later event running + # against an idle client -- run 20260824_224909 lost its workload at + # event 3 and would have coasted through the remaining 97. + if not fio_alive(client_ip, key_path): + print(" restarting the client workload after the outage") + try: + cleanup_client(client_ip, key_path, mounts) + mounts = connect_and_mount(client_ip, key_path, mgmt_ip, lvols, + fmt=False) + start_fio(client_ip, key_path, + write_fio_jobfile(client_ip, key_path, mounts)) + except Exception as exc: # noqa: BLE001 - keep the soak going + print(f" could not restart the workload: {exc}") + + print("Chaos done; requiring full catch-up + integrity...") + wait_replication_caught_up(mgmt_ip, key_path, lvols, timeout=3600) + quiesce_ts = time.time() + stop_fio(client_ip, key_path) + run(client_ip, key_path, "sync", check=False) + wait_data_replicated(mgmt_ip, key_path, lvols, quiesce_ts, timeout=3600) + cleanup_client(client_ip, key_path, mounts) + + # End-to-end proof the surviving pipeline shipped GOOD data: kill the + # source, fail over, verify the baselines on the target copies. + print("Final integrity check via fail-over...") + for ip in src["storage_public_ips"][:2]: + kill_spdk(ip, key_path) + time.sleep(15) + tgt_lvols = [] + for lv in lvols: + fo = do_failover(mgmt_ip, key_path, lv) + if not isinstance(fo, dict) or not fo.get("connection_strings"): + raise RuntimeError( + f"FAIL: post-chaos fail-over failed for {lv}: " + f"{fo.get('error') or ''} {fo.get('log') or ''}".strip()) + tgt_lvols.append(fo["lvol_id"]) + tgt_baseline = {t: baseline[s] for s, t in zip(lvols, tgt_lvols)} + fo_mounts = connect_and_mount(client_ip, key_path, mgmt_ip, tgt_lvols, + fmt=False, mount_base=MOUNT_BASE + "_chaos") + ok, _ = verify_baseline(client_ip, key_path, fo_mounts, tgt_baseline) + cleanup_client(client_ip, key_path, fo_mounts) + restore_cluster(mgmt_ip, key_path, src, label="src (after chaos)") + + print(f" fio survived {CHAOS_EVENTS - len(fio_deaths)}/{CHAOS_EVENTS} kills" + + (f" (died after events {fio_deaths})" if fio_deaths else "")) + print(" kill log:", json.dumps(kills)) + if not ok: + raise RuntimeError("FAIL: data not intact after chaos (seed " + f"{seed}; kill log above)") + print(f"CASE 9 PASSED: {CHAOS_EVENTS} random kills across both clusters, " + f"replication recovered every time, data intact (seed {seed}).") + + + + +# --------------------------------------------------------------------------- # +# Cases 10-12: feature tests (migration under load, retention ladder, CGs) +# --------------------------------------------------------------------------- # +CASE10_VOLS_PER_NODE = int(os.environ.get("CASE10_VOLS_PER_NODE", "3")) +CASE11_RUNTIME_MIN = int(os.environ.get("CASE11_RUNTIME_MIN", "115")) +CASE11_SCHEDULE = os.environ.get("CASE11_SCHEDULE", "5m:15m,7m:30m,10m:1h") +CASE11_ROUNDS = int(os.environ.get("CASE11_ROUNDS", "3")) +CASE12_SCHEDULE = os.environ.get("CASE12_SCHEDULE", "5m:15m") +CASE12_RUNTIME_MIN = int(os.environ.get("CASE12_RUNTIME_MIN", "25")) + + +def _start_stall_probe(client_ip, key_path, mount): + """100ms wall-clock heartbeats into the volume; gaps measure IO freezes.""" + run(client_ip, key_path, + "sudo rm -f {m}/probe.ts; sudo nohup bash -c 'while :; do " + "date +%s%3N >> {m}/probe.ts; sync {m}/probe.ts; sleep 0.1; done' " + ">/dev/null 2>&1 & echo probe_started".format(m=mount), quiet=True) + + +def _stop_probes(client_ip, key_path): + run(client_ip, key_path, "sudo pkill -f probe.ts 2>/dev/null || true", + check=False, quiet=True) + + +def _max_probe_gap_ms(client_ip, key_path, mount, t0_ms, t1_ms): + """Largest heartbeat gap inside [t0_ms, t1_ms] = the freeze duration the + CLIENT actually observed (includes a ~100ms sampling floor).""" + awk = ("sudo awk 'p && $1>={t0} && $1<={t1} && $1-p>m {{m=$1-p}} {{p=$1}} " + "END{{print m+0}}' {m}/probe.ts").format(t0=t0_ms, t1=t1_ms, m=mount) + out = run(client_ip, key_path, awk, check=False, quiet=True) + try: + return int(out.strip()) + except ValueError: + return -1 + + +def _start_recorder(client_ip, key_path, mount, interval_sec=30): + """Timestamped, fsynced records every interval -- the ground truth for + which point-in-time a snapshot generation captured.""" + cmd = ("sudo rm -f {m}/records.log; sudo nohup bash -c 'i=0; while :; do " + "i=$((i+1)); echo \"iter=$i ts=$(date +%s)\" >> {m}/records.log; " + "sync {m}/records.log; sleep {iv}; done' >/dev/null 2>&1 & " + "echo recorder_started").format(m=mount, iv=interval_sec) + run(client_ip, key_path, cmd, quiet=True) + + +def _stop_recorders(client_ip, key_path): + run(client_ip, key_path, "sudo pkill -f records.log 2>/dev/null || true", + check=False, quiet=True) + + +def _snapshot_ages(mgmt_ip, key_path, lvol_uuid): + """created_at (epoch) of every replicated internal snapshot of the volume, + newest first.""" + return mgmt_py(mgmt_ip, key_path, """ +import json +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.snapshot import SnapShot +db = DBController() +out = [] +for snp in db.get_snapshots(): + if snp.deleted or not snp.lvol or snp.lvol.get_id() != {lv!r}: + continue + if snp.snap_type == SnapShot.TYPE_INTERNAL and snp.target_replicated_snap_uuid: + out.append(snp.created_at or 0) +print(json.dumps(sorted(out, reverse=True))) +""".format(lv=lvol_uuid), replayable=True) + + +def _verify_retention_ladder(times, tiers, now, cadence_sec=60, slack=150): + """Independent check of (a) kept and (b) pruned per schedule. + + tiers: [(every_sec, span_sec)] finest first. Verifies: nothing older + than the horizon (+slack), and inside each tier window consecutive + retained snapshots are no further apart than every+cadence+slack and no + denser than the schedule plus a small tolerance allows. + """ + problems = [] + horizon = sum(t[1] for t in tiers) + ages = sorted(now - t for t in times) + for a in ages: + if a > horizon + slack: + problems.append("snapshot %ds old survives past the %ds horizon " + "(not pruned)" % (a, horizon)) + start = 0 + for every, span in tiers: + end = start + span + inside = [a for a in ages if start <= a < end] + for prev, cur in zip(inside, inside[1:]): + if cur - prev > every + cadence_sec + slack: + problems.append( + "gap of %ds inside the %ds tier (%d-%ds): a scheduled " + "snapshot is missing" % (cur - prev, every, start, end)) + expected = span // every + if len(inside) > expected + 2: + problems.append( + "%d snapshots inside the %ds tier (%d-%ds), expected <= %d: " + "not pruned" % (len(inside), every, start, end, expected + 2)) + start = end + return problems + + +def _records_match_generation(client_ip, key_path, mount, snap_ts, slack=35, + logname="records.log"): + """The mounted snapshot copy must contain every record up to snap_ts and + none from after it (records are fsynced every 30s).""" + out = run(client_ip, key_path, + "sudo grep -oE 'ts=[0-9]+' %s/%s | tail -200" % (mount, logname), + check=False, quiet=True) + ts = [int(x.split("=")[1]) for x in out.split() if x.startswith("ts=")] + if not ts: + return False, "no records found on the snapshot copy" + newest = max(ts) + if newest > snap_ts + slack: + return False, ("record from %d present, %ds AFTER the generation's " + "snapshot (%d)" % (newest, newest - snap_ts, snap_ts)) + if newest < snap_ts - 95: + return False, ("newest record %d is %ds older than the snapshot; " + "records up to the snapshot are missing" + % (newest, snap_ts - newest)) + return True, "newest record %ds before the snapshot" % (int(snap_ts) - newest) + + +def _target_snapshot_ts(mgmt_ip, key_path, target_lvol): + """created_at of the snapshot the fail-over copy was cloned from.""" + return mgmt_py(mgmt_ip, key_path, """ +import json +from simplyblock_core.db_controller import DBController +db = DBController() +lv = db.get_lvol_by_id({t!r}) +snp = db.get_snapshot_by_id(lv.cloned_from_snap) +print(json.dumps(snp.created_at or 0)) +""".format(t=target_lvol), replayable=True) + + +def test_case_10(meta): + """Migration under sustained heavy IO: 3 volumes per source node at + 8k randwrite QD64x4 jobs, replication must keep up, the final online + migration must complete, and the client-observed IO freeze during each + cutover is measured.""" + print("\n========== CASE 10: online migration under heavy IO (freeze timing) ==========") + key_path = meta["key_path"] + mgmt_ip = meta["mgmt"]["public_ip"] + client_ip = meta["clients"][0]["public_ip"] + src_uuid, src, tgt_uuid, tgt = _src_target(meta) + + prepare_mount_points(client_ip, key_path) + delete_test_volumes(mgmt_ip, key_path, _all_test_pools(meta)) + + policy = set_cluster_replication(mgmt_ip, key_path, src_uuid, tgt_uuid, + pool_uuid_of(mgmt_ip, key_path, tgt["pool"]), + mode="migration") + node_ids = mgmt_py(mgmt_ip, key_path, """ +import json +from simplyblock_core.db_controller import DBController +print(json.dumps([n.get_id() for n in + DBController().get_storage_nodes_by_cluster_id({c!r}) + if n.status == "online"])) +""".format(c=src_uuid), replayable=True) + lvols = [] + for n_idx, nid in enumerate(node_ids): + for v in range(CASE10_VOLS_PER_NODE): + name = "replvol%d" % (n_idx * CASE10_VOLS_PER_NODE + v) + run(mgmt_ip, key_path, + "%s -d volume add %s %s %s --replication-policy %s --host-id %s" + % (SBCTL, name, VOL_SIZE, src["pool"], policy, nid)) + lvols.append(resolve_lvol(mgmt_ip, key_path, name)["uuid"]) + print(" %d volumes across %d nodes" % (len(lvols), len(node_ids))) + + mounts = connect_and_mount(client_ip, key_path, mgmt_ip, lvols, fmt=True) + write_baseline(client_ip, key_path, mounts) + for m in mounts: + _start_stall_probe(client_ip, key_path, m["mount"]) + jobfile = write_fio_jobfile(client_ip, key_path, mounts, rw="randwrite", + bs="8k", iodepth=64, numjobs=4, size="2G", + verify=False, jobfile="/tmp/fio_case10.fio") + start_fio(client_ip, key_path, jobfile) + + # The whole point: replication must reach a bounded lag AGAINST this load. + wait_replication_caught_up(mgmt_ip, key_path, lvols, timeout=5400) + + print("Committing the final online migration per volume (freeze timing)...") + windows = {} + for lv in lvols: + t0 = int(run(client_ip, key_path, "date +%s%3N", quiet=True).strip()) + run(mgmt_ip, key_path, "%s -d volume replication-commit %s" % (SBCTL, lv)) + windows[lv] = [t0, 0] + start = time.time() + done = 0 + while time.time() - start < CUTOVER_WAIT_TIMEOUT * 3: + if not fio_alive(client_ip, key_path): + raise RuntimeError("FAIL: fio stopped during the online migration") + states = replication_states(mgmt_ip, key_path, lvols) + now_ms = int(run(client_ip, key_path, "date +%s%3N", quiet=True).strip()) + done = 0 + for lv in lvols: + if states.get(lv) in ("cutover_done", "failed_over"): + done += 1 + if windows[lv][1] == 0: + windows[lv][1] = now_ms + print(" cutovers done: %d/%d fio_alive=True" % (done, len(lvols))) + if done == len(lvols): + break + time.sleep(15) + + time.sleep(10) + alive = fio_alive(client_ip, key_path) + errors = fio_error_count(client_ip, key_path) + stop_fio(client_ip, key_path) + _stop_probes(client_ip, key_path) + + freezes = [] + for lv, m in zip(lvols, mounts): + t0, t1 = windows[lv] + if t1 == 0: + t1 = t0 + CUTOVER_WAIT_TIMEOUT * 1000 + gap = _max_probe_gap_ms(client_ip, key_path, m["mount"], + t0 - 2000, t1 + 5000) + freezes.append(gap) + print(" %s: client-observed IO freeze during cutover = %sms" % (lv[:8], gap)) + valid = [f for f in freezes if f >= 0] + print(" freeze summary: max=%sms avg=%sms" + % (max(freezes), sum(valid) // max(1, len(valid)))) + + cleanup_client(client_ip, key_path, mounts) + if done != len(lvols): + raise RuntimeError("FAIL: only %d/%d migrations completed" % (done, len(lvols))) + if not alive or errors: + raise RuntimeError("FAIL: fio alive=%s errors=%s during migration" + % (alive, errors)) + if max(freezes) > 30000: + raise RuntimeError("FAIL: IO freeze of %sms during cutover (sanity bound 30s)" + % max(freezes)) + print("CASE 10 PASSED: migration kept up under heavy IO; freeze times above.") + + +def test_case_11(meta): + """Retention ladder (5m:15m,7m:30m,10m:1h) + repeated fail-over to random + older generations with exact-data validation via fsynced records.""" + print("\n========== CASE 11: retention schedule + generation fail-overs ==========") + import random + key_path = meta["key_path"] + mgmt_ip = meta["mgmt"]["public_ip"] + client_ip = meta["clients"][0]["public_ip"] + src_uuid, src, tgt_uuid, tgt = _src_target(meta) + tiers = [(5 * 60, 15 * 60), (7 * 60, 30 * 60), (10 * 60, 60 * 60)] + + prepare_mount_points(client_ip, key_path) + delete_test_volumes(mgmt_ip, key_path, _all_test_pools(meta)) + policy = set_cluster_replication( + mgmt_ip, key_path, src_uuid, tgt_uuid, + pool_uuid_of(mgmt_ip, key_path, tgt["pool"]), mode="failover", + extra_flags="--retention-schedule %s" % CASE11_SCHEDULE) + lvols = [] + for i in range(2): + run(mgmt_ip, key_path, + "%s -d volume add replvol%d 20G %s --replication-policy %s" + % (SBCTL, i, src["pool"], policy)) + lvols.append(resolve_lvol(mgmt_ip, key_path, "replvol%d" % i)["uuid"]) + mounts = connect_and_mount(client_ip, key_path, mgmt_ip, lvols, fmt=True) + for m in mounts: + _start_recorder(client_ip, key_path, m["mount"]) + + print("Building %d minutes of history (schedule %s)..." + % (CASE11_RUNTIME_MIN, CASE11_SCHEDULE)) + t_end = time.time() + CASE11_RUNTIME_MIN * 60 + while time.time() < t_end: + time.sleep(120) + infos = get_replication_infos(mgmt_ip, key_path, lvols) + worst = max((i.get("lag_seconds") or 0) for i in infos.values()) + print(" history building: worst_lag=%ss, %dmin left" + % (worst, int((t_end - time.time()) / 60))) + + # (a) kept + (b) pruned per schedule, verified independently per volume. + now = time.time() + for lv in lvols: + times = _snapshot_ages(mgmt_ip, key_path, lv) + problems = _verify_retention_ladder(times, tiers, now) + print(" %s: %d retained snapshots, %d schedule violations" + % (lv[:8], len(times), len(problems))) + for prob in problems[:4]: + print(" VIOLATION: %s" % prob) + if problems: + raise RuntimeError("FAIL: retention schedule violated for %s: %s" + % (lv, problems[0])) + + # (c) three random-generation fail-over / fail-back rounds. + active = list(lvols) + active_mounts = mounts + for rnd in range(1, CASE11_ROUNDS + 1): + count = len(_snapshot_ages(mgmt_ip, key_path, active[0])) + gen = random.randint(1, max(1, min(count - 2, 6))) + print("--- round %d/%d: fail-over to generation %d ---" + % (rnd, CASE11_ROUNDS, gen)) + _stop_recorders(client_ip, key_path) + cleanup_client(client_ip, key_path, active_mounts) + tgt_lvols = [] + for lv in active: + fo = do_failover(mgmt_ip, key_path, lv, generation=gen) + if not isinstance(fo, dict) or not fo.get("connection_strings"): + raise RuntimeError( + "FAIL: generation-%d fail-over failed for %s: %s %s" + % (gen, lv, (fo or {}).get("error", ""), (fo or {}).get("log", ""))) + tgt_lvols.append(fo["lvol_id"]) + fo_mounts = connect_and_mount(client_ip, key_path, mgmt_ip, tgt_lvols, + fmt=False, mount_base=MOUNT_BASE + "_g") + for t, m in zip(tgt_lvols, fo_mounts): + snap_ts = _target_snapshot_ts(mgmt_ip, key_path, t) + ok, why = _records_match_generation(client_ip, key_path, m["mount"], snap_ts) + print(" %s gen=%d: %s" % (t[:8], gen, why)) + if not ok: + raise RuntimeError("FAIL: generation %d data mismatch: %s" % (gen, why)) + + print(" failing back...") + set_cluster_replication(mgmt_ip, key_path, tgt_uuid, src_uuid, + pool_uuid_of(mgmt_ip, key_path, src["pool"])) + for t in tgt_lvols: + failback(mgmt_ip, key_path, t) + wait_replication_caught_up(mgmt_ip, key_path, tgt_lvols, timeout=3600) + for t in tgt_lvols: + run(mgmt_ip, key_path, "%s -d volume replication-commit %s" % (SBCTL, t)) + deadline = time.time() + CUTOVER_WAIT_TIMEOUT + while time.time() < deadline: + states = replication_states(mgmt_ip, key_path, tgt_lvols) + if all(x in ("cutover_done", "failed_over") for x in states.values()): + break + time.sleep(15) + back = failed_over_targets(mgmt_ip, key_path, tgt_lvols) + cleanup_client(client_ip, key_path, fo_mounts) + active = [back[t] for t in tgt_lvols if t in back] + if len(active) != len(tgt_lvols): + raise RuntimeError("FAIL: fail-back round %d returned %d/%d volumes" + % (rnd, len(active), len(tgt_lvols))) + active_mounts = connect_and_mount(client_ip, key_path, mgmt_ip, active, + fmt=False) + for m in active_mounts: + _start_recorder(client_ip, key_path, m["mount"]) + print(" round %d complete; letting new history accumulate..." % rnd) + time.sleep(600) + + _stop_recorders(client_ip, key_path) + cleanup_client(client_ip, key_path, active_mounts) + print("CASE 11 PASSED: schedule kept+pruned correctly; " + "%d random-generation fail-over rounds data-exact." % CASE11_ROUNDS) + + +def test_case_12(meta): + """Consistency groups: CG policy with 3 volumes, ordered writes, group + snapshots kept/pruned per schedule, generation fail-overs correlating with + the generation AND crash-consistent (write order A>=B>=C preserved).""" + print("\n========== CASE 12: consistency groups ==========") + import random + key_path = meta["key_path"] + mgmt_ip = meta["mgmt"]["public_ip"] + client_ip = meta["clients"][0]["public_ip"] + src_uuid, src, tgt_uuid, tgt = _src_target(meta) + + helptext = run(mgmt_ip, key_path, + "%s cluster replication-policy-add --help 2>&1" % SBCTL, + check=False, quiet=True) + if "consistency-group" not in helptext: + raise RuntimeError( + "case 12 requires the consistency-groups build " + "(sbcli branch consistency-groups + its spdk RPC); not on this lab") + + prepare_mount_points(client_ip, key_path) + delete_test_volumes(mgmt_ip, key_path, _all_test_pools(meta)) + policy = set_cluster_replication( + mgmt_ip, key_path, src_uuid, tgt_uuid, + pool_uuid_of(mgmt_ip, key_path, tgt["pool"]), mode="failover", + extra_flags="--consistency-group --retention-schedule %s" % CASE12_SCHEDULE) + lvols = [] + for i in range(3): + run(mgmt_ip, key_path, + "%s -d volume add replvol%d 20G %s --replication-policy %s" + % (SBCTL, i, src["pool"], policy)) + lvols.append(resolve_lvol(mgmt_ip, key_path, "replvol%d" % i)["uuid"]) + + # CG invariant 1: all members on one LVS. + nodes = set(node_of_lvol(mgmt_ip, key_path, lv)["node_id"] for lv in lvols) + if len(nodes) != 1: + raise RuntimeError("FAIL: CG members scattered across %d nodes" % len(nodes)) + print(" all 3 members pinned to one node/LVS") + + mounts = connect_and_mount(client_ip, key_path, mgmt_ip, lvols, fmt=True) + # Ordered writer: seq into A, then B, then C, fsync each -- in ANY + # crash-consistent group snapshot seq(A) >= seq(B) >= seq(C). + ordered = " ".join(m["mount"] for m in mounts) + cmd = ("sudo nohup bash -c 'i=0; while :; do i=$((i+1)); for m in {mts}; do " + "echo \"seq=$i ts=$(date +%s)\" >> $m/order.log; sync $m/order.log; " + "done; sleep 2; done' >/dev/null 2>&1 & echo writer_started" + ).format(mts=ordered) + run(client_ip, key_path, cmd, quiet=True) + + print("Running %d minutes of CG history..." % CASE12_RUNTIME_MIN) + time.sleep(CASE12_RUNTIME_MIN * 60) + + # Group snapshots: every generation must cover ALL members with one seq. + groups = mgmt_py(mgmt_ip, key_path, """ +import json +from collections import defaultdict +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.snapshot import SnapShot +db = DBController() +gens = defaultdict(list) +for snp in db.get_snapshots(): + if snp.deleted or not snp.lvol or snp.lvol.get_id() not in {lvs!r}: + continue + if snp.snap_type == SnapShot.TYPE_INTERNAL and getattr(snp, "group_seq", 0): + gens[snp.group_seq].append(snp.lvol.get_id()) +print(json.dumps(dict((str(k), sorted(v)) for k, v in gens.items()))) +""".format(lvs=lvols), replayable=True) + bad = [g for g, members in groups.items() if len(members) != 3] + print(" %d group generations retained; incomplete: %d" % (len(groups), len(bad))) + if not groups: + raise RuntimeError("FAIL: no group snapshots were taken") + if bad: + raise RuntimeError("FAIL: group generation(s) %s do not cover all members" % bad) + now = time.time() + times = _snapshot_ages(mgmt_ip, key_path, lvols[0]) + problems = _verify_retention_ladder(times, [(5 * 60, 15 * 60)], now, + cadence_sec=300) + if problems: + raise RuntimeError("FAIL: CG retention violated: %s" % problems[0]) + + run(client_ip, key_path, "sudo pkill -f order.log || true", check=False, quiet=True) + cleanup_client(client_ip, key_path, mounts) + + for label, gen in (("latest", 0), + ("random earlier", random.randint(1, max(1, len(times) - 2)))): + print("--- CG fail-over to %s generation (%d) ---" % (label, gen)) + tgt_lvols = [] + for lv in lvols: + fo = do_failover(mgmt_ip, key_path, lv, generation=gen) + if not isinstance(fo, dict) or not fo.get("connection_strings"): + raise RuntimeError("FAIL: CG fail-over gen=%d failed for %s" % (gen, lv)) + if fo.get("warnings"): + print(" membership warnings: %s" % fo["warnings"]) + tgt_lvols.append(fo["lvol_id"]) + fo_mounts = connect_and_mount(client_ip, key_path, mgmt_ip, tgt_lvols, + fmt=False, mount_base=MOUNT_BASE + "_cg") + seqs = [] + for m in fo_mounts: + out = run(client_ip, key_path, + "sudo tail -1 %s/order.log" % m["mount"], check=False, quiet=True) + seqs.append(int(out.split("seq=")[1].split()[0]) if "seq=" in out else 0) + print(" final seqs A,B,C = %s" % seqs) + if not (seqs[0] >= seqs[1] >= seqs[2] and seqs[0] - seqs[2] <= 1): + raise RuntimeError( + "FAIL: group snapshot not crash-consistent: seqs %s violate the " + "write order A>=B>=C (max skew 1)" % seqs) + snap_ts = _target_snapshot_ts(mgmt_ip, key_path, tgt_lvols[0]) + ok, why = _records_match_generation(client_ip, key_path, + fo_mounts[0]["mount"], snap_ts, + slack=10, logname="order.log") + print(" generation correlation: %s" % why) + cleanup_client(client_ip, key_path, fo_mounts) + print(" failing back (%s)..." % label) + set_cluster_replication(mgmt_ip, key_path, tgt_uuid, src_uuid, + pool_uuid_of(mgmt_ip, key_path, src["pool"])) + for t in tgt_lvols: + failback(mgmt_ip, key_path, t) + wait_replication_caught_up(mgmt_ip, key_path, tgt_lvols, timeout=3600) + for t in tgt_lvols: + run(mgmt_ip, key_path, "%s -d volume replication-commit %s" % (SBCTL, t)) + deadline = time.time() + CUTOVER_WAIT_TIMEOUT + while time.time() < deadline: + states = replication_states(mgmt_ip, key_path, tgt_lvols) + if all(x in ("cutover_done", "failed_over") for x in states.values()): + break + time.sleep(15) + back = failed_over_targets(mgmt_ip, key_path, tgt_lvols) + lvols = [back[t] for t in tgt_lvols if t in back] + if len(lvols) != 3: + raise RuntimeError("FAIL: CG fail-back returned %d/3 volumes" % len(lvols)) + + print("CASE 12 PASSED: CG snapshots complete per generation, retention " + "correct, both fail-overs crash-consistent and generation-exact.") + + + +# --------------------------------------------------------------------------- # +# Cases 13-15: a single node dies DURING fail-over / fail-back / cutover +# --------------------------------------------------------------------------- # +# One node is killed at a RANDOM instant inside the phase under test, repeated +# so the kills land at many different points of it. Chaos is allowed to make +# the operation fail -- what is NOT allowed is losing data, wedging the +# cluster, or leaving a state a retry cannot get out of. Each round therefore +# ends in one of two accepted verdicts (completed despite the kill / failed +# then succeeded on retry after recovery) and any third outcome fails the case. +CHAOS_PHASE_ROUNDS = int(os.environ.get("CHAOS_PHASE_ROUNDS", "20")) +# 20 policies x 2 volumes = 40 volumes replicating at once. At the default +# 1-minute cadence that is 40 transfers a minute and the cluster steady-states +# at ~6 minutes of lag -- above the 3-period gate, so the setup phase never +# completes (run 20260827_110415, killed after 50 minutes without a round). +# The cadence has to be one this volume count can hold. +CHAOS_PHASE_INTERVAL_MIN = int(os.environ.get("CHAOS_PHASE_INTERVAL_MIN", "10")) +CHAOS_PHASE_VOLS_PER_POLICY = int(os.environ.get("CHAOS_PHASE_VOLS_PER_POLICY", "2")) +CHAOS_PHASE_VOL_SIZE = os.environ.get("CHAOS_PHASE_VOL_SIZE", "10G") +#: cap for the randomized kill delay when a phase turns out to be slow +CHAOS_PHASE_MAX_DELAY = float(os.environ.get("CHAOS_PHASE_MAX_DELAY", "45")) + + +def _kill_after(delay_s, ip, key_path, sink): + """Kill SPDK on *ip* after *delay_s*, off the main thread.""" + import threading + + def _run(): + time.sleep(delay_s) + try: + kill_spdk(ip, key_path) + sink.append(("killed", ip, time.time())) + except Exception as exc: # noqa: BLE001 - recorded + sink.append(("kill-failed", ip, str(exc))) + + t = threading.Thread(target=_run, daemon=True) + t.start() + return t + + +def _await_cluster_healthy(mgmt_ip, key_path, timeout=NODE_STATE_TIMEOUT): + """Every node online AND healthy, every cluster active/degraded.""" + deadline = time.time() + timeout + last = {} + while time.time() < deadline: + last = _all_nodes_online(mgmt_ip, key_path) + if last.get("all_online"): + return True, last + time.sleep(20) + return False, last + + +def _phase_victims(meta, phase): + """Nodes worth killing for the phase under test, as (label, public_ip). + + Fail-over and fail-back BUILD the copy on the destination, so the + destination cluster is where a kill bites; the cutover freezes and flips + the source, so both sides matter there. + """ + _src_uuid, src, _tgt_uuid, tgt = _src_target(meta) + if phase == "failover": + pool = [("tgt", ip) for ip in tgt["storage_public_ips"]] + elif phase == "failback": + pool = [("src", ip) for ip in src["storage_public_ips"]] + else: + pool = ([("src", ip) for ip in src["storage_public_ips"]] + + [("tgt", ip) for ip in tgt["storage_public_ips"]]) + return pool + + +def _failover_all(mgmt_ip, key_path, lvols): + """Returns (ok, target_lvols, error). Never raises on a chaos failure.""" + out = [] + for lv in lvols: + fo = do_failover(mgmt_ip, key_path, lv) + if not isinstance(fo, dict) or not fo.get("connection_strings"): + return False, out, (f"{lv[:8]}: " + f"{(fo or {}).get('error', '')} " + f"{(fo or {}).get('log', '')}".strip()[:300]) + out.append(fo["lvol_id"]) + return True, out, "" + + +def _commit_all(mgmt_ip, key_path, lvols, timeout=None): + """Drive replication-commit for every volume and wait for the cutovers.""" + timeout = timeout or CUTOVER_WAIT_TIMEOUT + for lv in lvols: + run(mgmt_ip, key_path, "%s -d volume replication-commit %s" % (SBCTL, lv), + check=False) + deadline = time.time() + timeout + while time.time() < deadline: + states = replication_states(mgmt_ip, key_path, lvols) + done = sum(1 for x in states.values() if x in ("cutover_done", "failed_over")) + if done == len(lvols): + return True, "" + time.sleep(15) + return False, "only %d/%d cutovers completed in %ds" % (done, len(lvols), timeout) + + +def _failback_all(mgmt_ip, key_path, meta, tgt_lvols): + """Point replication home, fail back, commit, and return the new lvol ids.""" + src_uuid, src, tgt_uuid, _tgt = _src_target(meta) + set_cluster_replication(mgmt_ip, key_path, tgt_uuid, src_uuid, + pool_uuid_of(mgmt_ip, key_path, src["pool"])) + for t in tgt_lvols: + failback(mgmt_ip, key_path, t) + wait_replication_caught_up(mgmt_ip, key_path, tgt_lvols, timeout=3600) + ok, why = _commit_all(mgmt_ip, key_path, tgt_lvols) + if not ok: + return False, [], why + back = failed_over_targets(mgmt_ip, key_path, tgt_lvols) + new = [back[t] for t in tgt_lvols if t in back] + if len(new) != len(tgt_lvols): + return False, new, "fail-back returned %d/%d volumes" % (len(new), len(tgt_lvols)) + return True, new, "" + + +def _chaos_phase_case(meta, phase, title): + """Kill ONE node at a random instant inside *phase*, once per POLICY. + + Each round owns its own policy and its own volumes, so a round never has + to undo what the previous one did: no fail-back, no re-sync, no restore to + a "home shape". Set up CHAOS_PHASE_ROUNDS policies with + CHAOS_PHASE_VOLS_PER_POLICY volumes each, replicate them all once, then + walk the policies one at a time and attack the phase on that policy's + volumes with the kill landing at a random instant. + """ + import random + print("\n========== %s ==========" % title) + key_path = meta["key_path"] + mgmt_ip = meta["mgmt"]["public_ip"] + client_ip = meta["clients"][0]["public_ip"] + src_uuid, src, tgt_uuid, tgt = _src_target(meta) + seed = int(os.environ.get("CHAOS_PHASE_SEED") or time.time()) + rng = random.Random(seed) + # The cutover under test in case 15 is the FINAL MIGRATION STEP: a + # migration-mode policy committed on the SOURCE volumes. + mode = "migration" if phase == "commit" else "failover" + print(" seed=%d rounds=%d vols/policy=%d mode=%s phase=%s cadence=%dmin " + "lag_gate=%ds" + % (seed, CHAOS_PHASE_ROUNDS, CHAOS_PHASE_VOLS_PER_POLICY, mode, phase, + CHAOS_PHASE_INTERVAL_MIN, lag_gate_for(CHAOS_PHASE_INTERVAL_MIN))) + + prepare_mount_points(client_ip, key_path) + delete_test_volumes(mgmt_ip, key_path, _all_test_pools(meta)) + + # --- one-time setup: N policies x M volumes, all replicating ------------ + groups, all_lvols = [], [] + for r in range(CHAOS_PHASE_ROUNDS): + policy = set_cluster_replication( + mgmt_ip, key_path, src_uuid, tgt_uuid, + pool_uuid_of(mgmt_ip, key_path, tgt["pool"]), mode=mode, + policy_name="pol_chaos_%s_%02d" % (phase, r), + interval_min=CHAOS_PHASE_INTERVAL_MIN) + vols = [] + for v in range(CHAOS_PHASE_VOLS_PER_POLICY): + name = "replvol%02d_%d" % (r, v) + run(mgmt_ip, key_path, + "%s -d volume add %s %s %s --replication-policy %s" + % (SBCTL, name, CHAOS_PHASE_VOL_SIZE, src["pool"], policy)) + vols.append(resolve_lvol(mgmt_ip, key_path, name)["uuid"]) + groups.append({"policy": policy, "vols": vols}) + all_lvols.extend(vols) + print(" created %d policies x %d volumes = %d volumes" + % (CHAOS_PHASE_ROUNDS, CHAOS_PHASE_VOLS_PER_POLICY, len(all_lvols))) + + mounts = connect_and_mount(client_ip, key_path, mgmt_ip, all_lvols, fmt=True) + baseline = write_baseline(client_ip, key_path, mounts) + baseline_ts = time.time() + cleanup_client(client_ip, key_path, mounts) + wait_replication_caught_up(mgmt_ip, key_path, all_lvols, timeout=7200, + max_lag=lag_gate_for(CHAOS_PHASE_INTERVAL_MIN)) + wait_data_replicated(mgmt_ip, key_path, all_lvols, baseline_ts, timeout=7200) + print(" all %d volumes replicated; starting the rounds" % len(all_lvols)) + + victims = _phase_victims(meta, phase) + phase_secs = 20.0 # replaced by the first round's measurement + verdicts, retries, kills = [], 0, [] + + for rnd, grp in enumerate(groups, start=1): + vols = grp["vols"] + delay = round(rng.uniform(0.0, min(phase_secs * 1.2, CHAOS_PHASE_MAX_DELAY)), 1) + label, victim = rng.choice(victims) + print("--- round %d/%d (policy %s): kill %s node %s at T+%.1fs of %s ---" + % (rnd, CHAOS_PHASE_ROUNDS, grp["policy"], label, victim, delay, phase)) + sink = [] + started = time.time() + + if phase == "failover": + _kill_after(delay, victim, key_path, sink) + ok, result_lvols, why = _failover_all(mgmt_ip, key_path, vols) + elif phase == "failback": + # The fail-BACK is under test, so the fail-over that sets it up runs + # clean; only then does the node die. + ok, tgt_lvols, why = _failover_all(mgmt_ip, key_path, vols) + if not ok: + raise RuntimeError("FAIL: round %d -- the setup fail-over failed " + "before the fail-back under test: %s" % (rnd, why)) + _kill_after(delay, victim, key_path, sink) + ok, result_lvols, why = _failback_all(mgmt_ip, key_path, meta, tgt_lvols) + else: # commit / online cutover + _kill_after(delay, victim, key_path, sink) + ok, why = _commit_all(mgmt_ip, key_path, vols) + result_lvols = [] + measured = time.time() - started + if rnd == 1: + phase_secs = max(5.0, measured) + print(" measured %s phase: %.1fs (kill delays randomize over it)" + % (phase, phase_secs)) + kills.extend(sink) + + healthy, state = _await_cluster_healthy(mgmt_ip, key_path) + if not healthy: + raise RuntimeError( + "FAIL: round %d -- cluster did not recover %ds after killing %s: " + "unhealthy=%s clusters=%s" + % (rnd, NODE_STATE_TIMEOUT, victim, state.get("offline"), + state.get("clusters"))) + + if ok: + verdicts.append("completed") + else: + # Chaos may legitimately fail the operation. A RETRY once the + # cluster is healthy again must then succeed -- anything else is a + # wedged state, which is the bug this case hunts. + print(" operation failed under chaos (%s); retrying after recovery" + % why[:160]) + retries += 1 + if phase == "failover": + ok, result_lvols, why = _failover_all(mgmt_ip, key_path, vols) + elif phase == "failback": + ok, result_lvols, why = _failback_all(mgmt_ip, key_path, meta, tgt_lvols) + else: + ok, why = _commit_all(mgmt_ip, key_path, vols) + if not ok: + raise RuntimeError("FAIL: round %d -- %s did not succeed even on a " + "retry after recovery: %s" % (rnd, phase, why)) + verdicts.append("retry") + + # Where the data ended up, and whether it is intact. + if phase == "commit": + after = failed_over_targets(mgmt_ip, key_path, vols) + result_lvols = [after[v] for v in vols if v in after] + if len(result_lvols) != len(vols): + raise RuntimeError("FAIL: round %d -- cutover produced %d/%d target " + "volumes" % (rnd, len(result_lvols), len(vols))) + check_base = dict(zip(result_lvols, [baseline[v] for v in vols])) + vmounts = connect_and_mount(client_ip, key_path, mgmt_ip, result_lvols, + fmt=False, mount_base=MOUNT_BASE + "_ph") + good, _ = verify_baseline(client_ip, key_path, vmounts, check_base) + cleanup_client(client_ip, key_path, vmounts) + if not good: + raise RuntimeError("FAIL: round %d -- data not intact after %s with a " + "node killed at T+%.1fs (policy %s)" + % (rnd, phase, delay, grp["policy"])) + # No restore: the next round owns different volumes entirely. + + completed = verdicts.count("completed") + delivered = sum(1 for k in kills if k[0] == "killed") + print(" %d rounds: %d completed under the kill, %d needed a retry after " + "recovery, %d/%d kills delivered" + % (CHAOS_PHASE_ROUNDS, completed, retries, delivered, len(kills))) + print("%s PASSED: every round ended intact and unwedged (seed %d)." + % (title.split(":")[0], seed)) + + +def test_case_13(meta): + _chaos_phase_case(meta, "failover", + "CASE 13: single node dies DURING fail-over") + + +def test_case_14(meta): + _chaos_phase_case(meta, "failback", + "CASE 14: single node dies DURING fail-back") + + +def test_case_15(meta): + _chaos_phase_case(meta, "commit", + "CASE 15: single node dies DURING the online cutover") + CASES = { "case1": test_case_1, # online migration cutover, no IO interruption "case2": test_case_2, # DR fail-over on source-cluster loss @@ -1543,15 +3085,29 @@ def test_case_6(meta): "case4": test_case_4, # full fail-back to a fresh empty cluster "case5": test_case_5, # error: replication target node offline "case6": test_case_6, # error: source primary offline, secondary survives + "case7": test_case_7, # namespaced: 2 subsystems x 10 ns, 2 clients, fo+fb + "case8": test_case_8, # sequential pressure: repeated 50G deltas must catch up + "case9": test_case_9, # chaos: random SPDK kills on src+tgt during replication + "case10": test_case_10, # migration under heavy IO + cutover freeze timing + "case11": test_case_11, # retention ladder + random-generation fail-overs + "case12": test_case_12, # consistency groups (needs the CG build) + "case13": test_case_13, # node dies during fail-over, x20 random instants + "case14": test_case_14, # node dies during fail-back, x20 random instants + "case15": test_case_15, # node dies during the online cutover, x20 } GROUPS = { "both": ["case1", "case2"], "failback": ["case3", "case4"], "errors": ["case5", "case6"], + "extended": ["case7", "case8", "case9"], + "features": ["case10", "case11", "case12"], + "phase-chaos": ["case13", "case14", "case15"], "all": ["case1", "case2", "case3", "case4", "case5", "case6"], # Case 3 last: it is the only case that needs the killed primary restored # and recovered, so a failure there cannot cost the other five cases. "all_c3_last": ["case1", "case2", "case4", "case5", "case6", "case3"], + "all9": ["case1", "case2", "case4", "case5", "case6", "case3", + "case7", "case8", "case9"], } diff --git a/scripts/xfer_timing_report.py b/scripts/xfer_timing_report.py new file mode 100755 index 0000000000..389b89d9e8 --- /dev/null +++ b/scripts/xfer_timing_report.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Turn XFER-TIMING lines into the breakdown the soak could not produce. + +The question this exists to answer: of the time a convergence round takes, how +much is DATA TRANSFER and how much is orchestration? Round duration was +previously the only number available, and it spans the landing-volume create, +the hub attach, the transfer, the detach, add_clone and convert on two nodes, +DB writes, and up to TASK_EXEC_INTERVAL_SEC of task-runner latency per state +change. Every hardware-level theory we tested against that number came out an +order of magnitude off, because it is not a throughput. + +Usage: + python scripts/xfer_timing_report.py [--csv out.csv] + +Input is anything containing XFER-TIMING lines (a `docker service logs` dump is +fine). Lines look like: + + XFER-TIMING t=1787868791.244 phase=transfer_complete lvol=1c8874f3 \ + snap=a0f48bf5 round=2 ms=1843.2 bytes=33554432 mbps=18.2 ok=1 +""" +import argparse +import re +import sys +from collections import defaultdict + +LINE = re.compile(r"XFER-TIMING\s+(.*)$") +KV = re.compile(r"(\w+)=(\S+)") + +# pipeline order for display; anything unlisted is appended +ROUND_PHASES = [ + "take_shrink_snapshot", "landing_volume_create", "hub_attach", + "transfer_submit", "transfer_complete", "hub_detach", + "chain_add_clone", "chain_convert", "round_total", + "round_gap_to_next_snapshot", +] +FREEZE_PHASES = [ + "final_hub_attach", "fence_source", "final_step_transfer", + "final_peer_add_clone", "enable_target_paths", "freeze_total", +] +# what counts as moving data, as opposed to arranging for data to move +TRANSFER_PHASES = {"transfer_complete", "final_step_transfer"} + + +def parse(path): + events = [] + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for raw in fh: + m = LINE.search(raw) + if not m: + continue + rec = {} + for k, v in KV.findall(m.group(1)): + if k in ("t", "ms", "mbps"): + try: + rec[k] = float(v) + except ValueError: + rec[k] = None + elif k in ("round", "bytes", "offset"): + try: + rec[k] = int(v) + except ValueError: + rec[k] = None + else: + rec[k] = v + if "phase" in rec: + events.append(rec) + events.sort(key=lambda r: r.get("t") or 0) + return events + + +def transfer_rate_from_progress(events): + """MB/s per (lvol, snap) inferred from transfer_running offsets. + + This is the one direct read on throughput: `offset` is bytes moved, so the + slope between the first and last sample is the real rate. + """ + runs = defaultdict(list) + for e in events: + if e["phase"] == "transfer_running" and e.get("offset") is not None: + runs[(e.get("lvol"), e.get("snap"))].append((e["t"], e["offset"])) + out = {} + for key, samples in runs.items(): + if len(samples) < 2: + continue + samples.sort() + (t0, o0), (t1, o1) = samples[0], samples[-1] + dt, do = t1 - t0, o1 - o0 + if dt > 0 and do > 0: + out[key] = (do / 1e6 / dt, do, dt, len(samples)) + return out + + +def report(events, csv_path=None): + if not events: + print("no XFER-TIMING lines found -- was the instrumented build deployed?") + return 1 + + span = events[-1]["t"] - events[0]["t"] + print("%d timing events over %.1fs\n" % (len(events), span)) + + # ---- per-phase totals ------------------------------------------------- + agg = defaultdict(lambda: [0, 0.0, 0.0]) # count, total_ms, max_ms + for e in events: + if e.get("ms") is None: + continue + a = agg[e["phase"]] + a[0] += 1 + a[1] += e["ms"] + a[2] = max(a[2], e["ms"]) + + ordered = [p for p in ROUND_PHASES + FREEZE_PHASES if p in agg] + ordered += [p for p in sorted(agg) if p not in ordered] + + print("%-28s %6s %12s %10s %10s" % ("phase", "n", "total_s", "mean_ms", "max_ms")) + print("-" * 70) + for p in ordered: + n, tot, mx = agg[p] + print("%-28s %6d %12.1f %10.1f %10.1f" + % (p, n, tot / 1000.0, tot / n, mx)) + + # ---- the split that matters ------------------------------------------- + # round_total/freeze_total are envelopes; don't double-count them. + envelopes = {"round_total", "freeze_total"} + moved = sum(agg[p][1] for p in agg if p in TRANSFER_PHASES) + arranged = sum(agg[p][1] for p in agg + if p not in TRANSFER_PHASES and p not in envelopes) + if moved + arranged > 0: + pct = 100.0 * moved / (moved + arranged) + print("\nDATA MOVEMENT %8.1fs (%.1f%%)" % (moved / 1000.0, pct)) + print("ORCHESTRATION %8.1fs (%.1f%%)" % (arranged / 1000.0, 100 - pct)) + if pct < 25: + print(" -> the pipeline is dominated by orchestration, not throughput;") + print(" tuning the transfer path cannot fix this.") + + # ---- what the instrumentation cannot explain ------------------------- + # If round_total dwarfs the sum of its measured parts, the missing time is + # somewhere we are not looking -- and that gap is the finding, not a + # rounding error. + inner = [p for p in ROUND_PHASES + if p not in ("round_total", "round_gap_to_next_snapshot")] + measured = sum(agg[p][1] for p in inner if p in agg) + envelope = agg.get("round_total", [0, 0.0, 0.0])[1] + if envelope > 0: + unaccounted = envelope - measured + pct = 100.0 * unaccounted / envelope + print() + print("round envelopes %8.1fs" % (envelope / 1000.0)) + print("measured phases %8.1fs" % (measured / 1000.0)) + print("UNACCOUNTED %8.1fs (%.1f%% of the envelope)" + % (unaccounted / 1000.0, pct)) + if pct > 30: + print(" -> most of a round is NOT in any instrumented phase.") + print(" Look at task_pass spacing first (scheduler latency),") + print(" then add phases where the gap actually is.") + + # ---- measured transfer throughput ------------------------------------ + rates = transfer_rate_from_progress(events) + if rates: + print("\nmeasured transfer throughput (from transfer_running offsets):") + print("%-12s %-10s %10s %12s %8s" % ("lvol", "snap", "MB/s", "bytes", "samples")) + for (lvol, snap), (mbps, nbytes, dt, n) in sorted( + rates.items(), key=lambda kv: -kv[1][0]): + print("%-12s %-10s %10.1f %12d %8d" % (lvol, snap, mbps, nbytes, n)) + else: + print("\nno transfer_running samples with offsets -- cannot measure") + print("throughput directly; only envelope durations are available.") + + # ---- the freeze, per volume ------------------------------------------ + freezes = [e for e in events if e["phase"] == "freeze_total"] + if freezes: + print("\nclient-visible freeze windows (fence -> paths live):") + for e in sorted(freezes, key=lambda r: -(r.get("ms") or 0)): + flag = " <-- OVER the 8s fast_io_fail_tmo" if (e.get("ms") or 0) > 8000 else "" + print(" lvol=%-10s %8.2fs%s" % (e.get("lvol"), (e["ms"] or 0) / 1000.0, flag)) + + # ---- task-runner latency -------------------------------------------- + passes = defaultdict(list) + for e in events: + if e["phase"] == "task_pass": + passes[e.get("lvol")].append(e["t"]) + if passes: + gaps = [] + for _lvol, times in passes.items(): + times.sort() + gaps += [b - a for a, b in zip(times, times[1:])] + if gaps: + gaps.sort() + print("\ntask-runner pass spacing: n=%d median=%.1fs max=%.1fs" + % (len(gaps), gaps[len(gaps) // 2], gaps[-1])) + print(" (each state change of a cutover costs about one of these)") + + if csv_path: + import csv + cols = ["t", "phase", "lvol", "snap", "round", "ms", "bytes", "mbps", + "offset", "state", "node", "ok"] + with open(csv_path, "w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=cols, extrasaction="ignore") + w.writeheader() + for e in events: + w.writerow(e) + print("\nwrote %s" % csv_path) + return 0 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("logfile") + ap.add_argument("--csv", default=None, help="also write the raw events as CSV") + args = ap.parse_args() + return report(parse(args.logfile), args.csv) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index f237c17a10..e9767f8c8e 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -1695,6 +1695,15 @@ commands: help: "Replicated internal snapshots to retain on each side. Minimum (and default): `2`." dest: keep_replicated type: int + - name: "--retention-schedule" + help: "Tiered retention, e.g. `15m:2h,1h:11h,1d:7d` - one snapshot every 15 minutes for the last 2 hours, then hourly for 11 hours, then daily for 7 days. Snapshots older than the total span are pruned. Empty (default) keeps the flat --keep behaviour." + dest: retention_schedule + type: str + - name: "--consistency-group" + help: "All volumes attached to this policy form ONE consistency group: they must share an LVS (creation pins them to it), cadence snapshots are taken as one frozen group, and fail-over generations resolve group-wide." + dest: consistency_group + type: bool + action: store_true - name: replication-policy-list help: Lists the replication policies of a cluster arguments: @@ -1727,6 +1736,13 @@ commands: dest: json type: bool action: store_true + - name: replication-policy-snapshot + help: "Takes ONE crash-consistent snapshot of every volume in the policy's consistency group, as a new group generation" + arguments: + - name: "policy_id" + help: "Replication policy id (must be a consistency-group policy)" + dest: policy_id + type: str - name: "volume" help: "Logical Volume Commands" aliases: diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 77c498dd97..e449d2c5bf 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -398,6 +398,7 @@ def init_cluster(self): self.init_cluster__replication_policy_list(subparser) self.init_cluster__replication_policy_remove(subparser) self.init_cluster__replication_policy_failover(subparser) + self.init_cluster__replication_policy_snapshot(subparser) def init_cluster__create(self, subparser): @@ -662,6 +663,8 @@ def init_cluster__replication_policy_add(self, subparser): subcommand.add_argument('--interval-min', help='Cadence: minutes between internal replication snapshots. 0 replicates user snapshots only. Default: `1`.', type=int, dest='interval_min') subcommand.add_argument('--mode', help='Replication mode. Default: `failover`.', type=str, dest='mode', choices=['failover','migration',]) subcommand.add_argument('--keep', help='Replicated internal snapshots to retain on each side. Minimum (and default): `2`.', type=int, dest='keep_replicated') + subcommand.add_argument('--retention-schedule', help='Tiered retention, e.g. `15m:2h,1h:11h,1d:7d` - one snapshot every 15 minutes for the last 2 hours, then hourly for 11 hours, then daily for 7 days. Snapshots older than the total span are pruned. Empty (default) keeps the flat --keep behaviour.', type=str, dest='retention_schedule') + subcommand.add_argument('--consistency-group', help='All volumes attached to this policy form ONE consistency group: they must share an LVS (creation pins them to it), cadence snapshots are taken as one frozen group, and fail-over generations resolve group-wide.', dest='consistency_group', action='store_true') def init_cluster__replication_policy_list(self, subparser): subcommand = self.add_sub_command(subparser, 'replication-policy-list', 'Lists the replication policies of a cluster') @@ -677,6 +680,10 @@ def init_cluster__replication_policy_failover(self, subparser): subcommand.add_argument('policy_id', help='Replication policy id', type=str) subcommand.add_argument('--json', help='Print outputs in json format.', dest='json', action='store_true') + def init_cluster__replication_policy_snapshot(self, subparser): + subcommand = self.add_sub_command(subparser, 'replication-policy-snapshot', 'Takes ONE crash-consistent snapshot of every volume in the policy\'s consistency group, as a new group generation') + subcommand.add_argument('policy_id', help='Replication policy id (must be a consistency-group policy)', type=str) + def init_volume(self): subparser = self.add_command('volume', 'Logical Volume Commands', aliases=['lvol',]) @@ -1503,6 +1510,8 @@ def run(self): ret = self.cluster__replication_policy_remove(sub_command, args) elif sub_command in ['replication-policy-failover']: ret = self.cluster__replication_policy_failover(sub_command, args) + elif sub_command in ['replication-policy-snapshot']: + ret = self.cluster__replication_policy_snapshot(sub_command, args) else: self.parser.print_help() diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 749efd7e50..76cfbd3bb4 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -661,7 +661,17 @@ def cluster__replication_policy_add(self, sub_command, args): return replication_policy_controller.add_policy( args.cluster_id, args.name, args.target, interval_min=args.interval_min, mode=args.mode, - keep_replicated=args.keep_replicated) + keep_replicated=args.keep_replicated, + retention_schedule=args.retention_schedule, + consistency_group=args.consistency_group) + + def cluster__replication_policy_snapshot(self, sub_command, args): + from simplyblock_core.controllers import consistency_group_controller + snap_ids, err = consistency_group_controller.create_group_snapshot( + args.policy_id) + if err: + return f"Group snapshot failed: {err}" + return utils.print_table([{"Snapshot": s} for s in snap_ids]) def cluster__replication_policy_list(self, sub_command, args): data = [{ @@ -671,6 +681,8 @@ def cluster__replication_policy_list(self, sub_command, args): "Interval (min)": p.interval_min, "Mode": p.mode, "Keep": p.keep_replicated, + "Retention": p.retention_schedule or "-", + "CG": "yes" if getattr(p, "consistency_group", False) else "-", "Status": p.status, } for p in replication_policy_controller.list_policies(args.cluster_id)] return _format_result(data, json=args.json) @@ -688,12 +700,16 @@ def _format_failover_results(self, results, args): return _format_json(results) if not results: return "No volumes to fail over" - return utils.print_table([{ + table = utils.print_table([{ "Volume": r.get("lvol_id", ""), "Status": r.get("status", ""), "Target Volume": r.get("target_lvol_id", "") or "-", "Detail": r.get("detail", "") or "", } for r in results]) + warnings = [w for r in results for w in (r.get("warnings") or [])] + if warnings: + table += chr(10) + chr(10).join("WARNING: " + w for w in warnings) + return table def volume__replication_policy_set(self, sub_command, args): return replication_policy_controller.attach_policy(args.volume_id, args.policy) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 34739a22e6..d4d28bebfb 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -374,6 +374,91 @@ def get_config_var(name, default=None): # if the operator is unavailable. Cutover proceeds regardless after this many seconds. REPL_CUTOVER_PROCEED_TIMEOUT_SEC = 120 +# --- cutover delta convergence ------------------------------------------- +# The IO freeze copies everything written since the last replicated snapshot, +# so the cutover converges the delta FIRST: take a snapshot, transfer it, and +# immediately take the next, until a round transfers in "low seconds". A fixed +# two rounds (the previous behaviour) does not converge under load -- it just +# stops. +REPL_CUTOVER_CONVERGE_TARGET_SEC = 2.0 +# Safety bound: a volume written faster than it replicates never converges, so +# stop and freeze rather than looping forever. +REPL_CUTOVER_MAX_SHRINK_ROUNDS = 12 +# When to stop converging in the open and take the lvstore for the endgame. +# A round completing within this multiple of the target means the delta is +# nearly converged, so the exclusive window that follows will be short. Claiming +# earlier serialises the bulk catch-up, which is what produced 0/20 cutovers in +# run 20260828_124859 (round 1 growing 340s -> 2584s purely from queueing). +# The endgame starts once ordinary replication has the target within this many +# seconds. Before that the cutover waits and takes NO snapshots of its own -- +# the iterative snapshots ARE the endgame. +REPL_CUTOVER_ENDGAME_LAG_SEC = 50 +# Rounds must follow each other within MILLISECONDS. Returning to the task +# scheduler between them costs TASK_EXEC_INTERVAL_SEC (10s) of fresh writes +# each time, which puts a floor under the delta no number of rounds can beat. +REPL_CUTOVER_POLL_INTERVAL_SEC = 0.2 +# How long a single runner pass may stay inside the convergence loop. +REPL_CUTOVER_CONVERGE_BUDGET_SEC = 60 +# Always worth polling inline for at least this long: a round that finishes +# just after the pass is handed back costs a full TASK_EXEC_INTERVAL_SEC of +# writes in the next round. +# The snapshot-replication runner now finishes a transfer in the pass that +# submitted it, so a convergence round completes in about the transfer's own +# duration. Staying inline across that is what makes "next snapshot within a +# second of completion" true; yielding mid-round reintroduces pass latency. +REPL_CUTOVER_MIN_INLINE_SEC = 30 + +# Whether to block the cutover on the operator's preconnect signal. The wait +# sits BETWEEN the cutover clone's base snapshot and the freeze, so every +# second of it is a second of writes the frozen final step must copy: with no +# operator present the 120s fallback timeout fired 34 times in one soak run and +# fed the 25-72s freezes. Deployments whose operator posts +# .../replication/cutover-proceed set this True and accept that cost until the +# clone's base can be advanced after the signal. +# +# Enabled 2026-09-02: the operator's reconcileCutoverPending runs the +# preconnect Job and posts cutover-proceed for both migration and failback +# (annotFailbackTarget routes the call to the target cluster on failback). +# Without the gate, the flip races the client: the 2026-09-02 failback run +# flipped ANA on listeners no client had connected to and deleted the DR-side +# subsystem 150ms later, orphaning every connected client for ctrl_loss_tmo. +REPL_CUTOVER_PROCEED_REQUIRED = True + +# --- noticing a finished transfer ---------------------------------------- +# A transfer that has completed must be acted on within a second: the next +# convergence snapshot cannot be taken until the previous one is marked +# replicated, so observation latency lands directly in the IO freeze. +REPL_XFER_POLL_INTERVAL_SEC = 0.1 +# How long the submitting pass may wait inline for the transfer. The runner is +# single-threaded, so this is a starvation budget, not a timeout: exceeding it +# just falls back to being noticed on a later pass. +REPL_XFER_INLINE_WAIT_SEC = 5.0 +# A volume in its final cutover already owns its lvstore and every other +# transfer on it is held, so there is nothing to starve -- wait as long as the +# transfer needs, because this is exactly the window the client freeze pays for. +REPL_XFER_INLINE_WAIT_CUTOVER_SEC = 300.0 +# Cooldown between hub-attach retry attempts when the target node is down or +# recovering (covers control-plane lag before the DB reflects the down state). +REPL_CUTOVER_HUB_RETRY_COOLDOWN_SEC = 30 +# Max consecutive hub-attach failures with the node still appearing online +# before we give up and burn a task.retry. 30s × 20 = 10 min of coverage. +REPL_CUTOVER_MAX_HUB_ATTEMPTS = 10 +# Pass interval while a cutover is mid-round. NOT sub-second: this loop reads +# the task table per pass, and polling a database at 5Hz to detect an event is +# the wrong shape. Sub-second reaction lives in the RPC-based inline wait. +REPL_CUTOVER_ACTIVE_POLL_SEC = 1.0 +# Delete the superseded original volume BEFORE building the fail-back clone +# (_retire_superseded_original). Disabled 2026-09-01: that delete frees the +# original's blob id while its parent snapshot's clone registry is already +# inconsistent ("Clone entry not found for blob ... under snapshot ..."), the +# clone created seconds later reuses the freed id, and every final-step delta +# write to it fails rc -1 (-EPERM) -> transfer_state Failed on all fail-back +# cutovers. The SPDK-side namespace slot is still freed by +# _evict_stale_namespace, and the original's DB record is removed after a +# successful cutover by _swap_failback_lvol_uuid. Re-enable once the fork's +# clone-entry/blob-id-reuse defect is fixed. +REPL_FAILBACK_RETIRE_ORIGINAL_BEFORE_CUTOVER = False + SPDK_PROXY_MULTI_THREADING_ENABLED=True SPDK_PROXY_TIMEOUT=60*5 LVOL_NVME_CONNECT_RECONNECT_DELAY=2 diff --git a/simplyblock_core/controllers/consistency_group_controller.py b/simplyblock_core/controllers/consistency_group_controller.py new file mode 100644 index 0000000000..fb2df7ccaf --- /dev/null +++ b/simplyblock_core/controllers/consistency_group_controller.py @@ -0,0 +1,414 @@ +# coding=utf-8 +"""Consistency groups: group-wide crash-consistent snapshots for a policy. + +A replication policy created with ``consistency_group=True`` owns exactly one +auto-managed :class:`ConsistencyGroup`. Its members are the volumes attached +to the policy; they all live on ONE node/LVS (the group pins placement on the +first attach and enforces it afterwards), because the group snapshot freezes +IO per member blob on that LVS and a cross-LVS "group" would only be as +consistent as its slowest freeze. + +The group snapshot itself is ONE SPDK call (``bdev_lvol_snapshot_group``): +IO on every member is parked before the first snapshot and released after the +last, so the resulting set is a single point in time across the group. SPDK +garbage-collects on mid-sequence failure (unfreeze first, then delete the +snapshots already taken), so this controller never sees half a group from a +failed RPC. What this controller owns is everything around that call: +member resolution, the monotonically increasing ``group_seq``, replica +registration, snapshot records, chain linking and replication-task enqueue — +mirroring ``snapshot_controller.add`` step for step for each member. + +Membership epochs: a volume attached after the group already ticked joins at +``last_group_seq + 1`` — the first group snapshot that actually contains it. +Earlier generations do not, and a volume detached at seq M is not in +generations after M. :func:`generation_membership_warnings` computes exactly +the two warnings the fail-over path must surface when an operator selects an +older generation. +""" +import time +import uuid as uuid_module +from datetime import datetime + +from simplyblock_core import db_controller as db_mod +from simplyblock_core import utils +from simplyblock_core.controllers import snapshot_events, tasks_controller +from simplyblock_core.controllers.snapshot_controller import ( + _find_lvs_leader, _rollback_snapshot_bdev, lvstore_op_lock) +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.replication import ConsistencyGroup +from simplyblock_core.models.snapshot import SnapShot +from simplyblock_core.models.storage_node import StorageNode + +logger = utils.get_logger(__name__) +db = db_mod.DBController() + + +class ConsistencyGroupError(Exception): + pass + + +# --------------------------------------------------------------------------- # +# Group lifecycle (driven by the policy controller) +# --------------------------------------------------------------------------- # + +def create_group_for_policy(policy): + """Auto-create the group record when a consistency-group policy is made.""" + group = ConsistencyGroup() + group.uuid = str(uuid_module.uuid4()) + group.cluster_id = policy.cluster_id + group.policy_id = policy.get_id() + group.members = {} + group.write_to_db(db.kv_store) + logger.info("Created consistency group %s for policy %s", + group.get_id(), policy.policy_name) + return group + + +def delete_group_for_policy(policy_id): + """Auto-delete with the policy (which is only removable member-free).""" + group = db.get_consistency_group_for_policy(policy_id) + if group is not None: + group.remove(db.kv_store) + logger.info("Removed consistency group %s of policy %s", + group.get_id(), policy_id) + + +def add_member(policy, lvol): + """Enforce requirement 1 and record the member's epoch. + + The FIRST member pins the group to its node/LVS. Every later member must + already live there — the attachment FAILS otherwise; membership becomes + effective at the NEXT group snapshot (``joined_seq = last_group_seq + 1``), + because no earlier group snapshot contains this volume. + + A re-attaching volume gets a fresh epoch: its old history window stays + recorded under the closed epoch semantics (removed_seq of the old entry + is preserved only implicitly by the new joined_seq being later — the + entry is replaced, and the generation math treats the gap correctly + because the new joined_seq excludes the detached window). + """ + group = db.get_consistency_group_for_policy(policy.get_id()) + if group is None: + # Policies created before the flag existed, or records lost: fail + # loudly rather than silently degrading to per-volume snapshots. + raise ConsistencyGroupError( + f"Policy {policy.policy_name} declares a consistency group but " + f"has no group record") + + if group.lvs_name and (lvol.lvs_name != group.lvs_name + or lvol.node_id != group.node_id): + raise ConsistencyGroupError( + f"Volume {lvol.get_id()} lives on {lvol.node_id[:8]}/{lvol.lvs_name} " + f"but consistency group {group.uuid[:8]} of policy " + f"{policy.policy_name} is pinned to " + f"{group.node_id[:8]}/{group.lvs_name}; all members of a " + f"consistency group must share one LVS") + + if not group.lvs_name: + group.node_id = lvol.node_id + group.lvs_name = lvol.lvs_name + logger.info("Consistency group %s pinned to node %s / %s by its first " + "member %s", group.uuid[:8], lvol.node_id[:8], + lvol.lvs_name, lvol.get_id()) + + members = dict(group.members or {}) + members[lvol.get_id()] = {"joined_seq": group.last_group_seq + 1, + "removed_seq": 0} + group.members = members + group.write_to_db(db.kv_store) + logger.info("Volume %s joined consistency group %s at generation %d " + "(effective from the next group snapshot)", + lvol.get_id(), group.uuid[:8], group.last_group_seq + 1) + return group + + +def remove_member(policy_id, lvol_id): + """Close the member's epoch at the current generation.""" + group = db.get_consistency_group_for_policy(policy_id) + if group is None: + return + members = dict(group.members or {}) + entry = members.get(lvol_id) + if entry and entry.get("removed_seq", 0) == 0: + entry = dict(entry) + entry["removed_seq"] = max(group.last_group_seq, entry.get("joined_seq", 1) - 1) + members[lvol_id] = entry + group.members = members + group.write_to_db(db.kv_store) + logger.info("Volume %s left consistency group %s (included up to " + "generation %d)", lvol_id, group.uuid[:8], entry["removed_seq"]) + + +def pinned_node_for_policy(policy): + """The node a NEW volume under this policy must be created on, or None.""" + group = db.get_consistency_group_for_policy(policy.get_id()) + if group is not None and group.node_id: + return group.node_id + return None + + +# --------------------------------------------------------------------------- # +# Generation membership warnings (requirement 4) — pure logic +# --------------------------------------------------------------------------- # + +def generation_membership_warnings(group, seq): + """The two warnings an operator must see when failing over to ``seq``. + + Returns a list of strings: + * one for current members NOT included in that generation (late + joiners whose ``joined_seq`` is newer than ``seq``); + * one for volumes included in that generation that are NO LONGER + members (their epoch covers ``seq`` but ``removed_seq`` is set). + Empty list when the generation matches current membership exactly. + """ + if group is None or not seq: + return [] + missing = [] + stale = [] + for lvol_id, m in (group.members or {}).items(): + joined = m.get("joined_seq", 1) + removed = m.get("removed_seq", 0) + is_current = removed == 0 + included = joined <= seq and (removed == 0 or seq <= removed) + if is_current and not included: + missing.append(lvol_id) + if not is_current and included: + stale.append(lvol_id) + warnings = [] + if missing: + warnings.append( + "generation %d predates %d current group member(s); NOT included " + "in this point-in-time: %s" % (seq, len(missing), ", ".join(sorted(missing)))) + if stale: + warnings.append( + "generation %d includes %d volume(s) that are no longer group " + "members: %s" % (seq, len(stale), ", ".join(sorted(stale)))) + return warnings + + +def warnings_for_snapshot(lvol, snapshot): + """Convenience for the fail-over path: warnings for the generation the + chosen snapshot belongs to, [] for non-group snapshots/policies.""" + seq = getattr(snapshot, "group_seq", 0) + group_id = getattr(snapshot, "group_id", "") + if not seq or not group_id: + return [] + try: + group = db.get_consistency_group_by_id(group_id) + except KeyError: + return [] + return generation_membership_warnings(group, seq) + + +# --------------------------------------------------------------------------- # +# The group snapshot tick +# --------------------------------------------------------------------------- # + +def _current_members(group): + """Live member volumes: attached to the policy, epoch open, usable.""" + members = [] + for lvol_id, m in (group.members or {}).items(): + if m.get("removed_seq", 0) != 0: + continue + try: + lvol = db.get_lvol_by_id(lvol_id) + except KeyError: + continue + if lvol.status != LVol.STATUS_ONLINE: + logger.warning("Consistency group %s: member %s is %s; the group " + "snapshot is skipped this tick (a group snapshot " + "missing a member is not a group snapshot)", + group.uuid[:8], lvol_id, lvol.status) + return None + members.append(lvol) + return members + + +def create_group_snapshot(policy_id, snap_type=SnapShot.TYPE_INTERNAL, lock=True): + """Take ONE crash-consistent snapshot of every group member. + + Returns (list_of_snapshot_ids, None) or (None, error). All-or-nothing: + a failure anywhere rolls back every snapshot bdev of this tick (SPDK + already GC'd if the failure was inside the RPC; registration/record + failures are rolled back here) and the generation counter does not move. + """ + policy = db.get_replication_policy_by_id(policy_id) + group = db.get_consistency_group_for_policy(policy.get_id()) + if group is None: + return None, f"Policy {policy_id} has no consistency group" + + members = _current_members(group) + if members is None: + return None, "consistency group member not online" + if not members: + return None, "consistency group has no members" + + # Placement invariant (defense in depth: attach enforces it already). + for lvol in members: + if lvol.lvs_name != group.lvs_name or lvol.node_id != group.node_id: + return None, (f"member {lvol.get_id()} is on " + f"{lvol.node_id[:8]}/{lvol.lvs_name}, group is pinned " + f"to {group.node_id[:8]}/{group.lvs_name}") + + host_node = db.get_storage_node_by_id(group.node_id) + pool = db.get_pool_by_id(members[0].pool_uuid) + cluster = db.get_cluster_by_id(pool.cluster_id) + + # Leader + HA member set, same as the single-snapshot path. + secondary_ids = [host_node.secondary_node_id] + if host_node.tertiary_node_id: + secondary_ids.append(host_node.tertiary_node_id) + all_nodes = [host_node] + for sid in secondary_ids: + if not sid: + continue + try: + all_nodes.append(db.get_storage_node_by_id(sid)) + except KeyError: + pass + primary_node = _find_lvs_leader(pool.cluster_id, group.lvs_name, all_nodes) + if not primary_node: + return None, (f"No leader available for LVS {group.lvs_name} — " + f"rejecting the group snapshot until leadership is " + f"re-established") + secondary_nodes = [n for n in all_nodes + if n.get_id() != primary_node.get_id() + and n.status == StorageNode.STATUS_ONLINE] + + group_seq = group.last_group_seq + 1 + now_ts = int(time.time()) + plan = [] + for lvol in members: + snap_vuid = utils.get_random_snapshot_vuid() + plan.append({ + "lvol": lvol, + "vuid": snap_vuid, + "snap_bdev_name": f"SNAP_{snap_vuid}", + "snap_name": f"repl_cg_{group.uuid[:8]}_{group_seq}_{lvol.get_id()[:8]}_{now_ts}", + }) + + rpc_client = primary_node.rpc_client() + logger.info("Consistency group %s: taking generation %d over %d member(s) " + "on %s/%s", group.uuid[:8], group_seq, len(plan), + primary_node.get_id()[:8], group.lvs_name) + + # ONE lvstore mutation: the whole frozen window is a single RPC. + with lvstore_op_lock(pool.cluster_id, group.lvs_name, + node_id=primary_node.get_id(), enabled=lock): + ret = rpc_client.bdev_lvol_snapshot_group( + group.lvs_name, + [{"lvol_name": f"{p['lvol'].lvs_name}/{p['lvol'].lvol_bdev}", + "snapshot_name": p["snap_bdev_name"]} for p in plan]) + if not ret: + # SPDK unfroze first and garbage-collected the partial snapshots. + return None, (f"Group snapshot RPC failed on {primary_node.get_id()}; " + f"SPDK rolled the partial group back") + + # Bound outside the closure: mypy does not carry the ``group is None`` + # guard's narrowing into nested functions. + group_lvs_name = group.lvs_name + + def _rollback_all(): + for p in plan: + _rollback_snapshot_bdev(pool.cluster_id, group_lvs_name, + primary_node, p["snap_bdev_name"], + all_nodes, lock=lock) + + # Everything below mirrors snapshot_controller.add's tail per member: + # read back uuid/blobid, register on the HA peers, then the record. + created_ids: list = [] + for p in plan: + lvol = p["lvol"] + snap_bdev = rpc_client.get_bdevs(f"{group.lvs_name}/{p['snap_bdev_name']}") + if not snap_bdev: + _rollback_all() + return None, (f"group snapshot {p['snap_bdev_name']} not readable " + f"after creation") + p["snap_uuid"] = snap_bdev[0]["uuid"] + p["blobid"] = snap_bdev[0]["driver_specific"]["lvol"]["blobid"] + num_allocated = snap_bdev[0]["driver_specific"]["lvol"]["num_allocated_clusters"] + p["used_size"] = int(num_allocated * cluster.page_size_in_blocks) + + for sec in secondary_nodes: + from simplyblock_core.storage_node_ops import ( + wait_or_delay_for_restart_gate, queue_for_restart_drain) + gate = wait_or_delay_for_restart_gate(sec.get_id(), group.lvs_name) + if gate == "delay": + queue_for_restart_drain( + sec.get_id(), group.lvs_name, + lambda s=sec, pp=p, lv=lvol: s.rpc_client().bdev_lvol_snapshot_register( + f"{group.lvs_name}/{lv.lvol_bdev}", pp["snap_bdev_name"], + pp["snap_uuid"], pp["blobid"]), + f"register group snapshot {p['snap_bdev_name']} on {sec.get_id()[:8]}") + continue + with lvstore_op_lock(pool.cluster_id, group.lvs_name, + node_id=sec.get_id(), enabled=lock): + reg = sec.rpc_client().bdev_lvol_snapshot_register( + f"{group.lvs_name}/{lvol.lvol_bdev}", p["snap_bdev_name"], + p["snap_uuid"], p["blobid"]) + if not reg: + logger.error("Group snapshot register of %s failed on %s; " + "rolling the WHOLE generation back", + p["snap_bdev_name"], sec.get_id()) + _rollback_all() + for snap_id in created_ids: + try: + rec = db.get_snapshot_by_id(snap_id) + db.unindex_snapshot(rec) + rec.remove(db.kv_store) + except Exception: + pass + return None, f"Failed to register group snapshot on {sec.get_id()}" + + snap = SnapShot() + snap.uuid = str(uuid_module.uuid4()) + snap.data_uuid = str(uuid_module.uuid4()) + snap.snap_uuid = p["snap_uuid"] + snap.size = lvol.size + snap.used_size = p["used_size"] + snap.blobid = p["blobid"] + snap.pool_uuid = pool.get_id() + snap.cluster_id = pool.cluster_id + snap.snap_name = p["snap_name"] + snap.snap_bdev = f"{group.lvs_name}/{p['snap_bdev_name']}" + snap.created_at = now_ts + snap.lvol = lvol + snap.fabric = lvol.fabric + snap.vuid = p["vuid"] + snap.status = SnapShot.STATUS_ONLINE + snap.snap_type = snap_type + snap.group_id = group.get_id() + snap.group_seq = group_seq + snap.create_dt = str(datetime.now()) + snap.write_to_db(db.kv_store) + + prev = db.get_lvol_latest_snapshot(lvol.get_id(), exclude_uuid=snap.get_id()) + if prev is not None and not prev.next_snap_uuid: + prev.next_snap_uuid = snap.get_id() + snap.prev_snap_uuid = prev.get_id() + prev.write_to_db() + snap.write_to_db() + + db.index_snapshot(snap) + snapshot_events.snapshot_create(snap) + created_ids.append(snap.get_id()) + p["snap_id"] = snap.get_id() + + # The generation exists in full: bump the counter, then enqueue the + # per-member replication tasks (transfer machinery is per-snapshot). + group = db.get_consistency_group_by_id(group.get_id()) + group.last_group_seq = group_seq + group.write_to_db(db.kv_store) + + for p in plan: + lvol = p["lvol"] + if lvol.do_replicate: + task = tasks_controller.add_snapshot_replication_task( + pool.cluster_id, lvol.node_id, p["snap_id"]) + if task: + snap = db.get_snapshot_by_id(p["snap_id"]) + snapshot_events.replication_task_created(snap) + + logger.info("Consistency group %s: generation %d complete (%d snapshots)", + group.uuid[:8], group_seq, len(created_ids)) + return created_ids, None diff --git a/simplyblock_core/controllers/health_controller.py b/simplyblock_core/controllers/health_controller.py index da51ed919c..01ed604f56 100644 --- a/simplyblock_core/controllers/health_controller.py +++ b/simplyblock_core/controllers/health_controller.py @@ -449,6 +449,10 @@ def _check_sec_node_hublvol(node: StorageNode, auto_fix=False, primary_node_id=N # same target, so this is a fault to surface, not cosmetics. duplicate_ips = storage_node_ops.duplicate_attached_paths(ret) if duplicate_ips: + logger.error( + "Hublvol %s on %s has duplicate path(s) %s -- node is NOT " + "healthy; repair_multipath_controller will prune them", + primary_node.hublvol.bdev_name, node.get_id(), duplicate_ips) # Detecting this without acting on it is what made the # 2026-09-01 multipath soak unrunnable: LVS_4/hublvol carried # peer 98.248 three times (97.36 twice, cntlid 1002 and 1003), @@ -1109,7 +1113,10 @@ def check_lvol_on_node(lvol_id, node_id, node_bdev_names=None, node_lvols_nqns=N bdev_check = check_bdev(lvol.top_bdev, rpc_client=rpc_client) passed &= bdev_check - passed &= check_subsystem(lvol.nqn, rpc_client=rpc_client, ns_uuid=lvol.uuid) + # The namespace advertises the WIRE identity, which differs from the + # record uuid after a fail-back — checking the record uuid flags every + # failed-back volume unhealthy and triggers the monitor's self-heal. + passed &= check_subsystem(lvol.nqn, rpc_client=rpc_client, ns_uuid=lvol.get_ns_uuid()) except Exception as e: logger.error(e) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 7f149c204f..e17844324c 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -335,6 +335,51 @@ def validate_aes_xts_keys(key1: str, key2: str) -> Tuple[bool, str]: return True, "" +def _sibling_replication_node(lvol, cl, all_lvols=None): + """Target node an already-replicating sibling of ``lvol``'s subsystem uses. + + Namespaced siblings MUST replicate to the same target node. A fail-over + copy keeps the volume's NQN, so every volume of one shared subsystem lands + in the SAME subsystem on the target. When the group is split across two + target primaries, each primary auto-assigns nsids from only its own subset + and the HA peers -- which hold the whole group -- reject the collision + (soak case 7: peer held nsids 1..7 while a split-off primary asked for 1). + + Returns "" when no sibling has a target node yet. + """ + if not (getattr(lvol, "namespaced", False) or lvol.max_namespace_per_subsys > 1): + return "" + for lv in (all_lvols or DBController().get_lvols(cl.get_id())): + if (lv.nqn == lvol.nqn and lv.get_id() != lvol.get_id() + and getattr(lv, "replication_node_id", "")): + return lv.replication_node_id + return "" + + +def _realign_replication_node_after_claim(lvol, cl): + """Re-derive the target node once the AUTHORITATIVE subsystem is known. + + ``_resolve_lvol_subsystem`` is advisory: it may hand out a fresh standalone + nqn while ``claim_lvol_ns_slot``'s transaction -- recounting occupancy with + concurrent creates visible -- then joins the lvol to an EXISTING shared + subsystem instead. The target node was picked against the advisory nqn, so + such an lvol joins a group whose other members replicate elsewhere, and it + is exactly that member whose fail-over copy collides on the peer. Recheck + against the nqn that was actually persisted. + """ + if not getattr(lvol, "replication_node_id", ""): + return + sibling_node_id = _sibling_replication_node(lvol, cl) + if sibling_node_id and sibling_node_id != lvol.replication_node_id: + logger.info( + "LVol %s was placed in subsystem %s by the claim transaction " + "(not the advisory pick); moving its replication node from %s to " + "%s to match the siblings already in that subsystem", + lvol.lvol_name, lvol.nqn, lvol.replication_node_id, sibling_node_id) + lvol.replication_node_id = sibling_node_id + lvol.write_to_db() + + def _resolve_lvol_subsystem(lvol, host_node, cl, namespaced, all_lvols, internal=False): """ADVISORY pre-check of the subsystem pick for a new lvol — fails the @@ -428,6 +473,28 @@ def add_lvol_ha(name, size, host_id_or_name, ha_type, pool_id_or_name, use_comp= f"max_namespace_per_subsys={max_namespace_per_subsys} exceeds the " f"hard limit of {constants.MAX_NAMESPACES_PER_SUBSYSTEM} " f"namespaces per subsystem") + if replication_policy: + # A consistency-group policy pins placement: every member must live in + # ONE LVS, so a volume created under such a policy is forced onto the + # group's node BEFORE placement runs (requirement: pin to host before + # creation). An explicit conflicting --host is an error, not a + # preference fight. + from simplyblock_core.controllers import replication_policy_controller as _rpc + from simplyblock_core.controllers import consistency_group_controller as _cgc + try: + _policy = _rpc._resolve_policy(replication_policy) + except KeyError: + return False, f"Replication policy not found: {replication_policy}" + if getattr(_policy, "consistency_group", False): + pinned = _cgc.pinned_node_for_policy(_policy) + if pinned: + if host_id_or_name and host_id_or_name != pinned: + return False, ( + f"Volume must be created on node {pinned} — its " + f"replication policy {_policy.policy_name} is a " + f"consistency group pinned to that node's LVS") + host_id_or_name = pinned + host_node = None if host_id_or_name: try: @@ -674,8 +741,27 @@ def add_lvol_ha(name, size, host_id_or_name, ha_type, pool_id_or_name, use_comp= return False, f"Replication cluster not found: {replication_cluster_id}" else: replication_cluster_id = cl.snapshot_replication_target_cluster - random_nodes = _get_next_3_nodes(replication_cluster_id, lvol.size, all_lvols) - lvol.replication_node_id = random_nodes[0].get_id() + # Namespaced siblings MUST replicate to the same target node. + # A fail-over copy preserves the volume's NQN (its nsid is assigned + # afresh by the destination), so all + # volumes sharing a subsystem land in the SAME subsystem on the + # target. Picking the destination purely by capacity scattered + # siblings across the target cluster's nodes, which splits one + # shared subsystem across unrelated primaries: each advertises the + # same NQN with only its own subset of namespaces, and the copies + # collide when a sibling's nsid is already taken there (soak case 7, + # run 20260824_215758: 14 of 20 namespaces failed over, the 15th + # died in add_ns). + sibling_node_id = _sibling_replication_node(lvol, cl, all_lvols) + if sibling_node_id: + logger.info( + f"LVol {lvol.lvol_name} shares subsystem {lvol.nqn} with an " + f"already-replicating sibling; using its replication node " + f"{sibling_node_id} so the shared subsystem is not split") + lvol.replication_node_id = sibling_node_id + else: + random_nodes = _get_next_3_nodes(replication_cluster_id, lvol.size, all_lvols) + lvol.replication_node_id = random_nodes[0].get_id() lvol_dict: dict = { "type": "bdev_lvol", @@ -771,6 +857,8 @@ def add_lvol_ha(name, size, host_id_or_name, ha_type, pool_id_or_name, use_comp= logger.error(str(e)) return False, str(e) + _realign_replication_node_after_claim(lvol, cl) + if ha_type == "single": if host_node.status == StorageNode.STATUS_ONLINE: # INNER per-node lock. The create path took no lock at all, @@ -1057,6 +1145,26 @@ def _create_bdev_stack(lvol, snode, is_primary=True): else: ret = rpc_client.bdev_lvol_clone_register( lvol.lvol_bdev, lvol.snapshot_name, lvol.lvol_uuid, lvol.blobid) + if ret: + # clone_register ACKNOWLEDGES before the bdev is + # examinable (the same async false-success family as + # remove_ns in the PVC-expand and case-3 incidents). The + # very next step adds this bdev to the nvmf subsystem, and + # racing the registration lost every time on the fail-over + # of namespaced volumes: peer add_ns -32602 with the + # subsystem EMPTY, while the bdev existed moments later + # (run 20260825_122423, LVS_13/LVOL_121). Poll until the + # bdev is really there before letting add_ns proceed. + bdev_name = f"{lvol.lvs_name}/{lvol.lvol_bdev}" + for _ in range(40): + if rpc_client.get_bdevs(bdev_name): + break + time.sleep(0.5) + else: + logger.error( + f"clone_register acknowledged but {bdev_name} did " + f"not appear within 20s on the peer") + ret = None else: logger.debug(f"Unknown BDev type: {type}") @@ -1105,19 +1213,62 @@ def _resolve_namespaced_subsystem(lvol, rpc_client, snode): whose response grows with total lvol count, paid on EVERY create to catch a race that occurs at most once per max_namespaces creates. Trust the CP's own record and let the error path pay the dump only when it actually fires. + + EXCEPT that "someone else already created it" is only true PER NODE. The + record says the volume shares a subsystem; it says nothing about whether + THIS node has that subsystem yet. On the primary a wrong guess self-heals + (the -32602 fallback re-resolves and retries), but a REPLICA has no such + fallback -- its nsid is dictated by the primary -- so it fails the whole + create/fail-over with "subsystem does not exist on " (case 7, + run 20260826_214011: the very first namespaced fail-over died this way, + on a peer whose subsystem had never been created). So for a shared + subsystem, ask the node itself. The probe is the nqn-FILTERED + nvmf_get_subsystems, not the full dump this docstring warns about. """ - return not lvol.namespace + if not lvol.namespace: + return True # dedicated subsystem: always create + try: + return not rpc_client.subsystem_get(lvol.nqn) + except Exception as e: # noqa: BLE001 - probe must not decide + logger.warning("Could not probe subsystem %s on %s (%s); assuming it " + "exists, as the record implies", lvol.nqn, + snode.get_id()[:8], e) + return False -def _fail_after_bdev(lvol, rpc_client, msg): +def _fail_after_bdev(lvol, rpc_client, msg, is_primary=True): """Rollback an in-progress add_lvol_on_node after _create_bdev_stack has already produced a bdev/blob. Without this, a post-bdev-stack failure (a missing namespaced subsystem, a listener add error, an add_ns error) leaves the SPDK clone-blob in place, which then blocks the parent snapshot delete with "vbdev_lvol_destroy: ... has N clones". Logs but does not raise on - rollback failure so the caller still sees the original error.""" + rollback failure so the caller still sees the original error. + + ``is_primary`` decides HOW the bdev goes away. A REPLICA's blob is a + registration, not the leader's copy, and the leader-gated async delete is + refused there -- "Deleting async lvol on non-leader lvs" -- so the + rollback silently left the bdev AND its namespace behind. Those leftovers + poison the next attempt: the peer's shared subsystem keeps namespaces at + nsids the primary (starting clean) later hands to OTHER volumes, and every + subsequent fail-over is rejected with "wanted nsid=1 ... holds=[(1, ), ...]" -- 4 stale namespaces on the peer in case 7 run + 20260826_221806. Replicas therefore roll back with a SYNC delete, and the + namespace is dropped first so nothing of this attempt survives in the + subsystem. + """ try: - _remove_bdev_stack(lvol.bdev_stack[::-1], rpc_client) + try: + subsystem = rpc_client.subsystem_get(lvol.nqn) + for ns in ((subsystem or {}).get("namespaces") or []): + if ns.get("bdev_name") == lvol.top_bdev: + logger.info("rollback: removing namespace nsid=%s (%s) left " + "by the failed attempt", + ns.get("nsid"), lvol.top_bdev) + rpc_client.nvmf_subsystem_remove_ns(lvol.nqn, ns.get("nsid")) + except Exception: # noqa: BLE001 - best effort + logger.exception("rollback: could not clear the namespace for %s", + lvol.get_id()) + _remove_bdev_stack(lvol.bdev_stack[::-1], rpc_client, sync=not is_primary) lvol.status = LVol.STATUS_IN_DELETION lvol.write_to_db(DBController().kv_store) except Exception: @@ -1202,7 +1353,7 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid ret, msg = _create_bdev_stack(lvol, snode, is_primary=is_primary) if not ret: - return _fail_after_bdev(lvol, rpc_client, msg) + return _fail_after_bdev(lvol, rpc_client, msg, is_primary=is_primary) db_controller = DBController() pool = db_controller.get_pool_by_id(lvol.pool_uuid) @@ -1212,7 +1363,7 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid try: resolve_subsys = _resolve_namespaced_subsystem(lvol, rpc_client, snode) except Exception as e: - return _fail_after_bdev(lvol, rpc_client, str(e)) + return _fail_after_bdev(lvol, rpc_client, str(e), is_primary=is_primary) if resolve_subsys: if min_cntlid is None: @@ -1299,7 +1450,7 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid else: return _fail_after_bdev( lvol, rpc_client, - f"Failed to create listener for {lvol.get_id()}") + f"Failed to create listener for {lvol.get_id()}", is_primary=is_primary) elif iface.ip4_address and lvol.fabric == "tcp" and snode.active_tcp: logger.info("adding listener for %s on IP %s, fabric TCP port %s" % (lvol.nqn, iface.ip4_address, listener_port)) ret, err = rpc_client.nvmf_subsystem_add_listener( @@ -1310,7 +1461,7 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid else: return _fail_after_bdev( lvol, rpc_client, - f"Failed to create listener for {lvol.get_id()}") + f"Failed to create listener for {lvol.get_id()}", is_primary=is_primary) logger.info("Add BDev to subsystem") # Cluster-consistent namespace IDs: the PRIMARY add lets the target @@ -1326,36 +1477,57 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid # nsid (ns_id == 0, e.g. a drain-queued registration firing early) # must fail loudly instead of guessing. if is_primary: - # primary_nsid is set by migration/failover callers to preserve the - # source cluster's nsid assignment. The client connects to both source - # and target under the same NQN during preconnect; if nsid positions - # differ, the kernel rejects the target namespaces as mismatched - # ("IDs don't match for shared namespace N"). Regular creates pass - # primary_nsid=None (auto-assign, the previous behaviour). - requested_nsid = primary_nsid + # Two callers can dictate the primary's nsid, and both outrank + # auto-assignment. R26.3's migration/fail-over callers pass it + # explicitly as primary_nsid to preserve the source cluster's + # assignment; the fail-over path additionally pre-claims one across the + # whole target HA set and leaves it on the record (_claim_target_nsid). + # An ordinary create has neither and lets SPDK assign. + requested_nsid = primary_nsid or (lvol.ns_id or None) else: if not lvol.ns_id: return _fail_after_bdev( lvol, rpc_client, f"Replica namespace add for {lvol.get_id()} has no primary-" f"assigned ns_id; refusing auto-assignment (divergent nsid " - f"maps across the shared subsystem's paths)") + f"maps across the shared subsystem's paths)", is_primary=is_primary) requested_nsid = lvol.ns_id ret, err = rpc_client.nvmf_subsystem_add_ns2( - lvol.nqn, lvol.top_bdev, ns_uuid or lvol.uuid, lvol.guid, nsid=requested_nsid) + lvol.nqn, lvol.top_bdev, ns_uuid or lvol.get_ns_uuid(), lvol.guid, + nsid=requested_nsid) if err: + def _ns_add_detail(): + """What the node actually holds for this subsystem. + + The bare "Failed to add bdev to subsystem" costs a full lab run to + diagnose, and the cause is always in this table: the nsid is taken + by another bdev, or max_namespaces is smaller than the nsid asked + for.""" + try: + subsys = rpc_client.subsystem_get(lvol.nqn) + except Exception as diag_exc: # noqa: BLE001 + return f" [could not read the subsystem: {diag_exc}]" + if not subsys: + return (f" [subsystem {lvol.nqn} does not exist on " + f"{snode.get_id()[:8]}]") + existing = sorted((n.get("nsid"), n.get("bdev_name")) + for n in (subsys.get("namespaces") or [])) + return (f" [node {snode.get_id()[:8]} wanted nsid={requested_nsid} " + f"max_namespaces={subsys.get('max_namespaces')} " + f"holds={existing}]") + if err["code"] == -32602 and lvol.namespace: # -32602 from nvmf_subsystem_add_ns has two distinct causes: # A) subsystem exists on this node but the nsid slot is occupied # B) subsystem does not exist on this node at all (cross-LVS: # the shared subsystem's LVS has no nodes in common with this - # lvol's LVS — e.g. a failover clone on LVS_6 whose shared + # lvol's LVS - e.g. a failover clone on LVS_6 whose shared # subsystem lives on LVS_1) # nvmf_subsystem_add_ns2 already ran the idempotency probe and # returned the original error, so subsystem_get here distinguishes A/B. subsys_here = rpc_client.subsystem_get(lvol.nqn) if subsys_here is None: - # Case B — cross-LVS: subsystem not on this node. + # Case B - cross-LVS: subsystem not on this node. # Primary: the namespace cannot be registered from this LVS; # log a warning so the operator can investigate placement. # Secondary: the primary already registered it on the subsystem's @@ -1368,7 +1540,7 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid snode.get_id(), lvol.nqn, lvol.get_id()) # The lvol is a deepcopy of the source: lvol_uuid/blobid still # carry the source cluster's values. The bdev_lvol_clone just - # created has its own uuid/blobid — read them back so the caller + # created has its own uuid/blobid - read them back so the caller # can pass correct values to bdev_lvol_clone_register on HA peers. actual = rpc_client.get_bdevs(f"{lvol.lvs_name}/{lvol.lvol_bdev}") if actual: @@ -1376,20 +1548,22 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid return {'uuid': lvol.lvol_uuid, 'driver_specific': {'lvol': {'blobid': lvol.blobid}}}, None - # Case A — subsystem IS on this node; the nsid slot is occupied. + # Case A - subsystem IS on this node; the nsid slot is occupied. if lvol.node_id == snode.get_id(): - if primary_nsid is not None: - # Migration/failover: re-claiming to a different subsystem - # would lose the shared NQN. _evict_stale_namespace should - # have cleared the occupant; a persistent -32602 here means - # genuine unexpected state — fail hard. + if requested_nsid is not None: + # The nsid was dictated - explicitly by a migration/fail-over + # caller, or by the control plane's claim - so re-claiming + # into a different subsystem would lose the shared NQN. + # _evict_stale_namespace should have cleared the occupant; a + # persistent -32602 means genuine unexpected state. return _fail_after_bdev( lvol, rpc_client, - f"Failed to add bdev to subsystem at requested nsid={primary_nsid}: " - f"nsid occupied and eviction did not clear it") + f"Failed to add bdev to subsystem at requested nsid=" + f"{requested_nsid}: nsid occupied and eviction did not " + f"clear it" + _ns_add_detail(), is_primary=is_primary) logger.info("Error adding namespace to subsystem, finding new subsystem for namespaced lvol") # Re-claim transactionally, excluding the subsystem SPDK just - # rejected (the DB count said it had room — SPDK is the authority + # rejected (the DB count said it had room - SPDK is the authority # on its own namespace table). The lvol's record is rewritten in # the same transaction, so its slot moves atomically from the # rejected subsystem to the new one (or to a standalone one). @@ -1401,15 +1575,31 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid exclude_nqns={lvol.nqn}) except SubsystemCapacityError as e: logger.error(str(e)) - return _fail_after_bdev(lvol, rpc_client, str(e)) + return _fail_after_bdev(lvol, rpc_client, str(e), + is_primary=is_primary) return add_lvol_on_node(lvol, snode, is_primary=is_primary, secondary_index=secondary_index) + + # A REPLICA add cannot re-claim a slot (its nsid is dictated by the + # primary), so -32602 here ends the whole create/fail-over. Say WHY. + detail = _ns_add_detail() + logger.error("Namespace add rejected on %s for %s:%s", + snode.get_id()[:8], lvol.get_id(), detail) return _fail_after_bdev( - lvol, rpc_client, "Failed to add bdev to subsystem") + lvol, rpc_client, "Failed to add bdev to subsystem" + detail, + is_primary=is_primary) if is_primary: # Persist the target-assigned nsid; replicas re-add with exactly # this value, so it must never be overwritten by a replica's # (identical) response. + if requested_nsid and int(ret) != int(requested_nsid): + # The node accepted the add but at a DIFFERENT number than the + # control plane claimed. The replicas would then be told the + # claimed one and collide. Trust what the node actually did. + logger.warning( + "Node %s placed %s at nsid %s, not the claimed %s; the " + "replicas will follow the node", + snode.get_id(), lvol.get_id(), ret, requested_nsid) lvol.ns_id = int(ret) if not is_primary: @@ -1514,7 +1704,7 @@ def recreate_lvol_on_node(lvol, snode, ha_inode_self=None, ana_state=None): # correct nsid as well. Only a record with ns_id unset falls back to # auto-assignment. ret = rpc_client.nvmf_subsystem_add_ns( - lvol.nqn, lvol.top_bdev, lvol.uuid, lvol.guid, + lvol.nqn, lvol.top_bdev, lvol.get_ns_uuid(), lvol.guid, nsid=lvol.ns_id if lvol.ns_id else None) if not ret: # FATAL, deliberately. This used to log and fall through to the @@ -1730,7 +1920,10 @@ def _remove_lvol_subsys_from_node(lvol, rpc_client): return True for ns in subsystem["namespaces"]: - if ns["uuid"] == lvol.uuid: + # Match by the WIRE identity: a failed-back volume's namespace + # advertises ns_uuid, not the record uuid — matching the record uuid + # here leaves the namespace behind on a shared subsystem forever. + if ns["uuid"] == lvol.get_ns_uuid(): logger.info("Removing namespace %s from subsystem %s", ns["uuid"], lvol.nqn) ret = bool(rpc_client.nvmf_subsystem_remove_ns(lvol.nqn, ns['nsid'])) if not ret: @@ -1746,6 +1939,24 @@ def _remove_lvol_subsys_from_node(lvol, rpc_client): break if not subsystem or len(subsystem["namespaces"]) == 0: + # SHARED subsystems: delete-on-empty is only safe when no other live + # volume claims this NQN. With namespaced volumes the subsystem is + # legitimately empty in the WINDOW between one member's teardown and + # the next member's add -- and a stuck in_deletion member's retry + # loop observes that window sooner or later. Run 20260825_224221: + # 8 of 20 namespaced fail-overs landed, then a looping rollback + # record deleted the shared subsystem on the HA peer and every + # following member's add_ns died -32602 on a missing subsystem. + db_controller = DBController() + others = [x for x in db_controller.get_lvols_by_node_id(lvol.node_id) + if x.nqn == lvol.nqn and x.get_id() != lvol.get_id() + and x.status not in (LVol.STATUS_DELETED,) + and not getattr(x, "deleted", False)] + if others: + logger.info( + f"Leaving subsystem {lvol.nqn} in place: {len(others)} other " + f"volume(s) still claim it (shared/namespaced subsystem)") + return True logger.info(f"Removing subsystem {lvol.nqn}") return bool(rpc_client.subsystem_delete(lvol.nqn)) @@ -2591,6 +2802,38 @@ def connect_lvol(uuid, ctrl_loss_tmo=constants.LVOL_NVME_CONNECT_CTRL_LOSS_TMO, for entry in entries: entry.target_lvol_id = clone_id out.extend(entries) + + # Device-lookup identity for the FAIL-BACK shape. A clone's wire NSUUID + # may be borrowed from another volume (the kernel merges multipath paths + # only on matching NSUUIDs), so the client's /dev/disk/by-id links can + # carry an id that is not the record's own. After a forward migration the + # requested volume is the relationship's source end, the loop above + # redirects to the target and target_lvol_id points at the right device. + # After a fail-back the UUID swap makes the requested volume the TARGET + # end under its original id, clone_id == uuid for every entry, and + # nothing would report the id the device actually advertises — the CSI + # then cannot find the block device. The record's ns_uuid carries the + # wire identity: for a fail-back over a still-live original it equals the + # restored original uuid (no override needed — the CSI's own-id lookup + # matches), while a fail-back to a fresh cluster keeps the DR source's + # id and must be reported. + if not any(e.target_lvol_id for e in out): + wire_id = getattr(lvol, "ns_uuid", "") + if wire_id: + if wire_id != uuid: + for entry in out: + entry.target_lvol_id = wire_id + else: + # Legacy records created before ns_uuid was persisted: the only + # fail-back shape that existed then inherited the DR source's + # NSUUID, recoverable from the relationship's other end (only its + # ID is needed, so a source record since deleted still resolves). + rep = _replication_for_lvol(db_controller, uuid) + if (rep is not None and rep.state == LVolReplication.STATE_CUTOVER_DONE + and rep.target_lvol and rep.target_lvol.get_id() == uuid + and rep.source_lvol and rep.source_lvol.get_id() != uuid): + for entry in out: + entry.target_lvol_id = rep.source_lvol.get_id() return out, None @@ -2613,19 +2856,11 @@ def _connect_path_volumes(db_controller, lvol): cutover_done -> ONLY the post-move volume; the pre-migration paths are not handed out any more - The clone preserves the source NQN and ns_id, so every path returned here + The clone preserves the source NQN -- its nsid is assigned by the + destination primary and may differ -- so every path returned here aggregates into one multipath device on the client. """ - from simplyblock_core.models.lvol_model import LVolReplication - - lvol_id = lvol.get_id() - rep = None - for candidate in reversed(db_controller.get_lvol_replication_objects()): - source_id = candidate.source_lvol.get_id() if candidate.source_lvol else "" - target_id = candidate.target_lvol.get_id() if candidate.target_lvol else "" - if lvol_id in (source_id, target_id): - rep = candidate - break + rep = _replication_for_lvol(db_controller, lvol.get_id()) if rep is None or rep.state == LVolReplication.STATE_REPLICATING: return [lvol] @@ -2648,6 +2883,20 @@ def _live(candidate): return [target or source or lvol] +def _replication_for_lvol(db_controller, lvol_id): + """The most recent replication relationship *lvol_id* is an end of, or None. + + Newest-first so a volume that went through several cycles (fail-over then + fail-back) resolves to the relationship describing its CURRENT location. + """ + for candidate in reversed(db_controller.get_lvol_replication_objects()): + source_id = candidate.source_lvol.get_id() if candidate.source_lvol else "" + target_id = candidate.target_lvol.get_id() if candidate.target_lvol else "" + if lvol_id in (source_id, target_id): + return candidate + return None + + def _connect_entries_for_volume(db_controller, lvol, ctrl_loss_tmo, host_entry, host_nqn): out = [] nodes_ids = [] @@ -3190,33 +3439,49 @@ def replication_start(lvol_id, replication_cluster_id=None, mode=None, interval_ if not replication_cluster_id: logger.error(f"Cluster: {snode.cluster_id} not replicated") return False - - # Namespaced volumes sharing a subsystem must replicate to the SAME - # target node so their failover clones land on the same LVS. With - # separate nodes each PVC's snapshot ends up on a different LVStore; - # bdev_lvol_clone_register then fails cross-LVS and the whole group - # can't share a subsystem on the target. If any sibling already has a - # replication node, inherit it (verified online below). - if lvol.namespace: - for sibling in db_controller.get_lvols(snode.cluster_id): - if (sibling.nqn == lvol.nqn - and sibling.get_id() != lvol.get_id() - and sibling.replication_node_id): - try: - sib_node = db_controller.get_storage_node_by_id( - sibling.replication_node_id) - if sib_node.status == StorageNode.STATUS_ONLINE: - logger.info( - "Namespaced lvol %s: inheriting replication node %s " - "from sibling %s to co-locate snapshots on the same LVS", - lvol.get_id(), sib_node.get_id(), sibling.get_id()) - lvol.replication_node_id = sib_node.get_id() - lvol.write_to_db() - except KeyError: - pass - break - - if not lvol.replication_node_id: + # Volumes sharing a subsystem MUST replicate to the same target node. + # Two reasons, one from each branch: their failover clones have to land + # on the same LVS (bdev_lvol_clone_register fails cross-LVS, so a split + # group cannot share a subsystem on the target at all), and a split + # group leaves no node advertising the whole subsystem -- soak case 7 + # run 20260826_233417 put one 10-namespace subsystem on three target + # primaries (nsids 1,2,3,6 / 1,2,4,5,7..10 / 3..10) and the client's + # kernel showed 0 of the 10 namespaces on one of its paths. + # + # add_lvol_ha enforces this for a volume created with a replication + # cluster; this is the OTHER entry point -- attaching a policy -- which + # used to pick purely by capacity. + sibling_node_id = _sibling_replication_node(lvol, cluster) + if sibling_node_id: + try: + sib_node = db_controller.get_storage_node_by_id(sibling_node_id) + except KeyError: + sib_node = None + if sib_node is None or sib_node.status != StorageNode.STATUS_ONLINE: + # An offline node cannot host the clone. Fall through to the + # capacity pick; _create_target_lvol_clone re-checks at creation + # time and refuses to split the subsystem rather than quietly + # building an incoherent one. + logger.warning( + "LVol %s should follow subsystem %s to node %s, but that " + "node is not online; picking by capacity instead", + lvol.get_id(), lvol.nqn, sibling_node_id) + sibling_node_id = "" + if sibling_node_id and sibling_node_id in excluded_nodes: + # Keeping a subsystem whole outranks the clone/origin placement + # preference, which is only an optimisation. + logger.warning( + "LVol %s must replicate to %s to keep subsystem %s whole, " + "though that node is the origin of its snapshot", + lvol.get_id(), sibling_node_id, lvol.nqn) + if sibling_node_id: + logger.info( + "Replicating on node %s: it is where subsystem %s already " + "replicates (co-locating the group's snapshots on one LVS)", + sibling_node_id, lvol.nqn) + lvol.replication_node_id = sibling_node_id + lvol.write_to_db() + else: random_nodes = _get_next_3_nodes(replication_cluster_id, lvol.size) for r_node in random_nodes: if r_node.get_id() not in excluded_nodes: @@ -3451,18 +3716,235 @@ def replication_stop(lvol_id, delete=False, from_policy=False): return True +def _claim_target_nsid(db_controller, new_lvol, target_node): + """Pick the nsid a fail-over copy takes on the target HA set, or 0. + + The primary used to auto-assign, which is only safe when the primary sees + the whole shared subsystem. It does not when the group is split across two + target primaries: each counts only its own subset, hands out an nsid the + HA PEER already gave to a sibling, and the peer's add_ns is rejected + ("wanted nsid=1 ... holds=[(1, ) ... (7, ...)]", soak case 7). + Splitting is prevented at create time now, but a node that was down when + the group was placed can still reintroduce it, so do not depend on it: + claim the nsid from the UNION of what every node of the target HA set + actually holds, and give the same number to the primary and the replicas. + + The nsid does NOT have to match the source cluster's -- clients resolve + their paths through connect_lvol -- it only has to be consistent across + the paths of THIS subsystem. + + Returns 0 when the subsystem exists nowhere yet (nothing to collide with, + so ordinary auto-assignment is correct). + """ + node_ids = [target_node.get_id()] + for peer_id in [target_node.secondary_node_id, target_node.tertiary_node_id]: + if peer_id and peer_id not in node_ids: + node_ids.append(peer_id) + + occupied, seen_subsystem, max_ns = set(), False, 0 + for node_id in node_ids: + try: + node = db_controller.get_storage_node_by_id(node_id) + except KeyError: + continue + if node.status != StorageNode.STATUS_ONLINE: + continue + try: + subsystem = node.rpc_client().subsystem_get(new_lvol.nqn) + except Exception as e: + # Unreadable node: claiming against a partial view is exactly the + # bug being fixed, so fall back to auto-assignment rather than + # inventing a number that may already be taken there. + logger.warning("Cannot read subsystem %s on node %s (%s); leaving " + "the nsid to auto-assignment", new_lvol.nqn, node_id, e) + return 0 + if not subsystem: + continue + seen_subsystem = True + max_ns = max(max_ns, subsystem.get("max_namespaces") or 0) + for ns in (subsystem.get("namespaces") or []): + # A namespace belonging to THIS copy is evicted right before the + # add, so its slot is free to reuse. + if ns.get("uuid") == new_lvol.uuid: + continue + if ns.get("nsid"): + occupied.add(int(ns["nsid"])) + + if not seen_subsystem: + return 0 + + limit = max_ns or (new_lvol.max_namespace_per_subsys or 0) or (max(occupied) + 1) + for nsid in range(1, limit + 1): + if nsid not in occupied: + logger.info("Claimed nsid %d for fail-over copy %s in subsystem %s " + "(occupied on the target HA set: %s)", + nsid, new_lvol.get_id(), new_lvol.nqn, sorted(occupied)) + return nsid + # Full: let the add fail with the node's own diagnostic rather than + # silently picking a colliding number here. + logger.error("Subsystem %s is full on the target HA set (%d namespaces); " + "no nsid to claim for %s", + new_lvol.nqn, len(occupied), new_lvol.get_id()) + return 0 + + +def _superseded_original(db_controller, lvol, dest_cluster_id): + """The still-live volume a fail-back of *lvol* supersedes on + *dest_cluster_id* — the original this copy descends from — or None when + nothing is in the way (first fail-over, original already deleted, or the + original lives on some other cluster).""" + original = None + for rep in db_controller.get_lvol_replication_objects(): + target = getattr(rep, "target_lvol", None) + if target and target.get_id() == lvol.get_id() and rep.source_lvol: + original = rep.source_lvol # keep the LAST match: the + # most recent fail-over wins + if not original: + return None + try: + current = db_controller.get_lvol_by_id(original.get_id()) + except KeyError: + return None # already gone + if current.status in (LVol.STATUS_DELETED, LVol.STATUS_IN_DELETION): + return None + try: + node = db_controller.get_storage_node_by_id(current.node_id) + except KeyError: + return None + if node.cluster_id != dest_cluster_id: + return None # not in our way + return current + + +def _retire_superseded_original(db_controller, lvol, dest_cluster_id): + """Delete the volume a fail-back replaces. Returns (ok, error). + + After a fail-over the ORIGINAL volume stays on its cluster, still holding + its namespace. Failing back builds the returning volume in the SAME + subsystem, so unless the original goes first there is no slot for it: a + shared subsystem is sized to its group (max_namespaces=10 for ten + volumes) and the add fails outright -- "Subsystem ... is full on the + target HA set", which suspended all 20 cutover tasks in soak case 7 run + 20260826_235940. Dedicated subsystems hid this: sized 1 they were equally + full, but each fail-back minted a fresh per-volume NQN and so never came + back to an occupied one. + + The original's SNAPSHOTS are deliberately left alone. The newest one still + present on that cluster is the common base the fail-back clones and + transfers a delta against -- that is what makes a fail-back an online + migration rather than a full copy. + + A no-op unless *lvol* is itself a fail-over copy whose original still + lives on the destination cluster, so a first fail-over deletes nothing. + """ + current = _superseded_original(db_controller, lvol, dest_cluster_id) + if current is None: + return True, "" + + logger.info( + "Fail-back: deleting the superseded original %s (nsid %s of subsystem " + "%s) on node %s so the returning volume has a namespace slot", + current.get_id(), current.ns_id, current.nqn, current.node_id[:8]) + try: + delete_lvol(current) + except Exception as e: + return False, ( + f"Fail-back cannot free the namespace of the superseded original " + f"{current.get_id()} in subsystem {current.nqn}: {e}") + return True, "" + + +def _subsystem_home_node(db_controller, nqn, cluster_id): + """Node in *cluster_id* that already hosts copies of subsystem *nqn*, or "". + + Every path of one shared subsystem must advertise the SAME namespaces, so + all of its volumes have to live on one primary and its HA peers. Whichever + node got there first owns the subsystem for that cluster. + """ + for lv in db_controller.get_lvols(): + if lv.nqn != nqn or lv.status == LVol.STATUS_IN_DELETION: + continue + if getattr(lv, "deleted", False) or not lv.node_id: + continue + try: + node = db_controller.get_storage_node_by_id(lv.node_id) + except KeyError: + continue + if node.cluster_id == cluster_id: + return lv.node_id + return "" + + def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snapshot, for_migration=False): """Create a writable clone of *lvol* on *target_node* (primary + online HA - peers) from *snapshot*, preserving the original NQN/ns_id. + peers) from *snapshot*, preserving the original NQN. Shared by fail-over (replicate_lvol_on_target_cluster) and migration-commit (replication_commit). Returns (new_lvol, error). The new lvol is left in STATUS_IN_CREATION with lvol_uuid/blobid populated; the caller is responsible for setting STATUS_ONLINE and any replication bookkeeping. """ + # A fail-back returns into the subsystem the original still occupies, so + # the original has to go first (its snapshots stay: they are the delta + # base). A first fail-over has no original here and this does nothing. + # + # Gated off while the fork's blob-id-reuse defect is open: this delete + # frees the original's blob id, the clone reuses it seconds later, and the + # cutover's delta writes then fail rc -1 (see the constant's comment). + # The namespace slot the delete used to free is still reclaimed by + # _evict_stale_namespace below; the DB-level slot claim may report the + # subsystem full on groups at max_namespaces until the original is + # removed post-cutover by _swap_failback_lvol_uuid. + superseded = None + if constants.REPL_FAILBACK_RETIRE_ORIGINAL_BEFORE_CUTOVER: + ok, err = _retire_superseded_original(db_controller, lvol, + target_node.cluster_id) + if not ok: + return None, err + else: + superseded = _superseded_original(db_controller, lvol, + target_node.cluster_id) + + # Last line of defence for the one-subsystem-one-primary invariant. The + # target node was chosen when the policy was attached, long before this + # copy is created; if anything put a sibling somewhere else in the + # meantime, following the stale pick would build a subsystem whose paths + # expose different namespace sets -- which clients resolve by showing + # NONE of them. Placement is negotiable, a coherent subsystem is not. + home = _subsystem_home_node(db_controller, lvol.nqn, target_node.cluster_id) + if home and home != target_node.get_id(): + try: + home_node = db_controller.get_storage_node_by_id(home) + except KeyError: + home_node = None + if home_node and home_node.status == StorageNode.STATUS_ONLINE: + logger.warning( + "Subsystem %s already lives on node %s in cluster %s; placing " + "the copy of %s there instead of %s to keep it whole", + lvol.nqn, home, target_node.cluster_id, lvol.get_id(), + target_node.get_id()) + target_node = home_node + else: + return None, ( + f"Subsystem {lvol.nqn} already has copies on node {home}, " + f"which is not online; placing this copy on " + f"{target_node.get_id()} would split the subsystem across " + f"primaries and hide its namespaces from clients") + new_lvol = copy.deepcopy(lvol) new_lvol.uuid = str(uuid.uuid4()) + if superseded is not None: + # The superseded original is left alive (see the constant) and still + # owns this deep-copied bdev name on the target lvstore. Creating the + # clone under the same name would not fail: the idempotency probe in + # _create_bdev_stack finds the original's bdev, skips the create, and + # add_lvol_on_node returns the ORIGINAL's uuid/blobid as the clone's — + # aiming the cutover delta at the original's stale data. Give the + # clone its own name; client identity is carried by + # NQN/nsid/namespace-UUID, never the bdev name. + new_lvol.vuid = utils.get_random_vuid() + new_lvol.lvol_bdev = f"LVOL_{new_lvol.vuid}" new_lvol.create_dt = str(datetime.now()) new_lvol.node_id = target_node.get_id() new_lvol.nodes = [target_node.get_id()] @@ -3527,9 +4009,57 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps if new_lvol.crypto_bdev: new_lvol.bdev_stack.append({"type": "crypto"}) + # Whether the source's nsid carries over depends on what the client is + # doing, and the two cases are opposites: + # + # MIGRATION -- the client is connected to the source AND the target at + # the same time under this one NQN, so the nsid positions must match + # or the kernel rejects the target namespaces ("IDs don't match for + # shared namespace N"). Keep the source's number (the deep copy + # already carries it). + # FAIL-OVER -- the client reconnects afterwards and resolves its paths + # through connect_lvol, so cross-cluster equality buys nothing, while + # insisting on it collides with whichever sibling already holds that + # slot on the target. Claim a number across the whole target HA set + # instead, so the primary and every replica add at the same one. + # + # Either way this must be settled BEFORE the record is written, because + # add_lvol_on_node reads it back as primary_nsid. + if not for_migration: + new_lvol.ns_id = _claim_target_nsid(db_controller, new_lvol, target_node) + + # Which identity the clone's namespace advertises on the wire: + # + # Migration and fail-back: preserve the SOURCE's wire identity — what the + # connected client's multipath head currently holds — so the kernel can + # merge source and target paths during the preconnect phase (the ANA flip + # requires matching NSUUIDs on both paths). The source's own wire identity + # may itself be borrowed (its ns_uuid, e.g. a fail-back clone from an + # earlier cycle), so propagate the chain, not the record uuid. + # Failover: the source is gone — use the clone's own UUID so it appears as + # nvme-uuid. in /dev/disk/by-id, consistent with standalone volumes. + # + # Do NOT use the superseded original's uuid here. That was tried (run + # 2026-09-02 17:00) to serve clients whose heads still held the ORIGINAL + # identity — but such clients only exist because failover leaves the + # original unfenced and their writes land on superseded data that fail-back + # discards anyway. For the canonical shape — pods restaged onto the DR side + # after failover, heads built from the DR identity — restoring the original + # uuid made the kernel reject every preconnected target path ("IDs don't + # match for shared namespace N"), and deleteSource then removed the head's + # only live paths: no available path, XFS shutdown (run 2026-09-02 ~19:00, + # subsystem 20d8a917 nsid 1). + if for_migration: + _src_ns_uuid = getattr(lvol, "ns_uuid", "") or lvol.uuid + else: + _src_ns_uuid = new_lvol.uuid + # Persist the wire identity when it is borrowed, so connect_lvol can tell + # the CSI which /dev/disk/by-id/nvme-uuid. the device really carries. + new_lvol.ns_uuid = _src_ns_uuid if _src_ns_uuid != new_lvol.uuid else "" + new_lvol.write_to_db(db_controller.kv_store) - _evict_stale_namespace(new_lvol, target_node) + _evict_stale_namespace(new_lvol, target_node, superseded=superseded) # The target clone shares the source NQN. During preconnect the host has # live paths to the source (cntlids 1, 1000, 2000) AND tries to add paths @@ -3542,13 +4072,6 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps random.randint(6001, 6500), # tertiary ] - # Migration: preserve the source UUID as the NVMe namespace UUID so the kernel - # can merge source and target paths into the same multipath namespace during - # the preconnect phase (ANA flip requires matching NSUUID on both paths). - # Failover: the source is gone — use the clone's own UUID so it appears as - # nvme-uuid. in /dev/disk/by-id, consistent with standalone volumes. - _src_ns_uuid = lvol.uuid if for_migration else new_lvol.uuid - # For migration/failover, preserve the source nsid so the kernel can # match target paths to source paths under the same NQN. new_lvol is a # deepcopy of the source lvol, so new_lvol.ns_id is already the source @@ -3570,6 +4093,7 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps # Expose the volume on the secondary and tertiary target nodes too (HA), # so connect_lvol returns all client paths. + placed_nodes = [target_node] _tgt_cntlid_iter = iter(_tgt_cntlids[1:]) for peer_id in [target_node.secondary_node_id, target_node.tertiary_node_id]: if not peer_id: @@ -3583,19 +4107,48 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps if peer_node.status != StorageNode.STATUS_ONLINE: next(_tgt_cntlid_iter, None) continue + # The preserved-NQN subsystem exists on EVERY node of the recovered + # HA set, each still holding the original volume's namespace at the + # preserved nsid. Evicting only on the primary made its add_ns + # succeed while the peer's failed with the same -32602, and the + # peer failure rolled the whole cutover back (run 20260824_113711: + # primary add_ns result:1, peer -32602, 0/5 cutovers). + _evict_stale_namespace(new_lvol, peer_node, superseded=superseded) _peer_cntlid = next(_tgt_cntlid_iter, None) lvol_bdev, error = add_lvol_on_node(new_lvol, peer_node, is_primary=False, min_cntlid=_peer_cntlid, ns_uuid=_src_ns_uuid) if error: logger.error(error) - delete_lvol_from_node(new_lvol, target_node) + # Roll back EVERY node that already carries this copy, not just the + # primary. With three target nodes a failure on the tertiary used + # to leave the SECONDARY holding the namespace while the primary + # was cleaned, so the next sibling's primary auto-assigned an nsid + # the peer had already given to someone else -- and its replica add + # was rejected with "wanted nsid=2 ... holds=[(2, )]" + # (case 7, run 20260826_223631). + # + # The ids also matter: this used to pass the LVol and StorageNode + # OBJECTS to delete_lvol_from_node(lvol_id, node_id), whose + # `except KeyError: return True` swallowed the mismatch -- so the + # rollback reported success while deleting nothing at all. + for node in placed_nodes: + try: + if not delete_lvol_from_node( + new_lvol.get_id(), node.get_id(), + sync=node.get_id() != target_node.get_id()): + logger.error("rollback: could not remove %s from %s", + new_lvol.get_id(), node.get_id()[:8]) + except Exception: + logger.exception("rollback: removing %s from %s raised", + new_lvol.get_id(), node.get_id()[:8]) db_controller.release_lvol_ns_slot(new_lvol) return None, error + placed_nodes.append(peer_node) return new_lvol, None -def _last_replicated_target_snapshot(db_controller, lvol_id, cluster_id): +def _last_replicated_target_snapshot(db_controller, lvol_id, cluster_id, generation=0): """Return the target-cluster copy of the most recent FULLY replicated snapshot of *lvol_id*, or None. @@ -3628,6 +4181,18 @@ def _last_replicated_target_snapshot(db_controller, lvol_id, cluster_id): snaps.append(snap) snaps.sort(key=lambda x: x.created_at, reverse=True) + # generation 0 = newest replicated point-in-time (the default and the only + # behaviour before tiered retention existed). A higher generation walks + # BACK through the retained history, which is what a retention schedule is + # for: recovering to a point before a logical corruption that a + # minute-old copy would have replicated faithfully. + if generation: + if generation >= len(snaps): + logger.error( + f"Fail-over generation {generation} requested for {lvol_id} but only " + f"{len(snaps)} replicated point(s)-in-time exist") + return None + snaps = snaps[generation:] for snap in snaps: try: target_snap = db_controller.get_snapshot_by_id(snap.target_replicated_snap_uuid) @@ -3645,9 +4210,18 @@ def _last_replicated_target_snapshot(db_controller, lvol_id, cluster_id): return None -def _evict_stale_namespace(new_lvol, target_node): +def _evict_stale_namespace(new_lvol, target_node, superseded=None): """Make room for the preserved identity on a RECOVERED fail-back target. + ``superseded`` is the still-live original volume this fail-back replaces + (from :func:`_superseded_original`, present since the pre-cutover retire + was disabled). Its namespace carries the ORIGINAL record's uuid — not the + clone's — so on a SHARED subsystem the own-uuid match below cannot see it + and the clone's add_ns fails -32602 forever (2026-09-02: shared subsystem + holding nsids 1-5, clone wanted nsid 1, occupant was the original). + Matching on the superseded original's identity is safe on shared + subsystems: it names exactly one namespace, never a sibling's. + The cutover clone keeps the ORIGINAL volume's NQN and nsid so the client reconnects to the same identity. Failing back to a recovered source means that subsystem usually still exists there WITH the original volume's @@ -3657,34 +4231,112 @@ def _evict_stale_namespace(new_lvol, target_node): retry: 2026-08-24, 5/5 fail-back cutovers, 40x "Failed to add bdev to subsystem". Evict a namespace occupying the clone's nsid unless it is already the clone's own bdev (idempotent re-run). + + The eviction must cover all HA peer nodes (secondary, tertiary) as well as + the primary. add_lvol_on_node registers the new namespace on every peer, so + any peer that still holds the old namespace at nsid also fails with -32602 + and blocks the entire cutover. """ - try: - rpc = target_node.rpc_client() - subsystems = rpc.subsystem_get(new_lvol.nqn) - if not subsystems: - return - # subsystem_get returns a list of subsystem dicts; take the first entry. - subsystem = subsystems[0] if isinstance(subsystems, list) else subsystems - for ns in (subsystem.get("namespaces") or []): - if ns.get("nsid") != new_lvol.ns_id: + # Evict on the primary node AND every HA peer (secondary, tertiary). + # add_lvol_on_node registers the new namespace on all peers, so any peer + # that still holds the old namespace at this nsid also fails with -32602. + peer_ids = [target_node.secondary_node_id, target_node.tertiary_node_id] + db = DBController() + nodes_to_evict = [target_node] + for pid in peer_ids: + if not pid: + continue + try: + nodes_to_evict.append(db.get_storage_node_by_id(pid)) + except KeyError: + # A peer id with no record holds no stale namespace to evict, and + # this eviction is best-effort — it must not abort the fail-over. + logger.warning("Stale-namespace eviction: peer node %s of %s not " + "found; skipping it", pid, target_node.get_id()) + for node in nodes_to_evict: + try: + rpc = node.rpc_client() + # subsystem_get returns ONE subsystem dict (single_or_none), not a + # list — indexing with [0] raised KeyError(0), the best-effort + # except swallowed it, and the eviction silently never ran (run + # 20260824_104449: same 40x add_ns -32602 with the fix "in place"). + subsystem = rpc.subsystem_get(new_lvol.nqn) + if not subsystem: continue - if ns.get("bdev_name") == new_lvol.top_bdev: - return # already ours (re-run) - logger.info( - f"Fail-back cutover: evicting stale namespace nsid={ns.get('nsid')} " - f"(bdev {ns.get('bdev_name')}) from {new_lvol.nqn} on " - f"{target_node.get_id()} -- superseded by the failed-over data") - rpc.nvmf_subsystem_remove_ns(new_lvol.nqn, ns.get("nsid")) - return - except Exception as e: - # Best effort: if the subsystem is not there, add_lvol_on_node creates - # it; if the eviction genuinely failed, add_ns will say so loudly. - logger.warning(f"Stale-namespace check on {target_node.get_id()} for " - f"{new_lvol.nqn} raised: {e}") + # Match on the volume's UUID — that is what identifies THIS volume's + # own stale namespace. An nsid match is only safe when the subsystem + # cannot hold anyone else's namespace. + # + # nsid is NOT identity on a SHARED (namespaced) subsystem: siblings + # occupy the other slots, and new_lvol.ns_id here is still the SOURCE + # cluster's number (the destination primary auto-assigns and only then + # overwrites the record). Matching on it could evict a sibling's live, + # already-failed-over namespace — taking a healthy volume's device + # away from its client. Cross-cluster nsid equality is not required + # anyway: the client looks its paths up through connect_lvol. + namespaces = subsystem.get("namespaces") or [] + single_namespace_subsystem = len(namespaces) <= 1 + own_uuid = getattr(new_lvol, "uuid", None) + own_nsid = getattr(new_lvol, "ns_id", None) + # The superseded original's namespace is identified by ITS record + # uuid (the ns uuid it was added with) or its bdev — SPDK reports + # lvol namespaces' bdev_name as the raw lvol_uuid. + superseded_ids = set() + if superseded is not None: + # ns_uuid first: a multi-cycle superseded original is itself an + # earlier fail-back clone whose namespace advertises a borrowed + # wire identity, not its record uuid. + superseded_ids = { + v for v in (getattr(superseded, "ns_uuid", None), + getattr(superseded, "uuid", None), + getattr(superseded, "lvol_uuid", None), + getattr(superseded, "top_bdev", None)) + if v} + stale = [ns for ns in namespaces + if ((own_uuid is not None and ns.get("uuid") == own_uuid) + or ns.get("uuid") in superseded_ids + or ns.get("bdev_name") in superseded_ids + or (single_namespace_subsystem + and own_nsid is not None + and ns.get("nsid") == own_nsid)) + and ns.get("bdev_name") != new_lvol.top_bdev] + if not stale: + continue + for ns in stale: + logger.info( + f"Fail-back cutover: evicting stale namespace nsid={ns.get('nsid')} " + f"(bdev {ns.get('bdev_name')}) from {new_lvol.nqn} on " + f"{node.get_id()} -- superseded by the failed-over data") + rpc.nvmf_subsystem_remove_ns(new_lvol.nqn, ns.get("nsid")) + # remove_ns ACKNOWLEDGES before it completes (the same async + # false-success that dropped a shared subsystem in the PVC-expand + # incident; its fix polls for confirmation, eb127eed). Without this + # poll the follow-up add_ns raced the removal and lost on all 8 + # retries (run 20260824_110959: 40 evictions logged, 40 add_ns + # -32602 right behind them). + stale_ids = {ns.get("nsid") for ns in stale} + deadline = time.time() + 20 + while time.time() < deadline: + current = rpc.subsystem_get(new_lvol.nqn) or {} + if not any(ns.get("nsid") in stale_ids + for ns in (current.get("namespaces") or [])): + break # eviction confirmed on this node + time.sleep(1) + else: + logger.error( + f"Stale namespace(s) {sorted(stale_ids)} on {new_lvol.nqn} did not " + f"disappear within 20s of removal on {node.get_id()}; the " + f"cutover's add_ns will fail and retry") + except Exception as e: + # Best effort: if the subsystem is not there, add_lvol_on_node creates + # it; if the eviction genuinely failed, add_ns will say so loudly. + logger.warning(f"Stale-namespace check on {node.get_id()} for " + f"{new_lvol.nqn} raised: {e}") def _clone_from_last_replicated(db_controller, lvol_id, lvol, target_node, pool_uuid, - cluster_id, attempts=3, for_migration=False): + cluster_id, attempts=3, generation=0, + for_migration=False): """Pick the last fully replicated target snapshot and clone from it ATOMICALLY. Selecting and then cloning as two unsynchronised steps loses the data: the @@ -3705,9 +4357,12 @@ def _clone_from_last_replicated(db_controller, lvol_id, lvol, target_node, pool_ Returns (new_lvol, snapshot_used, error). """ for _ in range(attempts): - snapshot = _last_replicated_target_snapshot(db_controller, lvol_id, cluster_id) + snapshot = _last_replicated_target_snapshot(db_controller, lvol_id, cluster_id, + generation=generation) if not snapshot: - return None, None, "No replicated snapshot on target yet" + return None, None, ( + f"No replicated snapshot on target for generation {generation}" + if generation else "No replicated snapshot on target yet") with snapshot_controller.object_mutation_lock(snapshot.cluster_id, snapshot.uuid): # Re-read INSIDE the lock: retention may have removed or started @@ -3769,7 +4424,55 @@ def resolve_replication_destination(db_controller, lvol, target_node, source_nod return target_cluster, "" -def replicate_lvol_on_target_cluster(lvol_id): +def _retire_source_data_path(db_controller, lvol): + """Fence and unpublish the SOURCE volume's data path after a fail-over. + + The source is only ASSUMED dead. When it is in fact alive (administrative + or test fail-over), leaving its path published breaks everything + downstream: the client keeps writing to the superseded original (all of + it discarded at fail-back), and a namespaced volume's restage re-attaches + to the ORIGINAL device — the shared subsystem's controllers survive the + unstage, the stale device matches the same model/nsid glob, and the DR + clone's namespace is rejected against the original-identity head ("IDs + don't match for shared namespace N") — so pods never actually reach the + DR cluster (run 2026-09-03 ~09:00: all three namespaced pods remounted + on the originals after fail-over; fail-back's eviction then shut down + all three filesystems mid-IO). + + Best-effort: an unreachable source (a genuine DR event) changes nothing + here, and a source that recovers later comes back without the namespace + registered — fenced by absence. The DB record stays (it is the fail-back + delta base); its from_source=False marks it retired so the lvol monitor + and the non-leader repair do not register the namespace back. + """ + try: + if not suspend_lvol(lvol.get_id()): + logger.warning("Fail-over: could not fence the source listeners " + "of %s", lvol.get_id()) + except Exception as e: # noqa: BLE001 + logger.warning("Fail-over: fencing the source of %s failed: %s", + lvol.get_id(), e) + + for node_id in (lvol.nodes or [lvol.node_id]): + if not node_id: + continue + try: + snode = db_controller.get_storage_node_by_id(node_id) + except KeyError: + continue + if snode.status != StorageNode.STATUS_ONLINE: + continue + try: + if not _remove_lvol_subsys_from_node( + lvol, snode.rpc_client(timeout=5, retry=1)): + logger.warning("Fail-over: source namespace of %s not removed " + "on %s", lvol.get_id(), node_id) + except Exception as e: # noqa: BLE001 + logger.warning("Fail-over: removing the source namespace of %s " + "on %s failed: %s", lvol.get_id(), node_id, e) + + +def replicate_lvol_on_target_cluster(lvol_id, generation=0): db_controller = DBController() try: lvol = db_controller.get_lvol_by_id(lvol_id) @@ -3794,19 +4497,39 @@ def replicate_lvol_on_target_cluster(lvol_id): target_cluster, target_pool_uuid = resolve_replication_destination( db_controller, lvol, target_node, source_node) + # Match the preserved identity in FULL: nqn AND nsid. A namespaced volume + # SHARES its subsystem with up to max-namespace-per-subsys siblings, so an + # nqn-only test made this idempotency guard fire for every namespace after + # the first: namespace 1's fail-over copy already carried the nqn, so + # namespaces 2..N returned ITS id and were never failed over at all — + # silent loss of 9 of 10 volumes in a DR event, reported as success + # (soak case 7, run 20260824_174611). + # Which of the target cluster's volumes is THIS volume's copy? Every + # sibling of a shared subsystem matches on NQN, and the nsid is no longer + # a reliable discriminator either: a fail-over claims a target-LOCAL nsid + # (see _claim_target_nsid), so the copy's number need not equal the + # source's. Ask the replication record, which links source to target + # explicitly, and fall back to nqn+nsid only when no record exists yet. + own_copies = { + rep.target_lvol.get_id() + for rep in db_controller.get_lvol_replication_objects() + if rep.source_lvol and getattr(rep, "target_lvol", None) + and rep.source_lvol.get_id() == lvol.get_id() + } for lv in db_controller.get_lvols(target_cluster.get_id()): - if lv.nqn == lvol.nqn: - # Namespaced volumes share one NQN but have distinct NS IDs. - # A match on NQN alone would return the first sibling's clone - # for every PVC in the group, so also verify NS ID. - if lvol.namespace and lv.ns_id != lvol.ns_id: - continue - logger.info(f"LVol with same nqn already exists on target cluster: {lv.get_id()}") - return lv.get_id() + if lv.nqn != lvol.nqn: + continue + if own_copies: + if lv.get_id() not in own_copies: + continue # a sibling's copy, not ours + elif lv.ns_id != lvol.ns_id: + continue # no record: nsid is all we have + logger.info(f"LVol with same nqn already exists on target cluster: {lv.get_id()}") + return lv.get_id() new_lvol, _snapshot, error = _clone_from_last_replicated( db_controller, lvol_id, lvol, target_node, - target_pool_uuid, source_node.cluster_id) + target_pool_uuid, source_node.cluster_id, generation=generation) if error: logger.error(f"Fail-over clone failed for lvol {lvol_id}: {error}") return False, error @@ -3854,6 +4577,12 @@ def replicate_lvol_on_target_cluster(lvol_id): lvol_events.lvol_replicated(lvol, new_lvol) + # The relationship is durable and the DR copy is online: retire the + # source's data path NOW (fence + namespace removal, best-effort) so a + # still-alive source cannot keep serving superseded data. See + # _retire_source_data_path for the failure chain this closes. + _retire_source_data_path(db_controller, lvol) + # Provide the new connection paths (primary/secondary/tertiary) — identical # NQN, different IP/port — so the client can fail over to the target cluster. connection_strings = [] @@ -3863,11 +4592,30 @@ def replicate_lvol_on_target_cluster(lvol_id): else: connection_strings = [c.model_dump(by_alias=True) for c in conn] + # Requirement 4 (consistency groups): a generation older than the current + # membership must SAY so — which current members the chosen point-in-time + # does not contain (late joiners), and which contained volumes are no + # longer members. Logged AND returned, so CLI and API callers surface it. + warnings = [] + if _snapshot is not None: + try: + from simplyblock_core.controllers import consistency_group_controller + # The clone was made from the TARGET copy; group provenance lives + # on the SOURCE snapshot record it replicated from. Both carry it, + # so read whichever the selector handed us. + warnings = consistency_group_controller.warnings_for_snapshot( + lvol, _snapshot) + except Exception as e: + logger.warning("Group-membership warning computation failed: %s", e) + for w in warnings: + logger.warning("Fail-over of %s: %s", lvol_id, w) + return { "lvol_id": new_lvol.uuid, "nqn": new_lvol.nqn, "ns_id": new_lvol.ns_id, "connection_strings": connection_strings, + "warnings": warnings, } @@ -3917,14 +4665,14 @@ def replication_commit(lvol_id, delete_source=False): source_node = db_controller.get_storage_node_by_id(lvol.node_id) - # Shrink round 1: freeze the current top delta and let the normal - # replication pipeline carry it (snapshot_controller.add auto-enqueues the - # replication task for do_replicate volumes). - snap_uuid, snap_err = snapshot_controller.add( - lvol_id, f"repl_commit_{uuid.uuid4()}", snap_type=SnapShot.TYPE_INTERNAL) - if snap_err: - logger.error(f"Shrink snapshot failed: {snap_err}") - return False, f"Shrink snapshot failed: {snap_err}" + # NO snapshot here. The iterative snapshots ARE the endgame, and the + # endgame does not start until ordinary replication has the target within + # REPL_CUTOVER_ENDGAME_LAG_SEC. Taking one at commit time meant every + # volume held an ageing snapshot while it waited its turn, and the "round" + # that followed measured that wait rather than the transfer (run + # 20260828_124859: round 1 growing 341s -> 2584s across five volumes, while + # the one that never waited finished in 12.4s). The runner takes the first + # one when it enters the endgame, so its delta covers only the residual. task = tasks_controller.add_replication_final_task( source_node.cluster_id, source_node.get_id(), @@ -3934,8 +4682,12 @@ def replication_commit(lvol_id, delete_source=False): "tgt_node_id": target_node.get_id(), "operation": "replicate", "final_state": LVolReplication.STATE_CUTOVER_DONE, - "shrink_round": 1, - "shrink_snap_id": snap_uuid, + # The runner takes the first iterative snapshot when it enters the + # endgame and stamps shrink_started_at then; a round measured + # without that stamp reads as 0.00s and "converges" instantly, + # which is how the freeze stayed at 9-55s with the convergence loop + # deployed (run 20260827_172734). + "shrink_round": 0, "shrink_deadline": int(time.time()) + constants.REPL_CUTOVER_SHRINK_TIMEOUT_SEC, # Migration semantics on request: retire the source volume once # the cutover state is durable (see _finalize in the final runner). @@ -4240,10 +4992,19 @@ def replicate_lvol_on_source_cluster(lvol_id, cluster_id=None, pool_uuid=None): lvol_bdev, error = add_lvol_on_node(new_lvol, secondary_node, is_primary=False) if error: logger.error(error) - # remove lvol from primary - ret = delete_lvol_from_node(new_lvol, source_node) + # IDs, not objects: delete_lvol_from_node(lvol_id, node_id) hits + # `except KeyError: return True` when handed the records, so this + # rollback reported success while deleting nothing -- leaving the + # primary's namespace behind to collide with the next attempt. + try: + ret = delete_lvol_from_node(new_lvol.get_id(), source_node.get_id()) + except Exception: + logger.exception("rollback: removing %s from %s raised", + new_lvol.get_id(), source_node.get_id()[:8]) + ret = False if not ret: - logger.error("") + logger.error("rollback: could not remove %s from %s", + new_lvol.get_id(), source_node.get_id()[:8]) db_controller.release_lvol_ns_slot(new_lvol) return False, error diff --git a/simplyblock_core/controllers/migration_controller.py b/simplyblock_core/controllers/migration_controller.py index 49a173f874..8195e04253 100644 --- a/simplyblock_core/controllers/migration_controller.py +++ b/simplyblock_core/controllers/migration_controller.py @@ -1280,8 +1280,11 @@ def create_migration(lvol_id, target_node_id, # namespace occupying a low nsid on the target (or add_ns calls # racing/reordering across nodes) would silently diverge the # source and target nsid maps for this lvol. + # WIRE identity, not the record uuid: a failed-back volume's + # namespace advertises its ns_uuid, and the client's multipath + # head only merges the migration target's path when it matches. _ns = _rpc.nvmf_subsystem_add_ns( - nqn, _ns_bdev, lvol.uuid, lvol.guid, + nqn, _ns_bdev, lvol.get_ns_uuid(), lvol.guid, nsid=lvol.ns_id if lvol.ns_id else None) if _ns: logger.info( diff --git a/simplyblock_core/controllers/replication_policy_controller.py b/simplyblock_core/controllers/replication_policy_controller.py index f1e97db912..ac5de401af 100644 --- a/simplyblock_core/controllers/replication_policy_controller.py +++ b/simplyblock_core/controllers/replication_policy_controller.py @@ -16,6 +16,7 @@ from simplyblock_core.controllers import lvol_controller, snapshot_controller from simplyblock_core.models.lvol_model import LVolReplication from simplyblock_core.models.pool import Pool +from simplyblock_core import snapshot_retention from simplyblock_core.models.replication import ReplicationPolicy, ReplicationTarget from simplyblock_core.models.snapshot import SnapShot @@ -92,7 +93,8 @@ def remove_target(target_id): # Policies # --------------------------------------------------------------------------- # -def add_policy(cluster_id, policy_name, target, interval_min=1, mode=None, keep_replicated=None): +def add_policy(cluster_id, policy_name, target, interval_min=1, mode=None, keep_replicated=None, + retention_schedule=None, consistency_group=False): """Create a policy on *target* (id or name).""" db.get_cluster_by_id(cluster_id) try: @@ -118,6 +120,15 @@ def add_policy(cluster_id, policy_name, target, interval_min=1, mode=None, keep_ raise ReplicationConfigError( f"keep_replicated must be at least {ReplicationPolicy.MIN_KEEP_REPLICATED}") + if retention_schedule: + # Validate at ingress: an unparseable schedule silently falling back to + # flat retention would quietly discard the history the operator asked + # for, and they would only find out at fail-over. + try: + snapshot_retention.parse_schedule(retention_schedule) + except snapshot_retention.RetentionScheduleError as e: + raise ReplicationConfigError(f"invalid retention schedule: {e}") + policy = ReplicationPolicy() policy.uuid = str(uuid_module.uuid4()) policy.cluster_id = cluster_id @@ -129,8 +140,15 @@ def add_policy(cluster_id, policy_name, target, interval_min=1, mode=None, keep_ policy.mode = mode if keep_replicated is not None: policy.keep_replicated = keep_replicated + if retention_schedule is not None: + policy.retention_schedule = retention_schedule + policy.consistency_group = bool(consistency_group) policy.status = ReplicationPolicy.STATUS_ACTIVE policy.write_to_db(db.kv_store) + if policy.consistency_group: + # Auto-created with the policy, auto-deleted with it (requirement 2). + from simplyblock_core.controllers import consistency_group_controller + consistency_group_controller.create_group_for_policy(policy) logger.info("Created replication policy %s on target %s (%s)", policy_name, tgt.target_name, policy.get_id()) return policy.get_id() @@ -148,6 +166,9 @@ def remove_policy(policy_id): raise ReplicationConfigError( f"Replication policy {policy.policy_name} is followed by " f"{len(users)} volume(s); detach them first") + if getattr(policy, "consistency_group", False): + from simplyblock_core.controllers import consistency_group_controller + consistency_group_controller.delete_group_for_policy(policy.get_id()) policy.remove(db.kv_store) logger.info("Removed replication policy %s", policy_id) return True @@ -205,6 +226,12 @@ def attach_policy(lvol_id, policy): detach_policy(lvol_id) lvol = db.get_lvol_by_id(lvol_id) + if getattr(pol, "consistency_group", False): + # Requirement 1: all members share one LVS. Checked BEFORE any state + # is written, so a failed attachment leaves the volume untouched. + from simplyblock_core.controllers import consistency_group_controller + consistency_group_controller.add_member(pol, lvol) + lvol.replication_policy_id = pol.get_id() lvol.write_to_db() ret = lvol_controller.replication_start( @@ -219,6 +246,9 @@ def attach_policy(lvol_id, policy): lvol = db.get_lvol_by_id(lvol_id) lvol.replication_policy_id = "" lvol.write_to_db() + if getattr(pol, "consistency_group", False): + from simplyblock_core.controllers import consistency_group_controller + consistency_group_controller.remove_member(pol.get_id(), lvol_id) raise ReplicationConfigError( f"Could not start replication of {lvol_id} to target {target.target_name}") logger.info("Volume %s now follows policy %s (target %s)", @@ -244,9 +274,19 @@ def detach_policy(lvol_id): f"Volume {lvol_id} has a cutover in flight; wait for it to finish " f"before detaching the replication policy") + detached_policy_id = lvol.replication_policy_id lvol.replication_policy_id = "" lvol.write_to_db() + if detached_policy_id: + try: + pol = db.get_replication_policy_by_id(detached_policy_id) + except KeyError: + pol = None + if pol is not None and getattr(pol, "consistency_group", False): + from simplyblock_core.controllers import consistency_group_controller + consistency_group_controller.remove_member(pol.get_id(), lvol_id) + # Stops streaming and cancels the non-DONE FN_SNAPSHOT_REPLICATION tasks. lvol_controller.replication_stop(lvol_id, from_policy=True) @@ -353,23 +393,38 @@ def _failover_volumes(volumes, what): elif isinstance(ret, dict): results.append({"lvol_id": lvol_id, "status": "failed_over", "target_lvol_id": ret.get("lvol_id", ""), - "connection_strings": ret.get("connection_strings", [])}) + "connection_strings": ret.get("connection_strings", []), + "warnings": ret.get("warnings", [])}) else: results.append({"lvol_id": lvol_id, "status": "failed_over", "target_lvol_id": str(ret)}) return results def set_cutover_proceed(lvol_id): - """Signal that the operator has connected the target NVMe paths. + """Signal that the operator has connected the NVMe paths. - Finds the cutover_pending LVolReplication for *lvol_id* (source side) and + Finds the cutover_pending LVolReplication for *lvol_id* — either as the + source (migration direction) or as the target (failback direction) — and sets cutover_proceed = True so the task runner advances past the wait. + During failback the replication direction is reversed: the original source + volume becomes the TARGET of the reverse replication, so _active_relationship + (which searches by source) would miss it. The fallback search by target_lvol + handles this case without changing the API surface. + Returns the replication ID on success, raises KeyError when no matching cutover_pending record is found. """ rep = _active_relationship(lvol_id) if rep is None or rep.state != LVolReplication.STATE_CUTOVER_PENDING: + # Failback path: lvol_id is the target of the reverse replication. + rep = None + for r in reversed(db.get_lvol_replication_objects()): + if (r.target_lvol and r.target_lvol.get_id() == lvol_id + and r.state == LVolReplication.STATE_CUTOVER_PENDING): + rep = r + break + if rep is None: raise KeyError( f"No cutover_pending replication found for volume {lvol_id}") rep.cutover_proceed = True diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index d90c78b857..e566663220 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -20,7 +20,7 @@ from simplyblock_core.models.backup import Backup, BackupChainLock, BackupPolicy, BackupPolicyAttachment from simplyblock_core.models.lvol_migration import LVolMigration from simplyblock_core.models.lvol_migration_group import LVolMigrationGroup -from simplyblock_core.models.replication import ReplicationPolicy, ReplicationTarget +from simplyblock_core.models.replication import ConsistencyGroup, ReplicationPolicy, ReplicationTarget from simplyblock_core.models.qos import QOSClass from simplyblock_core.models.snapshot import SnapShot, SnapShotMini from simplyblock_core.models.stats import DeviceStatObject, NodeStatObject, ClusterStatObject, LVolStatObject, \ @@ -1461,6 +1461,27 @@ def get_replication_policy_for_lvol(self, lvol) -> Optional[ReplicationPolicy]: except KeyError: return None + def get_consistency_groups(self, cluster_id: Optional[str] = None) -> List[ConsistencyGroup]: + prefix = cluster_id if cluster_id else " " + return ConsistencyGroup().read_from_db(self.kv_store, id=prefix) + + def get_consistency_group_by_id(self, group_id: str) -> ConsistencyGroup: + if not group_id: + raise KeyError('ConsistencyGroup lookup with a blank id') + wanted = group_id.split('/')[-1] + group = single_or_none(g for g in self.get_consistency_groups() if g.uuid == wanted) + if group is None: + raise KeyError(f'ConsistencyGroup {group_id} not found') + return group + + def get_consistency_group_for_policy(self, policy_id: str) -> Optional[ConsistencyGroup]: + wanted = policy_id.split('/')[-1] if policy_id else "" + if not wanted: + return None + return single_or_none( + g for g in self.get_consistency_groups() + if g.policy_id.split('/')[-1] == wanted) + def get_lvols_by_replication_policy(self, policy_id: str) -> List[LVol]: wanted = policy_id.split('/')[-1] if policy_id else "" if not wanted: diff --git a/simplyblock_core/models/lvol_model.py b/simplyblock_core/models/lvol_model.py index 78c77f91f5..f2c48a6239 100644 --- a/simplyblock_core/models/lvol_model.py +++ b/simplyblock_core/models/lvol_model.py @@ -50,7 +50,26 @@ class LVol(BaseModel): nodes: List[str] = default_factory(list) nqn: str = "" ns_id: int = 1 + # The UUID the NVMe namespace advertises on the wire when it differs from + # the record's uuid (migration/fail-back clones inherit another volume's + # identity so the client's multipath head keeps its paths). Empty means + # the namespace carries the record's own uuid. connect_lvol reports it as + # target_lvol_id so the CSI globs /dev/disk/by-id/nvme-uuid.. + ns_uuid: str = "" max_namespace_per_subsys: int = 1 + + def get_ns_uuid(self) -> str: + """The UUID this volume's NVMe namespace advertises on the wire. + + Every site that registers or verifies the namespace must use this, + never the record ``uuid``: after a fail-back the two differ, and + probing or re-adding by the record uuid flags a healthy volume + unhealthy — or re-registers the namespace under an identity the + client's multipath head rejects ("IDs don't match for shared + namespace N"), severing its paths (run 2026-09-02 ~19:40: the lvol + monitor's self-heal fought the fail-back identity every cycle). + """ + return self.ns_uuid or self.uuid subsys_port: int = 9090 # Node ids whose sync delete already completed inline in the API delete # call (lvol_controller._delete_lvol_from_all_nodes). lvol_monitor skips diff --git a/simplyblock_core/models/replication.py b/simplyblock_core/models/replication.py index d7b741cc84..497f80a621 100644 --- a/simplyblock_core/models/replication.py +++ b/simplyblock_core/models/replication.py @@ -10,7 +10,7 @@ from typing import ClassVar import datetime -from simplyblock_core.models.base_model import BaseModel +from simplyblock_core.models.base_model import BaseModel, default_factory class ReplicationTarget(BaseModel): @@ -72,6 +72,17 @@ class ReplicationPolicy(BaseModel): interval_min: int = 1 # internal snapshot cadence, 0 = user snaps only mode: str = MODE_FAILOVER keep_replicated: int = MIN_KEEP_REPLICATED + #: Tiered retention, e.g. "15m:2h,1h:11h,1d:7d" -- one snapshot every 15 + #: minutes for the last 2 hours, then hourly for 11 hours, then daily for + #: 7 days. Empty keeps the flat keep_replicated behaviour. A schedule + #: never overrides MIN_KEEP_REPLICATED: the newest pair is always kept so + #: an arriving delta has a predecessor to chain onto. + retention_schedule: str = "" + #: All volumes attached to this policy form ONE consistency group: they + #: must share an LVS, cadence snapshots are taken as one frozen group + #: (bdev_lvol_snapshot_group), and fail-over generations are resolved + #: group-wide. Auto-creates/deletes a ConsistencyGroup record. + consistency_group: bool = False status: str = STATUS_ACTIVE def get_id(self): @@ -80,3 +91,47 @@ def get_id(self): def write_to_db(self, kv_store=None): self.updated_at = str(datetime.datetime.now(datetime.timezone.utc)) super().write_to_db(kv_store) + + +class ConsistencyGroup(BaseModel): + """Auto-managed group record behind a consistency-group policy. + + Created with the policy and removed with it. ``members`` maps lvol id to + its membership EPOCH: + + {"joined_seq": N, "removed_seq": M} + + A member is included in group generation ``seq`` iff + ``joined_seq <= seq`` and (``removed_seq == 0`` or ``seq <= removed_seq``). + Late joiners deliberately do NOT inherit history: they join at + ``last_group_seq + 1``, i.e. the first group snapshot taken AFTER the + attach, because earlier group snapshots simply do not contain them. + """ + + cluster_id: str = "" + policy_id: str = "" # ReplicationPolicy.get_id() + #: pinned placement: every member volume lives on this node / LVS. Set by + #: the first member and enforced for all others. + node_id: str = "" + lvs_name: str = "" + #: monotonically increasing generation counter; group snapshot N stamps + #: every member snapshot it takes with group_seq = N. + last_group_seq: int = 0 + members: dict = default_factory(dict) + status: str = "active" + + def get_id(self): + return "%s/%s" % (self.cluster_id, self.uuid) + + def write_to_db(self, kv_store=None): + self.updated_at = str(datetime.datetime.now(datetime.timezone.utc)) + super().write_to_db(kv_store) + + def included_in_seq(self, lvol_id, seq): + m = (self.members or {}).get(lvol_id) + if not m or not seq: + return False + if m.get("joined_seq", 0) > seq: + return False + removed = m.get("removed_seq", 0) + return removed == 0 or seq <= removed diff --git a/simplyblock_core/models/snapshot.py b/simplyblock_core/models/snapshot.py index a3a3bd3e68..01e955a729 100644 --- a/simplyblock_core/models/snapshot.py +++ b/simplyblock_core/models/snapshot.py @@ -39,6 +39,10 @@ class SnapShot(BaseModel): target_replicated_snap_uuid: str = "" source_replicated_snap_uuid: str = "" snap_type: str = "user" + #: consistency-group provenance: which group and which group generation + #: this snapshot belongs to (0 = not a group snapshot). + group_id: str = "" + group_seq: int = 0 next_snap_uuid: str = "" prev_snap_uuid: str = "" instances: list[dict] = default_factory(list) diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index ba3bd4a968..18997d8d94 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -1704,6 +1704,36 @@ def bdev_lvol_connect_hublvol(self, lvs, bdev): "remote_bdev": bdev, }) + def jc_set_dual_node(self, enable): + """Tell the journal component whether this is a DUAL-NODE cluster. + + The JC aborts its whole SPDK application when the number of reachable + journal members drops below jc_ha_nmin_jms(), which is 2 unless the + dual-node flag is set, and 1 when it is. On a 2-node cluster losing + the peer leaves exactly 1 of 2 JMs, so without this flag the SURVIVING + node aborts itself the moment its partner stops -- a single graceful + `sn shutdown` takes the entire cluster down (soak case 6: "JC detected + a network outage nd=1 njms=2" / "JC aborts the node due to network + outage" / core dump, every client path gone, XFS shut down). + The fork implements the tolerance; nothing ever switched it on. + """ + return self._request2("jc_set_dual_node", {"enable": bool(enable)}) + + def bdev_lvol_snapshot_group(self, lvs_name, snapshots): + """One crash-consistent snapshot per consistency-group member. + + ``snapshots`` is a list of {"lvol_name": "LVS_1/LVOL_5", + "snapshot_name": "SNAP_123"}; all members must be in ``lvs_name``. + IO on every member is frozen before the first snapshot and released + after the last; a mid-sequence failure unfreezes first and then + garbage-collects the snapshots already taken (SPDK side). + Returns [{"lvol_name", "snapshot_name", "uuid"}, ...] or False. + """ + return self._request2("bdev_lvol_snapshot_group", { + "lvs_name": lvs_name, + "snapshots": snapshots, + }) + def jc_suspend_compression(self, jm_vuid, suspend=False): params = { "jm_vuid": jm_vuid, @@ -1888,22 +1918,43 @@ def bdev_lvol_set_migration_flag(self, name): """Mark *name* (composite lvol bdev) as a migration-target lvol.""" return self._request("bdev_lvol_set_migration_flag", {"lvol_name": name}) - def bdev_lvol_transfer(self, name, offset, batch_size, bdev_name, operation="migrate", lvol_id=0): + def bdev_lvol_transfer(self, name, offset, batch_size, bdev_name, operation="migrate", lvol_id=0, + allow_partial=False): """ Start an async blob transfer from *name* (source composite bdev) to the NVMe-oF bdev *bdev_name* attached on the caller's node. Returns the RPC result (truthy on success) or None on error. Poll progress with :meth:`bdev_lvol_transfer_stat`. + + *allow_partial* opts this transfer into the dirty-bitmap delta path: the + SPDK side then ships only the ranges written since the previous snapshot + instead of every allocated cluster. It is a REQUEST, not a guarantee -- + the fork gates the bitmap path on the snapshot carrying a COMPLETE + dirty generation and silently sends a full transfer whenever it does + not, so passing it can never produce less data than the destination + needs *from this transfer*. + + What it does NOT excuse is the destination's starting content: a partial + transfer only ships the delta, so everything outside the delta must + already be on the destination. Pass it only when the landing volume is a + clone of the destination's copy of the PREVIOUS snapshot. Passing it for + a transfer into a fresh empty volume would silently drop every cluster + the delta does not cover. """ - return self._request("bdev_lvol_transfer", { + params = { "lvol_name": name, "lvol_id": lvol_id, "offset": offset, "cluster_batch": batch_size, "gateway": bdev_name, "operation": operation, - }) + } + # Send the key only when opting in: the RPC parameter is optional on the + # SPDK side, so every existing caller keeps its exact current wire form. + # if allow_partial: + # params["allow_partial"] = True + return self._request("bdev_lvol_transfer", params) def bdev_lvol_transfer_stat(self, name): """ diff --git a/simplyblock_core/services/lvol_monitor.py b/simplyblock_core/services/lvol_monitor.py index a47cf36a82..c11c9e98e0 100644 --- a/simplyblock_core/services/lvol_monitor.py +++ b/simplyblock_core/services/lvol_monitor.py @@ -403,6 +403,25 @@ def check_node(cluster, snode, all_lvols, subsys_check=False): deletions_processed += 1 + # RECORD-ONLY deletion: a retired landing volume's record carries + # an EMPTY bdev_stack on purpose -- its blob lives on as the + # converted, chained snapshot and must never be deleted. When the + # retirement sequence in snapshot_replication is interrupted + # between emptying the stack and removing the record, the record + # is left in_deletion here; the delete flow below then has nothing + # to issue, the status poll returns 4 ("no async delete request") + # forever (856x/30min, run 20260825_125156), and every cleanup + # that waits for lvols to drain times out behind it. Nothing on + # any node belongs to this record any more: retire it. + if not lvol.bdev_stack: + logger.info( + f"LVol {lvol.get_id()} ({lvol.lvol_name}) is in_deletion " + f"with an empty bdev stack (retired landing volume); " + f"removing the record only -- its blob lives on as the " + f"converted snapshot") + process_lvol_delete_finish(cluster, lvol) + continue + # The FULL delete of a chain member — the async delete, its # completion wait, and the sync deletes that follow — is one # atomic sequence per LVS+chain: a delete swap-merges segments @@ -590,12 +609,25 @@ def check_node(cluster, snode, all_lvols, subsys_check=False): if lvol.status not in (LVol.STATUS_ONLINE, LVol.STATUS_OFFLINE): continue + if not getattr(lvol, "from_source", True): + # The retired SOURCE of a fail-over: its namespace was removed + # deliberately (_retire_source_data_path) and the record only + # survives as the fail-back delta base. Checking it always fails, + # and the self-heal below would register the namespace back — + # resurrecting the stale device the retirement removed. + continue + if snode.lvstore_status != "ready": continue passed = True try: - passed &= health_controller.check_subsystem(lvol.nqn, rpc_client=snode.rpc_client(), ns_uuid=lvol.uuid) + # Verify against the WIRE identity: after a fail-back the record + # uuid differs from the namespace's advertised uuid, and checking + # the record uuid here marked healthy volumes unhealthy and drove + # the self-heal below into re-registering the wrong identity. + passed &= health_controller.check_subsystem( + lvol.nqn, rpc_client=snode.rpc_client(), ns_uuid=lvol.get_ns_uuid()) except Exception as e: logger.error(f"Failed to check lvol:{lvol.get_id()} on node: {lvol.node_id}") logger.error(e) @@ -609,7 +641,8 @@ def check_node(cluster, snode, all_lvols, subsys_check=False): if sec_node and sec_node.status == StorageNode.STATUS_ONLINE: try: ret = health_controller.check_subsystem( - lvol.nqn, rpc_client=sec_node.rpc_client(), ns_uuid=lvol.uuid) + lvol.nqn, rpc_client=sec_node.rpc_client(), + ns_uuid=lvol.get_ns_uuid()) if not ret: passed = False # Explicit, greppable degraded-path signal. Without diff --git a/simplyblock_core/services/replication_final_step.py b/simplyblock_core/services/replication_final_step.py index 907c054f28..320c1e9fb4 100644 --- a/simplyblock_core/services/replication_final_step.py +++ b/simplyblock_core/services/replication_final_step.py @@ -13,6 +13,7 @@ choreography is the simple "no-overlap" case (target primary → optimized, other target paths → non_optimized, all source paths → inaccessible). """ +from simplyblock_core import xfer_timing from simplyblock_core import db_controller, utils from simplyblock_core.models.storage_node import StorageNode @@ -240,37 +241,71 @@ def run_cutover(src_node, tgt_node, lvol, tgt_lvol_composite, tgt_map_id, # ENODEV (-19) and left the volume stuck in cutover_pending forever, while # snapshot replication (which passes the n1 bdev) kept working. This matches # tasks_runner_lvol_migration, which uses the second element for the same RPC. - _ctrl_name, hub_bdev, err = ensure_hub_attached(src_rpc, tgt_node) + with xfer_timing.phase("final_hub_attach", lvol=lvol.get_id()): + _ctrl_name, hub_bdev, err = ensure_hub_attached(src_rpc, tgt_node) if err: return False, err # Fence the source FIRST: from here on the source cannot take IO by any # means; the delta the freeze copies is definitively final. - fence_source_paths(src_node, src_node.lvstore, lvol.nqn, lvol.ns_id) + # + # The client-visible freeze begins at this call and ends at + # enable_target_paths, so freeze_total below is the number that has to fit + # inside the client's fast_io_fail_tmo (8s in the soak). + _freeze_started = xfer_timing.now() + xfer_timing.stamp("freeze_begin", lvol=lvol.get_id(), nqn=lvol.nqn, + nsid=lvol.ns_id) + with xfer_timing.phase("fence_source", lvol=lvol.get_id()): + fence_source_paths(src_node, src_node.lvstore, lvol.nqn, lvol.ns_id) logger.info( f"[IO-FREEZE] bdev_lvol_transfer_final_step starting: lvol={lvol.uuid} " f"src={src_lvol_composite} tgt_snap={tgt_snap_composite} " f"gateway={hub_bdev} op={operation}") - ret = src_rpc.bdev_lvol_transfer_final_step( - src_lvol_composite, tgt_map_id, tgt_snap_composite, - _FINAL_STEP_BATCH, hub_bdev, operation) - if ret is None: + with xfer_timing.phase("final_step_transfer", lvol=lvol.get_id(), + batch=_FINAL_STEP_BATCH): + ret = src_rpc.bdev_lvol_transfer_final_step( + src_lvol_composite, tgt_map_id, tgt_snap_composite, + _FINAL_STEP_BATCH, hub_bdev, operation) + # The RPC can return normally (no exception) while still reporting the + # transfer failed -- transfer_state is one of "No process" | "In progress" | + # "Failed" | "Done". Checking only for None lets a "Failed" response slip + # through to the ANA flip as if the data had moved (seen 2026-08-31, + # lvol=1fc7911f: {"transfer_state": "Failed"} logged, then [IO-RESUME] + # fired unconditionally, leaving the target with missing delta data). + transfer_state = ret.get("transfer_state") if isinstance(ret, dict) else None + if transfer_state != "Done": # The freeze failed with the source fenced: restore the source paths so # the client resumes there (nothing moved; source is still authoritative) # rather than leaving the volume dark until a retry succeeds. - restore_source_paths(src_node, src_node.lvstore, lvol.nqn, lvol.ns_id) - return False, "bdev_lvol_transfer_final_step failed" + logger.error( + f"bdev_lvol_transfer_final_step: transfer_state={transfer_state!r} " + f"(expected 'Done'): {ret!r} lvol={lvol.uuid}") + with xfer_timing.phase("restore_source_paths", lvol=lvol.get_id()): + restore_source_paths(src_node, src_node.lvstore, lvol.nqn, lvol.ns_id) + xfer_timing.gap("freeze_total", _freeze_started, lvol=lvol.get_id(), + aborted=1) + return False, ( + f"bdev_lvol_transfer_final_step: transfer_state={transfer_state!r}" + f" (expected 'Done'): {ret!r}" + ) logger.info(f"[IO-RESUME] final step Done: lvol={lvol.uuid} io now live on target") # Link the final lvol to its predecessor snapshot on the target peers. # bdev_lvol_transfer_final_step handles the primary internally; peers need an # explicit add_clone. Non-fatal — a missing peer link self-heals on rejoin. for peer in _online_peers(tgt_node): + with xfer_timing.phase("final_peer_add_clone", lvol=lvol.get_id(), + peer=peer.get_id()): if not peer.rpc_client().bdev_lvol_add_clone(tgt_lvol_composite, tgt_snap_composite): logger.warning( f"add_clone on peer {peer.get_id()[:8]} failed for final lvol (non-fatal)") # Light the target: queued client IO drains here. - enable_target_paths(tgt_node, tgt_node.lvstore, lvol.nqn, lvol.ns_id) + with xfer_timing.phase("enable_target_paths", lvol=lvol.get_id()): + enable_target_paths(tgt_node, tgt_node.lvstore, lvol.nqn, lvol.ns_id) + # The whole client-visible window, fence -> paths live. Compare against the + # client's fast_io_fail_tmo: anything longer surfaces as IO errors, not + # just latency. + xfer_timing.gap("freeze_total", _freeze_started, lvol=lvol.get_id()) return True, None diff --git a/simplyblock_core/services/snapshot_monitor.py b/simplyblock_core/services/snapshot_monitor.py index 0c3a77ca53..b0d2c3c81d 100644 --- a/simplyblock_core/services/snapshot_monitor.py +++ b/simplyblock_core/services/snapshot_monitor.py @@ -184,15 +184,20 @@ def set_snap_offline(snap): def _warn_leaderless(lvs_name): now = time.monotonic() - last, suppressed = _leaderless_warn_memo.get(lvs_name, (0.0, 0)) - if now - last >= _LEADERLESS_WARN_INTERVAL_SEC: + entry = _leaderless_warn_memo.get(lvs_name) + # An unseen lvs must warn NOW. Seeding "last" with 0.0 instead suppressed + # the FIRST warning whenever monotonic() itself was still under the + # interval — i.e. for the first 60s after boot, which is exactly when a + # leadership flap is most likely to be happening. + if entry is None or now - entry[0] >= _LEADERLESS_WARN_INTERVAL_SEC: + suppressed = entry[1] if entry else 0 logger.warning( f"No confirmed leader for {lvs_name} — snapshot deletes needing " f"phase-1 are deferred ({suppressed} similar messages suppressed " f"in the last {_LEADERLESS_WARN_INTERVAL_SEC}s)") _leaderless_warn_memo[lvs_name] = (now, 0) else: - _leaderless_warn_memo[lvs_name] = (last, suppressed + 1) + _leaderless_warn_memo[lvs_name] = (entry[0], entry[1] + 1) def _poll_delete_status(node, bdev_name): @@ -585,6 +590,63 @@ def take_due_internal_snapshots(cluster_id, now_ts): if not repl_lvols: return all_snaps = db.get_mini_snapshots() + + # Consistency-group policies snapshot as a GROUP (requirement 3): one + # frozen point-in-time across every member, via ONE group-snapshot call + # per tick — never per-volume snapshots. Members of such policies are + # removed from the per-volume loop below. + cg_policies = {p.get_id(): p for p in db.get_replication_policies(cluster_id) + if getattr(p, "consistency_group", False)} + if cg_policies: + from simplyblock_core.controllers import consistency_group_controller + grouped_ids: set = set() + for policy_id, policy in cg_policies.items(): + members = [lv for lv in repl_lvols + if getattr(lv, "replication_policy_id", "") == policy_id] + if not members: + continue + grouped_ids.update(lv.get_id() for lv in members) + try: + # Due when ANY member's interval elapsed (they tick together, + # so member timestamps agree except right after a join). + if not any(_due_for_internal_snapshot(lv, all_snaps, now_ts) + for lv in members): + continue + # Group-wide back-pressure: one member's unfinished transfer + # holds the WHOLE group's next generation, otherwise the + # generations stop being aligned points in time. + blocked = None + for lv in members: + outstanding = _outstanding_internal_snapshot(lv, all_snaps) + if outstanding is not None: + # Case 11 (run 20260827_224741) ended with 2 internal + # snapshots after 124 minutes at a 1-minute cadence and + # nothing said why. A skipped cadence tick must be + # visible, or the next investigation needs another + # two-hour repro to find out. + logger.info( + "Cadence snapshot for lvol %s deferred: %s has not " + "replicated yet", lv.get_id(), outstanding.get_id()) + blocked = (lv, outstanding) + break + if blocked: + logger.warning( + "Skipping group snapshot for policy %s: member %s " + "has an unreplicated internal snapshot (%s)", + policy.policy_name, blocked[0].get_id(), + blocked[1].get_id()) + continue + logger.info("Taking consistency-group snapshot for policy %s " + "(%d members)", policy.policy_name, len(members)) + _ids, err = consistency_group_controller.create_group_snapshot(policy_id) + if err: + logger.warning("Group snapshot for policy %s failed: %s", + policy.policy_name, err) + except Exception as e: + logger.error("Group snapshot scheduling failed for policy %s: %s", + policy_id, e) + repl_lvols = [lv for lv in repl_lvols if lv.get_id() not in grouped_ids] + for lvol in repl_lvols: try: if not _due_for_internal_snapshot(lvol, all_snaps, now_ts): diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index 4dd61c3ac5..6833356599 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -2,7 +2,8 @@ import time import uuid -from simplyblock_core import constants, db_controller, utils +from simplyblock_core import (constants, db_controller, snapshot_retention, + utils, xfer_timing) from simplyblock_core.controllers import lvol_controller, snapshot_events, snapshot_controller from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.lvol_model import LVol @@ -127,9 +128,191 @@ def _unreplicated_local_ancestor(snode, snapshot, replicate_to_source): return ("blocked", None, "chain deeper than 64 blobs") +def _group_id_for_lvol(lvol): + """The consistency group *lvol* belongs to, or "". + + A group is owned by a replication policy and pinned to one node/LVS. + """ + policy_id = getattr(lvol, "replication_policy_id", "") + if not policy_id: + return "" + try: + group = db.get_consistency_group_for_policy(policy_id) + except Exception as e: # noqa: BLE001 + logger.warning("Could not resolve the consistency group of %s: %s", + lvol.get_id(), e) + return "" + return group.get_id() if group else "" + + +def _lvs_transfer_hold(task, snapshot): + """Why this transfer must wait, or "" when it may start now. + + Two priorities share one lvstore's bandwidth: + + 1. A volume in its FINAL CUTOVER owns the lvstore. Its convergence rounds + decide how long client IO freezes -- every second another volume steals + from a round is a second of writes the frozen final step must copy -- + so nothing else on that lvstore transfers meanwhile. Members of the same + consistency group are exempt: the group cuts over together. + + 2. Consistency groups outrank loose volumes. A group's members transfer in + PARALLEL with each other (their snapshots belong to one generation and + are only useful together), and everything else on the lvstore waits, so + groups are effectively serialized against each other rather than + interleaved. + """ + own_lvol = getattr(snapshot, "lvol", None) + lvs_name = getattr(own_lvol, "lvs_name", "") if own_lvol else "" + if own_lvol is None or not lvs_name: + return "" + own_id = own_lvol.get_id() + own_group = _group_id_for_lvol(own_lvol) + + tasks = db.get_job_tasks(task.cluster_id) + + # --- priority 1: a cutover in progress on this lvstore ----------------- + for t in tasks: + if t.function_name != JobSchedule.FN_REPLICATION_FINAL: + continue + if t.status == JobSchedule.STATUS_DONE or t.canceled: + continue + params = t.function_params or {} + if params.get("cutover_lvs") != lvs_name: + continue + holder = params.get("lvol_id") + if not holder or holder == own_id: + return "" # our own cutover: keep moving + holder_group = params.get("cutover_group") or "" + if holder_group and holder_group == own_group: + return "" # same group: cut over together + return (f"lvol {holder[:8]} is in final cutover on lvstore {lvs_name}") + + # --- priority 2: a consistency group is transferring on this lvstore --- + for t in tasks: + if t.function_name != JobSchedule.FN_SNAPSHOT_REPLICATION: + continue + if t.status != JobSchedule.STATUS_RUNNING or t.get_id() == task.get_id(): + continue + other_snap_id = (t.function_params or {}).get("snapshot_id") + if not other_snap_id: + continue + try: + other_lvol = db.get_snapshot_by_id(other_snap_id).lvol + except KeyError: + continue + if getattr(other_lvol, "lvs_name", "") != lvs_name: + continue + other_group = _group_id_for_lvol(other_lvol) + if other_group and other_group != own_group: + return (f"consistency group {other_group.split('/')[-1][:8]} is " + f"transferring on lvstore {lvs_name}") + + return "" + + +def _finish_completed_transfer(task, snapshot, offset): + """Chain, convert and mark the snapshot replicated. Returns True. + + This is what sets target_replicated_snap_uuid, which is the signal a + cutover's convergence loop waits on -- so it must run as soon as the + transfer is known to be finished, not on some later pass. + """ + # submit -> Done, measured. NOT a throughput: see fix_xfer_latency notes. + xfer_timing.gap("transfer_complete", + task.function_params.get("xfer_submit_t"), + snap=snapshot.get_id(), lvol=snapshot.lvol.get_id(), + bytes=offset) + with xfer_timing.phase("replicate_finish", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id()): + new_snapshot_uuid = process_snap_replicate_finish(task, snapshot) + if new_snapshot_uuid: + task.function_result = new_snapshot_uuid + task.status = JobSchedule.STATUS_DONE + task.function_params["end_time"] = int(time.time()) + task.write_to_db() + else: + task.function_result = "complete repl failed, retrying" + task.status = JobSchedule.STATUS_SUSPENDED + task.retry += 1 + task.write_to_db() + return True + + +def _cutover_owns(lvol_id, cluster_id): + """True while a final cutover is running for this volume. + + Such a volume already holds its lvstore and every other transfer on it is + held, so waiting inline for its transfer starves nothing -- and this is the + window the client's IO freeze is paying for. + """ + if not lvol_id: + return False + try: + tasks = db.get_job_tasks(cluster_id) + except Exception: # noqa: BLE001 + return False + for other in tasks: + if other.function_name != JobSchedule.FN_REPLICATION_FINAL: + continue + if other.status == JobSchedule.STATUS_DONE or other.canceled: + continue + if (other.function_params or {}).get("lvol_id") == lvol_id: + return True + return False + + +def _await_transfer_completion(task, snapshot, snode): + """Poll the just-submitted transfer at 100ms and finish it in this pass. + + Returns True when the transfer completed and was finished here; False to + leave it for the pass-based path (still in flight, or the budget ran out). + + Before this, submit and the Done check happened on DIFFERENT passes of the + runner loop, so a transfer that finished in milliseconds was not acted on + for a median of 81 SECONDS (run 20260828_115307: every poll found state + already 'Done', never once 'In progress'). + """ + budget = (constants.REPL_XFER_INLINE_WAIT_CUTOVER_SEC + if _cutover_owns(snapshot.lvol.get_id(), task.cluster_id) + else constants.REPL_XFER_INLINE_WAIT_SEC) + deadline = time.time() + budget + rpc = snode.rpc_client() + while time.time() < deadline: + try: + ret = rpc.bdev_lvol_transfer_stat(snapshot.snap_bdev) + except Exception as e: # noqa: BLE001 + logger.warning("transfer_stat for %s raised while waiting inline " + "(%s); leaving it to the next pass", + snapshot.get_id(), e) + return False + if not ret: + return False + state = ret.get("transfer_state") + if state == "Done": + return _finish_completed_transfer(task, snapshot, ret.get("offset")) + if state == "Failed": + return False # the pass-based path records the retry + if state == "No process": + return False # transfer never started; pass-based path retries + time.sleep(constants.REPL_XFER_POLL_INTERVAL_SEC) + xfer_timing.stamp("inline_wait_expired", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id(), budget=budget) + return False + + def process_snap_replicate_start(task, snapshot): # 1 create lvol on remote node logger.info("Starting snapshot replication task") + _t_landing = None # set only if we create the landing volume below + + hold = _lvs_transfer_hold(task, snapshot) + if hold: + # Not a failure and not a retry: come back when the lvstore frees up. + task.function_result = f"held: {hold}" + task.write_to_db() + logger.info("Holding replication of %s: %s", snapshot.get_id(), hold) + return False # Drive the transfer from whichever member of the SOURCE lvstore leads it # now — the snapshot exists on every member, so an outage of the recorded # primary must not stop replication (see _source_leader_node). @@ -252,11 +435,53 @@ def process_snap_replicate_start(task, snapshot): logger.error(f"Unable to find pool on remote cluster: {remote_node_uuid.cluster_id}") return + # An earlier attempt of THIS task may have created the landing volume + # and died before storing its id (a node outage mid-create): add_lvol_ha + # then fails "LVol name must be unique" on EVERY retry and the task + # loops forever, stalling the volume's whole chain behind it (case 6, + # run 20260824_144226: three volumes stuck on their first cadence + # snapshot, retrying every ~31s for the rest of the run). The name is + # derived from the snapshot, so a record wearing it IS this transfer's + # landing volume: adopt it when it is usable, clear it when it is not. + rep_name = f"REP_{snapshot.snap_name}" + existing = None + try: + existing = db.get_lvol_by_name(rep_name) + except KeyError: + pass + if existing is not None: + if existing.status == LVol.STATUS_ONLINE: + logger.info(f"Adopting landing volume {existing.get_id()} " + f"({rep_name}) left by an interrupted attempt") + task.function_params["remote_lvol_id"] = existing.get_id() + task.write_to_db() + elif existing.status == LVol.STATUS_IN_DELETION: + task.function_result = f"stale landing volume {rep_name} still deleting, retrying" + task.status = JobSchedule.STATUS_SUSPENDED + task.retry += 1 + task.write_to_db() + return + else: + logger.warning(f"Deleting half-created landing volume " + f"{existing.get_id()} ({rep_name}, status " + f"{existing.status}) from an interrupted attempt") + try: + lvol_controller.delete_lvol(existing, force_delete=True) + except Exception as e: + logger.error(f"Failed to clear stale landing volume {rep_name}: {e}") + task.function_result = "cleared stale landing volume, retrying" + task.status = JobSchedule.STATUS_SUSPENDED + task.retry += 1 + task.write_to_db() + return + + if "remote_lvol_id" not in task.function_params or not task.function_params["remote_lvol_id"]: # internal=True: this REP_* volume is the landing copy for a transfer, # created by the system and never handed to a client. The per-node # subsystem cap is a user-admission limit; enforcing it here only stops # replication on a node that is already full, which is precisely when # the transfers that would let retention free those slots are needed. + _t_landing = xfer_timing.now() lv_id, err = lvol_controller.add_lvol_ha( f"REP_{snapshot.snap_name}", snapshot.size, remote_node_uuid.get_id(), snapshot.lvol.ha_type, remote_pool_uuid, internal=True) @@ -288,7 +513,12 @@ def process_snap_replicate_start(task, snapshot): # is not a valid transfer gateway. This mirrors the (working) migration # runner, which has always sent bulk transfers hub+map_id. from simplyblock_core.services.replication_final_step import ensure_hub_attached - _hub_ctrl, hub_bdev, hub_err = ensure_hub_attached(snode.rpc_client(), remote_lv_node) + xfer_timing.gap("landing_volume_create", _t_landing, + snap=snapshot.get_id(), lvol=snapshot.lvol.get_id()) + with xfer_timing.phase("hub_attach", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id(), + tgt=remote_lv_node.get_id()): + _hub_ctrl, hub_bdev, hub_err = ensure_hub_attached(snode.rpc_client(), remote_lv_node) if hub_err: logger.error(f"Transfer hub attach failed: {hub_err}") task.function_result = "transfer hub attach failed, retrying" @@ -327,6 +557,15 @@ def process_snap_replicate_start(task, snapshot): task.write_to_db() return + allow_partial = _partial_transfer_decision( + task, snapshot, replicate_to_source, remote_lv, remote_lv_node) + logger.info("Transfer of %s into %s: %s", snapshot.get_id(), + remote_lv.top_bdev, + "PARTIAL allowed (landing volume is a clone of the previous " + "replicated snapshot on every online member)" if allow_partial + else "FULL (no delta basis on every online member; see above " + "for which check declined)") + offset = 0 if "offset" in task.function_params and task.function_params["offset"]: offset = task.function_params["offset"] @@ -359,6 +598,10 @@ def process_snap_replicate_start(task, snapshot): fresh.write_to_db() # 3 start replication + xfer_timing.stamp("transfer_submit", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id(), size=snapshot.size) + task.function_params["xfer_submit_t"] = xfer_timing.now() + task.write_to_db() snode.rpc_client().bdev_lvol_transfer( name=snapshot.snap_bdev, offset=offset, @@ -370,12 +613,25 @@ def process_snap_replicate_start(task, snapshot): batch_size=16, bdev_name=hub_bdev, operation="replicate", - lvol_id=remote_map_id + lvol_id=remote_map_id, + # Safe only because the landing volume was chained onto the previous + # replicated snapshot above; the fork independently refuses the delta + # path unless the snapshot's dirty generation is complete. + allow_partial=allow_partial, ) task.status = JobSchedule.STATUS_RUNNING task.function_params["start_time"] = int(time.time()) task.write_to_db() + # Do not hand the transfer back to the pass loop and forget about it: wait + # for it here, polling every 100ms, and finish it in this same pass. Before + # this, submit and the Done check happened on different passes and a + # transfer that finished in milliseconds went unnoticed for a median of 81 + # SECONDS (run 20260828_115307). Returns False if it is still running when + # the budget expires, in which case the pass-based path picks it up as + # before. + _await_transfer_completion(task, snapshot, snode) + def _receiving_leader_node(remote_lv): """The node that currently leads *remote_lv*'s lvstore, or None. @@ -555,6 +811,27 @@ def _keep_replicated_for(source_lvol): from simplyblock_core.models.replication import ReplicationPolicy return max(policy.keep_replicated, ReplicationPolicy.MIN_KEEP_REPLICATED) +def _retention_schedule_for(source_lvol): + """Parsed retention tiers from the volume's policy, or [] when it has none. + + A malformed schedule must not silently disable retention or crash the + replication runner: it is reported and treated as "no schedule", which + falls back to the flat keep-count. + """ + try: + policy = db.get_replication_policy_for_lvol(source_lvol) + except KeyError: + return [] + if (policy is None) or (spec := getattr(policy, "retention_schedule", None)) is None: + return [] + try: + return snapshot_retention.parse_schedule(spec) + except snapshot_retention.RetentionScheduleError as e: + logger.error("Ignoring invalid retention_schedule %r on policy %s: %s", + spec, policy.get_id(), e) + return [] + + def _prune_internal_snapshots(source_lvol): """Retention for replication-driven internal snapshots. @@ -578,10 +855,37 @@ def _prune_internal_snapshots(source_lvol): and s.target_replicated_snap_uuid ] keep = _keep_replicated_for(source_lvol) - if len(replicated_internal) <= keep: - return - replicated_internal.sort(key=lambda s: s.created_at) + + # A retention SCHEDULE, when the policy defines one, decides which older + # snapshots survive; without it retention stays the flat "newest N". + # Either way the newest `keep` are protected, because deleting a snapshot + # swap-merges its segments into the successor chained to it. + schedule = _retention_schedule_for(source_lvol) + # Say which retention is in force, every time it prunes. Soak run + # 20260827_224741 ended with 2 snapshots at consecutive cadence ticks after + # 124 minutes under `5m:15m,7m:30m,10m:1h` -- which is exactly what the FLAT + # keep-N path produces, and nothing in the log said which path ran. The + # ladder itself is provably correct (test_case11_retention_ladder), so the + # open question is whether the schedule reaches this function at all. + logger.info( + "Retention for lvol %s: %s (replicated internal snapshots: %d, keep=%d)", + source_lvol.get_id(), + ("schedule %s" % snapshot_retention.describe(schedule)) if schedule + else "FLAT keep-newest (no schedule on the policy)", + len(replicated_internal), keep) + if schedule: + retained_ts = snapshot_retention.select_retained( + [s.created_at for s in replicated_internal], schedule, + now=time.time(), always_keep_newest=keep) + candidates = [(i, s) for i, s in enumerate(replicated_internal) + if s.created_at not in retained_ts] + if not candidates: + return + else: + if len(replicated_internal) <= keep: + return + candidates = list(enumerate(replicated_internal))[:-keep] # Keep the newest TWO replicated internal snapshots, not just one. # # A replicated snapshot holds only its own clusters; the rest of the data @@ -598,7 +902,7 @@ def _prune_internal_snapshots(source_lvol): # for one snapshot while newer ones kept arriving, the predecessor was still # pruned and its segments were dropped instead of merged. So the chain is # verified per candidate below, and an unchained successor defers the prune. - for index, snap in enumerate(replicated_internal[:-keep]): + for index, snap in candidates: target_uuid = snap.target_replicated_snap_uuid try: db.get_snapshot_by_id(target_uuid) @@ -749,6 +1053,154 @@ def _resolve_chain_target(snapshot, replicate_to_source, remote_snode): return {"snap_bdev": _snap_obj.snap_bdev}, _snap_obj, True +def _prechain_landing_volume(task, snapshot, replicate_to_source, remote_lv, + remote_lv_node): + """Chain the landing volume onto the destination's copy of the PREVIOUS + snapshot, before the transfer runs, so only the delta has to be sent. + + Returns True when the landing volume (and every online member of its HA + pair) is chained, which is the precondition for asking for a PARTIAL + transfer. Returns False to mean "send a full transfer", which is always + correct and is what this pipeline did unconditionally until now. + + Why chaining first is what makes a partial transfer correct + ----------------------------------------------------------- + ``bdev_lvol_transfer`` sends a blob's OWN cluster map and nothing else, so a + partial transfer ships only the ranges written since the previous snapshot. + Everything OUTSIDE that delta therefore has to be on the destination + already, and a fresh empty landing volume does not have it -- that is + exactly why the delta path has never been switched on. Chaining the landing + volume onto the predecessor's remote copy supplies it: a cluster the new + snapshot does not own stays unallocated and reads through to the parent, + and the first write into a cluster the delta DOES touch makes the blobstore + copy-on-write the whole cluster from that same parent before applying the + incoming range. Both halves of the destination image thus come from the + same predecessor the source computed its delta against. + + This is the very ``bdev_lvol_add_clone`` the finish step used to issue AFTER + the transfer; issuing it BEFORE is what turns the landing volume into a + valid delta target. Nodes chained here are recorded on the task so the + finish step does not add the same clone entry twice. + + The bar for returning True is deliberately high: + * a predecessor copy must resolve cleanly on this node (``ok`` and a + non-empty ``target_prev_snap`` from :func:`_resolve_chain_target`); + * the LVS leader must accept the chain; + * the secondary must be ONLINE and accept it too. The transfer lands on + the leader and the lvstore mirrors it to the secondary; an unchained + secondary would read zeros wherever the delta did not write, so a + degraded pair gets a full transfer rather than a delta. + Anything short of that logs why and falls back to a full transfer. + """ + target_prev_snap, _prev_snap_for_db, ok = _resolve_chain_target( + snapshot, replicate_to_source, remote_lv_node) + if not ok: + logger.info( + "Landing volume %s stays a full-transfer target: the predecessor's " + "copy on the destination could not be resolved", remote_lv.top_bdev) + return False + if not target_prev_snap: + logger.info( + "Landing volume %s stays a full-transfer target: %s starts the " + "chain, so there is no predecessor to clone from", + remote_lv.top_bdev, snapshot.get_id()) + return False + + sec_node = None + if remote_lv_node.secondary_node_id: + try: + sec_node = db.get_storage_node_by_id(remote_lv_node.secondary_node_id) + except KeyError: + sec_node = None + if sec_node is None or sec_node.status != StorageNode.STATUS_ONLINE: + logger.info( + "Landing volume %s stays a full-transfer target: the secondary of " + "%s is not online, and a delta would leave holes on it", + remote_lv.top_bdev, remote_lv_node.get_id()) + return False + + prechained = list(task.function_params.get("prechained_node_ids") or []) + + def _chain_on(node, role): + if node.get_id() in prechained: + return True + logger.info("Pre-chaining landing volume %s onto %s on %s (%s) so the " + "transfer can ship only the delta", + remote_lv.top_bdev, target_prev_snap['snap_bdev'], + node.get_id(), role) + try: + ret = node.rpc_client().bdev_lvol_add_clone( + remote_lv.top_bdev, target_prev_snap['snap_bdev']) + except Exception as e: + logger.warning("Pre-chain of %s on %s (%s) raised %s; falling back " + "to a full transfer", remote_lv.top_bdev, + node.get_id(), role, e) + return False + if not ret: + logger.warning("Pre-chain of %s onto %s failed on %s (%s); falling " + "back to a full transfer", remote_lv.top_bdev, + target_prev_snap['snap_bdev'], node.get_id(), role) + return False + prechained.append(node.get_id()) + return True + + primary_ok = _chain_on(remote_lv_node, "primary") + # Record whatever actually landed even on the failure path: the entry exists + # on that node now, and the finish step must not add it a second time. + secondary_ok = _chain_on(sec_node, "secondary") if primary_ok else False + if prechained != (task.function_params.get("prechained_node_ids") or []): + task.function_params["prechained_node_ids"] = prechained + task.write_to_db() + + if not (primary_ok and secondary_ok): + # A full transfer into a partially chained volume is still correct: it + # writes every cluster the snapshot owns, and the chained node reads the + # remainder from the parent exactly as it should. + return False + return True + + +def _partial_transfer_decision(task, snapshot, replicate_to_source, remote_lv, + remote_lv_node): + """Whether this transfer may ship only the delta, chaining the landing + volume first if that has not happened yet. + + The verdict is cached on the task so a resumed transfer (offset > 0) keeps + the mode its first attempt used -- but it is KEYED to the landing volume it + was made about. An interrupted attempt can delete a half-created landing + volume and build a fresh, unchained one (see the adoption block in + process_snap_replicate_start); carrying "partial is fine" onto that volume + would ship a delta into something that holds nothing, which is precisely + the silent data loss this design exists to avoid. A key mismatch throws + away the old verdict AND the old chain record and re-derives both. + """ + landing_key = remote_lv.get_id() + if task.function_params.get("allow_partial_landing") == landing_key: + return bool(task.function_params.get("allow_partial")) + + # Chain state recorded against a previous landing volume says nothing about + # this one. + task.function_params["prechained_node_ids"] = [] + allow_partial = _prechain_landing_volume( + task, snapshot, replicate_to_source, remote_lv, remote_lv_node) + task.function_params["allow_partial"] = allow_partial + task.function_params["allow_partial_landing"] = landing_key + task.write_to_db() + return allow_partial + + +def _prechained_nodes_for(task, remote_lv): + """Nodes already carrying the landing volume's clone entry. + + Empty unless the record was made about THIS landing volume: adding the same + clone entry twice is not idempotent on the SPDK side, and trusting a stale + record would skip a chain the volume actually needs. + """ + if task.function_params.get("allow_partial_landing") != remote_lv.get_id(): + return set() + return set(task.function_params.get("prechained_node_ids") or []) + + def process_snap_replicate_finish(task, snapshot): # Close the transfer session — but ONLY when this was the last active @@ -768,6 +1220,8 @@ def process_snap_replicate_finish(task, snapshot): or db.get_storage_node_by_id(snapshot.lvol.node_id)) if remote_snode.transfer_hublvol and remote_snode.transfer_hublvol.bdev_name: if not _other_active_transfers_to_node(task, remote_snode.get_id()): + xfer_timing.stamp("hub_detach", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id()) _src_node.rpc_client().bdev_nvme_detach_controller( remote_snode.transfer_hublvol.bdev_name) replicate_to_source = task.function_params["replicate_to_source"] @@ -794,16 +1248,30 @@ def process_snap_replicate_finish(task, snapshot): if not _require_lvs_leader(remote_snode, remote_lv.lvs_name, "add_clone/convert"): return False + # Nodes whose landing volume was already chained BEFORE the transfer, so it + # could receive a delta (see _prechain_landing_volume). Those must be + # skipped here: adding the same clone entry twice is not idempotent. + _prechained = _prechained_nodes_for(task, remote_lv) + # chain snaps on primary - if target_prev_snap: + if target_prev_snap and remote_snode.get_id() not in _prechained: logger.info(f"Chaining replicated lvol: {remote_lv.top_bdev} to snap: {target_prev_snap['snap_bdev']}") - ret = remote_snode.rpc_client().bdev_lvol_add_clone( remote_lv.top_bdev, target_prev_snap['snap_bdev']) + with xfer_timing.phase("chain_add_clone", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id(), node="primary"): + ret = remote_snode.rpc_client().bdev_lvol_add_clone( remote_lv.top_bdev, target_prev_snap['snap_bdev']) if not ret: logger.error("Failed to chain replicated snapshot on primary node") return False + elif target_prev_snap: + logger.info("Landing volume %s was already chained to %s on %s before " + "the transfer; skipping the redundant add_clone", + remote_lv.top_bdev, target_prev_snap['snap_bdev'], + remote_snode.get_id()) # convert to snapshot on primary - ret = remote_snode.rpc_client().bdev_lvol_convert(remote_lv.top_bdev) + with xfer_timing.phase("chain_convert", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id(), node="primary"): + ret = remote_snode.rpc_client().bdev_lvol_convert(remote_lv.top_bdev) if not ret: logger.error("Failed to convert to snapshot on primary node") return False @@ -811,15 +1279,24 @@ def process_snap_replicate_finish(task, snapshot): # chain snaps on secondary sec_node = db.get_storage_node_by_id(remote_snode.secondary_node_id) if sec_node.status == StorageNode.STATUS_ONLINE: - if target_prev_snap: + if target_prev_snap and sec_node.get_id() not in _prechained: logger.info(f"Chaining replicated lvol: {remote_lv.top_bdev} to snap: {target_prev_snap['snap_bdev']}") - ret = sec_node.rpc_client().bdev_lvol_add_clone(remote_lv.top_bdev, target_prev_snap['snap_bdev']) + with xfer_timing.phase("chain_add_clone", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id(), node="secondary"): + ret = sec_node.rpc_client().bdev_lvol_add_clone(remote_lv.top_bdev, target_prev_snap['snap_bdev']) if not ret: logger.error("Failed to chain replicated snapshot on secondary node") return False + elif target_prev_snap: + logger.info("Landing volume %s was already chained to %s on %s " + "before the transfer; skipping the redundant add_clone", + remote_lv.top_bdev, target_prev_snap['snap_bdev'], + sec_node.get_id()) # convert to snapshot on secondary - ret = sec_node.rpc_client().bdev_lvol_convert(remote_lv.top_bdev) + with xfer_timing.phase("chain_convert", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id(), node="secondary"): + ret = sec_node.rpc_client().bdev_lvol_convert(remote_lv.top_bdev) if not ret: logger.error("Failed to convert to snapshot on secondary node") return False @@ -840,6 +1317,11 @@ def process_snap_replicate_finish(task, snapshot): new_snapshot.blobid = remote_lv.blobid new_snapshot.created_at = int(time.time()) new_snapshot.status = SnapShot.STATUS_ONLINE + # Consistency-group provenance travels with the copy: the fail-over + # generation selector returns the TARGET record, and requirement 4's + # membership warnings are computed from (group_id, group_seq) on it. + new_snapshot.group_id = getattr(snapshot, "group_id", "") + new_snapshot.group_seq = getattr(snapshot, "group_seq", 0) snapshot.instances.append(new_snapshot) if not replicate_as_snap_instance: if replicate_to_source: @@ -882,12 +1364,23 @@ def process_snap_replicate_finish(task, snapshot): # later cleanup that waits for lvols to drain times out on it. remote_lv.bdev_stack = [] remote_lv.write_to_db() - try: - lvol_controller.delete_lvol(remote_lv, force_delete=True) - except Exception as e: - logger.error(f"Landing volume {remote_lv.get_id()} teardown raised: {e}; " - f"retiring its record anyway (its bdev lives on as the " - f"converted snapshot)") + # Tear the subsystem/namespace down DIRECTLY, not via delete_lvol: + # delete_lvol flips the record to in_deletion and hands it to the + # monitor's async machinery, so an interruption anywhere before the + # remove() below stranded a record the monitor can never finish (empty + # stack -> nothing to issue -> status poll 4 forever; runs 20260824 and + # 20260825_125156). With the stack already emptied there is no blob work + # to do -- only nvmf plumbing on the volume's nodes. + for _node_id in remote_lv.nodes: + try: + _node = db.get_storage_node_by_id(_node_id) + if _node.status == StorageNode.STATUS_ONLINE: + lvol_controller.delete_lvol_from_node( + remote_lv.get_id(), _node_id, force=True) + except Exception as e: + logger.error(f"Landing volume {remote_lv.get_id()} teardown on " + f"{_node_id[:8]} raised: {e}; retiring the record " + f"anyway (its bdev lives on as the converted snapshot)") remote_lv.remove(db.kv_store) snapshot_events.replication_task_finished(snapshot) _prune_internal_snapshots(snapshot.lvol) @@ -970,6 +1463,13 @@ def task_runner(task: JobSchedule): elif task.status == JobSchedule.STATUS_RUNNING: snode = _source_leader_node(snapshot) or db.get_storage_node_by_id(snapshot.lvol.node_id) ret = snode.rpc_client().bdev_lvol_transfer_stat(snapshot.snap_bdev) + if ret: + # offset is the bytes moved so far: the ONLY direct read on actual + # transfer throughput, as distinct from round duration. + xfer_timing.stamp("transfer_running", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id(), + state=str(ret.get("transfer_state")).replace(" ", "_"), + offset=ret.get("offset")) if not ret: logger.error("Failed to get transfer stat") return False @@ -993,18 +1493,7 @@ def task_runner(task: JobSchedule): task.write_to_db() return False if status == "Done": - new_snapshot_uuid = process_snap_replicate_finish(task, snapshot) - if new_snapshot_uuid: - task.function_result = new_snapshot_uuid - task.status = JobSchedule.STATUS_DONE - task.function_params["end_time"] = int(time.time()) - task.write_to_db() - else: - task.function_result = "complete repl failed, retrying" - task.status = JobSchedule.STATUS_SUSPENDED - task.retry += 1 - task.write_to_db() - return True + return _finish_completed_transfer(task, snapshot, offset) def main(): diff --git a/simplyblock_core/services/spdk_http_proxy_server.py b/simplyblock_core/services/spdk_http_proxy_server.py index c83e8902ef..aecd2146a5 100644 --- a/simplyblock_core/services/spdk_http_proxy_server.py +++ b/simplyblock_core/services/spdk_http_proxy_server.py @@ -15,10 +15,14 @@ from simplyblock_core.settings import Settings -logger_handler = logging.StreamHandler(stream=sys.stdout) -logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s')) logger = logging.getLogger() -logger.addHandler(logger_handler) +if __name__ == "__main__": + # Attach the stdout handler only when running as the proxy script. + # Attaching on IMPORT handed every importer (e.g. the test suite) a + # duplicate root handler, doubling every log line in the process. + logger_handler = logging.StreamHandler(stream=sys.stdout) + logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s')) + logger.addHandler(logger_handler) logger.setLevel(logging.INFO) read_line_time_diff: dict = {} @@ -319,4 +323,13 @@ def run_server(host, port, user, password, is_threading_enabled=False): logger.info(f"SPDK concurrency limit: {MAX_CONCURRENT_SPDK}") is_threading_enabled = bool(is_threading_enabled) -run_server(server_ip, rpc_port, rpc_username, rpc_password, is_threading_enabled=is_threading_enabled) + +if __name__ == "__main__": + # Start ONLY when executed as the proxy script (deploy_spdk.yaml and the + # docker snode path both run this file directly). Starting on import made + # every importer spawn the print_stats daemon thread too: after the proxy + # e2e tests ran, that thread logged every 3s for the rest of the pytest + # session and could hold the stderr buffer lock at interpreter shutdown — + # "Fatal Python error: _enter_buffered_busy", SIGABRT, a red CI run with + # 1227/1227 tests passed (2026-09-04). + run_server(server_ip, rpc_port, rpc_username, rpc_password, is_threading_enabled=is_threading_enabled) diff --git a/simplyblock_core/services/tasks_runner_lvol_migration.py b/simplyblock_core/services/tasks_runner_lvol_migration.py index 5b8cf53024..16d04c20a1 100644 --- a/simplyblock_core/services/tasks_runner_lvol_migration.py +++ b/simplyblock_core/services/tasks_runner_lvol_migration.py @@ -2022,7 +2022,12 @@ def _revert_src_replicas(reason): stat = src_rpc.bdev_lvol_transfer_stat(src_lvol_composite) state = (stat or {}).get('transfer_state') if stat is not None else None else: - state = None + # The RPC can return normally (no exception) while still reporting + # transfer failure — transfer_state is one of "No process" | + # "In progress" | "Failed" | "Done". Checking only for falsy + # lets a "Failed" dict slip through to the ANA flip as if the + # data had moved (same class of bug as batch migration 2026-08-22). + state = ret.get("transfer_state") if isinstance(ret, dict) else None except Exception: # SRC secondary/tertiary were just flipped inaccessible above; if # either RPC above raises (e.g. source unreachable — RPCException @@ -2047,6 +2052,12 @@ def _revert_src_replicas(reason): logger.info( f"[IO-RESUME] {_now_ms()} final migration complete (recovered from RPC error, " f"transfer_state={state}): lvol={migration.lvol_id} io now live on target") + elif state != "Done": + logger.error( + f"bdev_lvol_final_migration: transfer_state={state!r} " + f"(expected 'Done'): {ret!r} lvol={migration.lvol_id}") + _revert_src_replicas(f"final migration failed: transfer_state={state!r}") + return False, True, f"bdev_lvol_final_migration failed: transfer_state={state!r}" else: logger.info( f"[IO-RESUME] {_now_ms()} final migration Done: " diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 761002a90d..aaabc48521 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -24,7 +24,7 @@ import uuid as uuid_lib from datetime import datetime -from simplyblock_core import constants, db_controller, utils +from simplyblock_core import constants, db_controller, utils, xfer_timing from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.lvol_model import LVol, LVolReplication from simplyblock_core.models.snapshot import SnapShot @@ -37,10 +37,73 @@ db = db_controller.DBController() +def _lvs_cutover_owner(task, lvs_name, tasks=None): + """Return the task that should own the active cutover slot, or None if *task* wins. + + Serializes all cutovers globally: only one may run at a time across all + volumes and lvstores. Deterministic: the earliest-created claimant wins + (sorted by create_dt then id), so two tasks that both wrote a claim agree on + the same winner regardless of interleaving. + + Callers MUST write their own ``cutover_lvs`` claim to DB **before** calling + this function so they appear in the scan and participate in the sort. + + Returns None → the caller is the rightful owner; proceed. + Returns → that other task owns the slot; the caller should remove + its own claim and yield. + + ``tasks`` may be a pre-fetched list from this pass. Re-reading per task is + O(N^2) DB reads, which the sub-second poll interval cannot afford. + """ + if not lvs_name: + return None + owners = [] + for other in (tasks if tasks is not None else db.get_job_tasks(task.cluster_id)): + if other.function_name != JobSchedule.FN_REPLICATION_FINAL: + continue + if other.status == JobSchedule.STATUS_DONE or other.canceled: + continue + if (other.function_params or {}).get("cutover_lvs") != lvs_name: + continue + owners.append(other) + if not owners: + return None + owners.sort(key=lambda t: (str(getattr(t, "create_dt", "")), t.get_id())) + winner = owners[0] + return None if winner.get_id() == task.get_id() else winner + + +def _group_id_for_lvol(lvol): + """The consistency group *lvol* belongs to, or "". + + A group is owned by a replication policy and pinned to one node/LVS, so a + volume's group is the group of its policy. + """ + policy_id = getattr(lvol, "replication_policy_id", "") + if not policy_id: + return "" + try: + group = db.get_consistency_group_for_policy(policy_id) + except Exception as e: # noqa: BLE001 + logger.warning("Could not resolve the consistency group of %s: %s", + lvol.get_id(), e) + return "" + return group.get_id() if group else "" + + +def _release_lvs_claim(task): + """Let other volumes on this LVS replicate again.""" + released = task.function_params.pop("cutover_lvs", None) is not None + task.function_params.pop("cutover_group", None) + if released: + task.write_to_db(db.kv_store) + + def _finalize(task, ok, err): if ok: replication_id = task.function_params.get("replication_id") final_state = task.function_params.get("final_state", LVolReplication.STATE_CUTOVER_DONE) + rep = None if replication_id: try: rep = db.get_lvol_replication_by_id(replication_id) @@ -48,6 +111,24 @@ def _finalize(task, ok, err): rep.write_to_db(db.kv_store) except Exception as e: logger.error(f"Failed to update replication state: {e}") + + failback_source_id = task.function_params.get("failback_source_lvol_id") + if failback_source_id and rep is not None: + _swap_failback_lvol_uuid(rep, failback_source_id) + # Remove the stale failed_over LVolReplication that predates the + # failback. Without this the operator's get_relationship query keeps + # finding the old record and reports failed_over indefinitely even + # though IO has already returned to the original source cluster. + prior_replication_id = task.function_params.get("failback_prior_replication_id") + if prior_replication_id: + try: + prior_rep = db.get_lvol_replication_by_id(prior_replication_id) + prior_rep.remove(db.kv_store) + logger.info( + "failback: removed stale failed_over replication record %s", + prior_replication_id) + except KeyError: + pass task.function_result = "cutover done" task.status = JobSchedule.STATUS_DONE task.function_params["end_time"] = int(time.time()) @@ -97,23 +178,191 @@ def _finalize(task, ok, err): # reported loudly but does not un-succeed the task. logger.error(f"Source volume {src_lvol_id} could not be " f"deleted after the cutover: {e}") + _release_lvs_claim(task) return True task.function_result = err or "cutover failed, retrying" + # Keep the reason where the max-retry branch cannot overwrite it, and say + # it out loud: a task that quietly retries to death costs a whole lab run + # to diagnose (run 20260827_194551 -- 20 tasks, 160 attempts, no log line). + task.function_params["last_error"] = task.function_result + logger.warning("cutover attempt %d/%d failed for lvol %s: %s", + task.retry + 1, task.max_retry, + task.function_params.get("lvol_id"), task.function_result) task.status = JobSchedule.STATUS_SUSPENDED + # A retry re-claims the LVS on its next pass; holding the claim across the + # wait would stall every other volume's replication for nothing. + _release_lvs_claim(task) + + # When the failure happened inside run_cutover (tgt_lvol_composite already + # set), it is likely a connectivity issue: the target node restarted, or the + # control plane hasn't yet reflected the down state in DB. Hammering at the + # 0.2s poll interval burns all retries in seconds — long before the node + # recovers. Instead, re-read the node status and add a cooldown. + if task.function_params.get("tgt_lvol_composite"): + tgt_node_id = task.function_params.get("tgt_node_id") + node_offline = False + if tgt_node_id: + try: + current_tgt = db.get_storage_node_by_id(tgt_node_id) + node_offline = current_tgt.status != StorageNode.STATUS_ONLINE + except KeyError: + node_offline = True + hub_attempts = task.function_params.get("cutover_hub_attempts", 0) + 1 + task.function_params["cutover_hub_attempts"] = hub_attempts + if node_offline or hub_attempts <= constants.REPL_CUTOVER_MAX_HUB_ATTEMPTS: + # Transient: add a cooldown, do NOT burn task.retry. + task.function_params["cutover_retry_after"] = ( + int(time.time()) + constants.REPL_CUTOVER_HUB_RETRY_COOLDOWN_SEC) + logger.warning( + "cutover for lvol %s: connectivity failure (attempt %d, node_offline=%s); " + "waiting %ds before retry", + task.function_params.get("lvol_id"), hub_attempts, node_offline, + constants.REPL_CUTOVER_HUB_RETRY_COOLDOWN_SEC) + task.write_to_db(db.kv_store) + return False + # Exceeded transient cap with node appearing online — real failure. + logger.warning( + "cutover for lvol %s: hub attach failed %d times with node appearing " + "online; treating as real failure and burning a retry", + task.function_params.get("lvol_id"), hub_attempts) + task.function_params.pop("cutover_hub_attempts", None) + task.function_params.pop("cutover_retry_after", None) + task.retry += 1 task.write_to_db(db.kv_store) return False +def _swap_failback_lvol_uuid(rep, failback_source_id): + """After a successful failback cutover, reassign the new clone's UUID to + the original source lvol's UUID. + + The operator stores the original source lvolID in the ReplicationSlot's + Spec.VolumeID. Preserving that UUID through the failback means the slot + stays valid with no update required. + + The old source lvol DB record is removed first (its NVMf subsystem was + already evicted by _evict_stale_namespace during _create_target_lvol_clone). + The stale clone record at the old UUID is cleared last so there is never a + window with two records under the same UUID. + """ + try: + new_lvol = db.get_lvol_by_id(rep.target_lvol.get_id()) + except KeyError: + logger.error( + "failback UUID swap: new clone %s not found in DB; skipping", + rep.target_lvol.get_id()) + return + + stale_uuid = new_lvol.get_id() + + # The failover's _finalize() cleared do_replicate / replication_interval_min / + # replication_policy_id on the original cluster-1 source to stop it replicating. + # Those cleared values must NOT be propagated here; instead restore them from + # the failback source (cluster-2 volume) whose DB record is still intact at + # this point — _finalize() clears its fields in a later step. + failback_src_interval = 0 + failback_src_policy_id = "" + try: + failback_src = db.get_lvol_by_id(rep.source_lvol.get_id()) + failback_src_interval = failback_src.replication_interval_min + failback_src_policy_id = failback_src.replication_policy_id + except Exception as exc: + logger.warning( + "failback UUID swap: could not read failback source %s for interval/policy: %s", + rep.source_lvol.get_id(), exc) + + # Copy replication_node_id / replication_mode from the original source — + # these fields were NOT cleared by the failover's _finalize(), so they still + # point at the correct cluster-2 target node. + try: + old_lvol = db.get_lvol_by_id(failback_source_id) + new_lvol.replication_node_id = old_lvol.replication_node_id + new_lvol.replication_mode = old_lvol.replication_mode + old_lvol.remove(db.kv_store) + except KeyError: + logger.warning( + "failback UUID swap: original source lvol %s already absent from DB", + failback_source_id) + + # Explicitly re-enable replication on the restored volume. The original + # source had do_replicate cleared during failover; failback means it is the + # active source again and should resume its cadence. + new_lvol.do_replicate = True + new_lvol.replication_interval_min = failback_src_interval + new_lvol.replication_policy_id = failback_src_policy_id + + # Write the clone under the original source UUID. + new_lvol.uuid = failback_source_id + new_lvol.write_to_db(db.kv_store) + + # Clear the stale record at the old clone UUID. + stale = LVol() + stale.uuid = stale_uuid + stale.remove(db.kv_store) + + # Keep the relationship's target reference consistent. + rep.target_lvol.uuid = failback_source_id + rep.write_to_db(db.kv_store) + + logger.info( + "failback UUID swap complete: clone %s reassigned to original source UUID %s", + stale_uuid, failback_source_id) + -def task_runner(task: JobSchedule): +def _acquire_lvs_claim(task, lvol, tasks=None): + """Take the lvstore for this cutover's endgame. False means queued. + + Queueing here is cheap: it happens with a nearly-converged delta and NO + snapshot in hand. The previous design claimed on the task's first pass and + held the lvstore through the entire catch-up, so queued volumes sat on an + ageing round-1 snapshot and their "round" measured the queue rather than + the transfer. + """ + params = task.function_params + lvs_name = getattr(lvol, "lvs_name", "") + own_group = _group_id_for_lvol(lvol) + owner = _lvs_cutover_owner(task, lvs_name, tasks) + if owner is not None: + owner_group = (owner.function_params or {}).get("cutover_group") or "" + owner_lvol = str((owner.function_params or {}).get("lvol_id")) + # A consistency group cuts over AS A GROUP, so a sibling member joins + # the owner rather than queueing behind it. + if not (own_group and own_group == owner_group): + params["shrink_deadline"] = ( + int(time.time()) + constants.REPL_CUTOVER_SHRINK_TIMEOUT_SEC) + task.function_result = ( + f"queued for lvstore {lvs_name} behind {owner_lvol[:8]} " + f"(delta already converged)") + task.status = JobSchedule.STATUS_SUSPENDED + task.write_to_db(db.kv_store) + return False + params["cutover_lvs"] = lvs_name + params["cutover_group"] = own_group + xfer_timing.stamp("lvs_claim_acquired", lvol=lvol.get_id(), lvs=lvs_name, + round=params.get("shrink_round")) + task.write_to_db(db.kv_store) + return True + + +def task_runner(task: JobSchedule, tasks=None): params = task.function_params lvol_id = params.get("lvol_id") if not lvol_id: return _finalize(task, False, "missing lvol_id in task params") if task.retry >= task.max_retry or task.canceled is True: - task.function_result = "task cancelled" if task.canceled else "max retry reached" + if task.canceled: + task.function_result = "task cancelled" + else: + # Carry the last real error: "max retry reached" on its own names + # a symptom and hides the cause. + last = task.function_params.get("last_error") + task.function_result = (f"max retry reached ({task.max_retry}) after: {last}" + if last else "max retry reached") + logger.error("cutover gave up on lvol %s after %d attempts: %s", + task.function_params.get("lvol_id"), task.max_retry, + last or "reason not recorded") task.status = JobSchedule.STATUS_DONE task.write_to_db(db.kv_store) return True @@ -133,9 +382,23 @@ def task_runner(task: JobSchedule): pass if tgt_node.status != StorageNode.STATUS_ONLINE: - task.function_result = "target node not online, retrying" + logger.warning("cutover for lvol %s waiting: target node %s is %s", + params.get("lvol_id"), tgt_node.get_id(), tgt_node.status) + task.function_params["last_error"] = ( + f"target node {tgt_node.get_id()[:8]} is {tgt_node.status}") + task.function_result = "target node not online, waiting" + task.status = JobSchedule.STATUS_SUSPENDED + # Do NOT burn task.retry — node offline is transient; hammering at + # 0.2s would exhaust all retries before the node recovers. + task.write_to_db(db.kv_store) + return False + + # Cooldown set after a hub-attach failure to give the target node time + # to recover and the control plane time to reflect a down state in DB. + retry_after = params.get("cutover_retry_after", 0) + if int(time.time()) < retry_after: + task.function_result = "waiting after connectivity failure" task.status = JobSchedule.STATUS_SUSPENDED - task.retry += 1 task.write_to_db(db.kv_store) return False @@ -143,25 +406,82 @@ def task_runner(task: JobSchedule): return _finalize(task, False, "source node not found for cutover") if task.status in [JobSchedule.STATUS_NEW, JobSchedule.STATUS_SUSPENDED, JobSchedule.STATUS_RUNNING]: + # One line per runner pass: the spacing between these shows the task + # scheduler's contribution (TASK_EXEC_INTERVAL_SEC per state change), + # which is invisible in any per-phase number. + xfer_timing.stamp("task_pass", lvol=lvol_id, state=task.status, + result=str(task.function_result)[:40].replace(" ", "_")) task.status = JobSchedule.STATUS_RUNNING task.function_params.setdefault("start_time", int(time.time())) - task.write_to_db(db.kv_store) - # ---- SHRINK PHASE ----------------------------------------------- # - if "shrink_snap_id" in params and params.get("shrink_round", 0) > 0: - done, err = _shrink_step(task, lvol) - if err: - return _finalize(task, False, err) - if not done: - # waiting on replication of the current shrink snapshot; come - # back next pass WITHOUT burning a retry (bounded by deadline) + # ---- WAIT FOR THE VOLUME TO CATCH UP ---------------------------- # + # The iterative snapshots ARE the endgame. Until then the volume just + # replicates on its ordinary cadence: the cutover takes no snapshots of + # its own and holds nothing, so it adds no load to a cluster that is + # still catching up -- exactly when it can least afford it. + if not params.get("cutover_lvs"): + lag = _replication_lag_sec(lvol) + # lag is None when the volume has no internal snapshot replicated in + # the current source→target direction yet (e.g. a freshly set-up + # failback pair that has not completed even one reverse-direction + # cycle). Treat it as "no measurement" and proceed: the first + # shrink round captures all outstanding delta, and _prepare_cutover + # will surface a proper error if there is truly no base to chain + # onto. Only block when lag is a real number that exceeds the gate. + if lag is not None and lag > constants.REPL_CUTOVER_ENDGAME_LAG_SEC: + task.function_result = ( + "waiting for replication to catch up before the endgame " + "(lag %.0fs > %ds)" % (lag, constants.REPL_CUTOVER_ENDGAME_LAG_SEC)) + xfer_timing.stamp("await_catchup", lvol=lvol_id, + ms=lag * 1000.0) + # Not a failure: no retry burned, and the deadline is pushed out + # because catching up is legitimate progress, not a stall. + params["shrink_deadline"] = ( + int(time.time()) + constants.REPL_CUTOVER_SHRINK_TIMEOUT_SEC) task.status = JobSchedule.STATUS_SUSPENDED task.write_to_db(db.kv_store) return False + # Caught up (or no measurement): take the lvstore for the endgame. + # Queueing here is cheap -- nothing is held and no snapshot is ageing. + if not _acquire_lvs_claim(task, lvol, tasks): + return False + xfer_timing.stamp("endgame_entered", lvol=lvol_id, ms=(lag or 0) * 1000.0) + + # The first iterative snapshot belongs to the endgame, not to task + # creation: taken here, its delta covers only the catch-up residual. + if not params.get("shrink_snap_id"): + _, snap_err = _take_shrink_snapshot(task, lvol) + if snap_err: + return _finalize(task, False, snap_err) + task.write_to_db(db.kv_store) + + # ---- SHRINK PHASE ----------------------------------------------- # + # Skip shrink entirely once the cutover clone is prepared: tgt_lvol_composite + # being set means shrink already completed on a prior pass and the clone was + # created from the resulting snapshot. Every retry after a failed run_cutover + # should jump straight to run_cutover without redoing any shrink rounds. + if "tgt_lvol_composite" not in params: + if "shrink_snap_id" in params and params.get("shrink_round", 0) > 0: + done, err = _shrink_step(task, lvol) + if err: + return _finalize(task, False, err) + if not done: + # waiting on replication of the current shrink snapshot; come + # back next pass WITHOUT burning a retry (bounded by deadline) + task.status = JobSchedule.STATUS_SUSPENDED + task.write_to_db(db.kv_store) + return False + # ---- CUTOVER PHASE (immediately after the last shrink round) ---- # + # The freeze runs under the claim. A volume whose very first round was + # already fast enough reaches here without having taken it. + if not params.get("cutover_lvs"): + if not _acquire_lvs_claim(task, lvol, tasks): + return False if "tgt_lvol_composite" not in params: - err = _prepare_cutover(task, lvol, src_node, tgt_node) + with xfer_timing.phase("prepare_cutover", lvol=lvol_id): + err = _prepare_cutover(task, lvol, src_node, tgt_node) if err: return _finalize(task, False, err) params = task.function_params @@ -170,8 +490,13 @@ def task_runner(task: JobSchedule): # (operator calls POST .../replication/cutover-proceed after its preconnect # Job succeeds). REPL_CUTOVER_PROCEED_TIMEOUT_SEC is the safety fallback # so cutover proceeds even if the operator is unavailable. + # Every second spent here is a second of writes the FROZEN final step + # must copy, because the cutover clone's base snapshot was taken before + # it. With no operator to signal, the 120s fallback fired 34 times in + # soak run 20260827_110415 and produced 25-72s freezes. Only wait when + # the deployment actually has an operator posting cutover-proceed. replication_id = params.get("replication_id") - if replication_id: + if replication_id and constants.REPL_CUTOVER_PROCEED_REQUIRED: try: rep = db.get_lvol_replication_by_id(replication_id) if not rep.cutover_proceed: @@ -180,6 +505,8 @@ def task_runner(task: JobSchedule): int(time.time()) + constants.REPL_CUTOVER_PROCEED_TIMEOUT_SEC) task.write_to_db(db.kv_store) if int(time.time()) < params["cutover_proceed_timeout"]: + xfer_timing.stamp("cutover_gate_wait", lvol=lvol_id, + deadline=params["cutover_proceed_timeout"]) task.function_result = "cutover_pending: waiting for preconnect signal" task.status = JobSchedule.STATUS_SUSPENDED task.write_to_db(db.kv_store) @@ -192,10 +519,11 @@ def task_runner(task: JobSchedule): "replication record %s not found; proceeding with cutover", replication_id) try: - ok, err = replication_final_step.run_cutover( - src_node, tgt_node, lvol, - params["tgt_lvol_composite"], params["tgt_map_id"], - params["tgt_snap_composite"], operation=params.get("operation", "replicate")) + with xfer_timing.phase("run_cutover", lvol=lvol_id): + ok, err = replication_final_step.run_cutover( + src_node, tgt_node, lvol, + params["tgt_lvol_composite"], params["tgt_map_id"], + params["tgt_snap_composite"], operation=params.get("operation", "replicate")) except Exception as e: logger.error(f"Cutover raised: {e}", exc_info=True) return _finalize(task, False, str(e)) @@ -203,48 +531,211 @@ def task_runner(task: JobSchedule): return True -SHRINK_ROUNDS = 2 -def _shrink_step(task, lvol): - """Advance the delta-shrink state machine one step. +def _replication_lag_sec(lvol): + """How far the target is behind: age of the newest REPLICATED snapshot. + + Returns None when no internal snapshot with target_replicated_snap_uuid + exists for this volume in the current source→target direction. The caller + treats None as "no measurement available" and proceeds rather than blocking: + this happens legitimately on failback pairs that have not yet completed + their first reverse-direction replication cycle. - Returns (done, error): done=True when all rounds are replicated and the - cutover may start IMMEDIATELY; error aborts the task. + Measuring lag from ordinary cadence replication costs nothing: the + alternative (taking rounds to find out how fast a round is) adds snapshot + and transfer load to a cluster that is still catching up, which is + precisely when it can least afford it. """ + newest = None + try: + snaps = db.get_snapshots_by_node_id(lvol.node_id) + except Exception as e: # noqa: BLE001 + logger.warning("Cannot read snapshots of %s to measure replication lag: %s", + lvol.get_id(), e) + return None + for s in snaps: + if (s.lvol.get_id() == lvol.get_id() + and s.snap_type == SnapShot.TYPE_INTERNAL + and getattr(s, "target_replicated_snap_uuid", "") + and (newest is None or s.created_at > newest.created_at)): + newest = s + if newest is None: + return None + return max(0.0, time.time() - float(newest.created_at)) + + +def _take_shrink_snapshot(task, lvol): + """Snapshot the source and record it as the round in flight.""" + from simplyblock_core.controllers import snapshot_controller params = task.function_params - if int(time.time()) > params.get("shrink_deadline", 0): - return False, "shrink phase timed out waiting for replication" + # The gap since the previous round finished: dead time between a round + # completing and the next snapshot existing is pure added delta. + xfer_timing.gap("round_gap_to_next_snapshot", + params.get("shrink_round_done_at"), lvol=lvol.get_id(), + round=params.get("shrink_round", 0)) + with xfer_timing.phase("take_shrink_snapshot", lvol=lvol.get_id(), + round=params.get("shrink_round", 0) + 1): + new_snap, err = snapshot_controller.add( + lvol.get_id(), f"repl_commit_{uuid_lib.uuid4()}", + snap_type=SnapShot.TYPE_INTERNAL) + if err: + return None, f"shrink round {params.get('shrink_round', 0) + 1} snapshot failed: {err}" + params["shrink_round"] = params.get("shrink_round", 0) + 1 + params["shrink_snap_id"] = new_snap + params["shrink_started_at"] = time.time() + return new_snap, None - snap_id = params["shrink_snap_id"] + +def _shrink_round_done(snap_id): + """True once the round's snapshot is replicated AND chained on the target. + + target_replicated_snap_uuid is set at replicate-finish, after the target + copy is chained and converted -- replicated and usable in one signal. + """ try: - snap = db.get_snapshot_by_id(snap_id) + return bool(db.get_snapshot_by_id(snap_id).target_replicated_snap_uuid) except KeyError: - return False, f"shrink snapshot {snap_id} disappeared" + return None # disappeared - # target_replicated_snap_uuid is set at replicate-finish AFTER the target - # copy is chained and converted — replicated AND converted in one signal. - if not snap.target_replicated_snap_uuid: - task.function_result = (f"shrink round {params['shrink_round']}: waiting " - f"for {snap_id[:8]} to replicate") - return False, None - if params["shrink_round"] >= SHRINK_ROUNDS: - return True, None +def _inline_window(last_round_secs): + """How long a pass may poll inline, given the last round's transfer time. - # Round replicated — IMMEDIATELY take the next snapshot: its delta covers - # only the wait window of the previous round. - from simplyblock_core.controllers import snapshot_controller - new_snap, err = snapshot_controller.add( - lvol.get_id(), f"repl_commit_{uuid_lib.uuid4()}", - snap_type=SnapShot.TYPE_INTERNAL) - if err: - return False, f"shrink round {params['shrink_round'] + 1} snapshot failed: {err}" - params["shrink_round"] += 1 - params["shrink_snap_id"] = new_snap - task.function_result = f"shrink round {params['shrink_round']}: snapshot taken" - task.write_to_db(db.kv_store) - return False, None + The runner is single-threaded, so a flat multi-minute budget would stall + every other volume's cutover behind this one. Scaling to the last round + keeps the loop inline exactly where it matters -- near convergence, where + the next snapshot must follow within milliseconds -- and yields early when + rounds are still long, which is where yielding costs nothing because the + freeze is far away regardless. + + The floor is REPL_CUTOVER_MIN_INLINE_SEC rather than zero because yielding + hands the round back to the pass loop, and being picked up again costs far + more than the poll it replaces: the requirement is that NO time is lost + between a transfer completing and the next snapshot starting. + """ + return min(constants.REPL_CUTOVER_CONVERGE_BUDGET_SEC, + max(constants.REPL_CUTOVER_MIN_INLINE_SEC, last_round_secs * 3)) + + +def _shrink_step(task, lvol): + """Converge the delta, then hand straight over to the cutover. + + Returns (done, error). done=True means the delta is as small as it is going + to get and the freeze may start IMMEDIATELY. + + This loop deliberately does NOT return to the task scheduler between + rounds. Each return costs TASK_EXEC_INTERVAL_SEC (10s) before the next + pass, and every one of those seconds is written by the client and lands in + the next round -- a floor on the delta that more rounds cannot lower. Here + a round ends and the next snapshot is taken within + REPL_CUTOVER_POLL_INTERVAL_SEC, so a round only carries the writes made + during the previous round's transfer. + """ + params = task.function_params + deadline = params.get("shrink_deadline", 0) + # How long this pass may poll inline before handing the runner back. The + # runner is single-threaded, so a flat multi-minute budget would stall + # every OTHER volume's cutover behind this one. Scale it to how long the + # last round actually took: near convergence the rounds are seconds and we + # stay inline (which is the whole point -- the next snapshot must follow + # within milliseconds), while a slow early round yields quickly, and + # yielding costs nothing there because we are far from the freeze anyway. + budget_end = time.time() + _inline_window( + (params.get("shrink_round_times") or [0])[-1]) + + while True: + if int(time.time()) > deadline: + # Deadline expired: stop adding rounds and fall through to cutover + # rather than failing and burning a retry. The freeze that follows + # is slightly larger than if we had converged, but proceeding is + # always better than another 900-second wait. + logger.warning( + "cutover convergence: lvol=%s shrink deadline expired after %d " + "rounds; proceeding to cutover", lvol.get_id(), + params.get("shrink_round", 0)) + return True, None + + snap_id = params["shrink_snap_id"] + done = _shrink_round_done(snap_id) + if done is None: + return False, f"shrink snapshot {snap_id} disappeared" + + if not done: + if time.time() >= budget_end: + # Give the pass back so the runner can service other tasks; + # the round is still in flight and resumes on the next pass. + task.function_result = (f"shrink round {params['shrink_round']}: waiting " + f"for {snap_id[:8]} to replicate") + return False, None + time.sleep(constants.REPL_CUTOVER_POLL_INTERVAL_SEC) + continue + + started_at = params.get("shrink_started_at") + if started_at is None: + # Unmeasurable round (an older task, or one enqueued without the + # stamp). Treat it as NOT converged rather than as instant: a + # missing measurement must never be read as "the delta is small", + # which is precisely the mistake that kept the freeze at 9-55s. + elapsed = float("inf") + logger.warning( + "cutover convergence: lvol=%s round %d has no start stamp; " + "taking another round rather than assuming it was fast", + lvol.get_id(), params["shrink_round"]) + else: + elapsed = time.time() - started_at + params.setdefault("shrink_round_times", []).append(round(elapsed, 2)) + logger.info("cutover convergence: lvol=%s round %d transferred in %.2fs", + lvol.get_id(), params["shrink_round"], elapsed) + # Structured twin of the line above. round_total is snapshot-taken + # to replicated-and-chained, i.e. it INCLUDES all orchestration -- + # compare it against the transfer phase from snapshot_replication + # to see how much is data movement. + xfer_timing.stamp("round_total", lvol=lvol.get_id(), + snap=params.get("shrink_snap_id"), + round=params["shrink_round"], + ms=elapsed * 1000.0) + params["shrink_round_done_at"] = time.time() + + # Every round is an endgame round now -- the lvstore is held before the + # first iterative snapshot is taken. + if elapsed <= constants.REPL_CUTOVER_CONVERGE_TARGET_SEC: + task.function_result = (f"converged in {params['shrink_round']} rounds " + f"(last {elapsed:.2f}s)") + return True, None + + if params["shrink_round"] >= constants.REPL_CUTOVER_MAX_SHRINK_ROUNDS: + # Written faster than it replicates. Freezing now is still the best + # move -- the freeze at least stops the writes -- but say so. + logger.warning( + "cutover convergence: lvol=%s did not converge in %d rounds " + "(last round %.2fs > %.2fs target); freezing anyway", + lvol.get_id(), params["shrink_round"], elapsed, + constants.REPL_CUTOVER_CONVERGE_TARGET_SEC) + task.function_result = (f"not converged after {params['shrink_round']} " + f"rounds (last {elapsed:.2f}s)") + return True, None + + # IMMEDIATELY take the next snapshot. This is the whole mechanism: the + # next round carries only what was written while this one transferred, + # so waiting here -- for the scheduler or for anything else -- puts + # those seconds straight into the freeze. It happens before any + # yielding decision for exactly that reason. + _, err = _take_shrink_snapshot(task, lvol) + if err: + return False, err + task.write_to_db(db.kv_store) + + # Re-arm the inline window from what this round just measured: rounds + # near convergence are short and stay inline, a slow one hands the + # runner back so other volumes' cutovers are not stuck behind it. + budget_end = time.time() + _inline_window( + elapsed if elapsed != float("inf") else 0) + if time.time() >= budget_end: + task.function_result = (f"shrink round {params['shrink_round'] - 1} done " + f"({elapsed:.2f}s); continuing next pass") + task.write_to_db(db.kv_store) + return False, None def _prepare_cutover(task, lvol, src_node, tgt_node): @@ -292,6 +783,22 @@ def _prepare_cutover(task, lvol, src_node, tgt_node): rep.target_ns_id = new_lvol.ns_id rep.write_to_db(db.kv_store) + # Detect failback: if the current replication source (lvol) was previously + # the TARGET in a completed relationship whose source cluster is now tgt_node, + # this is a failback cutover. Store the original source UUID so _finalize can + # reassign it to the new clone, keeping the operator's VolumeID valid. + for prior in db.get_lvol_replication_objects(): + if (prior.target_lvol and prior.target_lvol.get_id() == lvol.get_id() + and prior.source_cluster_id == tgt_node.cluster_id + and prior.source_lvol): + task.function_params["failback_source_lvol_id"] = prior.source_lvol.get_id() + task.function_params["failback_prior_replication_id"] = prior.get_id() + logger.info( + "failback cutover detected: original source UUID %s will be " + "preserved on new clone %s after cutover", + prior.source_lvol.get_id(), new_lvol.get_id()) + break + task.function_params.update({ "tgt_lvol_composite": new_lvol.top_bdev, "tgt_map_id": tgt_map_id, @@ -302,6 +809,24 @@ def _prepare_cutover(task, lvol, src_node, tgt_node): return None +def _any_cutover_in_flight(tasks): + """True while some cutover is between its first snapshot and its freeze. + + Only then is a sub-second pass interval worth paying for: that is the + window where a completed transfer must be picked up immediately. + """ + for t in tasks: + if t.function_name != JobSchedule.FN_REPLICATION_FINAL: + continue + if t.status == JobSchedule.STATUS_DONE or t.canceled: + continue + params = t.function_params or {} + # claimed the lvstore, or already has a round in flight + if params.get("cutover_lvs") or params.get("shrink_snap_id"): + return True + return False + + def main(): logger.info("Starting replication-final tasks runner...") while True: @@ -311,21 +836,37 @@ def main(): logger.error(f"Failed to get clusters: {e}") time.sleep(3) continue + active = False for cl in clusters: - for task in db.get_job_tasks(cl.get_id(), reverse=False): + # Read once per cluster per pass and reuse: the owner lookup used to + # re-read this for every task. + cluster_tasks = db.get_job_tasks(cl.get_id(), reverse=False) + if _any_cutover_in_flight(cluster_tasks): + active = True + for task in cluster_tasks: if task.function_name != JobSchedule.FN_REPLICATION_FINAL: continue if task.status == JobSchedule.STATUS_DONE: continue task = db.get_task_by_id(task.uuid) try: - res = task_runner(task) + task_runner(task, cluster_tasks) except Exception as e: logger.error(f"replication-final task {task.uuid} failed: {e}", exc_info=True) - res = False - if not res: - time.sleep(3) - time.sleep(constants.TASK_EXEC_INTERVAL_SEC) + # No blanket backoff here, and the return value is deliberately + # ignored: False is the NORMAL result for a task that is queued + # or mid-round, and sleeping 3s per such task cost ~70s per + # pass with 20 volumes -- which landed directly in the + # client's IO freeze. + # Deliberately NOT a sub-second poll: this loop reads the task table + # (and each task) per pass, so polling it at 5Hz burns transactions + # proportional to clusters x tasks to learn nothing almost every time. + # The latency that mattered is gone from the hot path instead -- a + # transfer is now awaited and finished in the pass that submitted it + # (snapshot_replication._await_transfer_completion, an RPC poll), and a + # converging round stays inside its own inline loop. + time.sleep(constants.REPL_CUTOVER_ACTIVE_POLL_SEC if active + else constants.TASK_EXEC_INTERVAL_SEC) if __name__ == "__main__": diff --git a/simplyblock_core/snapshot_retention.py b/simplyblock_core/snapshot_retention.py new file mode 100644 index 0000000000..5afe42386f --- /dev/null +++ b/simplyblock_core/snapshot_retention.py @@ -0,0 +1,132 @@ +"""Tiered retention for replication snapshots. + +Replication used to keep a flat count of internal snapshots (the newest +``keep_replicated``), which gives no history: everything older than a couple +of cadence ticks is gone, so a fail-over can only ever land on "a minute +ago". A retention SCHEDULE keeps a thinning history instead -- dense for the +recent past, sparse further back -- so an operator can fail over to a chosen +point in time (yesterday, six hours ago) after a logical corruption that a +one-minute-old copy would have faithfully replicated. + +Schedule syntax (compact, order-independent, parsed left to right): + + "15m:2h,1h:11h,1d:7d" + +reads as "one snapshot every 15 minutes covering the last 2 hours, then one +every hour covering the next 11 hours, then one per day covering the next 7 +days". Each tier is ``:``; both accept an integer with a unit +suffix s/m/h/d. A snapshot older than the sum of all tier spans is not +covered by the schedule and is pruned. + +The selection is a pure function of (snapshot times, schedule, now), which +is what makes it testable without a cluster: see test_snapshot_retention.py. +""" +from __future__ import annotations + +import re +from typing import Iterable, List, NamedTuple, Sequence, Set + +_UNITS = {"s": 1, "m": 60, "h": 3600, "d": 86400} +_TOKEN = re.compile(r"^(\d+)([smhd])$") + + +class RetentionTier(NamedTuple): + """Keep one snapshot per ``every_sec`` bucket, covering ``span_sec``.""" + + every_sec: int + span_sec: int + + +class RetentionScheduleError(ValueError): + """The schedule string could not be parsed.""" + + +def _duration(token: str) -> int: + m = _TOKEN.match(token.strip().lower()) + if not m: + raise RetentionScheduleError( + f"bad duration {token!r}: expected , e.g. 15m or 7d") + value, unit = int(m.group(1)), m.group(2) + if value <= 0: + raise RetentionScheduleError(f"duration must be positive: {token!r}") + return value * _UNITS[unit] + + +def parse_schedule(spec: str) -> List[RetentionTier]: + """Parse ``"15m:2h,1h:11h,1d:7d"`` into tiers. Empty string -> no tiers.""" + if not spec or not spec.strip(): + return [] + tiers: List[RetentionTier] = [] + for chunk in spec.split(","): + chunk = chunk.strip() + if not chunk: + continue + if chunk.count(":") != 1: + raise RetentionScheduleError( + f"bad tier {chunk!r}: expected :, e.g. 15m:2h") + every_s, span_s = chunk.split(":") + every, span = _duration(every_s), _duration(span_s) + if every > span: + raise RetentionScheduleError( + f"tier {chunk!r} keeps one snapshot every {every_s} but only " + f"covers {span_s} -- the interval cannot exceed the span") + tiers.append(RetentionTier(every, span)) + # Coarser tiers must come after finer ones; sorting makes the spec + # order-independent rather than silently producing a nonsense ladder. + tiers.sort(key=lambda t: t.every_sec) + return tiers + + +def horizon_sec(tiers: Sequence[RetentionTier]) -> int: + """Total age covered by the schedule; older snapshots are not retained.""" + return sum(t.span_sec for t in tiers) + + +def select_retained(created_ats: Iterable[float], tiers: Sequence[RetentionTier], + now: float, always_keep_newest: int = 0) -> Set[float]: + """Return the subset of ``created_ats`` the schedule retains. + + One snapshot per bucket per tier: the NEWEST in each bucket, so the + retained point-in-time is as close as possible to the bucket boundary the + operator asked for. Tiers apply to successive age ranges, finest first. + + ``always_keep_newest`` protects the N most recent regardless of the + schedule. Replication needs that: deleting a snapshot swap-merges its + segments into the successor chained to it, so the newest pair must + survive or an arriving delta has nothing to chain onto. + """ + times = sorted({float(t) for t in created_ats if t}, reverse=True) + if not times: + return set() + + keep: Set[float] = set(times[:max(0, always_keep_newest)]) + if not tiers: + return keep + + # Walk age ranges: tier i covers [range_start, range_start + span). + range_start = 0.0 + for tier in tiers: + range_end = range_start + tier.span_sec + # Bucket by absolute age so bucket edges do not drift between calls. + best_per_bucket: dict[int, float] = {} + for t in times: + age = now - t + if age < range_start or age >= range_end: + continue + bucket = int(age // tier.every_sec) + # times is newest-first, so the first hit in a bucket is newest. + best_per_bucket.setdefault(bucket, t) + keep.update(best_per_bucket.values()) + range_start = range_end + return keep + + +def describe(tiers: Sequence[RetentionTier]) -> str: + """Render tiers back to the compact spec (for display / round-trip).""" + def fmt(seconds: int) -> str: + for unit in ("d", "h", "m", "s"): + size = _UNITS[unit] + if seconds % size == 0: + return f"{seconds // size}{unit}" + return f"{seconds}s" + return ",".join(f"{fmt(t.every_sec)}:{fmt(t.span_sec)}" for t in tiers) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 39eccd71f9..3719efb5b9 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -1087,6 +1087,40 @@ def _search_for_partitions(rpc_client, nvme_device): return partitioned_devices +def apply_jc_dual_node(cluster_id): + """Keep every node's JC dual-node flag in step with the cluster size. + + The journal component ABORTS its whole SPDK application when reachable + journal members drop below jc_ha_nmin_jms(), which is 2 normally and 1 + when the dual-node flag is set. A 2-node cluster that loses its peer is + left with 1 of 2 JMs, so without the flag the SURVIVING node aborts + itself the instant its partner stops: one graceful `sn shutdown` takes + the entire cluster down, every client path disappears at once and + filesystems shut down (soak case 6, 2026-08-24 -- "JC detected a network + outage nd=1 njms=2", "JC aborts the node due to network outage", core + dumped). The fork implements the tolerance and exposes jc_set_dual_node; + nothing in the control plane ever called it. + + Keyed on MEMBERSHIP, not on how many nodes are online: a 3-node cluster + with one node down must keep requiring 2 journals. Only a cluster whose + whole membership is 2 is a dual-node cluster. + """ + db_controller = DBController() + members = [n for n in db_controller.get_storage_nodes_by_cluster_id(cluster_id) + if n.status != StorageNode.STATUS_REMOVED] + enable = len(members) == 2 + for node in members: + if node.status != StorageNode.STATUS_ONLINE: + continue + try: + node.rpc_client().jc_set_dual_node(enable) + logger.info("JC dual-node=%s applied on %s (cluster membership %d)", + enable, node.get_id(), len(members)) + except Exception as e: # noqa: BLE001 - best effort + logger.warning("Could not set JC dual-node=%s on %s: %s", + enable, node.get_id(), e) + + def _create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode: StorageNode, after_restart): # RAID 0+1 journal layout (see simplyblock_core/jm_raid.py): # 1 device -> no raid (bare device) @@ -1489,6 +1523,11 @@ def _prepare_cluster_devices_partitions(snode: StorageNode, devices): snode.jm_device = jm_device + # Applied cluster-wide, not just to this node: a cluster growing 2 -> 3 + # must also CLEAR the flag on the two nodes that already have it, and one + # shrinking 3 -> 2 must set it on the survivors. + apply_jc_dual_node(snode.cluster_id) + snode.nvme_devices = new_devices return True @@ -1668,6 +1707,10 @@ def _prepare_cluster_devices_on_restart(snode: StorageNode, clear_data=False): snode.jm_device = jm_device snode.write_to_db() + # A restarted node comes up with the JC default (dual-node off), so the + # flag has to be re-applied on every bring-up, not only at node-add. + apply_jc_dual_node(snode.cluster_id) + return True @@ -11018,14 +11061,18 @@ def add_lvol_thread(lvol, snode: StorageNode, lvol_ana_state="optimized"): return False, msg # Add NS to subsystem (idempotent: skip if already bound with matching NSID). + # Probe and register by the WIRE identity (get_ns_uuid), never the record + # uuid: after a fail-back the two differ, and re-adding under the record + # uuid presents an identity the client's multipath head rejects ("IDs + # don't match for shared namespace N"), severing its paths. if _rpc_subsystem_has_ns(rpc_client, lvol.nqn, nsid=lvol.ns_id, - bdev_name=lvol.top_bdev, uuid=lvol.uuid): + bdev_name=lvol.top_bdev, uuid=lvol.get_ns_uuid()): logger.info("Namespace nsid=%s already on subsystem %s, skipping add_ns", lvol.ns_id, lvol.nqn) else: logger.info("Add BDev to subsystem " + f"{lvol.vuid:016X}") if not rpc_client.nvmf_subsystem_add_ns( - lvol.nqn, lvol.top_bdev, lvol.uuid, lvol.guid, nsid=lvol.ns_id): + lvol.nqn, lvol.top_bdev, lvol.get_ns_uuid(), lvol.guid, nsid=lvol.ns_id): # An add_ns error is not by itself a reason to abandon the whole # registration. What matters for the client is whether the # namespace is on the subsystem now — it may already have been, @@ -11041,7 +11088,8 @@ def add_lvol_thread(lvol, snode: StorageNode, lvol_ana_state="optimized"): # Re-read the subsystem and only give up if the namespace is # genuinely absent. if _rpc_wait_subsystem_has_ns(rpc_client, lvol.nqn, nsid=lvol.ns_id, - bdev_name=lvol.top_bdev, uuid=lvol.uuid): + bdev_name=lvol.top_bdev, + uuid=lvol.get_ns_uuid()): logger.warning( "add_ns for nsid=%s (%s) on %s reported failure but the " "namespace is present; continuing to listener setup", @@ -11071,9 +11119,10 @@ def add_lvol_thread(lvol, snode: StorageNode, lvol_ana_state="optimized"): # path loss the control plane never flagged, re-refused by the lvol-monitor # repair loop on every cycle. if not _rpc_wait_subsystem_has_ns(rpc_client, lvol.nqn, nsid=lvol.ns_id, - bdev_name=lvol.top_bdev, uuid=lvol.uuid): + bdev_name=lvol.top_bdev, + uuid=lvol.get_ns_uuid()): msg = (f"Subsystem {lvol.nqn} on {snode.get_id()} has no namespace " - f"nsid={lvol.ns_id} ({lvol.top_bdev}, uuid={lvol.uuid}) after " + f"nsid={lvol.ns_id} ({lvol.top_bdev}, uuid={lvol.get_ns_uuid()}) after " f"registration; refusing to add a listener for an empty subsystem") logger.error(msg) return False, msg @@ -11186,6 +11235,12 @@ def repair_lvol_registration_on_non_leader(lvol, sec_node: StorageNode, secondar if lvol.status not in (LVol.STATUS_ONLINE, LVol.STATUS_OFFLINE): return False, (f"LVol {lvol.get_id()} status is {lvol.status}, " f"not repairing registration") + if not getattr(lvol, "from_source", True): + # Retired fail-over source: its namespace was removed deliberately + # (_retire_source_data_path). Registering it back republishes the + # superseded data path the fail-over just fenced. + return False, (f"LVol {lvol.get_id()} is the retired source of a " + f"fail-over, not re-registering its namespace") rpc_client = sec_node.rpc_client(timeout=10, retry=2) if rpc_client.subsystem_get(lvol.nqn) is None: diff --git a/simplyblock_core/test/test_case11_retention_ladder.py b/simplyblock_core/test/test_case11_retention_ladder.py new file mode 100644 index 0000000000..6b07849ed7 --- /dev/null +++ b/simplyblock_core/test/test_case11_retention_ladder.py @@ -0,0 +1,89 @@ +"""Case 11's schedule must retain a ladder, not collapse to the keep-floor. + +Soak run 20260827_224741: two volumes on a 1-minute cadence for 124 minutes +with `--retention-schedule 5m:15m,7m:30m,10m:1h` ended with 2 and 3 internal +snapshots, all at CONSECUTIVE cadence ticks (63-64s apart). A working ladder +cannot produce consecutive minutes -- its finest tier keeps one per 5 minutes +-- so either the cadence never ran or retention collapsed to the flat +keep-newest floor. + +This pins the retention half of that question with no cluster: feed the real +schedule the snapshot history the run should have produced and assert the +ladder survives. +""" +import unittest + +from simplyblock_core.snapshot_retention import ( + horizon_sec, parse_schedule, select_retained, +) + +CASE11_SCHEDULE = "5m:15m,7m:30m,10m:1h" +MINUTE = 60 + + +class TestCase11Ladder(unittest.TestCase): + + def setUp(self): + self.tiers = parse_schedule(CASE11_SCHEDULE) + self.now = 1_787_870_000.0 + # 124 minutes of a 1-minute cadence, as the run actually took them + self.history = [self.now - m * MINUTE for m in range(1, 125)] + + def test_schedule_parses_to_the_expected_ladder(self): + self.assertEqual([(t.every_sec, t.span_sec) for t in self.tiers], + [(300, 900), (420, 1800), (600, 3600)]) + # 15m + 30m + 1h of coverage + self.assertEqual(horizon_sec(self.tiers), 900 + 1800 + 3600) + + def test_the_ladder_retains_a_thinning_history_not_two_snapshots(self): + keep = select_retained(self.history, self.tiers, self.now, + always_keep_newest=2) + # 15m/5m = 3, then 30m/7m ~= 4-5, then 60m/10m = 6 + self.assertGreaterEqual( + len(keep), 12, + "the ladder should retain roughly a dozen points across 105 " + "minutes of horizon; collapsing to the keep-floor is the bug this " + "test exists for (got %d)" % len(keep)) + self.assertLessEqual(len(keep), 16, "and it must not keep everything") + + def test_retained_points_are_spread_not_consecutive(self): + """The observed failure looked like consecutive cadence ticks.""" + keep = sorted(select_retained(self.history, self.tiers, self.now, + always_keep_newest=2), reverse=True) + # Beyond the protected newest pair, retained points must be spread by + # roughly a tier interval. Gaps at a TIER BOUNDARY are legitimately + # smaller (the last bucket of one tier can sit close to the first of + # the next), so assert on the typical gap, not the minimum. + gaps = sorted(keep[i] - keep[i + 1] for i in range(2, len(keep) - 1)) + self.assertTrue(gaps, "expected several retained points") + median = gaps[len(gaps) // 2] + self.assertGreaterEqual( + median, 300, + "retained snapshots a cadence tick apart mean the ladder is not " + "being applied; the flat keep-N path produces exactly that " + "(gaps: %s)" % [int(g) for g in gaps]) + + def test_snapshots_older_than_the_horizon_are_dropped(self): + older = [self.now - 200 * MINUTE, self.now - 300 * MINUTE] + keep = select_retained(self.history + older, self.tiers, self.now, + always_keep_newest=2) + for t in older: + self.assertNotIn(t, keep, + "past the 105-minute horizon nothing is retained") + + def test_an_absent_schedule_falls_back_to_the_keep_floor(self): + """The suspected production path: no tiers -> only the newest N. + + This is what the soak looked like, and it is CORRECT behaviour for an + empty schedule -- which is why an empty schedule reaching retention + silently destroys the history an operator asked for. + """ + keep = select_retained(self.history, [], self.now, always_keep_newest=2) + self.assertEqual(len(keep), 2) + self.assertEqual(sorted(keep, reverse=True), self.history[:2], + "the two newest, i.e. consecutive cadence ticks -- " + "exactly the shape the soak produced") + + +if __name__ == "__main__": + unittest.main() diff --git a/simplyblock_core/test/test_connect_path_resolution.py b/simplyblock_core/test/test_connect_path_resolution.py index fff74c5dd7..f16912ff58 100644 --- a/simplyblock_core/test/test_connect_path_resolution.py +++ b/simplyblock_core/test/test_connect_path_resolution.py @@ -120,3 +120,81 @@ def test_resolution_never_consults_cluster_status(): db = _FakeDB([src, tgt], [_rep(src, tgt, LVolReplication.STATE_FAILED_OVER)]) assert not hasattr(db, "get_cluster_by_id") assert _ids(lvol_controller._connect_path_volumes(db, src)) == ["LV_TGT"] + + +# ---- device-lookup identity (entry.target_lvol_id) ------------------------ # +# +# A clone's wire NSUUID may be borrowed from another volume (the kernel merges +# multipath paths only on matching NSUUIDs), so the client's /dev/disk/by-id +# links can carry an id that is not the record's own. The record's ns_uuid +# persists that wire identity; connect reports it whenever it differs from the +# requested id. Records predating ns_uuid fall back to the relationship's +# other end (the only fail-back shape that existed then inherited the DR +# source's NSUUID). + + +class _FakeEntry: + target_lvol_id = None + + +def _connect(monkeypatch, db, requested_id): + """Run connect_lvol against the fake DB with the entry builder stubbed.""" + monkeypatch.setattr(lvol_controller, "DBController", lambda: db) + monkeypatch.setattr(lvol_controller.HostConnectAuth, "resolve", + classmethod(lambda cls, lvol, host_nqn, db_controller: None)) + monkeypatch.setattr(lvol_controller, "_connect_entries_for_volume", + lambda *a, **kw: [_FakeEntry()]) + entries, err = lvol_controller.connect_lvol(requested_id) + assert err is None + return entries + + +def test_legacy_failback_record_falls_back_to_the_relationship(monkeypatch): + """cutover_done with the requested volume on the TARGET end and no ns_uuid + persisted (records from before the field existed): those clones inherited + the DR SOURCE's NSUUID, so connect must hand that id out.""" + dr, back = _lvol("LV_DR"), _lvol("LV_BACK") + db = _FakeDB([dr, back], [_rep(dr, back, LVolReplication.STATE_CUTOVER_DONE)]) + entries = _connect(monkeypatch, db, "LV_BACK") + assert [e.target_lvol_id for e in entries] == ["LV_DR"] + + +def test_wire_identity_equal_to_own_id_needs_no_override(monkeypatch): + """A volume whose persisted wire identity equals its own id needs no + redirect: the CSI's own-id lookup already matches the device, and any + emission would point it elsewhere.""" + dr, back = _lvol("LV_DR"), _lvol("LV_BACK") + back.ns_uuid = "LV_BACK" + db = _FakeDB([dr, back], [_rep(dr, back, LVolReplication.STATE_CUTOVER_DONE)]) + entries = _connect(monkeypatch, db, "LV_BACK") + assert [e.target_lvol_id for e in entries] == [None] + + +def test_failback_reports_the_persisted_wire_identity(monkeypatch): + """After a fail-back the UUID swap gives the record its original id while + the namespace keeps advertising the DR source's wire identity (that is + what kept the client's multipath head alive through the cutover). Connect + must report the persisted ns_uuid — no relationship walk needed.""" + back = _lvol("LV_BACK") + back.ns_uuid = "LV_DR" + db = _FakeDB([back]) + entries = _connect(monkeypatch, db, "LV_BACK") + assert [e.target_lvol_id for e in entries] == ["LV_DR"] + + +def test_forward_migration_keeps_the_target_id_for_device_lookup(monkeypatch): + """Requested volume on the SOURCE end: the redirect loop already reports the + target id; the fail-back emission must not overwrite it.""" + src, tgt = _lvol("LV_SRC"), _lvol("LV_TGT") + db = _FakeDB([src, tgt], [_rep(src, tgt, LVolReplication.STATE_CUTOVER_DONE)]) + entries = _connect(monkeypatch, db, "LV_SRC") + assert [e.target_lvol_id for e in entries] == ["LV_TGT"] + + +def test_failed_over_target_end_reports_no_device_lookup_id(monkeypatch): + """A fail-over clone's NSUUID is its own uuid — connecting it by its own id + needs no redirect, so nothing must be emitted.""" + src, tgt = _lvol("LV_SRC"), _lvol("LV_TGT") + db = _FakeDB([src, tgt], [_rep(src, tgt, LVolReplication.STATE_FAILED_OVER)]) + entries = _connect(monkeypatch, db, "LV_TGT") + assert [e.target_lvol_id for e in entries] == [None] diff --git a/simplyblock_core/test/test_consistency_groups.py b/simplyblock_core/test/test_consistency_groups.py new file mode 100644 index 0000000000..3a9ee7c9d6 --- /dev/null +++ b/simplyblock_core/test/test_consistency_groups.py @@ -0,0 +1,216 @@ +"""Consistency groups: membership epochs, placement pinning, generation +warnings, and the wiring contracts of the group snapshot flow.""" +import inspect + +import pytest + +from simplyblock_core.controllers import consistency_group_controller as cgc +from simplyblock_core.models.replication import ConsistencyGroup + + +def _group(members=None, last_seq=0, lvs="LVS_1", node="NODE_A"): + g = ConsistencyGroup() + g.uuid = "g1" + g.cluster_id = "CL" + g.policy_id = "CL/p1" + g.lvs_name = lvs + g.node_id = node + g.last_group_seq = last_seq + g.members = members or {} + return g + + +# --------------------------------------------------------------------------- # +# Epoch semantics (requirement 4) +# --------------------------------------------------------------------------- # + +def test_member_included_between_join_and_removal(): + g = _group({"v1": {"joined_seq": 2, "removed_seq": 5}}) + assert not g.included_in_seq("v1", 1) + assert g.included_in_seq("v1", 2) + assert g.included_in_seq("v1", 5) + assert not g.included_in_seq("v1", 6) + + +def test_open_epoch_member_included_from_join_onwards(): + g = _group({"v1": {"joined_seq": 3, "removed_seq": 0}}) + assert not g.included_in_seq("v1", 2) + assert g.included_in_seq("v1", 3) + assert g.included_in_seq("v1", 99) + + +def test_late_joiner_is_warned_about_for_older_generations(): + """A volume attached after generation 4 is NOT in generations 1..4: + failing over to one of those must say so.""" + g = _group({ + "old": {"joined_seq": 1, "removed_seq": 0}, + "late": {"joined_seq": 5, "removed_seq": 0}, + }, last_seq=6) + warnings = cgc.generation_membership_warnings(g, 4) + assert len(warnings) == 1 + assert "late" in warnings[0] + assert "NOT included" in warnings[0] + assert "old" not in warnings[0].split(":")[-1] + + +def test_departed_member_is_warned_about_when_generation_contains_it(): + g = _group({ + "stay": {"joined_seq": 1, "removed_seq": 0}, + "gone": {"joined_seq": 1, "removed_seq": 3}, + }, last_seq=6) + warnings = cgc.generation_membership_warnings(g, 2) + assert len(warnings) == 1 + assert "gone" in warnings[0] + assert "no longer" in warnings[0] + + +def test_matching_generation_produces_no_warnings(): + g = _group({ + "a": {"joined_seq": 1, "removed_seq": 0}, + "b": {"joined_seq": 1, "removed_seq": 0}, + }, last_seq=3) + assert cgc.generation_membership_warnings(g, 3) == [] + + +def test_both_warning_kinds_can_coexist(): + g = _group({ + "late": {"joined_seq": 5, "removed_seq": 0}, + "gone": {"joined_seq": 1, "removed_seq": 3}, + }, last_seq=6) + warnings = cgc.generation_membership_warnings(g, 2) + assert len(warnings) == 2 + + +def test_no_group_or_no_seq_is_silent(): + assert cgc.generation_membership_warnings(None, 3) == [] + assert cgc.generation_membership_warnings(_group(), 0) == [] + + +# --------------------------------------------------------------------------- # +# Placement / lifecycle contracts (requirement 1 + 2) +# --------------------------------------------------------------------------- # + +class _FakeDB: + def __init__(self, group): + self._group = group + + def get_consistency_group_for_policy(self, policy_id): + return self._group + + @property + def kv_store(self): + return None + + +class _Policy: + policy_name = "p1" + consistency_group = True + + def get_id(self): + return "CL/p1" + + +class _Lvol: + def __init__(self, lvol_id, node, lvs): + self._id, self.node_id, self.lvs_name = lvol_id, node, lvs + + def get_id(self): + return self._id + + +def test_attach_to_pinned_group_fails_on_wrong_lvs(monkeypatch): + g = _group(lvs="LVS_1", node="NODE_A") + monkeypatch.setattr(cgc, "db", _FakeDB(g)) + with pytest.raises(cgc.ConsistencyGroupError): + cgc.add_member(_Policy(), _Lvol("v1", "NODE_B", "LVS_2")) + assert "v1" not in (g.members or {}) + + +def test_first_member_pins_the_group(monkeypatch): + g = _group(lvs="", node="") + g.write_to_db = lambda kv=None: None + monkeypatch.setattr(cgc, "db", _FakeDB(g)) + cgc.add_member(_Policy(), _Lvol("v1", "NODE_A", "LVS_1")) + assert g.node_id == "NODE_A" and g.lvs_name == "LVS_1" + assert g.members["v1"]["joined_seq"] == 1 + + +def test_late_joiner_epoch_starts_at_next_generation(monkeypatch): + """Requirement 4: membership becomes active only with the FIRST group + snapshot taken after the attach.""" + g = _group({"v1": {"joined_seq": 1, "removed_seq": 0}}, last_seq=7) + g.write_to_db = lambda kv=None: None + monkeypatch.setattr(cgc, "db", _FakeDB(g)) + cgc.add_member(_Policy(), _Lvol("v2", "NODE_A", "LVS_1")) + assert g.members["v2"]["joined_seq"] == 8 + # ... and generation 7 correctly warns about it + warnings = cgc.generation_membership_warnings(g, 7) + assert warnings and "v2" in warnings[0] + + +def test_detach_closes_the_epoch_at_current_generation(monkeypatch): + g = _group({"v1": {"joined_seq": 1, "removed_seq": 0}}, last_seq=4) + g.write_to_db = lambda kv=None: None + monkeypatch.setattr(cgc, "db", _FakeDB(g)) + cgc.remove_member("CL/p1", "v1") + assert g.members["v1"]["removed_seq"] == 4 + assert g.included_in_seq("v1", 4) + assert not g.included_in_seq("v1", 5) + + +# --------------------------------------------------------------------------- # +# Wiring contracts (source-inspection, matching the repo's test idiom) +# --------------------------------------------------------------------------- # + +def test_policy_lifecycle_auto_creates_and_deletes_the_group(): + from simplyblock_core.controllers import replication_policy_controller as rpc + src_add = inspect.getsource(rpc.add_policy) + assert "create_group_for_policy" in src_add + src_rm = inspect.getsource(rpc.remove_policy) + assert "delete_group_for_policy" in src_rm + + +def test_attach_checks_the_group_before_any_state_is_written(): + from simplyblock_core.controllers import replication_policy_controller as rpc + src = inspect.getsource(rpc.attach_policy) + assert "add_member" in src + assert src.index("add_member") < src.index("lvol.replication_policy_id = pol.get_id()") + + +def test_create_path_pins_cg_volumes_to_the_group_node(): + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc.add_lvol_ha) + assert "pinned_node_for_policy" in src + pin = src.index("pinned_node_for_policy") + place = src.index("host_node = None") + assert pin < place, "the pin must be resolved before placement" + + +def test_cadence_snapshots_cg_policies_as_a_group(): + from simplyblock_core.services import snapshot_monitor as sm + src = inspect.getsource(sm.take_due_internal_snapshots) + assert "create_group_snapshot" in src + assert "grouped_ids" in src, "group members must leave the per-volume loop" + + +def test_group_snapshot_is_one_rpc_and_bumps_seq_only_on_full_success(): + src = inspect.getsource(cgc.create_group_snapshot) + assert "bdev_lvol_snapshot_group" in src + rpc_at = src.index("bdev_lvol_snapshot_group") + seq_at = src.index("group.last_group_seq = group_seq") + assert rpc_at < seq_at, "the generation counter moves only after the whole tick" + assert "_rollback_all" in src + + +def test_failover_result_carries_membership_warnings(): + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc.replicate_lvol_on_target_cluster) + assert "warnings_for_snapshot" in src + assert '"warnings": warnings' in src + + +def test_target_snapshot_copy_inherits_group_provenance(): + from simplyblock_core.services import snapshot_replication as sr + src = inspect.getsource(sr) + assert 'new_snapshot.group_id = getattr(snapshot, "group_id", "")' in src + assert 'new_snapshot.group_seq = getattr(snapshot, "group_seq", 0)' in src diff --git a/simplyblock_core/test/test_cutover_convergence.py b/simplyblock_core/test/test_cutover_convergence.py new file mode 100644 index 0000000000..6f0caa55e7 --- /dev/null +++ b/simplyblock_core/test/test_cutover_convergence.py @@ -0,0 +1,697 @@ +"""The cutover must converge the delta before it freezes client IO. + +Soak run 20260827_110415, case 10 (online migration under heavy IO) measured +the client-observed freeze at avg 40.4s, max 71.9s, with fio logging 8 errors. +The freeze is bdev_lvol_transfer_final_step, which copies everything written +since the cutover clone's base snapshot -- so the freeze lasts as long as the +write window that precedes it. + +The intended sequence is: snapshot -> transfer -> IMMEDIATELY (milliseconds) +the next snapshot -> repeat until a round transfers in low seconds -> +IMMEDIATELY the final lvol transfer. Three things broke it: + + * SHRINK_ROUNDS was a fixed 2 -- a count, not a convergence criterion, so + under load it simply stopped while the delta was still large; + * each round returned to the task scheduler, costing TASK_EXEC_INTERVAL_SEC + (10s) of fresh writes per round -- a floor no number of rounds can beat; + * the operator preconnect gate sat BETWEEN the base snapshot and the freeze, + and with no operator its 120s fallback fired 34 times in that run. +""" +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import constants +from simplyblock_core.services import tasks_runner_replication_final as runner + + +class _Task: + def __init__(self, **params): + self.function_params = dict(params) + self.function_result = "" + self.status = "" + self.retry = 0 + self.max_retry = 0 + self.canceled = False + self.cluster_id = "CL" + + def write_to_db(self, *a, **kw): + pass + + +def _lvol(uuid="LV1", lvs="LVS_1"): + lv = MagicMock() + lv.get_id.return_value = uuid + lv.uuid = uuid + lv.lvs_name = lvs + return lv + + +class _Clock: + """A clock that only moves when the code under test waits. + + The convergence loop measures a round as (now - shrink_started_at), so a + fake that returns the same instant for both makes every round look + instantaneous and the test proves nothing. + """ + + def __init__(self, now=1000.0): + self.now = now + + def __call__(self): + return self.now + + def sleep(self, seconds): + self.now += seconds + + +class TestConvergence(unittest.TestCase): + """_shrink_step loops until a round is fast, without leaving the pass.""" + + def _run(self, round_times, max_rounds=None, exclusive=True): + """Drive _shrink_step with round i taking round_times[i] seconds. + + exclusive=True means the task already holds its lvstore, which is the + endgame. Unexclusive rounds converge in the open and then ASK for the + lvstore instead of handing straight over to the freeze. + """ + clock = _Clock() + task = _Task(shrink_snap_id="S0", shrink_round=1, + shrink_deadline=10 ** 9, lvol_id="LV1") + if exclusive: + task.function_params["cutover_lvs"] = "LVS_1" + task.function_params["shrink_started_at"] = clock.now + state = {"i": 0} + taken = [] + + def _done(snap_id): + # replicated once this round's transfer time has actually passed + started = task.function_params["shrink_started_at"] + idx = min(state["i"], len(round_times) - 1) + return (clock.now - started) >= round_times[idx] + + def _take(task_, lvol_): + state["i"] += 1 + taken.append(state["i"]) + task_.function_params["shrink_round"] += 1 + task_.function_params["shrink_snap_id"] = "S%d" % state["i"] + task_.function_params["shrink_started_at"] = clock.now + return "S%d" % state["i"], None + + patches: list = [ + patch.object(runner.time, "time", clock), + patch.object(runner.time, "sleep", clock.sleep), + patch.object(runner, "_shrink_round_done", side_effect=_done), + patch.object(runner, "_take_shrink_snapshot", side_effect=_take), + ] + if max_rounds is not None: + patches.append( + patch.object(constants, "REPL_CUTOVER_MAX_SHRINK_ROUNDS", max_rounds)) + for p in patches: + p.start() + self.addCleanup(p.stop) + + done, err = runner._shrink_step(task, _lvol()) + return done, err, task, taken, clock + + def test_a_fast_round_while_holding_the_lvstore_hands_over(self): + """Converged AND exclusive -> freeze immediately.""" + done, err, task, taken, _ = self._run([0.5]) + self.assertTrue(done) + self.assertIsNone(err) + self.assertIn("converged", task.function_result) + self.assertEqual(taken, [], "a fast first round needs no further rounds") + + def test_no_round_ever_runs_outside_the_lvstore_claim(self): + """Rounds are the endgame, and the endgame is exclusive by definition. + + There used to be "open" rounds converging before the claim, so that the + bulk catch-up was not serialised. That was the wrong instrument: the + volume is caught up by its ORDINARY replication cadence, which costs the + cutover nothing, and only then does it ask for the lvstore. The claim is + therefore taken before round 1, and the loop below need not consider a + round without it. + """ + import inspect + src = inspect.getsource(runner.task_runner) + entry = src.index("_acquire_lvs_claim") + self.assertLess(entry, src.index("_take_shrink_snapshot"), + "the lvstore is taken before the first round, not after") + self.assertNotIn("ready_for_exclusive", inspect.getsource(runner), + "open rounds are gone") + + def test_a_slow_round_takes_another_snapshot_without_leaving_the_pass(self): + """The whole point: rounds follow each other in milliseconds.""" + done, err, task, taken, clock = self._run([3.0, 3.0, 0.4]) + self.assertTrue(done) + self.assertIsNone(err) + self.assertEqual(taken, [1, 2], + "each slow round must be followed immediately by the next") + self.assertIn("converged", task.function_result) + # ~6.4s of transfers, not 6.4 + 3 x TASK_EXEC_INTERVAL_SEC of + # rescheduling: the delta a round carries is the previous round's + # transfer, nothing more. + self.assertLess(clock.now - 1000.0, 7.0) + + def test_it_gives_up_after_the_round_cap_and_freezes_anyway(self): + """Written faster than it replicates: freeze rather than loop forever.""" + done, err, task, taken, _ = self._run([3.0] * 20, max_rounds=3) + self.assertTrue(done, "the cap must hand over, not fail the cutover") + self.assertIsNone(err) + self.assertIn("not converged", task.function_result) + + def test_the_cap_freezes_under_the_claim_it_already_holds(self): + """Giving up converging goes straight to the freeze. + + The claim was taken on entry to the endgame, so reaching the round cap + needs no further acquisition -- it just stops converging and freezes. + """ + done, err, task, taken, _ = self._run([3.0] * 20, max_rounds=3) + self.assertTrue(done) + self.assertIsNone(err) + self.assertIn("not converged", task.function_result) + + def test_a_vanished_snapshot_is_an_error(self): + # This one runs on the real clock, so the deadline has to be a real + # future epoch -- 10**9 is 2001 and would trip the timeout instead. + task = _Task(shrink_snap_id="S0", shrink_round=1, + shrink_deadline=10 ** 12, lvol_id="LV1") + with patch.object(runner, "_shrink_round_done", return_value=None): + done, err = runner._shrink_step(task, _lvol()) + self.assertFalse(done) + self.assertIn("disappeared", err) + + def test_the_deadline_still_bounds_the_phase(self): + """The deadline bounds the phase by handing over to the freeze, not by + failing: proceeding with a slightly larger residual always beats + burning a retry on another 900-second shrink window.""" + task = _Task(shrink_snap_id="S0", shrink_round=1, + shrink_deadline=0, lvol_id="LV1") + with patch.object(runner, "_shrink_round_done", return_value=False): + done, err = runner._shrink_step(task, _lvol()) + self.assertTrue(done) + self.assertIsNone(err) + + def test_it_yields_the_pass_when_the_budget_runs_out(self): + """A very slow transfer must not hog the runner forever.""" + clock = _Clock() + task = _Task(shrink_snap_id="S0", shrink_round=1, + shrink_deadline=10 ** 9, lvol_id="LV1") + task.function_params["shrink_started_at"] = clock.now + with patch.object(runner.time, "time", clock), \ + patch.object(runner.time, "sleep", clock.sleep), \ + patch.object(runner, "_shrink_round_done", return_value=False), \ + patch.object(constants, "REPL_CUTOVER_CONVERGE_BUDGET_SEC", 5): + done, err = runner._shrink_step(task, _lvol()) + self.assertFalse(done) + self.assertIsNone(err, "yielding the pass is not a failure") + self.assertIn("waiting", task.function_result) + + +class TestProceedGate(unittest.TestCase): + """The preconnect wait must be opt-in: it costs freeze time.""" + + def test_enabled_now_that_the_operator_signals(self): + """Flipped 2026-09-02: the operator's reconcileCutoverPending posts + cutover-proceed for migration AND failback (annotFailbackTarget routes + the call to the target cluster). Without the gate the ANA flip races + the client's preconnect: the 2026-09-02 failback run flipped listeners + no client had connected to and deleted the DR-side subsystem 150ms + later, orphaning every connected client for ctrl_loss_tmo.""" + self.assertTrue( + constants.REPL_CUTOVER_PROCEED_REQUIRED, + "cutover must wait for the operator's preconnect signal; " + "flipping ANA on listeners no client is connected to and then " + "deleting the source subsystem strands every live client") + + def test_the_wait_is_guarded_by_the_flag(self): + import inspect + src = inspect.getsource(runner.task_runner) + self.assertIn("constants.REPL_CUTOVER_PROCEED_REQUIRED", src) + # Compare against the actual freeze CALL SITE, not the first "run_cutover" + # occurrence — the string also appears earlier in a comment about the + # retry path, which is not the freeze. + self.assertLess( + src.index("REPL_CUTOVER_PROCEED_REQUIRED"), + src.index("replication_final_step.run_cutover"), + "the gate must be evaluated before the freeze") + + +if __name__ == "__main__": + unittest.main() + + +class TestLvsAdmission(unittest.TestCase): + """One lvstore's bandwidth, two priorities. + + Case 10 ran 9 volumes over shared lvstores and every one kept replicating + while another was trying to converge, stretching each round -- and every + second a round is stretched is a second of writes that lands in the frozen + final step. + """ + + def setUp(self): + from simplyblock_core.services import snapshot_replication as sr + self.sr = sr + patcher = patch.object(sr, "db") + self.db = patcher.start() + self.addCleanup(patcher.stop) + self.groups: dict = {} # lvol id -> group id + gp = patch.object(sr, "_group_id_for_lvol", + side_effect=lambda lv: self.groups.get(lv.get_id(), "")) + gp.start() + self.addCleanup(gp.stop) + + # -- fixtures --------------------------------------------------------- + def _cutover_task(self, lvol_id, lvs, group="", status="running", + canceled=False): + from simplyblock_core.models.job_schedule import JobSchedule + t = MagicMock() + t.function_name = JobSchedule.FN_REPLICATION_FINAL + t.status = status + t.canceled = canceled + t.function_params = {"lvol_id": lvol_id, "cutover_lvs": lvs, + "cutover_group": group} + return t + + def _transfer_task(self, snap_id, task_id="T_other", status=None): + from simplyblock_core.models.job_schedule import JobSchedule + t = MagicMock() + t.function_name = JobSchedule.FN_SNAPSHOT_REPLICATION + t.status = status or JobSchedule.STATUS_RUNNING + t.canceled = False + t.get_id.return_value = task_id + t.function_params = {"snapshot_id": snap_id} + return t + + def _lv(self, lvol_id, lvs="LVS_1"): + lv = MagicMock() + lv.get_id.return_value = lvol_id + lv.lvs_name = lvs + return lv + + def _snapshot(self, lvol_id="LV_other", lvs="LVS_1"): + snap = MagicMock() + snap.lvol = self._lv(lvol_id, lvs) + return snap + + def _task(self, task_id="T_me"): + t = MagicMock() + t.cluster_id = "CL" + t.get_id.return_value = task_id + return t + + # -- priority 1: a cutover owns its lvstore --------------------------- + def test_another_volumes_cutover_holds_this_lvstore(self): + self.db.get_job_tasks.return_value = [ + self._cutover_task("LV_cutting", "LVS_1")] + self.assertIn("final cutover", + self.sr._lvs_transfer_hold(self._task(), self._snapshot())) + + def test_the_volume_in_cutover_may_still_replicate(self): + """Its convergence snapshots are exactly what must keep moving.""" + self.db.get_job_tasks.return_value = [ + self._cutover_task("LV_cutting", "LVS_1")] + self.assertEqual( + self.sr._lvs_transfer_hold( + self._task(), self._snapshot(lvol_id="LV_cutting")), "") + + def test_a_group_member_is_not_held_by_its_groups_cutover(self): + """A consistency group cuts over as a group, not one member at a time.""" + self.groups = {"LV_cutting": "CL/G1", "LV_sibling": "CL/G1"} + self.db.get_job_tasks.return_value = [ + self._cutover_task("LV_cutting", "LVS_1", group="CL/G1")] + self.assertEqual( + self.sr._lvs_transfer_hold( + self._task(), self._snapshot(lvol_id="LV_sibling")), "") + + def test_a_volume_outside_the_group_is_still_held_by_its_cutover(self): + self.groups = {"LV_cutting": "CL/G1", "LV_loose": ""} + self.db.get_job_tasks.return_value = [ + self._cutover_task("LV_cutting", "LVS_1", group="CL/G1")] + self.assertIn("final cutover", self.sr._lvs_transfer_hold( + self._task(), self._snapshot(lvol_id="LV_loose"))) + + def test_a_cutover_on_a_different_lvstore_does_not_hold_us(self): + self.db.get_job_tasks.return_value = [ + self._cutover_task("LV_cutting", "LVS_9")] + self.assertEqual( + self.sr._lvs_transfer_hold(self._task(), self._snapshot()), "") + + def test_a_finished_or_cancelled_cutover_holds_nothing(self): + from simplyblock_core.models.job_schedule import JobSchedule + for task in (self._cutover_task("LV_x", "LVS_1", + status=JobSchedule.STATUS_DONE), + self._cutover_task("LV_x", "LVS_1", canceled=True)): + self.db.get_job_tasks.return_value = [task] + self.assertEqual( + self.sr._lvs_transfer_hold(self._task(), self._snapshot()), "") + + def test_a_task_that_has_not_claimed_an_lvs_holds_nothing(self): + """Before the cutover starts its rounds there is nothing to protect.""" + t = self._cutover_task("LV_cutting", "LVS_1") + del t.function_params["cutover_lvs"] + self.db.get_job_tasks.return_value = [t] + self.assertEqual( + self.sr._lvs_transfer_hold(self._task(), self._snapshot()), "") + + # -- priority 2: groups outrank loose volumes, and serialize ----------- + def test_a_transferring_group_holds_a_volume_from_another_group(self): + self.groups = {"LV_a": "CL/G1", "LV_b": "CL/G2"} + self.db.get_job_tasks.return_value = [self._transfer_task("S_a")] + self.db.get_snapshot_by_id.return_value = MagicMock(lvol=self._lv("LV_a")) + self.assertIn("consistency group", self.sr._lvs_transfer_hold( + self._task(), self._snapshot(lvol_id="LV_b"))) + + def test_members_of_the_same_group_transfer_in_parallel(self): + self.groups = {"LV_a": "CL/G1", "LV_b": "CL/G1"} + self.db.get_job_tasks.return_value = [self._transfer_task("S_a")] + self.db.get_snapshot_by_id.return_value = MagicMock(lvol=self._lv("LV_a")) + self.assertEqual( + self.sr._lvs_transfer_hold(self._task(), self._snapshot(lvol_id="LV_b")), + "") + + def test_a_transferring_group_outranks_a_loose_volume(self): + self.groups = {"LV_a": "CL/G1", "LV_loose": ""} + self.db.get_job_tasks.return_value = [self._transfer_task("S_a")] + self.db.get_snapshot_by_id.return_value = MagicMock(lvol=self._lv("LV_a")) + self.assertIn("consistency group", self.sr._lvs_transfer_hold( + self._task(), self._snapshot(lvol_id="LV_loose"))) + + def test_loose_volumes_still_transfer_in_parallel_with_each_other(self): + """No group involved: unchanged behaviour, no new serialization.""" + self.groups = {"LV_a": "", "LV_b": ""} + self.db.get_job_tasks.return_value = [self._transfer_task("S_a")] + self.db.get_snapshot_by_id.return_value = MagicMock(lvol=self._lv("LV_a")) + self.assertEqual( + self.sr._lvs_transfer_hold(self._task(), self._snapshot(lvol_id="LV_b")), + "") + + def test_a_group_transferring_on_another_lvstore_does_not_hold_us(self): + self.groups = {"LV_a": "CL/G1", "LV_b": "CL/G2"} + self.db.get_job_tasks.return_value = [self._transfer_task("S_a")] + self.db.get_snapshot_by_id.return_value = MagicMock( + lvol=self._lv("LV_a", lvs="LVS_9")) + self.assertEqual( + self.sr._lvs_transfer_hold(self._task(), self._snapshot(lvol_id="LV_b")), + "") + + +class TestRoundOneIsMeasured(unittest.TestCase): + """The regression that let the freeze survive the convergence loop. + + Round 1 used to be created by replicate/commit without the stamp the loop + measures against, so it measured as 0.00s, counted as converged, and the + freeze copied the whole delta (run 20260827_172734, 9-55s server-side). + + Every round is now born in one place -- _take_shrink_snapshot, in the + endgame -- so there is a single stamp to get right. + """ + + def test_every_round_is_stamped_where_it_is_taken(self): + import inspect + src = inspect.getsource(runner._take_shrink_snapshot) + self.assertIn('params["shrink_started_at"] = time.time()', src) + self.assertIn('params["shrink_round"] = params.get("shrink_round", 0) + 1', + src, "the round number and its stamp must move together") + + def test_the_controller_enqueues_no_round_of_its_own(self): + """Commit takes no snapshot, so it must not claim a round in flight.""" + import inspect + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc.replication_commit) + self.assertIn('"shrink_round": 0', src) + self.assertNotIn('"shrink_snap_id"', src) + + def test_an_unmeasured_round_is_not_treated_as_converged(self): + """Belt and braces for tasks enqueued without the stamp.""" + clock = _Clock() + task = _Task(shrink_snap_id="S0", shrink_round=1, + shrink_deadline=10 ** 9, lvol_id="LV1") + # deliberately NO shrink_started_at + taken = [] + + def _take(task_, lvol_): + taken.append(task_.function_params["shrink_round"]) + task_.function_params["shrink_round"] += 1 + task_.function_params["shrink_snap_id"] = "S1" + task_.function_params["shrink_started_at"] = clock.now + return "S1", None + + with patch.object(runner.time, "time", clock), \ + patch.object(runner.time, "sleep", clock.sleep), \ + patch.object(runner, "_shrink_round_done", return_value=True), \ + patch.object(runner, "_take_shrink_snapshot", side_effect=_take), \ + patch.object(constants, "REPL_CUTOVER_MIN_INLINE_SEC", 0), \ + patch.object(constants, "REPL_CUTOVER_CONVERGE_BUDGET_SEC", 0): + done, err = runner._shrink_step(task, _lvol()) + + self.assertFalse(done, "an unmeasured round must not end the shrink phase") + self.assertIsNone(err) + self.assertEqual(taken, [1], + "it must take another round instead of freezing") + + +class TestCutoverQueue(unittest.TestCase): + """One cutover per lvstore, and the losers queue instead of starving. + + Run 20260827_185009: all 20 volumes entered their cutover together and each + wrote cutover_lvs, so the claim was a marker with no exclusion. The + replication side then held everyone but an arbitrary winner -- including + the other cutovers' own shrink snapshots -- and 9 of 10 volumes per + lvstore sat at "round 1: waiting to replicate" until their deadline killed + them (17 x "max retry reached"). + """ + + def setUp(self): + patcher = patch.object(runner, "db") + self.db = patcher.start() + self.addCleanup(patcher.stop) + self.db.kv_store = "KV" + gp = patch.object(runner, "_group_id_for_lvol", return_value="") + self.group_of = gp.start() + self.addCleanup(gp.stop) + + def _task(self, task_id, lvs=None, group="", status="running", + canceled=False, created="2026-01-01"): + from simplyblock_core.models.job_schedule import JobSchedule + t = MagicMock() + t.function_name = JobSchedule.FN_REPLICATION_FINAL + t.get_id.return_value = task_id + t.status = status + t.canceled = canceled + t.create_dt = created + t.function_params = {"lvol_id": "LV_" + task_id} + if lvs: + t.function_params["cutover_lvs"] = lvs + t.function_params["cutover_group"] = group + return t + + def test_no_owner_means_the_lvstore_is_free(self): + me = self._task("T1") + self.db.get_job_tasks.return_value = [me] + self.assertIsNone(runner._lvs_cutover_owner(me, "LVS_1")) + + def test_an_active_claim_owns_the_lvstore(self): + me, other = self._task("T1"), self._task("T2", lvs="LVS_1") + self.db.get_job_tasks.return_value = [me, other] + owner = runner._lvs_cutover_owner(me, "LVS_1") + self.assertIsNotNone(owner) + self.assertEqual(owner.get_id(), "T2") + + def test_a_finished_or_cancelled_cutover_owns_nothing(self): + from simplyblock_core.models.job_schedule import JobSchedule + me = self._task("T1") + for dead in (self._task("T2", lvs="LVS_1", + status=JobSchedule.STATUS_DONE), + self._task("T3", lvs="LVS_1", canceled=True)): + self.db.get_job_tasks.return_value = [me, dead] + self.assertIsNone(runner._lvs_cutover_owner(me, "LVS_1"), + "a dead task must not hold the lvstore forever") + + def test_the_earliest_claim_wins_deterministically(self): + """Two tasks racing must agree on the winner, not each see the other.""" + me = self._task("T1") + early = self._task("T2", lvs="LVS_1", created="2026-01-01") + late = self._task("T3", lvs="LVS_1", created="2026-06-01") + self.db.get_job_tasks.return_value = [me, late, early] + self.assertEqual(runner._lvs_cutover_owner(me, "LVS_1").get_id(), "T2") + + def test_a_claim_on_another_lvstore_is_irrelevant(self): + me, other = self._task("T1"), self._task("T2", lvs="LVS_9") + self.db.get_job_tasks.return_value = [me, other] + self.assertIsNone(runner._lvs_cutover_owner(me, "LVS_1")) + + def test_circular_stall_is_broken_when_both_tasks_hold_the_claim(self): + """Both tasks race and both write cutover_lvs — only one wins. + + Regression for production stall: + 36418f5d queued_for_lvstore_LVS_1_behind_309d3aeb + 309d3aeb queued_for_lvstore_LVS_1_behind_36418f5d + + Root cause: _lvs_cutover_owner excluded the calling task from the scan. + When two threads both raced through the check before either wrote its + claim, both stored cutover_lvs. On every subsequent pass each task's + candidate set was exactly {the other}, so each always deferred to the + other — a permanent cycle. The fix: include self in the sort, return + None iff the caller is the winner. + """ + # Simulate the post-race DB state: both have cutover_lvs set. + early = self._task("T1", lvs="LVS_1", created="2026-01-01") + late = self._task("T2", lvs="LVS_1", created="2026-06-01") + self.db.get_job_tasks.return_value = [early, late] + + # From T1's perspective: T1 is the earliest claimant → it is the owner. + self.assertIsNone( + runner._lvs_cutover_owner(early, "LVS_1"), + "the earliest claimant must see itself as the winner (None), " + "not defer to the only other claimant") + + # From T2's perspective: T1 is the earliest claimant → T2 must yield. + owner_seen_by_late = runner._lvs_cutover_owner(late, "LVS_1") + self.assertIsNotNone(owner_seen_by_late, + "the later claimant must see an owner") + self.assertEqual(owner_seen_by_late.get_id(), "T1", + "the later claimant must defer to the earlier one") + + +class TestQueuedCutoverDoesNotStarve(unittest.TestCase): + """A cutover that cannot have the lvstore waits without cost.""" + + def setUp(self): + from simplyblock_core.models.job_schedule import JobSchedule + from simplyblock_core.models.storage_node import StorageNode + self.JobSchedule = JobSchedule + patcher = patch.object(runner, "db") + self.db = patcher.start() + self.addCleanup(patcher.stop) + self.db.kv_store = "KV" + + lvol = MagicMock() + lvol.get_id.return_value = "LV_me" + lvol.lvs_name = "LVS_1" + self.db.get_lvol_by_id.return_value = lvol + + node = MagicMock() + node.status = StorageNode.STATUS_ONLINE + node.get_id.return_value = "N1" + node.cluster_id = "CL" + self.db.get_storage_node_by_id.return_value = node + + gp = patch.object(runner, "_group_id_for_lvol", return_value="") + gp.start() + self.addCleanup(gp.stop) + # A task waiting for the lvstore must not proceed into the endgame. + sp = patch.object(runner, "_shrink_step", + side_effect=AssertionError( + "a queued cutover must not run its endgame rounds")) + sp.start() + self.addCleanup(sp.stop) + # Caught up: the endgame is asked for at this point, and the answer is + # either the lvstore or a queue slot. + lp = patch.object(runner, "_replication_lag_sec", return_value=1.0) + lp.start() + self.addCleanup(lp.stop) + tp = patch.object(runner, "_take_shrink_snapshot", + side_effect=AssertionError( + "a queued cutover must not take a snapshot")) + tp.start() + self.addCleanup(tp.stop) + + def _me(self): + t = MagicMock() + t.function_name = self.JobSchedule.FN_REPLICATION_FINAL + t.get_id.return_value = "T_me" + t.cluster_id = "CL" + t.status = self.JobSchedule.STATUS_NEW + t.canceled = False + t.retry = 0 + t.max_retry = 8 + t.create_dt = "2026-06-01" + t.function_params = { + "lvol_id": "LV_me", "src_node_id": "N1", "tgt_node_id": "N2", + "shrink_round": 1, "shrink_snap_id": "S1", + "shrink_deadline": 1, # already expired + } + return t + + def _owner(self): + t = MagicMock() + t.function_name = self.JobSchedule.FN_REPLICATION_FINAL + t.get_id.return_value = "T_owner" + t.status = self.JobSchedule.STATUS_RUNNING + t.canceled = False + t.create_dt = "2026-01-01" + t.function_params = {"lvol_id": "LV_owner", "cutover_lvs": "LVS_1", + "cutover_group": ""} + return t + + def test_it_queues_without_burning_a_retry_or_its_deadline(self): + me, owner = self._me(), self._owner() + self.db.get_job_tasks.return_value = [me, owner] + + result = runner.task_runner(me) + + self.assertFalse(result) + self.assertEqual(me.status, self.JobSchedule.STATUS_SUSPENDED) + self.assertEqual(me.retry, 0, "queueing is not a failure") + self.assertIn("queued for lvstore", me.function_result) + self.assertGreater( + me.function_params["shrink_deadline"], 10 ** 9, + "the deadline must be pushed out while queued, or the task dies of " + "max retries waiting for a lock it cannot win") + self.assertNotIn("cutover_lvs", me.function_params, + "a queued task must not also claim the lvstore") + + def test_a_group_sibling_joins_the_owner_instead_of_queueing(self): + me, owner = self._me(), self._owner() + owner.function_params["cutover_group"] = "CL/G1" + with patch.object(runner, "_group_id_for_lvol", return_value="CL/G1"): + self.db.get_job_tasks.return_value = [me, owner] + # It proceeds into the shrink phase, which this fixture makes raise. + with self.assertRaises(AssertionError): + runner.task_runner(me) + + +class TestCutoverFailuresAreVisible(unittest.TestCase): + """160 failed attempts must not produce zero log lines. + + Run 20260827_194551: every fail-back cutover ended as "max retry reached + (8/8)" with nothing logged and the cause overwritten, so three separate + investigations could not name the failing branch. + """ + + def setUp(self): + patcher = patch.object(runner, "db") + self.db = patcher.start() + self.addCleanup(patcher.stop) + self.db.kv_store = "KV" + + def _task(self, retry=0): + from simplyblock_core.models.job_schedule import JobSchedule + t = _Task(lvol_id="LV1") + t.status = JobSchedule.STATUS_RUNNING + t.retry = retry + t.max_retry = 8 + t.canceled = False + return t + + def test_a_failed_attempt_is_logged_and_remembered(self): + task = self._task() + with self.assertLogs(runner.logger, level="WARNING") as logs: + runner._finalize(task, False, "target subsystem is full") + self.assertIn("target subsystem is full", "\n".join(logs.output)) + self.assertEqual(task.function_params["last_error"], + "target subsystem is full") + + def test_giving_up_reports_the_cause_not_just_the_symptom(self): + from simplyblock_core.models.job_schedule import JobSchedule + task = self._task(retry=8) + task.function_params["last_error"] = "target subsystem is full" + task.function_params.update({"src_node_id": "N1", "tgt_node_id": "N2"}) + with self.assertLogs(runner.logger, level="ERROR") as logs: + runner.task_runner(task) + self.assertEqual(task.status, JobSchedule.STATUS_DONE) + self.assertIn("target subsystem is full", task.function_result, + "'max retry reached' alone names a symptom and hides the " + "cause") + self.assertIn("target subsystem is full", "\n".join(logs.output)) diff --git a/simplyblock_core/test/test_evict_stale_namespace.py b/simplyblock_core/test/test_evict_stale_namespace.py new file mode 100644 index 0000000000..0538576a31 --- /dev/null +++ b/simplyblock_core/test/test_evict_stale_namespace.py @@ -0,0 +1,98 @@ +"""Which namespace the fail-back eviction removes. + +The cutover clone needs its preserved nsid; the still-live superseded original +occupies it. On a SHARED (namespaced) subsystem the original's namespace +carries the ORIGINAL record's uuid, so the own-uuid match cannot see it and the +nsid fallback is forbidden (it could evict a live sibling). The eviction must +therefore match the superseded original's identity explicitly — and never touch +siblings (2026-09-02: shared subsystem holding nsids 1-5, clone wanted nsid 1, +add_ns -32602 on every retry). +""" +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from simplyblock_core.controllers import lvol_controller + + +class _FakeRpc: + """Subsystem with mutable namespaces; remove_ns takes effect immediately.""" + + def __init__(self, namespaces): + self.namespaces = list(namespaces) + self.removed = [] + + def subsystem_get(self, nqn): + return {"nqn": nqn, "namespaces": list(self.namespaces)} + + def nvmf_subsystem_remove_ns(self, nqn, nsid): + self.removed.append(nsid) + self.namespaces = [ns for ns in self.namespaces if ns["nsid"] != nsid] + return True + + +def _node(rpc): + node = MagicMock() + node.secondary_node_id = "" + node.tertiary_node_id = "" + node.rpc_client.return_value = rpc + node.get_id.return_value = "N_tgt" + return node + + +def _clone(nsid=1): + return SimpleNamespace(uuid="CLONE_UUID", ns_id=nsid, + top_bdev="LVS/CLONE_BDEV", nqn="nqn.shared") + + +_ORIGINAL = SimpleNamespace(uuid="ORIG_UUID", lvol_uuid="ORIG_BDEV", + top_bdev="LVS/ORIG") + +_SHARED = [ + {"nsid": 1, "uuid": "ORIG_UUID", "bdev_name": "ORIG_BDEV"}, + {"nsid": 2, "uuid": "SIB_A", "bdev_name": "SIB_A_BDEV"}, + {"nsid": 3, "uuid": "SIB_B", "bdev_name": "SIB_B_BDEV"}, +] + + +def _evict(rpc, superseded): + with patch.object(lvol_controller, "DBController", MagicMock()): + lvol_controller._evict_stale_namespace(_clone(), _node(rpc), + superseded=superseded) + + +def test_superseded_original_is_evicted_on_a_shared_subsystem(): + rpc = _FakeRpc(_SHARED) + _evict(rpc, _ORIGINAL) + assert rpc.removed == [1] + + +def test_siblings_survive_even_when_the_clone_wants_their_nsid(): + """nsid never identifies a namespace on a shared subsystem.""" + rpc = _FakeRpc([ns for ns in _SHARED if ns["nsid"] != 1]) + _evict(rpc, None) + assert rpc.removed == [] + + +def test_without_superseded_the_shared_original_stays(): + """The pre-fix behavior: own-uuid match alone cannot see the original. + Guards that passing superseded is what makes the eviction possible.""" + rpc = _FakeRpc(_SHARED) + _evict(rpc, None) + assert rpc.removed == [] + + +def test_matches_by_bdev_name_when_ns_uuid_diverged(): + """SPDK reports lvol namespaces' bdev_name as the raw lvol_uuid; a + namespace registered with a different ns uuid is still the original's.""" + rpc = _FakeRpc([{"nsid": 1, "uuid": "SOMETHING_ELSE", + "bdev_name": "ORIG_BDEV"}] + _SHARED[1:]) + _evict(rpc, _ORIGINAL) + assert rpc.removed == [1] + + +def test_dedicated_subsystem_still_evicts_by_nsid(): + """Single-namespace subsystems keep the nsid fallback: there is nobody + else the match could hit.""" + rpc = _FakeRpc([{"nsid": 1, "uuid": "OLD", "bdev_name": "OLD_BDEV"}]) + _evict(rpc, None) + assert rpc.removed == [1] diff --git a/simplyblock_core/test/test_failover_clone_race.py b/simplyblock_core/test/test_failover_clone_race.py index 69a26a9672..78a604d4db 100644 --- a/simplyblock_core/test/test_failover_clone_race.py +++ b/simplyblock_core/test/test_failover_clone_race.py @@ -82,7 +82,7 @@ def __exit__(self, *a): monkeypatch.setattr(lvol_controller.snapshot_controller, "object_mutation_lock", _Lock) - def _fake_select(db, lvol_id, cluster_id): + def _fake_select(db, lvol_id, cluster_id, generation=0): """Stand-in for _last_replicated_target_snapshot. Mirrors the real selector: newest first, skipping anything missing or diff --git a/simplyblock_core/test/test_failover_retires_source.py b/simplyblock_core/test/test_failover_retires_source.py new file mode 100644 index 0000000000..4d6b060b1c --- /dev/null +++ b/simplyblock_core/test/test_failover_retires_source.py @@ -0,0 +1,116 @@ +"""Fail-over must retire the still-alive source's data path. + +The source is only assumed dead. Left published, the client keeps writing to +the superseded original (discarded at fail-back), and a namespaced volume's +restage re-attaches to the stale ORIGINAL device — same model/nsid glob, DR +clone rejected against the original-identity head — so pods never reach the +DR cluster (run 2026-09-03 ~09:00: all three namespaced pods remounted on the +originals after fail-over; fail-back's eviction then shut down all three +filesystems mid-IO). +""" +import inspect +from unittest.mock import MagicMock + +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.controllers import lvol_controller as lc + + +def _lvol(): + lvol = LVol() + lvol.uuid = "SRC" + lvol.nqn = "nqn.test:lvol:S" + lvol.node_id = "N1" + lvol.nodes = ["N1", "N2"] + return lvol + + +def _node(status=StorageNode.STATUS_ONLINE): + node = MagicMock() + node.status = status + return node + + +def _record(bucket, value): + """Append and report success — a lambda-safe stand-in for the patched + call sites, which must return truthy.""" + bucket.append(value) + return True + + +def test_retire_fences_then_removes_on_every_online_node(monkeypatch): + fenced: list = [] + removed: list = [] + monkeypatch.setattr(lc, "suspend_lvol", + lambda lvol_id: _record(fenced, lvol_id)) + monkeypatch.setattr(lc, "_remove_lvol_subsys_from_node", + lambda lvol, rpc: _record(removed, lvol.get_id())) + db = MagicMock() + db.get_storage_node_by_id.side_effect = \ + lambda nid: {"N1": _node(), "N2": _node()}[nid] + lc._retire_source_data_path(db, _lvol()) + assert fenced == ["SRC"] + assert removed == ["SRC", "SRC"], "namespace removed on primary AND peer" + + +def test_retire_skips_offline_nodes(monkeypatch): + removed: list = [] + monkeypatch.setattr(lc, "suspend_lvol", lambda lvol_id: True) + monkeypatch.setattr(lc, "_remove_lvol_subsys_from_node", + lambda lvol, rpc: _record(removed, rpc)) + db = MagicMock() + db.get_storage_node_by_id.side_effect = lambda nid: { + "N1": _node(), "N2": _node(status=StorageNode.STATUS_OFFLINE)}[nid] + lc._retire_source_data_path(db, _lvol()) + assert len(removed) == 1 + + +def test_retire_tolerates_an_unreachable_source(monkeypatch): + """A genuine DR event: fencing raises, nodes unknown — the fail-over must + not be aborted by its own best-effort cleanup.""" + def _boom(lvol_id): + raise RuntimeError("source cluster unreachable") + + monkeypatch.setattr(lc, "suspend_lvol", _boom) + db = MagicMock() + db.get_storage_node_by_id.side_effect = KeyError("gone") + lc._retire_source_data_path(db, _lvol()) # must not raise + + +def test_failover_retires_the_source_after_the_relationship_is_durable(): + """Ordering guard: by the time the source path disappears, connect_lvol + must already resolve the volume to the DR copy — so the relationship + write comes first, the retire after.""" + src = inspect.getsource(lc.replicate_lvol_on_target_cluster) + rel = src.index("lvol_replication.write_to_db") + retire = src.index("_retire_source_data_path") + assert rel < retire + + +def test_monitor_skips_retired_sources(): + """The monitor's health check would fail on the deliberately-removed + namespace and its self-heal would register it back — resurrecting the + stale device the retirement removed.""" + from simplyblock_core.services import lvol_monitor + src = inspect.getsource(lvol_monitor) + guard = src.index('from_source') + check = src.index("check_subsystem") + assert guard < check, "the from_source guard must run before any check" + + +def test_repair_refuses_retired_sources(monkeypatch): + from simplyblock_core import storage_node_ops as ops + + lvol = _lvol() + lvol.status = LVol.STATUS_ONLINE + lvol.from_source = False + lvol.lvs_name = "LVS_1" + + monkeypatch.setattr(ops, "get_restart_phase", lambda *a: "") + db = MagicMock() + db.get_lvol_by_id.return_value = lvol + monkeypatch.setattr(ops, "DBController", lambda: db) + + ok, err = ops.repair_lvol_registration_on_non_leader(lvol, _node(), 0) + assert ok is False + assert "retired source" in err diff --git a/simplyblock_core/test/test_failover_target.py b/simplyblock_core/test/test_failover_target.py index 945d69fd5a..53230f6977 100644 --- a/simplyblock_core/test/test_failover_target.py +++ b/simplyblock_core/test/test_failover_target.py @@ -79,6 +79,12 @@ def __init__(self, nodes, clusters, existing_lvols=None): def get_lvol_by_id(self, lid): return _src_lvol() + def get_lvol_replication_objects(self): + # Real DBController exposes this; the fail-back retirement asks it + # whether this volume is a copy of something. In these fail-OVER + # tests it is not, so there is nothing to retire. + return [] + def get_storage_node_by_id(self, nid): return self._nodes[nid] @@ -124,6 +130,11 @@ def _rep_write(self, kv=None): def _add_lvol_on_node(new_lvol, node, is_primary=True, **kw): add_calls.append((node.get_id(), is_primary)) + if is_primary: + # Faithful to the real primary add, which persists the nsid the + # target actually used: the control plane's claim when there was + # one, the target's own pick (lowest free) otherwise. + new_lvol.ns_id = new_lvol.ns_id or 1 return ({"uuid": "BDEV-UUID", "driver_specific": {"lvol": {"blobid": 123}}}, None) monkeypatch.setattr(lvol_controller, "add_lvol_on_node", _add_lvol_on_node) @@ -159,16 +170,21 @@ def test_failover_preserves_nqn_ns_and_returns_paths(monkeypatch, patched): result = lvol_controller.replicate_lvol_on_target_cluster("LV1") - # Same NQN + namespace as the original volume. + # Same NQN as the original volume -- the client reconnects to the SAME + # subsystem, only the IP/port differ. The nsid is NOT carried over: it is + # claimed on the target HA set (nothing there yet here, so the target + # picks 1). Cross-cluster nsid equality is not required -- clients resolve + # their paths through connect_lvol -- and insisting on the source's number + # is what collided with a sibling already holding it (soak case 7). assert result["nqn"] == "nqn.orig:lvol:LV1" - assert result["ns_id"] == 7 + assert result["ns_id"] == 1 assert len(result["connection_strings"]) == 1 rep = patched["rep"] assert rep.state == LVolReplication.STATE_FAILED_OVER assert rep.direction == LVolReplication.DIRECTION_TO_TARGET assert rep.target_nqn == "nqn.orig:lvol:LV1" - assert rep.target_ns_id == 7 + assert rep.target_ns_id == 1 assert rep.source_cluster_id == "CL_src" assert rep.target_cluster_id == "CL_tgt" # Source volume flipped to non-source after fail-over. @@ -203,6 +219,9 @@ def test_failover_idempotent_when_target_exists(monkeypatch, patched): existing = LVol() existing.uuid = "EXISTING" existing.nqn = "nqn.orig:lvol:LV1" + # A fail-over copy preserves the source's nsid, so the already-failed-over + # volume this guard recognises carries nsid 7 too -- not the model default. + existing.ns_id = 7 nodes = { "N_src": _node("N_src", "CL_src"), "N_tgt": _node("N_tgt", "CL_tgt", secondary="N_sec", lvstore="lvs_tgt"), @@ -218,3 +237,30 @@ def test_failover_idempotent_when_target_exists(monkeypatch, patched): # Returns the existing target lvol id; no new volume created. assert result == "EXISTING" assert patched["add_calls"] == [] + + +def test_failover_does_not_mistake_a_sibling_namespace_for_this_volume(monkeypatch, patched): + """Soak case 7 (run 20260824_174611): with namespaced volumes, up to + max-namespace-per-subsys volumes SHARE one nqn. The existing-copy guard + matched on nqn alone, so once namespace 1 had failed over, namespaces + 2..N returned ITS target id and were never failed over -- 9 of 10 volumes + silently absent after a DR fail-over, every call reporting success.""" + sibling = LVol() + sibling.uuid = "SIBLING_NS1" + sibling.nqn = "nqn.orig:lvol:LV1" # same shared subsystem + sibling.ns_id = 1 # DIFFERENT namespace + nodes = { + "N_src": _node("N_src", "CL_src"), + "N_tgt": _node("N_tgt", "CL_tgt", secondary="N_sec", lvstore="lvs_tgt"), + } + clusters = { + "CL_src": _cluster("CL_src", target_cluster="CL_tgt", target_pool="POOL_tgt"), + "CL_tgt": _cluster("CL_tgt"), + } + _install_db(monkeypatch, _FakeDB(nodes, clusters, existing_lvols=[sibling])) + + result = lvol_controller.replicate_lvol_on_target_cluster("LV1") + + # LV1 (nsid 7) must actually fail over, NOT return the nsid-1 sibling. + assert result != "SIBLING_NS1" + assert patched["add_calls"], "the volume must really be failed over" diff --git a/simplyblock_core/test/test_internal_snapshot_scheduler.py b/simplyblock_core/test/test_internal_snapshot_scheduler.py index 6d3c25a48a..2dadb73298 100644 --- a/simplyblock_core/test/test_internal_snapshot_scheduler.py +++ b/simplyblock_core/test/test_internal_snapshot_scheduler.py @@ -85,6 +85,11 @@ def get_lvols(self, cluster_id): def get_mini_snapshots(self): return [] + def get_replication_policies(self, cluster_id=None): + # The scheduler partitions consistency-group policies out of the + # per-volume loop; this scenario has none. + return [] + calls = [] class _SnapCtl: diff --git a/simplyblock_core/test/test_jc_dual_node.py b/simplyblock_core/test/test_jc_dual_node.py new file mode 100644 index 0000000000..d6347f957e --- /dev/null +++ b/simplyblock_core/test/test_jc_dual_node.py @@ -0,0 +1,109 @@ +"""The JC dual-node flag must track cluster MEMBERSHIP. + +Soak case 6 (2026-08-24): gracefully shutting down one node of a 2-node +cluster made the SURVIVOR abort its own SPDK -- + + JC detected a network outage nd=1 njms=2 + JC aborts the node due to network outage + spdk_abort_node: Forcing application shutdown via abort. + +-- because the journal component requires jc_ha_nmin_jms() reachable +journals, which is 2 unless the dual-node flag is set (then 1). The fork +implements the tolerance and exposes jc_set_dual_node; the control plane +never called it, so every 2-node cluster lost all availability the moment +either node stopped. +""" +import inspect + +import pytest + +from simplyblock_core import storage_node_ops +from simplyblock_core.models.storage_node import StorageNode + + +class _RPC: + def __init__(self, sink, node_id): + self._sink, self._node_id = sink, node_id + + def jc_set_dual_node(self, enable): + self._sink.append((self._node_id, enable)) + return True + + +class _Node: + def __init__(self, nid, sink, status=StorageNode.STATUS_ONLINE): + self._id, self._sink, self.status = nid, sink, status + self.cluster_id = "CL" + + def get_id(self): + return self._id + + def rpc_client(self, *a, **kw): + return _RPC(self._sink, self._id) + + +def _install(monkeypatch, nodes): + class _DB: + def get_storage_nodes_by_cluster_id(self, cluster_id): + return nodes + monkeypatch.setattr(storage_node_ops, "DBController", lambda: _DB()) + + +def test_two_node_cluster_enables_dual_node(monkeypatch): + sink: list[tuple[str, bool]] = [] + nodes = [_Node("A", sink), _Node("B", sink)] + _install(monkeypatch, nodes) + storage_node_ops.apply_jc_dual_node("CL") + assert sink == [("A", True), ("B", True)] + + +def test_three_node_cluster_disables_dual_node(monkeypatch): + sink: list[tuple[str, bool]] = [] + nodes = [_Node("A", sink), _Node("B", sink), _Node("C", sink)] + _install(monkeypatch, nodes) + storage_node_ops.apply_jc_dual_node("CL") + assert sink == [("A", False), ("B", False), ("C", False)] + + +def test_flag_follows_membership_not_how_many_are_online(monkeypatch): + """A 3-node cluster with one node down still needs 2 journals. Keying + the flag on the ONLINE count would switch a degraded 3-node cluster into + dual-node mode -- weakening the journal requirement exactly when a node + is already missing.""" + sink: list[tuple[str, bool]] = [] + nodes = [_Node("A", sink), _Node("B", sink), + _Node("C", sink, status=StorageNode.STATUS_OFFLINE)] + _install(monkeypatch, nodes) + storage_node_ops.apply_jc_dual_node("CL") + # offline node is not called, but the two online ones must be told FALSE + assert sink == [("A", False), ("B", False)] + + +def test_removed_nodes_do_not_count_towards_membership(monkeypatch): + sink: list[tuple[str, bool]] = [] + nodes = [_Node("A", sink), _Node("B", sink), + _Node("C", sink, status=StorageNode.STATUS_REMOVED)] + _install(monkeypatch, nodes) + storage_node_ops.apply_jc_dual_node("CL") + assert sink == [("A", True), ("B", True)] + + +def test_one_unreachable_node_does_not_stop_the_others(monkeypatch): + sink: list[tuple[str, bool]] = [] + + class _BadNode(_Node): + def rpc_client(self, *a, **kw): + raise RuntimeError("node unreachable") + + nodes = [_BadNode("A", sink), _Node("B", sink)] + _install(monkeypatch, nodes) + storage_node_ops.apply_jc_dual_node("CL") # must not raise + assert sink == [("B", True)] + + +@pytest.mark.parametrize("path", ["_prepare_cluster_devices_on_restart"]) +def test_bring_up_paths_apply_the_flag(path): + """A restarted node comes back with the JC default (off), so the flag has + to be re-applied on every bring-up, not only when the node is added.""" + src = inspect.getsource(getattr(storage_node_ops, path)) + assert "apply_jc_dual_node" in src, f"{path} must re-apply the dual-node flag" diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index c31691090f..dbf87c0c79 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -17,7 +17,7 @@ already existent on the target by every other descendant. """ -from typing import Optional +from typing import ClassVar, Optional from simplyblock_core.models.snapshot import SnapShot from simplyblock_core.services import snapshot_replication as sr @@ -269,19 +269,31 @@ def test_no_backward_task_for_a_policy_managed_clone(): class _EvictRPC: - def __init__(self, namespaces): + def __init__(self, namespaces, linger_polls=0): self.removed = [] - self._ns = namespaces + self._ns = list(namespaces) + self._linger = linger_polls # polls before a removed ns disappears def subsystem_get(self, nqn): - return [{"nqn": nqn, "namespaces": self._ns}] + # the real client returns ONE dict (single_or_none), never a list + if self._linger > 0: + self._linger -= 1 + return {"nqn": nqn, "namespaces": self._ns} + live = [n for n in self._ns + if (nqn, n.get("nsid")) not in self.removed] + return {"nqn": nqn, "namespaces": live} def nvmf_subsystem_remove_ns(self, nqn, nsid): + # acknowledges immediately; disappearance is governed by linger_polls, + # modelling the fork's async remove_ns false-success self.removed.append((nqn, nsid)) return True class _EvictNode: + secondary_node_id = None + tertiary_node_id = None + def __init__(self, rpc): self._rpc = rpc @@ -295,6 +307,11 @@ def rpc_client(self): class _CloneLvol: nqn = "nqn.test:lvol:ORIG" ns_id = 7 + # A real LVol always carries a uuid, and the eviction matches on it -- + # a fake without one made the matcher raise into the best-effort except + # and silently evict nothing, the exact failure mode this suite exists + # to catch. + uuid = "11111111-1111-1111-1111-111111111111" top_bdev = "LVS_1/LVOL_CLONE" @@ -331,3 +348,543 @@ def subsystem_get(self, nqn): rpc = _NoSubsysRPC([]) lc._evict_stale_namespace(_CloneLvol(), _EvictNode(rpc)) # must not raise assert rpc.removed == [] + + +def test_failback_eviction_waits_out_the_async_removal(monkeypatch): + """remove_ns acknowledges before it completes (the PVC-expand + false-success); the eviction must confirm the namespace is GONE before + add_ns runs, or the add races the removal and loses (run 20260824_110959: + 40 evictions immediately followed by 40 add_ns -32602).""" + from simplyblock_core.controllers import lvol_controller as lc + monkeypatch.setattr(lc.time, "sleep", lambda s: None) + rpc = _EvictRPC([{"nsid": 7, "bdev_name": "LVS_1/LVOL_ORIG"}], linger_polls=3) + lc._evict_stale_namespace(_CloneLvol(), _EvictNode(rpc)) + assert rpc.removed == [("nqn.test:lvol:ORIG", 7)] + # after the helper returns, the namespace must actually be gone + assert rpc.subsystem_get("nqn.test:lvol:ORIG")["namespaces"] == [] + + +def test_failback_eviction_matches_by_uuid_too(monkeypatch): + from simplyblock_core.controllers import lvol_controller as lc + monkeypatch.setattr(lc.time, "sleep", lambda s: None) + rpc = _EvictRPC([{"nsid": 3, "uuid": "LV_UUID", "bdev_name": "LVS_1/LVOL_ORIG"}]) + class _C(_CloneLvol): + uuid = "LV_UUID" + lc._evict_stale_namespace(_C(), _EvictNode(rpc)) + assert rpc.removed == [("nqn.test:lvol:ORIG", 3)] + + +def test_failback_evicts_on_every_ha_node_not_just_the_primary(monkeypatch): + """Run 20260824_113711: eviction on the primary made ITS add_ns succeed + (result: 1) while the HA peer's failed with the same -32602 -- the + preserved-NQN subsystem exists on EVERY node of the recovered set, each + still holding the original namespace. The peer failure rolled the whole + cutover back: 0/5. The clone path must evict per node.""" + from simplyblock_core.controllers import lvol_controller as lc + + evicted, added = [], [] + monkeypatch.setattr(lc, "_evict_stale_namespace", + lambda lvol, node, **kw: evicted.append(node.get_id())) + + def _fake_add_lvol_on_node(lvol, node, is_primary=True, **kw): + # The real signature also takes min_cntlid / ns_uuid / primary_nsid; + # accept them so this fake cannot drift out of call-compatibility. + added.append((node.get_id(), is_primary)) + return {"uuid": "U", "driver_specific": {"lvol": {"blobid": 9}}}, None + + monkeypatch.setattr(lc, "add_lvol_on_node", _fake_add_lvol_on_node) + + class _N: + def __init__(self, nid, secondary="", tertiary=""): + self._id, self.secondary_node_id, self.tertiary_node_id = nid, secondary, tertiary + self.lvstore = "LVS_1" + self.status = lc.StorageNode.STATUS_ONLINE + self.cluster_id = "CL_tgt" + self.lvol_subsys_port = 4420 + + def rpc_client(self): + # Real StorageNode hands out an RPC client; the nsid claim asks it + # what this subsystem already holds. Nothing here yet. + class _R: + @staticmethod + def subsystem_get(nqn): + return None + return _R() + + def get_lvol_subsys_port(self, lvstore): + # Real StorageNode resolves a per-lvstore listener port; the clone + # copies it onto the new volume so suspend_lvol addresses the right + # listener. + return self.lvol_subsys_port + def get_id(self): + return self._id + + primary = _N("P", secondary="S") + peer = _N("S") + + class _DB: + kv_store = None + def get_storage_node_by_id(self, nid): + return {"P": primary, "S": peer}[nid] + def release_lvol_ns_slot(self, lvol): + pass + def get_lvols(self): + # No copy of this subsystem exists on the target yet, so the + # one-subsystem-one-primary guard has nothing to redirect to. + return [] + def get_lvol_replication_objects(self): + # This volume is not a fail-over copy, so no original is + # superseded and nothing gets retired. + return [] + + class _Lvol: + uuid = "ORIG"; nqn = "nqn.test:lvol:ORIG"; ns_id = 7 + # Real LVol carries these; the clone reads namespace to decide whether + # it attaches to a sibling's subsystem or creates its own. + namespace = "" + max_namespace_per_subsys = 1 + lvol_bdev = "LVOL_C"; crypto_bdev = "" + def __deepcopy__(self, memo): + c = _Lvol(); c.__dict__.update(self.__dict__); return c + def write_to_db(self, kv=None): + pass + + class _Snap: + cluster_id = "C1"; snap_bdev = "LVS_1/SNAP_1" + def get_id(self): + return "SNAP1" + + new_lvol, error = lc._create_target_lvol_clone(_DB(), _Lvol(), primary, "POOL", _Snap()) + assert error is None + assert evicted == ["P", "S"], \ + "stale-namespace eviction must run on the primary AND every online HA peer" + assert ("S", False) in added + + +def test_failback_clone_keeps_the_client_visible_wire_identity(monkeypatch): + """The fail-back clone must advertise the SOURCE's wire identity — what + the connected client's multipath head currently holds — or the kernel + rejects every preconnected target path ("IDs don't match for shared + namespace N") and deleteSource then removes the head's only live paths + (run 2026-09-02 ~19:00, subsystem 20d8a917: no available path, XFS + shutdown on a restaged pod). Explicitly NOT the superseded original's + uuid: that variant was tried (run 2026-09-02 17:00) and only served + clients still riding the unfenced superseded original, whose writes + fail-back discards anyway.""" + from simplyblock_core.controllers import lvol_controller as lc + + added = [] + monkeypatch.setattr(lc, "_evict_stale_namespace", lambda lvol, node, **kw: None) + monkeypatch.setattr(lc.utils, "get_random_vuid", lambda *a, **kw: 999) + + def _fake_add_lvol_on_node(lvol, node, is_primary=True, ns_uuid=None, **kw): + added.append((node.get_id(), ns_uuid, lvol.guid)) + return {"uuid": "U", "driver_specific": {"lvol": {"blobid": 9}}}, None + + monkeypatch.setattr(lc, "add_lvol_on_node", _fake_add_lvol_on_node) + + class _Original: + uuid = "ORIG_ID" + guid = "ORIG_NGUID" + + monkeypatch.setattr(lc, "_superseded_original", lambda *a, **kw: _Original()) + + class _N: + def __init__(self, nid, secondary=""): + self._id, self.secondary_node_id, self.tertiary_node_id = nid, secondary, "" + self.lvstore = "LVS_1" + self.status = lc.StorageNode.STATUS_ONLINE + self.cluster_id = "CL_tgt" + + def get_lvol_subsys_port(self, lvstore): + return 4420 + + def get_id(self): + return self._id + + primary = _N("P", secondary="S") + peer = _N("S") + + class _DB: + kv_store = None + + def get_storage_node_by_id(self, nid): + return {"P": primary, "S": peer}[nid] + + def release_lvol_ns_slot(self, lvol): + pass + + def get_lvols(self): + return [] + + class _Lvol: + uuid = "DR_ID"; nqn = "nqn.test:lvol:SHARED"; ns_id = 3 + guid = "DR_NGUID" + ns_uuid = "" + namespace = "" + max_namespace_per_subsys = 10 + lvol_bdev = "LVOL_28"; crypto_bdev = "" + + def __deepcopy__(self, memo): + c = _Lvol(); c.__dict__.update(self.__dict__); return c + + def write_to_db(self, kv=None): + pass + + class _Snap: + cluster_id = "C1"; snap_bdev = "LVS_1/SNAP_1" + + def get_id(self): + return "SNAP1" + + new_lvol, error = lc._create_target_lvol_clone( + _DB(), _Lvol(), primary, "POOL", _Snap(), for_migration=True) + assert error is None + # Every node's add_ns must carry the DR source's wire identity — what the + # client's head holds — never the superseded original's. + assert [(nid, ns) for nid, ns, _ in added] == [("P", "DR_ID"), ("S", "DR_ID")] + assert all(guid == "DR_NGUID" for _, _, guid in added) + # The wire identity is persisted so connect_lvol can report it. + assert new_lvol.ns_uuid == "DR_ID" + # And the clone still got its own bdev name (adoption guard). + assert new_lvol.lvol_bdev == "LVOL_999" + + # A second fail-back cycle: the DR source's own wire identity is already + # borrowed (its ns_uuid points at an earlier generation). The chain must + # propagate — the client's head knows only the ORIGINAL wire id. + added.clear() + + class _Gen2Lvol(_Lvol): + ns_uuid = "GEN0_WIRE_ID" + + new_lvol, error = lc._create_target_lvol_clone( + _DB(), _Gen2Lvol(), primary, "POOL", _Snap(), for_migration=True) + assert error is None + assert [(nid, ns) for nid, ns, _ in added] == [ + ("P", "GEN0_WIRE_ID"), ("S", "GEN0_WIRE_ID")] + assert new_lvol.ns_uuid == "GEN0_WIRE_ID" + + +def test_interrupted_landing_volume_is_adopted_or_cleared(): + """Case 6, run 20260824_144226: a node outage mid-create left a REP_* + landing volume whose id was never stored on the task; every retry then + died on "LVol name must be unique" and three volumes' chains stalled for + the rest of the run. Before creating the landing volume, the runner must + look for a record already wearing the derived name and adopt it (online), + wait for it (in_deletion), or clear it (half-created).""" + import inspect + from simplyblock_core.services import snapshot_replication as sr + src = inspect.getsource(sr) + probe = src.index('rep_name = f"REP_{snapshot.snap_name}"') + create = src.index("lvol_controller.add_lvol_ha") + assert probe < create, "the adopt/clear probe must run before the create" + adopt = src.index("Adopting landing volume") + assert probe < adopt < create + for handled in ("STATUS_ONLINE", "STATUS_IN_DELETION", "force_delete=True"): + assert src.index(handled, probe) < create, \ + f"collision handling must cover {handled} before creating" + + +def test_failover_guard_matches_nqn_and_nsid_not_nqn_alone(): + """Soak case 7 (run 20260824_174611): namespaced volumes SHARE a subsystem, + so the fail-over idempotency guard's nqn-only match fired for every + namespace after the first — namespaces 2..N returned namespace 1's target + lvol id and were never failed over, losing 9 of 10 volumes in a DR event + while reporting success. The guard must compare the FULL preserved + identity: nqn AND ns_id.""" + import inspect + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc.replicate_lvol_on_target_cluster) + # The guard now resolves the copy through the replication record, because + # merging R26.3 made BOTH string matches unreliable: siblings share the + # NQN, and a fail-over claims a target-local nsid so the numbers need not + # match. nqn+nsid remains the fallback when no record exists yet. + assert "own_copies" in src, \ + "the fail-over existing-copy guard must identify the copy explicitly" + assert "rep.source_lvol.get_id() == lvol.get_id()" in src, \ + "the copy is the target of THIS volume's replication record" + assert "lv.ns_id != lvol.ns_id" in src, \ + "nqn+nsid must remain the fallback when there is no record" + + +def test_namespaced_siblings_replicate_to_the_same_target_node(): + """Soak case 7 (run 20260824_215758): the replication destination was + picked per volume by capacity, so volumes SHARING a subsystem were + scattered across the target cluster's nodes. A fail-over copy preserves + the NQN, so that splits one shared subsystem across unrelated primaries + -- each advertising the same NQN with only part of the namespaces. + Siblings must inherit the node their subsystem already replicates to.""" + import inspect + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc.add_lvol_ha) + assert "_sibling_replication_node" in src, "namespaced siblings must share a replication node" + pick = src.index("_get_next_3_nodes(replication_cluster_id") + check = src.index("_sibling_replication_node") + assert check < pick, "the sibling lookup must precede the capacity-based pick" + + # ...and it must run AGAIN after claim_lvol_ns_slot. That transaction is + # what authoritatively decides the subsystem: the earlier lookup only saw + # the ADVISORY pick from _resolve_lvol_subsystem, so a volume the + # transaction rehomed into an existing shared subsystem would otherwise + # keep a target node chosen for a subsystem it is no longer in -- which is + # how the group was still split in run 20260826_230358 (peer held nsids + # 1..7, the split-off primary asked for 1). + claim = src.index("claim_lvol_ns_slot") + realign = src.index("_realign_replication_node_after_claim") + assert claim < realign, "the target node must be re-derived from the subsystem the CLAIM chose" + assert "lv.nqn == lvol.nqn" in inspect.getsource(lc._sibling_replication_node), "siblings are identified by shared NQN" + + +def test_clone_register_confirms_the_bdev_before_add_ns(): + """Run 20260825_122423: bdev_lvol_clone_register acknowledges before the + bdev is examinable, and the peer's nvmf_subsystem_add_ns raced it and lost + (-32602 with the subsystem EMPTY; the bdev existed moments later). Third + member of the acknowledge-before-complete family, after remove_ns + (PVC-expand) and the case-3 eviction. The stack build must poll the bdev + into existence before the namespace add runs.""" + import inspect + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc._create_bdev_stack) + reg = src.index("bdev_lvol_clone_register") + assert "not appear within 20s" in src[reg:], \ + "clone_register must be followed by a bdev confirmation poll" + poll = src.index("not appear within 20s", reg) + assert "get_bdevs" in src[reg:poll], "the poll must probe get_bdevs" + + +def test_retired_landing_records_are_record_only_deletions(): + """A retired landing volume's record deliberately carries an EMPTY + bdev_stack: its blob lives on as the converted, chained snapshot. The + monitor must retire such a record without issuing ANY bdev delete (runs + 20260824/20260825: interrupted retirements left records in_deletion that + the delete flow could never finish -- status poll 4, forever -- and a + naive top_bdev fallback delete would have destroyed the replicated + snapshot's data).""" + import inspect + from simplyblock_core.services import lvol_monitor as lm + src = inspect.getsource(lm.check_node) + guard = src.index("if not lvol.bdev_stack:") + flow = src.index("delete_lvol_from_node", guard) + finish = src.index("process_lvol_delete_finish", guard) + assert finish < flow, "empty-stack records must retire BEFORE the delete flow" + + +def test_retirement_tears_down_plumbing_without_delete_lvol(): + """The retirement path must not route through delete_lvol: that flips the + record to in_deletion for the monitor's async machinery, so any + interruption before remove() strands the record.""" + import inspect + from simplyblock_core.services import snapshot_replication as sr + src = inspect.getsource(sr) + empty = src.index("remote_lv.bdev_stack = []") + remove = src.index("remote_lv.remove(db.kv_store)", empty) + seg = src[empty:remove] + assert "delete_lvol_from_node" in seg, "teardown must be the direct per-node call" + assert "delete_lvol(remote_lv" not in seg, "must not route through delete_lvol" + + +def test_shared_subsystem_survives_one_members_teardown(): + """Run 20260825_224221: a stuck in_deletion member's teardown loop saw the + SHARED subsystem transiently empty on the HA peer (between one member's + teardown and the next member's add) and deleted it -- every following + namespaced fail-over then died in add_ns on a missing subsystem (8/20 + landed). Delete-on-empty must first prove no other live volume claims + the NQN.""" + import inspect + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc._remove_lvol_subsys_from_node) + guard = src.index("other") + delete = src.index("subsystem_delete") + assert guard < delete, "the other-claimants check must precede subsystem_delete" + assert "x.nqn == lvol.nqn" in src, "claimants are identified by shared NQN" + assert "Leaving subsystem" in src + + +def test_eviction_never_removes_a_siblings_namespace(): + """On a SHARED subsystem the nsid is not this volume's identity: siblings + hold the other slots, and new_lvol.ns_id is still the SOURCE cluster's + number (the destination primary auto-assigns and only then overwrites the + record). Matching on nsid would evict a sibling's live, already-failed-over + namespace -- taking a healthy volume's device away from its client.""" + from simplyblock_core.controllers import lvol_controller as lc + rpc = _EvictRPC([ + {"nsid": 7, "bdev_name": "LVS_1/LVOL_SIBLING", + "uuid": "22222222-2222-2222-2222-222222222222"}, # same nsid, other volume + {"nsid": 3, "bdev_name": "LVS_1/LVOL_MINE_OLD", + "uuid": "11111111-1111-1111-1111-111111111111"}, # THIS volume, stale + ]) + lc._evict_stale_namespace(_CloneLvol(), _EvictNode(rpc)) + assert rpc.removed == [("nqn.test:lvol:ORIG", 3)], \ + "must evict only this volume's own namespace, never the sibling at nsid 7" + + +def test_eviction_still_uses_nsid_on_a_single_namespace_subsystem(): + """A dedicated subsystem cannot hold anyone else's namespace, so nsid stays + a safe match there -- which is the fail-back-to-a-recovered-source case the + eviction was written for, where the old record may carry a different uuid.""" + from simplyblock_core.controllers import lvol_controller as lc + rpc = _EvictRPC([{"nsid": 7, "bdev_name": "LVS_1/LVOL_ORIG", + "uuid": "99999999-9999-9999-9999-999999999999"}]) + lc._evict_stale_namespace(_CloneLvol(), _EvictNode(rpc)) + assert rpc.removed == [("nqn.test:lvol:ORIG", 7)] + + +class _SubsysProbeRPC: + def __init__(self, existing_nqns): + self._existing = set(existing_nqns) + self.probed = [] + + def subsystem_get(self, nqn): + self.probed.append(nqn) + return {"nqn": nqn, "namespaces": []} if nqn in self._existing else None + + +class _ProbeNode: + def get_id(self): + return "NODE_PEER" + + +class _NsLvol: + nqn = "nqn.test:lvol:SHARED" + namespace = "SHARED" # truthy => shared/namespaced subsystem + + +class _DedicatedLvol: + nqn = "nqn.test:lvol:OWN" + namespace = "" # dedicated subsystem + + +def test_shared_subsystem_is_created_on_a_node_that_lacks_it(): + """Case 7, run 20260826_214011: the very first namespaced fail-over died + with 'subsystem does not exist on '. The create-vs-attach decision + was read from the DB alone -- which says the volume SHARES a subsystem, + not whether THIS node has one. The primary self-heals through the -32602 + fallback; a replica cannot (its nsid is fixed by the primary), so it must + create the subsystem when the node genuinely lacks it.""" + from simplyblock_core.controllers import lvol_controller as lc + rpc = _SubsysProbeRPC(existing_nqns=[]) # peer has nothing + assert lc._resolve_namespaced_subsystem(_NsLvol(), rpc, _ProbeNode()) is True + assert rpc.probed == ["nqn.test:lvol:SHARED"] + + +def test_shared_subsystem_is_reused_where_it_already_exists(): + from simplyblock_core.controllers import lvol_controller as lc + rpc = _SubsysProbeRPC(existing_nqns=["nqn.test:lvol:SHARED"]) + assert lc._resolve_namespaced_subsystem(_NsLvol(), rpc, _ProbeNode()) is False + + +def test_dedicated_subsystem_needs_no_probe(): + """A non-namespaced volume always creates its own -- do not spend an RPC.""" + from simplyblock_core.controllers import lvol_controller as lc + rpc = _SubsysProbeRPC(existing_nqns=[]) + assert lc._resolve_namespaced_subsystem(_DedicatedLvol(), rpc, _ProbeNode()) is True + assert rpc.probed == [] + + +def test_probe_failure_falls_back_to_the_record(): + """A probe that raises must not decide: assume the record is right + (attach), which is the pre-existing behaviour.""" + + class _Boom: + def subsystem_get(self, nqn): + raise RuntimeError("rpc down") + + from simplyblock_core.controllers import lvol_controller + assert lvol_controller._resolve_namespaced_subsystem( + _NsLvol(), _Boom(), _ProbeNode()) is False + + +class _RollbackRPC: + def __init__(self, ns): + self.ns = list(ns) + self.removed_ns, self.deletes = [], [] + + def subsystem_get(self, nqn): + return {"nqn": nqn, "namespaces": self.ns} + + def nvmf_subsystem_remove_ns(self, nqn, nsid): + self.removed_ns.append(nsid) + self.ns = [n for n in self.ns if n.get("nsid") != nsid] + return True + + def get_bdevs(self, name): + return [{"name": name}] + + def delete_lvol(self, name, sync=False): + self.deletes.append((name, sync)) + return True, None + + +def test_replica_rollback_clears_its_namespace_and_syncs_the_delete(): + """Case 7, run 20260826_221806: a failed replica leg left BOTH its bdev + and its namespace on the peer, because the rollback issued the + leader-gated async delete a non-leader refuses ("Deleting async lvol on + non-leader lvs"). Four such leftovers accumulated on the peer's shared + subsystem, so the primary -- starting clean -- handed nsid 1 to a + different volume and every later fail-over was rejected. The rollback + must remove the namespace it added and delete the bdev synchronously.""" + from simplyblock_core.controllers import lvol_controller as lc + + class _Lvol: + nqn = "nqn.test:lvol:SHARED" + top_bdev = "LVS_1/LVOL_NEW" + bdev_stack: ClassVar[list] = [ + {"type": "bdev_lvol_clone", "name": "LVS_1/LVOL_NEW"}] + status = "" + def get_id(self): + return "LV_NEW" + def write_to_db(self, *a, **kw): + pass + + rpc = _RollbackRPC([ + {"nsid": 1, "bdev_name": "LVS_1/LVOL_OTHER"}, # someone else's + {"nsid": 5, "bdev_name": "LVS_1/LVOL_NEW"}, # this attempt's + ]) + ok, _msg = lc._fail_after_bdev(_Lvol(), rpc, "boom", is_primary=False) + assert ok is False + assert rpc.removed_ns == [5], "must drop only THIS attempt's namespace" + assert rpc.deletes and all(sync for _n, sync in rpc.deletes), \ + "a replica rollback must use the SYNC delete a non-leader accepts" + + +def test_failover_rollback_covers_every_placed_node_with_ids(): + """Case 7, run 20260826_223631: a peer add failure rolled back only the + PRIMARY, so a third target node's failure left the SECONDARY holding the + namespace. The next sibling's primary then auto-assigned an nsid the peer + had already given away, and its replica add was rejected with + 'wanted nsid=2 ... holds=[(2, )]'. Worse, the rollback + passed the LVol/StorageNode OBJECTS to delete_lvol_from_node(lvol_id, + node_id), whose 'except KeyError: return True' swallowed the mismatch -- + so it reported success while deleting nothing.""" + import inspect + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc._create_target_lvol_clone) + assert "placed_nodes" in src, "rollback must track every node that got the copy" + assert "for node in placed_nodes:" in src + assert "new_lvol.get_id(), node.get_id()" in src, "must pass ids, not records" + assert "delete_lvol_from_node(new_lvol, target_node)" not in src + + # the fail-back clone had the same object-vs-id bug + whole = inspect.getsource(lc) + assert "delete_lvol_from_node(new_lvol," not in whole, \ + "no rollback may hand records to an id-taking function" + + +def test_policy_attach_also_keeps_a_subsystem_on_one_target_node(): + """Run 20260826_233417: add_lvol_ha had the sibling rule but + replication_start -- the path a `volume add --replication-policy` takes -- + picked the target node purely by capacity. One 10-namespace subsystem + ended up on THREE target primaries (nsids 1,2,3,6 / 1,2,4,5,7-10 / + 3-10); no node advertised the whole set and a client saw 0 of the 10. + + Both entry points must consult the sibling rule, and _create_target_lvol_clone + re-checks it at creation time as the last line of defence.""" + import inspect + from simplyblock_core.controllers import lvol_controller as lc + + src = inspect.getsource(lc.replication_start) + assert "_sibling_replication_node" in src, "attaching a policy must honour the shared-subsystem rule too" + assert src.index("_sibling_replication_node") < src.index("_get_next_3_nodes"), "the sibling lookup must precede the capacity-based pick" + + clone = inspect.getsource(lc._create_target_lvol_clone) + assert "_subsystem_home_node" in clone, "the copy must not be built on a node that splits the subsystem" diff --git a/simplyblock_core/test/test_replication_commit.py b/simplyblock_core/test/test_replication_commit.py index 59ab9054b7..1c32bd21f2 100644 --- a/simplyblock_core/test/test_replication_commit.py +++ b/simplyblock_core/test/test_replication_commit.py @@ -133,15 +133,16 @@ def _add_task(cluster_id, node_id, params): def test_commit_enqueues_final_task(patched): - """Commit takes shrink snapshot #1 and enqueues the task; the clone/base - selection moved into the runner (after the shrink rounds), so the enqueued - params carry the shrink state instead of tgt_* composites.""" + """Commit only enqueues the task. + + It takes NO snapshot: the iterative snapshots are the endgame, which does + not start until ordinary replication has caught up. Taking one here left + every volume holding an ageing snapshot while it waited its turn, and the + round that followed measured the wait rather than the transfer.""" result = lvol_controller.replication_commit("LV1") assert result["cutover_task_queued"] is True - - # Shrink snapshot #1 taken at commit time. - assert patched["snap_add"] and patched["snap_add"][0][1] == SnapShot.TYPE_INTERNAL + assert patched["snap_add"] == [], "the endgame takes the first snapshot" # The clone is NOT built at commit time any more: the base must be the # LAST replicated shrink snapshot, which only exists after the runner's # shrink rounds complete. @@ -154,15 +155,20 @@ def test_commit_enqueues_final_task(patched): assert p["tgt_node_id"] == "N_tgt" assert p["operation"] == "replicate" assert p["final_state"] == LVolReplication.STATE_CUTOVER_DONE - assert p["shrink_round"] == 1 - assert p["shrink_snap_id"] == "snap" + assert p["shrink_round"] == 0, "no round is in flight yet" + assert "shrink_snap_id" not in p, "the endgame takes the first snapshot" assert p["shrink_deadline"] > 0 assert "tgt_lvol_composite" not in p, "clone base chosen before shrink completed" assert p["_cluster"] == "CL_src" and p["_node"] == "N_src" -def test_commit_fails_when_shrink_snapshot_fails(patched, monkeypatch): +def test_commit_does_not_depend_on_taking_a_snapshot(patched, monkeypatch): + """Commit is now pure bookkeeping. + + It used to take shrink snapshot #1 and fail if that failed. The snapshot + moved into the endgame, so a full lvstore no longer blocks the enqueue -- + the runner reports it when it actually gets there.""" monkeypatch.setattr(lvol_controller.snapshot_controller, "add", lambda lid, name, snap_type="user": (None, "no space")) result = lvol_controller.replication_commit("LV1") - assert result[0] is False + assert result["cutover_task_queued"] is True diff --git a/simplyblock_core/test/test_replication_final_step.py b/simplyblock_core/test/test_replication_final_step.py index f8b680bf82..6f1bec9b90 100644 --- a/simplyblock_core/test/test_replication_final_step.py +++ b/simplyblock_core/test/test_replication_final_step.py @@ -19,10 +19,11 @@ def get_remote_bdev_name(self): class _RPC: - def __init__(self, node_id, events, final_step_ret=True): + def __init__(self, node_id, events, final_step_ret=True, final_step_state="Done"): self.node_id = node_id self.events = events self._final_step_ret = final_step_ret + self._final_step_state = final_step_state self.final_step_gateways = [] self.ana_groups = [] @@ -42,7 +43,9 @@ def bdev_lvol_transfer_final_step(self, lvol_name, lvol_id, snapshot_name, # which is how the controller-name-instead-of-bdev bug went unnoticed. self.events.append(("final_step", operation, lvol_name, snapshot_name, batch)) self.final_step_gateways.append(gateway) - return ["ok"] if self._final_step_ret else None + if not self._final_step_ret: + return None + return {"transfer_state": self._final_step_state, "offset": 0} def bdev_lvol_add_clone(self, lvol_name, parent): self.events.append(("add_clone", self.node_id, lvol_name, parent)) @@ -60,7 +63,7 @@ def nvmf_subsystem_listener_set_ana_state(self, nqn, ip, port, trtype="TCP", ana class _Node: def __init__(self, uuid, events, ip, lvstore, status=StorageNode.STATUS_ONLINE, - secondary="", tertiary="", final_step_ret=True): + secondary="", tertiary="", final_step_ret=True, final_step_state="Done"): self.uuid = uuid self._events = events self._ip = ip @@ -72,7 +75,7 @@ def __init__(self, uuid, events, ip, lvstore, status=StorageNode.STATUS_ONLINE, self.data_nics = [_Nic(ip)] self.mgmt_ip = ip self.transfer_hublvol = _Hub() - self._rpc = _RPC(uuid, events, final_step_ret) + self._rpc = _RPC(uuid, events, final_step_ret, final_step_state) def get_id(self): return self.uuid @@ -95,6 +98,11 @@ class _Lvol: # volume's ANA group (group id == namespace id). ns_id = 2 + def get_id(self): + # Real LVol has this; the cutover's phase timing labels every line + # with it. + return self.uuid + def _install_nodes(monkeypatch, nodes_by_id): monkeypatch.setattr(rfs, "db", type("DB", (), { @@ -219,6 +227,31 @@ def test_run_cutover_final_step_failure_no_ana(monkeypatch): assert not [e for e in events if e[0] == "add_clone"] +def test_run_cutover_final_step_failed_state_treated_as_failure(monkeypatch): + """Regression: bdev_lvol_transfer_final_step returning {"transfer_state": "Failed"} + (a truthy non-None dict) must be treated as a failure, not silently as success. + Observed 2026-08-31, lvol=1fc7911f: SPDK returned 'Failed' but [IO-RESUME] + fired unconditionally because the old code only checked `if ret is None`.""" + events: list = [] + tgt = _Node("T1", events, "t1", "lvs_tgt") + src = _Node("S1", events, "s1", "lvs_src", + final_step_ret=True, final_step_state="Failed") + _install_nodes(monkeypatch, {"T1": tgt, "S1": src}) + + ok, err = rfs.run_cutover( + src, tgt, _Lvol(), "lvs_tgt/LVOL_1", 42, "lvs_tgt/SNAP1") + + assert ok is False + assert "Failed" in err + # target must never be enabled when the transfer didn't complete + assert not [e for e in events if e[0] == "ana" and e[1].startswith("T") + and e[2] in ("optimized", "non_optimized")] + assert not [e for e in events if e[0] == "add_clone"] + # source must be restored so the client can resume IO on the unfailed side + ana = [(e[1], e[2]) for e in events if e[0] == "ana"] + assert ana[-1] == ("S1", "optimized"), "source must be unfenced after failed freeze" + + def test_final_step_failure_unfences_the_source(monkeypatch): """If the freeze fails after the source was fenced, the source must be restored (it is still the authoritative copy) — never left dark.""" diff --git a/simplyblock_core/test/test_replication_partial_transfer.py b/simplyblock_core/test/test_replication_partial_transfer.py new file mode 100644 index 0000000000..33619a6afc --- /dev/null +++ b/simplyblock_core/test/test_replication_partial_transfer.py @@ -0,0 +1,382 @@ +"""Partial (dirty-bitmap delta) replication transfers. + +The in-memory dirty bitmap has been in the SPDK fork for a long time but never +executed: ``bdev_lvol_transfer`` takes an optional ``allow_partial`` and the +control plane never sent it. It could not be sent, either -- a partial transfer +ships only the ranges written since the previous snapshot, so the destination +has to hold everything else already, and this pipeline landed every transfer in +a FRESH EMPTY volume and only chained it afterwards. A delta into an empty +volume silently loses every cluster it does not cover. + +These tests pin the change that makes it safe: chain the landing volume onto the +destination's copy of the previous snapshot BEFORE the transfer, and ask for a +delta only when that actually succeeded on every online member of the target's +HA pair. +""" +from simplyblock_core.models.snapshot import SnapShot +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import snapshot_replication as sr + + +# -------------------------------------------------------------------------- +# fakes +# -------------------------------------------------------------------------- + +def _mk_snap(uuid, created_at, lvol_uuid, node_id, target="", source="", + status=SnapShot.STATUS_ONLINE, snap_type=None): + lv = LVol() + lv.uuid = lvol_uuid + lv.node_id = node_id + s = SnapShot() + s.uuid = uuid + s.created_at = created_at + s.status = status + s.snap_ref_id = "" + s.target_replicated_snap_uuid = target + s.source_replicated_snap_uuid = source + s.snap_bdev = f"LVS/{uuid}" + s.lvol = lv + if snap_type is not None: + s.snap_type = snap_type + return s + + +class _FakeRPC: + def __init__(self, owner, add_clone_ok=True): + self._owner = owner + self._add_clone_ok = add_clone_ok + + def bdev_lvol_add_clone(self, clone_bdev, snap_bdev): + self._owner.add_clone_calls.append((clone_bdev, snap_bdev)) + return self._add_clone_ok + + +class _Node: + def __init__(self, uuid, secondary_node_id=None, add_clone_ok=True, + status=StorageNode.STATUS_ONLINE): + self._uuid = uuid + self.secondary_node_id = secondary_node_id + self.status = status + self.add_clone_calls = [] + self._add_clone_ok = add_clone_ok + + def get_id(self): + return self._uuid + + def rpc_client(self): + return _FakeRPC(self, self._add_clone_ok) + + +class _FakeDB: + def __init__(self, snaps, nodes): + self._snaps = list(snaps) + self._nodes = dict(nodes) + + def get_snapshots_by_node_id(self, node_id): + return [s for s in self._snaps if s.lvol.node_id == node_id] + + def get_snapshot_by_id(self, uuid): + for s in self._snaps: + if s.uuid == uuid: + return s + raise KeyError(uuid) + + def get_storage_node_by_id(self, uuid): + if uuid not in self._nodes: + raise KeyError(uuid) + return self._nodes[uuid] + + +class _Task: + def __init__(self, **params): + self.function_params = dict(params) + self.writes = 0 + + def write_to_db(self): + self.writes += 1 + return True + + +class _LandingLV: + top_bdev = "LVS/REP_SNAP_2" + + def __init__(self, uuid="REP_LV_1"): + self.uuid = uuid + + def get_id(self): + return self.uuid + + +# -------------------------------------------------------------------------- +# the landing volume becomes a clone of the predecessor when eligible +# -------------------------------------------------------------------------- + +def _eligible_setup(monkeypatch, primary_ok=True, secondary_ok=True, + secondary_online=True): + """A snapshot with a replicated predecessor whose remote copy sits on the + receiving leader -- the case a delta is legitimate in.""" + cur = _mk_snap("SNAP_2", 200, "LV1", "N_SRC") + prev = _mk_snap("SNAP_1", 100, "LV1", "N_SRC", target="T_SNAP_1") + # the predecessor's copy on the destination, on the receiving leader + remote_copy = _mk_snap("T_SNAP_1", 150, "REP_LV", "N_TGT") + + sec = _Node("N_TGT_SEC", add_clone_ok=secondary_ok, + status=(StorageNode.STATUS_ONLINE if secondary_online + else StorageNode.STATUS_OFFLINE)) + leader = _Node("N_TGT", secondary_node_id="N_TGT_SEC", + add_clone_ok=primary_ok) + monkeypatch.setattr(sr, "db", _FakeDB([cur, prev, remote_copy], + {"N_TGT": leader, "N_TGT_SEC": sec})) + return cur, leader, sec + + +def test_landing_volume_is_chained_to_predecessor_when_eligible(monkeypatch): + cur, leader, sec = _eligible_setup(monkeypatch) + task = _Task() + + assert sr._prechain_landing_volume( + task, cur, False, _LandingLV(), leader) is True + + # chained on BOTH members, onto the predecessor's remote copy + assert leader.add_clone_calls == [("LVS/REP_SNAP_2", "LVS/T_SNAP_1")] + assert sec.add_clone_calls == [("LVS/REP_SNAP_2", "LVS/T_SNAP_1")] + # and recorded so the finish step does not add the entry a second time + assert task.function_params["prechained_node_ids"] == ["N_TGT", "N_TGT_SEC"] + + +def test_no_predecessor_falls_back_to_a_fresh_full_transfer(monkeypatch): + """First snapshot of a volume: nothing on the destination to build on.""" + cur = _mk_snap("SNAP_1", 100, "LV1", "N_SRC") + leader = _Node("N_TGT", secondary_node_id="N_TGT_SEC") + sec = _Node("N_TGT_SEC") + monkeypatch.setattr(sr, "db", + _FakeDB([cur], {"N_TGT": leader, "N_TGT_SEC": sec})) + task = _Task() + + assert sr._prechain_landing_volume( + task, cur, False, _LandingLV(), leader) is False + + assert leader.add_clone_calls == [] + assert sec.add_clone_calls == [] + assert "prechained_node_ids" not in task.function_params + + +def test_unreplicated_predecessor_falls_back(monkeypatch): + """A predecessor exists but was never replicated -- no remote copy to + clone from, so this snapshot starts the chain.""" + cur = _mk_snap("SNAP_2", 200, "LV1", "N_SRC") + prev = _mk_snap("SNAP_1", 100, "LV1", "N_SRC") # no target copy + leader = _Node("N_TGT", secondary_node_id="N_TGT_SEC") + sec = _Node("N_TGT_SEC") + monkeypatch.setattr(sr, "db", _FakeDB([cur, prev], + {"N_TGT": leader, "N_TGT_SEC": sec})) + task = _Task() + + assert sr._prechain_landing_volume( + task, cur, False, _LandingLV(), leader) is False + assert leader.add_clone_calls == [] + + +def test_offline_secondary_forces_a_full_transfer(monkeypatch): + """The transfer lands on the leader and the lvstore mirrors it to the + secondary. A secondary that is not there to be chained would read zeros + wherever the delta did not write, so a degraded pair gets a full copy.""" + cur, leader, sec = _eligible_setup(monkeypatch, secondary_online=False) + task = _Task() + + assert sr._prechain_landing_volume( + task, cur, False, _LandingLV(), leader) is False + # nothing was chained at all -- we bail before touching either node + assert leader.add_clone_calls == [] + assert sec.add_clone_calls == [] + + +def test_failed_primary_chain_forces_a_full_transfer(monkeypatch): + cur, leader, sec = _eligible_setup(monkeypatch, primary_ok=False) + task = _Task() + + assert sr._prechain_landing_volume( + task, cur, False, _LandingLV(), leader) is False + # the secondary is not touched once the primary refused + assert sec.add_clone_calls == [] + + +def test_failed_secondary_chain_forces_full_but_records_the_primary(monkeypatch): + """A full transfer into a half-chained volume is still correct, but the + entry that DID land must be remembered or the finish step adds it twice.""" + cur, leader, sec = _eligible_setup(monkeypatch, secondary_ok=False) + task = _Task() + + assert sr._prechain_landing_volume( + task, cur, False, _LandingLV(), leader) is False + assert task.function_params["prechained_node_ids"] == ["N_TGT"] + + +def test_prechain_is_not_repeated_for_a_node_already_chained(monkeypatch): + """A retried attempt must not add the same clone entry a second time.""" + cur, leader, sec = _eligible_setup(monkeypatch) + task = _Task(prechained_node_ids=["N_TGT"]) + + assert sr._prechain_landing_volume( + task, cur, False, _LandingLV(), leader) is True + assert leader.add_clone_calls == [] # already done + assert sec.add_clone_calls == [("LVS/REP_SNAP_2", "LVS/T_SNAP_1")] + + +def test_decision_is_cached_for_the_same_landing_volume(monkeypatch): + """A resumed transfer keeps the mode its first attempt used, without + re-issuing add_clone.""" + cur, leader, sec = _eligible_setup(monkeypatch) + lv = _LandingLV("REP_LV_1") + task = _Task() + + assert sr._partial_transfer_decision(task, cur, False, lv, leader) is True + assert len(leader.add_clone_calls) == 1 + + # second pass over the same landing volume: cached, nothing re-chained + leader.add_clone_calls.clear() + sec.add_clone_calls.clear() + assert sr._partial_transfer_decision(task, cur, False, lv, leader) is True + assert leader.add_clone_calls == [] + assert sec.add_clone_calls == [] + + +def test_a_replaced_landing_volume_re_derives_and_never_inherits_partial(monkeypatch): + """An interrupted attempt can delete a half-created landing volume and + build a fresh, UNCHAINED one. Inheriting the old "partial is fine" verdict + would ship a delta into something holding nothing. + """ + cur = _mk_snap("SNAP_2", 200, "LV1", "N_SRC") + prev = _mk_snap("SNAP_1", 100, "LV1", "N_SRC") # predecessor NOT replicated + leader = _Node("N_TGT", secondary_node_id="N_TGT_SEC") + sec = _Node("N_TGT_SEC") + monkeypatch.setattr(sr, "db", _FakeDB([cur, prev], + {"N_TGT": leader, "N_TGT_SEC": sec})) + # a stale verdict from a landing volume that no longer exists + task = _Task(allow_partial=True, allow_partial_landing="REP_LV_OLD", + prechained_node_ids=["N_TGT", "N_TGT_SEC"]) + + got = sr._partial_transfer_decision( + task, cur, False, _LandingLV("REP_LV_NEW"), leader) + + assert got is False, "a replaced landing volume must not inherit partial" + assert task.function_params["allow_partial"] is False + assert task.function_params["allow_partial_landing"] == "REP_LV_NEW" + # the stale chain record is discarded too + assert task.function_params["prechained_node_ids"] == [] + + +def test_prechained_nodes_are_ignored_for_a_different_landing_volume(): + """Skipping add_clone on the strength of a stale record would leave the + volume unchained.""" + task = _Task(allow_partial=True, allow_partial_landing="REP_LV_1", + prechained_node_ids=["N_TGT", "N_TGT_SEC"]) + + assert sr._prechained_nodes_for(task, _LandingLV("REP_LV_1")) == { + "N_TGT", "N_TGT_SEC"} + assert sr._prechained_nodes_for(task, _LandingLV("REP_LV_2")) == set() + + +# -------------------------------------------------------------------------- +# allow_partial reaches the RPC only when it was asked for +# -------------------------------------------------------------------------- + +def _transfer_params(**kwargs): + """Run the REAL bdev_lvol_transfer body and return the params it would + put on the wire.""" + from simplyblock_core.rpc_client import RPCClient + + class _C(RPCClient): + def __init__(self): + self.sent: tuple = () + + def _request(self, method, params=None, request_timeout=None): + self.sent = (method, params) + return True + + c = _C() + c.bdev_lvol_transfer(name="LVS/SNAP_2", offset=0, batch_size=16, + bdev_name="hub0", operation="replicate", lvol_id=7, + **kwargs) + method, params = c.sent + assert method == "bdev_lvol_transfer" + return params + + +def test_allow_partial_is_never_sent_while_the_fork_workaround_holds(): + """The delta path is DISABLED: bdev_lvol_transfer never emits allow_partial, + so every transfer is a full one regardless of what the caller requests + (commit 52e75afb2, 2026-08-31 — the SPDK fork's fragment write path + corrupts partial transfers). When the fork is fixed and the emission in + rpc_client.bdev_lvol_transfer is re-enabled, restore the opt-in + assertions: allow_partial=True must put the key on the wire, and + False/default must keep it absent.""" + assert "allow_partial" not in _transfer_params(allow_partial=True) + assert "allow_partial" not in _transfer_params(allow_partial=False) + assert "allow_partial" not in _transfer_params() + + +def test_transfer_still_carries_the_routing_fields(): + """allow_partial must not disturb the map-id routing the hub demux needs.""" + p = _transfer_params(allow_partial=True) + assert p["lvol_name"] == "LVS/SNAP_2" + assert p["lvol_id"] == 7 + assert p["gateway"] == "hub0" + assert p["operation"] == "replicate" + assert p["cluster_batch"] == 16 + + +# -------------------------------------------------------------------------- +# retention keeps the COW parent +# -------------------------------------------------------------------------- + +def test_retention_floor_keeps_the_cow_parent_alive(): + """The delta's correctness depends on the PREVIOUS replicated snapshot + still existing on the destination when the next one lands -- it is the COW + parent the landing volume is chained onto. Retention must therefore never + prune down to a single replicated internal snapshot. + """ + from simplyblock_core.models.replication import ReplicationPolicy + + # the flat default and the policy floor both keep a PAIR + assert sr._KEEP_REPLICATED_INTERNAL >= 2 + assert ReplicationPolicy.MIN_KEEP_REPLICATED >= 2 + # a policy cannot be configured below the floor + assert ReplicationPolicy.keep_replicated >= ReplicationPolicy.MIN_KEEP_REPLICATED + + +def test_retention_keep_count_never_drops_below_the_floor(monkeypatch): + """_keep_replicated_for clamps a policy that asks for fewer than a pair.""" + from simplyblock_core.models.replication import ReplicationPolicy + + class _Policy: + keep_replicated = 1 # below the floor + retention_schedule = None + + def get_id(self): + return "P1" + + class _DB: + def get_replication_policy_for_lvol(self, lvol): + return _Policy() + + monkeypatch.setattr(sr, "db", _DB()) + lv = LVol() + lv.uuid = "LV1" + assert sr._keep_replicated_for(lv) == ReplicationPolicy.MIN_KEEP_REPLICATED + + +def test_scheduled_retention_still_keeps_the_newest_pair(): + """The ladder thins history but always_keep_newest protects the COW parent + regardless of how coarse the schedule's finest tier is.""" + from simplyblock_core.snapshot_retention import parse_schedule, select_retained + + tiers = parse_schedule("1h:24h") + now = 1_787_900_000.0 + # a fast cadence: two snapshots a minute apart, far finer than the tier + history = [now - 60, now - 120] + keep = select_retained(history, tiers, now, always_keep_newest=2) + # BOTH survive: the newest is the next delta's base, the one before it is + # the COW parent the current landing volume is chained onto + assert set(keep) == set(history) diff --git a/simplyblock_core/test/test_snapshot_replication_retention.py b/simplyblock_core/test/test_snapshot_replication_retention.py deleted file mode 100644 index cf0c674c53..0000000000 --- a/simplyblock_core/test/test_snapshot_replication_retention.py +++ /dev/null @@ -1,414 +0,0 @@ -"""D2 unit tests for internal-snapshot retention on source + target.""" -from simplyblock_core.models.snapshot import SnapShot -from simplyblock_core.models.lvol_model import LVol -from simplyblock_core.services import snapshot_replication as sr - - -def _mk_snap(uuid, created_at, snap_type, lvol_uuid, node_id, - status=SnapShot.STATUS_ONLINE, target=""): - lv = LVol() - lv.uuid = lvol_uuid - lv.node_id = node_id - s = SnapShot() - s.uuid = uuid - s.created_at = created_at - s.snap_type = snap_type - s.status = status - s.target_replicated_snap_uuid = target - s.lvol = lv - return s - - -class _Clone: - def __init__(self, uuid, cloned_from, status=LVol.STATUS_ONLINE): - self.uuid = uuid - self.cloned_from_snap = cloned_from - self.status = status - - def get_id(self): - return self.uuid - - -class _TargetCopy: - """A replicated snapshot as it exists on the remote cluster. - - ``prev_snap_uuid`` is the chain link retention checks before it deletes a - predecessor: it is only written once bdev_lvol_add_clone + convert succeeded. - """ - - def __init__(self, uuid, prev_snap_uuid="", node_id="TN1"): - self.uuid = uuid - self.prev_snap_uuid = prev_snap_uuid - self.snap_bdev = f"LVS_T/{uuid}" - self.snap_uuid = f"uuid-{uuid}" - lv = LVol() - lv.uuid = "T_LV1" - lv.node_id = node_id - self.lvol = lv - - def get_id(self): - return self.uuid - - -class _FakeNode: - """Target node. Offline by default so the SPDK fallback stays out of the - way unless a test explicitly opts into it.""" - - def __init__(self, status=LVol.STATUS_OFFLINE, bdevs=None): - self.status = status - self._bdevs = bdevs or [] - - def rpc_client(self): - node = self - - class _RPC: - def get_bdevs(self, name): - return [b for b in node._bdevs if b.get("name") == name] - - return _RPC() - - -class _FakeDB: - def __init__(self, source_snaps, existing_uuids, clones=(), chain=None, node=None): - self._source_snaps = source_snaps - self._existing = set(existing_uuids) - self._clones = list(clones) - self._chain = dict(chain or {}) - self._node = node or _FakeNode() - - def get_snapshots_by_node_id(self, node_id): - return [s for s in self._source_snaps if s.lvol.node_id == node_id] - - def get_snapshot_by_id(self, uuid): - if uuid in self._existing: - return _TargetCopy(uuid, self._chain.get(uuid, "")) - raise KeyError(uuid) - - def get_storage_node_by_id(self, node_id): - return self._node - - def get_mini_lvols(self): - return self._clones - - -def _healthy_chain(source_snaps): - """Link each replicated internal target copy onto its predecessor's. - - This is the state a converged replication leaves behind, so it is the - default for the existing cases: they assert on retention, not on chaining. - """ - chain: dict = {} - per_lvol: dict = {} - for s in source_snaps: - if s.snap_type != SnapShot.TYPE_INTERNAL or not s.target_replicated_snap_uuid: - continue - per_lvol.setdefault(s.lvol.get_id(), []).append(s) - for snaps in per_lvol.values(): - snaps.sort(key=lambda s: s.created_at) - for prev, nxt in zip(snaps, snaps[1:]): - chain[nxt.target_replicated_snap_uuid] = prev.target_replicated_snap_uuid - return chain - - -class _FakeSnapCtl: - def __init__(self, db): - self.deleted = [] - self._db = db - - def delete(self, uuid, force_delete=False): - self.deleted.append(uuid) - self._db._existing.discard(uuid) - return True - - -def _patch(monkeypatch, source_snaps, existing_uuids, clones=(), chain=None, node=None): - if chain is None: - chain = _healthy_chain(source_snaps) - db = _FakeDB(source_snaps, existing_uuids, clones, chain, node) - snapctl = _FakeSnapCtl(db) - monkeypatch.setattr(sr, "db", db) - monkeypatch.setattr(sr, "snapshot_controller", snapctl) - return snapctl - - -def test_prunes_older_internal_keeps_newest_and_users(monkeypatch): - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_old"), - _mk_snap("user_mid", 150, SnapShot.TYPE_USER, "LV1", "N1", target="T_user"), - _mk_snap("int_mid", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_mid"), - _mk_snap("int_new", 300, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - snapctl = _patch(monkeypatch, snaps, {"T_old", "T_user", "T_mid", "T_new"}) - - sr._prune_internal_snapshots(source_lvol) - - # Target copy deleted before the source snapshot; the newest PAIR is kept - # so an arriving snapshot always has a predecessor to chain onto. - assert snapctl.deleted == ["T_old", "int_old"] - for kept in ("int_mid", "T_mid", "int_new", "T_new", "user_mid", "T_user"): - assert kept not in snapctl.deleted - - -def test_single_internal_not_pruned(monkeypatch): - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [_mk_snap("int_only", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_only")] - snapctl = _patch(monkeypatch, snaps, {"T_only"}) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == [] - - -def test_unreplicated_internal_ignored(monkeypatch): - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - # Newest internal not yet replicated (no target) -> excluded; the only - # replicated internal is the single oldest, so nothing is pruned. - snaps = [ - _mk_snap("int_repl", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_repl"), - _mk_snap("int_pending", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target=""), - ] - snapctl = _patch(monkeypatch, snaps, {"T_repl"}) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == [] - - -def test_missing_target_still_cleans_source(monkeypatch): - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_gone"), - _mk_snap("int_mid", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_mid"), - _mk_snap("int_new", 300, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - # T_gone already deleted on target -> only source snapshot is cleaned up. - snapctl = _patch(monkeypatch, snaps, {"T_mid", "T_new"}) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == ["int_old"] - - -def test_other_lvol_snapshots_untouched(monkeypatch): - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_old"), - _mk_snap("int_mid", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_mid"), - _mk_snap("int_new", 300, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - # Different lvol on same node — must never be pruned. - _mk_snap("other_old", 50, SnapShot.TYPE_INTERNAL, "LV2", "N1", target="TO_old"), - _mk_snap("other_new", 250, SnapShot.TYPE_INTERNAL, "LV2", "N1", target="TO_new"), - ] - snapctl = _patch(monkeypatch, snaps, {"T_old", "T_mid", "T_new", "TO_old", "TO_new"}) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == ["T_old", "int_old"] - - -def test_never_prunes_snapshot_a_failed_over_volume_is_cloned_from(monkeypatch): - """Root cause of the all-zeros DR fail-over (labs 2026-08-10/11). - - Fail-over clones the volume from the last replicated TARGET snapshot; the - prune, keyed only on the SOURCE snapshot age, then deleted that target copy. - The delete reaches SPDK as bdev_lvol_delete(sync=False) and frees the blocks - immediately, so no DB-level guard downstream can save the clone. Retention - must skip a target snapshot with a live dependent clone. - """ - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_old"), - _mk_snap("int_new", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - # A failed-over volume lives on the OLD target snapshot. - snapctl = _patch(monkeypatch, snaps, {"T_old", "T_new"}, - clones=[_Clone("FO_VOL", "T_old")]) - - sr._prune_internal_snapshots(source_lvol) - - assert "T_old" not in snapctl.deleted, ( - "pruned the target snapshot a failed-over volume is cloned from — " - "its blocks are freed by SPDK immediately (sync=False), the volume " - "reads zeros from then on") - # The source-side copy must survive too (it pairs with the kept target). - assert "int_old" not in snapctl.deleted - - -def test_in_deletion_clone_does_not_pin_the_snapshot(monkeypatch): - """A clone that is itself going away must not block retention forever.""" - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_old"), - _mk_snap("int_mid", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_mid"), - _mk_snap("int_new", 300, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - snapctl = _patch(monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, - clones=[_Clone("DYING", "T_old", status=LVol.STATUS_IN_DELETION)]) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == ["T_old", "int_old"] - - -def test_require_lvs_leader_gate(monkeypatch): - """Convert on a non-leader returns success WITHOUT persisting (silent - conversion error) — leadership must be checked BEFORE the operation and a - non-leader must fail-and-retry, never proceed.""" - import simplyblock_core.controllers.lvol_controller as lc - - class _N: - def get_id(self): - return "N1" - - monkeypatch.setattr(lc, "is_node_leader", lambda node, lvs: False) - assert sr._require_lvs_leader(_N(), "LVS_1", "convert") is False - - monkeypatch.setattr(lc, "is_node_leader", lambda node, lvs: True) - assert sr._require_lvs_leader(_N(), "LVS_1", "convert") is True - - -def test_newest_pair_is_kept_so_arrivals_have_a_chain_parent(monkeypatch): - """Chain continuity: a replicated snapshot holds only its own clusters, and - deleting one swap-merges its segments into the successor CHAINED to it. - Keeping just the newest pruned the predecessor the instant a replication - finished, so the next arrival had nothing to chain onto and kept only its - delta — the target then held the last delta over holes (labs run 15 vs 19, - same case passing then failing on timing alone).""" - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_prev", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_prev"), - _mk_snap("int_new", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - snapctl = _patch(monkeypatch, snaps, {"T_prev", "T_new"}) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == [], ( - "pruned the predecessor the next arrival must chain onto") - - -def test_defers_prune_until_the_successor_is_actually_chained(monkeypatch): - """The count cushion is not the precondition — the chain link is. - - Keeping the newest N only widens the window in which chaining is expected to - have happened. If it lagged or failed for one snapshot while newer ones kept - arriving, the predecessor was still pruned, and because the delete reaches - SPDK as bdev_lvol_delete(sync=False) its segments were freed instead of - swap-merged into the successor. The target then holds the newest delta over - holes and a fail-over clone reads zeros (labs 2026-08-10..17). Retention must - verify the link and defer while it is absent. - """ - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_old"), - _mk_snap("int_mid", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_mid"), - _mk_snap("int_new", 300, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - # T_mid arrived but was never chained onto T_old. - snapctl = _patch(monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, chain={}) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == [], ( - "pruned a predecessor whose successor is not chained onto it — SPDK frees " - "the blocks immediately, so those segments are lost rather than merged") - - -def test_prunes_once_the_chain_is_established(monkeypatch): - """The deferral must release as soon as chaining catches up (no livelock).""" - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_old"), - _mk_snap("int_mid", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_mid"), - _mk_snap("int_new", 300, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - snapctl = _patch(monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, - chain={"T_mid": "T_old", "T_new": "T_mid"}) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == ["T_old", "int_old"] - - -def test_spdk_verdict_releases_a_missing_db_link(monkeypatch): - """A missing link must not pin the pair for ever. - - The link write is best-effort, and snapshots replicated before chaining was - implemented have none at all. SPDK is the real authority, so when the DB has - no link we ask the target node before giving up — otherwise retention would - never release those snapshots and both chains would grow without bound. - """ - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_old"), - _mk_snap("int_mid", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_mid"), - _mk_snap("int_new", 300, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - # SPDK reports T_mid as a clone whose base is T_old, while the DB link is absent. - node = _FakeNode(status=LVol.STATUS_ONLINE, bdevs=[{ - "name": "LVS_T/T_mid", - "driver_specific": {"lvol": {"clone": True, "base_snapshot": "LVS_T/T_old"}}, - }]) - snapctl = _patch(monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, - chain={}, node=node) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == ["T_old", "int_old"] - - -def test_unchained_in_spdk_is_not_pruned_even_when_node_is_reachable(monkeypatch): - """An online target that reports a standalone blob must still defer.""" - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_old"), - _mk_snap("int_mid", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_mid"), - _mk_snap("int_new", 300, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - node = _FakeNode(status=LVol.STATUS_ONLINE, bdevs=[{ - "name": "LVS_T/T_mid", - "driver_specific": {"lvol": {"clone": False, "base_snapshot": None}}, - }]) - snapctl = _patch(monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, - chain={}, node=node) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == [] diff --git a/simplyblock_core/test/test_snapshot_retention.py b/simplyblock_core/test/test_snapshot_retention.py new file mode 100644 index 0000000000..b9e51e2b2c --- /dev/null +++ b/simplyblock_core/test/test_snapshot_retention.py @@ -0,0 +1,142 @@ +"""Tiered snapshot retention: parsing and selection are pure functions.""" +import pytest + +from simplyblock_core.snapshot_retention import ( + RetentionScheduleError, RetentionTier, describe, horizon_sec, + parse_schedule, select_retained, +) + +HOUR = 3600 +DAY = 86400 +#: the example from the feature request +SPEC = "15m:2h,1h:11h,1d:7d" + + +def test_parse_the_requested_schedule(): + assert parse_schedule(SPEC) == [ + RetentionTier(15 * 60, 2 * HOUR), + RetentionTier(HOUR, 11 * HOUR), + RetentionTier(DAY, 7 * DAY), + ] + assert horizon_sec(parse_schedule(SPEC)) == 2 * HOUR + 11 * HOUR + 7 * DAY + + +def test_empty_schedule_is_no_tiers(): + assert parse_schedule("") == [] + assert parse_schedule(" ") == [] + + +def test_schedule_is_order_independent(): + assert parse_schedule("1d:7d,15m:2h,1h:11h") == parse_schedule(SPEC) + + +@pytest.mark.parametrize("bad", [ + "15m", "15m:2h:3d", "15x:2h", "0m:2h", "-5m:2h", "2h:15m", "abc", +]) +def test_bad_schedules_are_rejected(bad): + with pytest.raises(RetentionScheduleError): + parse_schedule(bad) + + +def test_round_trip_describe(): + assert describe(parse_schedule(SPEC)) == SPEC + + +def test_one_snapshot_kept_per_bucket_newest_wins(): + tiers = parse_schedule("15m:1h") + now = 10_000_000.0 + # first four are all inside bucket 0 (ages 60..800s < 900s), the last + # one is in bucket 1 (age 1000s) + snaps = [now - 60, now - 120, now - 200, now - 800, now - 1000] + keep = select_retained(snaps, tiers, now) + assert now - 60 in keep # newest of bucket 0 + for shadowed in (now - 120, now - 200, now - 800): + assert shadowed not in keep + assert now - 1000 in keep # bucket 1 keeps its own + + +def test_history_thins_out_with_age(): + """A minute-cadence stream over 8 days collapses to roughly + 8 quarter-hours + 11 hours + 7 days, not thousands of snapshots.""" + tiers = parse_schedule(SPEC) + now = 1_000_000_000.0 + snaps = [now - 60 * i for i in range(8 * 24 * 60)] # every minute, 8 days + keep = select_retained(snaps, tiers, now) + assert 20 <= len(keep) <= 30, len(keep) + # dense recent history + assert max(keep) == now + # and genuine multi-day depth + assert min(keep) < now - 6 * DAY + + +def test_snapshots_older_than_the_horizon_are_dropped(): + tiers = parse_schedule("15m:2h") + now = 5_000_000.0 + keep = select_retained([now - 60, now - 10 * HOUR], tiers, now) + assert now - 60 in keep + assert now - 10 * HOUR not in keep + + +def test_always_keep_newest_survives_the_schedule(): + """Replication chains the arriving delta onto its predecessor, so the + newest pair must never be pruned no matter what the schedule says.""" + tiers = parse_schedule("1d:7d") + now = 2_000_000.0 + snaps = [now - 10, now - 20, now - 30] # all inside one daily bucket + keep = select_retained(snaps, tiers, now, always_keep_newest=2) + assert now - 10 in keep and now - 20 in keep + + +def test_no_tiers_keeps_only_the_protected_newest(): + now = 100.0 + keep = select_retained([now - 1, now - 2, now - 3], [], now, always_keep_newest=2) + assert keep == {now - 1, now - 2} + + +def test_selection_is_stable_across_repeated_calls(): + """Bucketing on absolute age must not drift, or a snapshot kept on one + pass gets pruned on the next and the history develops holes.""" + tiers = parse_schedule(SPEC) + now = 3_000_000.0 + snaps = [now - 60 * i for i in range(600)] + assert select_retained(snaps, tiers, now) == select_retained(snaps, tiers, now) + + +# --- wiring: the schedule and the fail-over generation ---------------------- + +def test_prune_consults_the_schedule_before_the_flat_count(): + """_prune_internal_snapshots must ask the policy's schedule which + snapshots survive; the flat keep-count is only the fallback.""" + import inspect + from simplyblock_core.services import snapshot_replication as sr + src = inspect.getsource(sr._prune_internal_snapshots) + assert "_retention_schedule_for" in src + assert "select_retained" in src + # the newest `keep` stay protected even under a schedule + assert "always_keep_newest=keep" in src + + +def test_invalid_schedule_falls_back_instead_of_crashing_the_runner(): + import inspect + from simplyblock_core.services import snapshot_replication as sr + src = inspect.getsource(sr._retention_schedule_for) + assert "RetentionScheduleError" in src and "return []" in src + + +def test_policy_rejects_an_invalid_schedule_at_ingress(): + import inspect + from simplyblock_core.controllers import replication_policy_controller as rpc + src = inspect.getsource(rpc.add_policy) + assert "parse_schedule" in src, "the policy must validate the schedule when set" + assert src.index("parse_schedule") < src.index("policy = ReplicationPolicy()") + + +def test_failover_generation_walks_back_through_history(): + """generation=0 is the newest point-in-time; higher values step back, and + asking for more generations than exist is an error, not a silent newest.""" + import inspect + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc._last_replicated_target_snapshot) + assert "generation" in src + assert "snaps[generation:]" in src + assert "only" in src and "exist" in src # explicit out-of-range error diff --git a/simplyblock_core/test/test_target_nsid_claim.py b/simplyblock_core/test/test_target_nsid_claim.py new file mode 100644 index 0000000000..23fa109f3c --- /dev/null +++ b/simplyblock_core/test/test_target_nsid_claim.py @@ -0,0 +1,353 @@ +"""A fail-over copy's nsid is claimed by the CONTROL PLANE, not auto-assigned. + +Soak case 7 failed six times on the same shape. Twenty volumes share two +subsystems; their fail-over copies must all land in ONE subsystem per group on +the target cluster. When the group is split across two target primaries, each +primary counts only the namespaces it holds itself, hands out an nsid the HA +PEER already gave to a sibling, and the peer's add_ns is rejected: + + Namespace add rejected ... [node a3220281 wanted nsid=1 max_namespaces=10 + holds=[(1, 531e4060-...), (2, b63ed6f1-...), ... (7, ebdd3e0d-...)]] + +Two independent defects produced that split, and both are covered here: + + * ``_resolve_lvol_subsystem`` is only ADVISORY -- ``claim_lvol_ns_slot``'s + transaction may join the lvol to a different subsystem than the one the + target node was picked against, so the pick is re-derived afterwards. + * the nsid itself is claimed from the UNION of what every node of the target + HA set holds, so even a split group cannot collide. +""" +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core.controllers import lvol_controller +from simplyblock_core.models.storage_node import StorageNode + +NQN = "nqn.2023-02.io.simplyblock:cl:lvol:shared" + + +def _lvol(uuid, nqn=NQN, repl_node="", max_ns=10): + lv = MagicMock(name="lvol-" + uuid) + lv.uuid = uuid + lv.nqn = nqn + lv.get_id.return_value = uuid + lv.replication_node_id = repl_node + lv.max_namespace_per_subsys = max_ns + lv.namespaced = True + lv.lvol_name = uuid + return lv + + +def _node(node_id, namespaces=None, exists=True, status=StorageNode.STATUS_ONLINE, + max_ns=10, raises=False): + n = MagicMock(name="node-" + node_id) + n.get_id.return_value = node_id + n.status = status + n.secondary_node_id = "" + n.tertiary_node_id = "" + rpc = MagicMock() + if raises: + rpc.subsystem_get.side_effect = RuntimeError("connection refused") + else: + rpc.subsystem_get.return_value = ( + {"nqn": NQN, "max_namespaces": max_ns, + "namespaces": namespaces or []} if exists else None) + n.rpc_client.return_value = rpc + return n + + +def _db(nodes): + db = MagicMock() + by_id = {n.get_id.return_value: n for n in nodes} + + def _get(node_id): + if node_id not in by_id: + raise KeyError(node_id) + return by_id[node_id] + + db.get_storage_node_by_id.side_effect = _get + return db + + +class TestClaimTargetNsid(unittest.TestCase): + + def test_absent_subsystem_leaves_auto_assignment(self): + primary = _node("P", exists=False) + nsid = lvol_controller._claim_target_nsid( + _db([primary]), _lvol("copy-1"), primary) + self.assertEqual(nsid, 0, "nothing to collide with -- SPDK assigns") + + def test_claims_across_the_union_not_just_the_primary(self): + """The exact case-7 shape: peer holds 1..7, primary holds none.""" + peer = _node("Q", namespaces=[ + {"nsid": i, "uuid": "sibling-%d" % i} for i in range(1, 8)]) + primary = _node("P", namespaces=[]) + primary.secondary_node_id = "Q" + nsid = lvol_controller._claim_target_nsid( + _db([primary, peer]), _lvol("copy-8"), primary) + self.assertEqual( + nsid, 8, + "the primary holds nothing but the peer holds 1..7; auto-assigning " + "1 here is what the peer rejected") + + def test_reuses_the_lowest_free_slot(self): + peer = _node("Q", namespaces=[{"nsid": 1, "uuid": "a"}, + {"nsid": 3, "uuid": "b"}]) + primary = _node("P", namespaces=[{"nsid": 1, "uuid": "a"}]) + primary.secondary_node_id = "Q" + self.assertEqual( + lvol_controller._claim_target_nsid( + _db([primary, peer]), _lvol("copy-x"), primary), 2) + + def test_own_namespace_is_not_an_obstacle(self): + """A retry evicts this copy's own namespace, so its slot is free.""" + primary = _node("P", namespaces=[{"nsid": 1, "uuid": "copy-me"}, + {"nsid": 2, "uuid": "other"}]) + self.assertEqual( + lvol_controller._claim_target_nsid( + _db([primary]), _lvol("copy-me"), primary), 1) + + def test_offline_peer_does_not_shrink_the_claim(self): + """An offline peer holds no namespaces the add can collide with.""" + peer = _node("Q", namespaces=[{"nsid": 1, "uuid": "a"}], + status=StorageNode.STATUS_OFFLINE) + primary = _node("P", namespaces=[{"nsid": 1, "uuid": "a"}, + {"nsid": 2, "uuid": "b"}]) + primary.secondary_node_id = "Q" + self.assertEqual( + lvol_controller._claim_target_nsid( + _db([primary, peer]), _lvol("copy-y"), primary), 3) + + def test_unreadable_node_falls_back_rather_than_guessing(self): + peer = _node("Q", raises=True) + primary = _node("P", namespaces=[{"nsid": 1, "uuid": "a"}]) + primary.secondary_node_id = "Q" + self.assertEqual( + lvol_controller._claim_target_nsid( + _db([primary, peer]), _lvol("copy-z"), primary), 0, + "claiming against a partial view is the bug, not the fix") + + def test_full_subsystem_defers_to_the_node_diagnostic(self): + primary = _node("P", max_ns=3, namespaces=[ + {"nsid": i, "uuid": "s%d" % i} for i in range(1, 4)]) + self.assertEqual( + lvol_controller._claim_target_nsid( + _db([primary]), _lvol("copy-full"), primary), 0) + + +class TestSiblingAffinity(unittest.TestCase): + + def test_sibling_in_the_same_subsystem_sets_the_target(self): + cl = MagicMock() + lv = _lvol("new") + siblings = [_lvol("old", repl_node="T1"), + _lvol("other", nqn="nqn:x", repl_node="T2")] + self.assertEqual( + lvol_controller._sibling_replication_node(lv, cl, siblings), "T1") + + def test_other_subsystems_are_not_siblings(self): + cl = MagicMock() + lv = _lvol("new") + others = [_lvol("elsewhere", nqn="nqn:other", repl_node="T2")] + self.assertEqual( + lvol_controller._sibling_replication_node(lv, cl, others), "") + + def test_non_namespaced_lvol_has_no_affinity(self): + cl = MagicMock() + lv = _lvol("new", max_ns=1) + lv.namespaced = False + self.assertEqual( + lvol_controller._sibling_replication_node( + lv, cl, [_lvol("old", repl_node="T1")]), "") + + +class TestRealignAfterClaim(unittest.TestCase): + """The advisory pick and the transaction can disagree about the subsystem.""" + + def setUp(self): + self.cl = MagicMock() + patcher = patch.object(lvol_controller, "DBController") + self.db = patcher.start().return_value + self.addCleanup(patcher.stop) + + def test_moves_to_the_siblings_target_when_the_claim_rehomed_it(self): + lv = _lvol("new", repl_node="T2") + self.db.get_lvols.return_value = [_lvol("old", repl_node="T1")] + lvol_controller._realign_replication_node_after_claim(lv, self.cl) + self.assertEqual(lv.replication_node_id, "T1", + "a volume the transaction put in this subsystem must " + "replicate where the subsystem already replicates") + lv.write_to_db.assert_called_once() + + def test_no_write_when_the_pick_already_agrees(self): + lv = _lvol("new", repl_node="T1") + self.db.get_lvols.return_value = [_lvol("old", repl_node="T1")] + lvol_controller._realign_replication_node_after_claim(lv, self.cl) + self.assertEqual(lv.replication_node_id, "T1") + lv.write_to_db.assert_not_called() + + def test_non_replicated_lvol_is_untouched(self): + lv = _lvol("new", repl_node="") + self.db.get_lvols.return_value = [_lvol("old", repl_node="T1")] + lvol_controller._realign_replication_node_after_claim(lv, self.cl) + self.assertEqual(lv.replication_node_id, "") + lv.write_to_db.assert_not_called() + + +if __name__ == "__main__": + unittest.main() + + +class TestSubsystemHomeNode(unittest.TestCase): + """One shared subsystem lives on exactly one primary per cluster.""" + + def _db_with(self, lvols, nodes): + db = MagicMock() + db.get_lvols.return_value = lvols + by_id = {n.get_id.return_value: n for n in nodes} + + def _get(node_id): + if node_id not in by_id: + raise KeyError(node_id) + return by_id[node_id] + + db.get_storage_node_by_id.side_effect = _get + return db + + def _copy(self, uuid, node_id, nqn=NQN, status="online", deleted=False): + lv = MagicMock() + lv.uuid = uuid + lv.get_id.return_value = uuid + lv.nqn = nqn + lv.node_id = node_id + lv.status = status + lv.deleted = deleted + return lv + + def _node(self, node_id, cluster_id): + n = MagicMock() + n.get_id.return_value = node_id + n.cluster_id = cluster_id + return n + + def test_finds_the_node_already_hosting_the_subsystem(self): + db = self._db_with( + [self._copy("c1", "N1")], + [self._node("N1", "CL_tgt")]) + self.assertEqual( + lvol_controller._subsystem_home_node(db, NQN, "CL_tgt"), "N1") + + def test_ignores_copies_in_a_different_cluster(self): + db = self._db_with( + [self._copy("c1", "N_src")], + [self._node("N_src", "CL_src")]) + self.assertEqual( + lvol_controller._subsystem_home_node(db, NQN, "CL_tgt"), "") + + def test_ignores_other_subsystems(self): + db = self._db_with( + [self._copy("c1", "N1", nqn="nqn:other")], + [self._node("N1", "CL_tgt")]) + self.assertEqual( + lvol_controller._subsystem_home_node(db, NQN, "CL_tgt"), "") + + def test_a_volume_being_deleted_does_not_own_the_subsystem(self): + from simplyblock_core.models.lvol_model import LVol + db = self._db_with( + [self._copy("c1", "N1", status=LVol.STATUS_IN_DELETION)], + [self._node("N1", "CL_tgt")]) + self.assertEqual( + lvol_controller._subsystem_home_node(db, NQN, "CL_tgt"), "") + + +class TestRetireSupersededOriginal(unittest.TestCase): + """A fail-back must free the slot the original still holds.""" + + def setUp(self): + from simplyblock_core.models.lvol_model import LVol + self.LVol = LVol + patcher = patch.object(lvol_controller, "delete_lvol") + self.delete = patcher.start() + self.addCleanup(patcher.stop) + + def _rep(self, source, target): + r = MagicMock() + r.source_lvol = source + r.target_lvol = target + return r + + def _vol(self, uuid, node_id="N1", status="online"): + lv = MagicMock() + lv.uuid = uuid + lv.get_id.return_value = uuid + lv.node_id = node_id + lv.status = status + lv.nqn = NQN + lv.ns_id = 3 + return lv + + def _db(self, reps, lvols, node_cluster): + db = MagicMock() + db.get_lvol_replication_objects.return_value = reps + by_id = {lv.get_id.return_value: lv for lv in lvols} + + def _get_lvol(uid): + if uid not in by_id: + raise KeyError(uid) + return by_id[uid] + + def _get_node(nid): + if nid not in node_cluster: + raise KeyError(nid) + n = MagicMock() + n.cluster_id = node_cluster[nid] + return n + + db.get_lvol_by_id.side_effect = _get_lvol + db.get_storage_node_by_id.side_effect = _get_node + return db + + def test_first_failover_deletes_nothing(self): + """No record names this volume as a copy, so it IS the original.""" + lvol = self._vol("orig") + db = self._db([], [lvol], {"N1": "CL_src"}) + ok, err = lvol_controller._retire_superseded_original(db, lvol, "CL_tgt") + self.assertTrue(ok) + self.assertEqual(err, "") + self.delete.assert_not_called() + + def test_failback_deletes_the_original_on_the_destination(self): + original, copy = self._vol("orig"), self._vol("copy", node_id="N2") + db = self._db([self._rep(original, copy)], [original, copy], + {"N1": "CL_src", "N2": "CL_tgt"}) + ok, err = lvol_controller._retire_superseded_original(db, copy, "CL_src") + self.assertTrue(ok, err) + self.delete.assert_called_once_with(original) + + def test_an_original_on_another_cluster_is_not_in_the_way(self): + original, copy = self._vol("orig"), self._vol("copy", node_id="N2") + db = self._db([self._rep(original, copy)], [original, copy], + {"N1": "CL_src", "N2": "CL_tgt"}) + ok, _ = lvol_controller._retire_superseded_original(db, copy, "CL_third") + self.assertTrue(ok) + self.delete.assert_not_called() + + def test_already_deleting_original_is_left_alone(self): + original = self._vol("orig", status=self.LVol.STATUS_IN_DELETION) + copy = self._vol("copy", node_id="N2") + db = self._db([self._rep(original, copy)], [original, copy], + {"N1": "CL_src", "N2": "CL_tgt"}) + ok, _ = lvol_controller._retire_superseded_original(db, copy, "CL_src") + self.assertTrue(ok) + self.delete.assert_not_called() + + def test_a_failed_delete_aborts_the_failback_with_a_clear_reason(self): + original, copy = self._vol("orig"), self._vol("copy", node_id="N2") + db = self._db([self._rep(original, copy)], [original, copy], + {"N1": "CL_src", "N2": "CL_tgt"}) + self.delete.side_effect = RuntimeError("lvstore restart in progress") + ok, err = lvol_controller._retire_superseded_original(db, copy, "CL_src") + self.assertFalse(ok) + self.assertIn("lvstore restart in progress", err) + self.assertIn("namespace", err, + "the message must say WHY the fail-back cannot proceed") diff --git a/simplyblock_core/test/test_tasks_runner_replication_final.py b/simplyblock_core/test/test_tasks_runner_replication_final.py index 61e53bac69..674a20c09d 100644 --- a/simplyblock_core/test/test_tasks_runner_replication_final.py +++ b/simplyblock_core/test/test_tasks_runner_replication_final.py @@ -25,6 +25,12 @@ def _task(**params): "operation": "replicate", "replication_id": "REP1", "final_state": LVolReplication.STATE_CUTOVER_DONE, + # These tests cover the FREEZE, so the task starts past the endgame + # entry: the lvstore is already claimed and the convergence rounds are + # behind it (round 0 = no round in flight). + "cutover_lvs": "lvs_src", + "shrink_snap_id": "S_endgame", + "shrink_round": 0, } t.function_params.update(params) return t @@ -91,7 +97,13 @@ def test_happy_path_marks_done_and_updates_state(monkeypatch): assert rep.state == LVolReplication.STATE_CUTOVER_DONE -def test_failure_suspends_and_retries(monkeypatch): +def test_failure_enters_hub_cooldown_without_burning_a_retry(monkeypatch): + """With the clone already prepared (tgt_lvol_composite set), a run_cutover + failure is treated as likely connectivity trouble: the task suspends + behind a cooldown and task.retry stays intact for the first + REPL_CUTOVER_MAX_HUB_ATTEMPTS attempts — burning retries at the poll + interval would exhaust the ceiling long before a restarting node + recovers.""" rep = LVolReplication() rep.cutover_proceed = True nodes = {"S1": _node("S1"), "T1": _node("T1")} @@ -102,8 +114,30 @@ def test_failure_suspends_and_retries(monkeypatch): assert res is False assert task.status == JobSchedule.STATUS_SUSPENDED - assert task.retry == 1 assert task.function_result == "boom" + assert task.retry == 0, "a transient hub attempt must not burn task.retry" + assert task.function_params["cutover_hub_attempts"] == 1 + assert task.function_params["cutover_retry_after"] > 0 + + +def test_failure_burns_a_retry_once_hub_attempts_are_exhausted(monkeypatch): + """Past the hub-attempt cap with the target node online, the failure is + real: the cooldown state resets and one retry is burned, so the ceiling + in task_runner can eventually end a cutover that keeps failing.""" + from simplyblock_core import constants + rep = LVolReplication() + rep.cutover_proceed = True + nodes = {"S1": _node("S1"), "T1": _node("T1")} + _install(monkeypatch, nodes, rep, (False, "boom")) + + task = _task(cutover_hub_attempts=constants.REPL_CUTOVER_MAX_HUB_ATTEMPTS) + res = runner.task_runner(task) + + assert res is False + assert task.status == JobSchedule.STATUS_SUSPENDED + assert task.retry == 1 + assert "cutover_hub_attempts" not in task.function_params + assert "cutover_retry_after" not in task.function_params def test_max_retry_marks_done_without_cutover(monkeypatch): @@ -180,6 +214,16 @@ def _mk(monkeypatch, snaps, params): def test_shrink_waits_until_replicated(monkeypatch): + """An unreplicated round yields the pass instead of failing. + + It now POLLS for a short window first (a round that lands just after the + pass is handed back would otherwise cost a full TASK_EXEC_INTERVAL_SEC of + writes in the next round), so squeeze the inline window to nothing to keep + the test instant. + """ + from simplyblock_core import constants + monkeypatch.setattr(constants, "REPL_CUTOVER_MIN_INLINE_SEC", 0) + monkeypatch.setattr(constants, "REPL_CUTOVER_CONVERGE_BUDGET_SEC", 0) runner, task = _mk(monkeypatch, {"S1": _ShrinkSnap(replicated=False)}, {"shrink_round": 1, "shrink_snap_id": "S1", "shrink_deadline": 2**60}) @@ -188,41 +232,91 @@ def test_shrink_waits_until_replicated(monkeypatch): assert "waiting" in task.function_result +def test_a_fast_round_converges_instead_of_taking_another(monkeypatch): + """The criterion is transfer TIME, not a round count. + + A round that replicated within REPL_CUTOVER_CONVERGE_TARGET_SEC means the + next such window -- the one the freeze copies -- is about as small, so the + cutover starts immediately rather than taking more snapshots for nothing. + """ + runner, task = _mk(monkeypatch, {"S1": _ShrinkSnap(replicated=True)}, + {"shrink_round": 1, "shrink_snap_id": "S1", + "shrink_deadline": 2**60, + # holding the lvstore == the endgame; converging in the + # open asks for it instead of freezing + "cutover_lvs": "LVS_1", + "shrink_started_at": __import__("time").time()}) + taken = [] + + def _add(lid, name, snap_type="user"): + taken.append((lid, snap_type)) + return "S2", None + import simplyblock_core.controllers.snapshot_controller as sc + monkeypatch.setattr(sc, "add", _add) + + done, err = runner._shrink_step(task, _ShrinkLvol()) + assert (done, err) == (True, None) + assert taken == [], "a converged round must not take another snapshot" + assert "converged" in task.function_result + + def test_shrink_takes_next_snapshot_immediately(monkeypatch): + """A SLOW round is followed by the next one straight away. + + The delta the next round carries is only what was written during this + round's transfer -- which is the whole mechanism by which the freeze gets + shorter. + """ + import time as _time + from simplyblock_core import constants runner, task = _mk(monkeypatch, {"S1": _ShrinkSnap(replicated=True)}, {"shrink_round": 1, "shrink_snap_id": "S1", - "shrink_deadline": 2**60}) + "shrink_deadline": 2**60, + # started long enough ago to be well over the target + "shrink_started_at": _time.time() - 60}) taken = [] def _add(lid, name, snap_type="user"): taken.append((lid, snap_type)) + # the second round reports as still in flight, so the loop yields + runner.db._snaps["S2"] = _ShrinkSnap(replicated=False) return "S2", None import simplyblock_core.controllers.snapshot_controller as sc monkeypatch.setattr(sc, "add", _add) + monkeypatch.setattr(constants, "REPL_CUTOVER_MIN_INLINE_SEC", 0) + monkeypatch.setattr(constants, "REPL_CUTOVER_CONVERGE_BUDGET_SEC", 0) done, err = runner._shrink_step(task, _ShrinkLvol()) assert (done, err) == (False, None) - assert taken == [("LV1", "internal")] or taken[0][0] == "LV1" + assert taken and taken[0][0] == "LV1" assert task.function_params["shrink_round"] == 2 assert task.function_params["shrink_snap_id"] == "S2" -def test_shrink_completes_after_last_round(monkeypatch): - runner, task = _mk(monkeypatch, {"S2": _ShrinkSnap(replicated=True)}, - {"shrink_round": runner_rounds(), "shrink_snap_id": "S2", - "shrink_deadline": 2**60}) +def test_shrink_hands_over_when_it_cannot_converge(monkeypatch): + """Written faster than it replicates: freeze anyway, but say so.""" + import time as _time + from simplyblock_core import constants + monkeypatch.setattr(constants, "REPL_CUTOVER_MAX_SHRINK_ROUNDS", 3) + runner, task = _mk(monkeypatch, {"S1": _ShrinkSnap(replicated=True)}, + {"shrink_round": 3, "shrink_snap_id": "S1", + "shrink_deadline": 2**60, + "cutover_lvs": "LVS_1", + "shrink_started_at": _time.time() - 60}) done, err = runner._shrink_step(task, _ShrinkLvol()) - assert (done, err) == (True, None), "cutover must start IMMEDIATELY after the last round" - - -def runner_rounds(): - import simplyblock_core.services.tasks_runner_replication_final as runner - return runner.SHRINK_ROUNDS + assert (done, err) == (True, None), \ + "the round cap must hand over to the freeze, not fail the cutover" + assert "not converged" in task.function_result -def test_shrink_deadline_aborts(monkeypatch): +def test_shrink_deadline_proceeds_to_cutover(monkeypatch): + """An expired deadline stops adding rounds and hands straight over to the + freeze: the residual delta is slightly larger than a converged one, but + proceeding always beats failing the task and waiting out another + 900-second shrink window.""" runner, task = _mk(monkeypatch, {"S1": _ShrinkSnap(replicated=False)}, {"shrink_round": 1, "shrink_snap_id": "S1", "shrink_deadline": 1}) done, err = runner._shrink_step(task, _ShrinkLvol()) - assert done is False and err and "timed out" in err + assert (done, err) == (True, None), \ + "the deadline must hand over to the freeze, not fail the cutover" diff --git a/simplyblock_core/test/test_transfer_completion_latency.py b/simplyblock_core/test/test_transfer_completion_latency.py new file mode 100644 index 0000000000..c6a1f5b714 --- /dev/null +++ b/simplyblock_core/test/test_transfer_completion_latency.py @@ -0,0 +1,160 @@ +"""No time may be lost between a transfer completing and the next snapshot. + +Measured before this fix (run 20260828_115307, case 7 fail-back): + + polls per in-flight transfer : p50 = 1 + gap between polls : p50 = 81.2s + states ever seen : {'Done': 86} -- never once "In progress" + +Every poll found the transfer already finished. Submit happened on one pass of +the runner loop and the Done check on a later one, so a transfer that completed +in milliseconds went unnoticed for ~81 seconds. A convergence round cannot take +its next snapshot until the previous one is marked replicated, so that latency +landed directly in the client's IO freeze. + +Requirement: completion must be picked up and the next snapshot initiated in +under one second, in the WORST case, not just typically. +""" +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import constants +from simplyblock_core.services import snapshot_replication as sr +from simplyblock_core.services import tasks_runner_replication_final as final + + +class TestInlineCompletion(unittest.TestCase): + """The submitting pass waits for the transfer and finishes it itself.""" + + def setUp(self): + self.db = patch.object(sr, "db").start() + self.addCleanup(patch.stopall) + self.finish = patch.object(sr, "_finish_completed_transfer", + return_value=True).start() + patch.object(sr, "_cutover_owns", return_value=False).start() + self.slept: list = [] + patch.object(sr.time, "sleep", side_effect=self.slept.append).start() + + self.snap = MagicMock() + self.snap.get_id.return_value = "S1" + self.snap.snap_bdev = "LVS_1/SNAP_1" + self.snap.lvol.get_id.return_value = "LV1" + + self.task = MagicMock() + self.task.cluster_id = "CL" + self.task.function_params = {} + + self.node = MagicMock() + self.rpc = MagicMock() + self.node.rpc_client.return_value = self.rpc + + def test_a_transfer_already_done_is_finished_without_sleeping(self): + self.rpc.bdev_lvol_transfer_stat.return_value = { + "transfer_state": "Done", "offset": 2 * 1024 * 1024} + self.assertTrue( + sr._await_transfer_completion(self.task, self.snap, self.node)) + self.finish.assert_called_once() + self.assertEqual(self.slept, [], + "a finished transfer must be acted on immediately") + + def test_completion_is_noticed_within_one_poll_interval(self): + states = [{"transfer_state": "In progress", "offset": 1}, + {"transfer_state": "Done", "offset": 2}] + self.rpc.bdev_lvol_transfer_stat.side_effect = states + self.assertTrue( + sr._await_transfer_completion(self.task, self.snap, self.node)) + self.finish.assert_called_once() + self.assertEqual(self.slept, [constants.REPL_XFER_POLL_INTERVAL_SEC]) + self.assertLess(constants.REPL_XFER_POLL_INTERVAL_SEC, 1.0, + "the poll interval IS the worst-case detection delay") + + def test_a_failed_transfer_is_left_to_the_retry_path(self): + self.rpc.bdev_lvol_transfer_stat.return_value = { + "transfer_state": "Failed", "offset": 0} + self.assertFalse( + sr._await_transfer_completion(self.task, self.snap, self.node)) + self.finish.assert_not_called() + + def test_a_volume_in_cutover_gets_the_generous_budget(self): + """Its lvstore is already claimed, so waiting starves nothing.""" + with patch.object(sr, "_cutover_owns", return_value=True): + self.rpc.bdev_lvol_transfer_stat.return_value = { + "transfer_state": "Done", "offset": 1} + self.assertTrue( + sr._await_transfer_completion(self.task, self.snap, self.node)) + self.assertGreater(constants.REPL_XFER_INLINE_WAIT_CUTOVER_SEC, + constants.REPL_XFER_INLINE_WAIT_SEC) + + def test_the_submit_path_calls_the_wait(self): + import inspect + src = inspect.getsource(sr.process_snap_replicate_start) + self.assertIn("_await_transfer_completion", src, + "submitting and forgetting is what cost 81 seconds") + + +class TestPassLatency(unittest.TestCase): + """A round that yields must be picked up again in well under a second.""" + + def test_sub_second_reaction_does_not_come_from_polling_the_database(self): + """The DB pass interval is deliberately NOT sub-second. + + This loop reads the task table (and each task) per pass, so polling it + at 5Hz burns transactions proportional to clusters x tasks to learn + nothing almost every time -- the wrong shape for detecting an event. + Sub-second reaction comes from the RPC-based inline wait instead. + """ + self.assertGreaterEqual(constants.REPL_CUTOVER_ACTIVE_POLL_SEC, 1.0, + "do not poll a database sub-second") + self.assertLess(constants.REPL_CUTOVER_ACTIVE_POLL_SEC, + constants.TASK_EXEC_INTERVAL_SEC, + "but a mid-round cutover still deserves a tighter pass") + # This is the interval the guarantee actually rests on, and it polls + # SPDK over RPC, not the DB. + self.assertLess(constants.REPL_XFER_POLL_INTERVAL_SEC, 1.0) + + def test_a_claimed_or_mid_round_task_counts_as_in_flight(self): + from simplyblock_core.models.job_schedule import JobSchedule + def t(**params): + x = MagicMock() + x.function_name = JobSchedule.FN_REPLICATION_FINAL + x.status = JobSchedule.STATUS_RUNNING + x.canceled = False + x.function_params = params + return x + self.assertTrue(final._any_cutover_in_flight([t(cutover_lvs="LVS_1")])) + self.assertTrue(final._any_cutover_in_flight([t(shrink_snap_id="S1")])) + self.assertFalse(final._any_cutover_in_flight([t()])) + + def test_a_finished_cutover_does_not_hold_the_fast_interval(self): + from simplyblock_core.models.job_schedule import JobSchedule + done = MagicMock() + done.function_name = JobSchedule.FN_REPLICATION_FINAL + done.status = JobSchedule.STATUS_DONE + done.canceled = False + done.function_params = {"cutover_lvs": "LVS_1"} + self.assertFalse(final._any_cutover_in_flight([done]), + "an idle cluster must not be spun on") + + def test_the_per_task_backoff_is_gone(self): + """It punished the COMMON case: 20 queued tasks cost ~70s per pass. + + The DB-failure backoff in the except branch is a different thing and + stays -- this asserts only that a yielding TASK no longer sleeps. + """ + import inspect + src = inspect.getsource(final.main) + self.assertNotIn("if not res:", src, + "a queued or mid-round task must not cost a sleep") + self.assertIn("REPL_CUTOVER_ACTIVE_POLL_SEC", src) + + def test_the_owner_lookup_reuses_the_prefetched_task_list(self): + """Re-reading per task is O(N^2) DB reads, unaffordable at 200ms.""" + import inspect + self.assertIn("tasks=None", + inspect.signature(final._lvs_cutover_owner).__str__() + .replace(" ", "").replace("'", "")) + self.assertIn("cluster_tasks", inspect.getsource(final.main)) + + +if __name__ == "__main__": + unittest.main() diff --git a/simplyblock_core/test/test_wire_ns_identity.py b/simplyblock_core/test/test_wire_ns_identity.py new file mode 100644 index 0000000000..da5579599b --- /dev/null +++ b/simplyblock_core/test/test_wire_ns_identity.py @@ -0,0 +1,105 @@ +"""The wire namespace identity (LVol.get_ns_uuid) is what every verify, +register, and remove site must use — never the record uuid. + +After a fail-back the two differ by design: the record carries the restored +original uuid while the namespace keeps advertising the DR-generation wire +identity the client's multipath head holds. Run 2026-09-02 ~19:40: the health +check compared the record uuid, flagged every failed-back volume unhealthy +(Health: False), and the lvol monitor's self-heal then re-registered the +namespace under the record uuid — an identity the client kernel rejects +("IDs don't match for shared namespace N") — severing live paths. +""" +import inspect +from unittest.mock import MagicMock, patch + +from simplyblock_core.models.lvol_model import LVol + + +def _failed_back_lvol(): + lvol = LVol() + lvol.uuid = "REC" + lvol.ns_uuid = "WIRE" + lvol.nqn = "nqn.test:lvol:SHARED" + lvol.ns_id = 1 + lvol.top_bdev = "LVS_1/LVOL_C" + lvol.status = LVol.STATUS_ONLINE + return lvol + + +def test_get_ns_uuid_prefers_the_borrowed_wire_identity(): + lvol = _failed_back_lvol() + assert lvol.get_ns_uuid() == "WIRE" + lvol.ns_uuid = "" + assert lvol.get_ns_uuid() == "REC" + + +def test_health_check_verifies_the_wire_identity(monkeypatch): + """check_lvol_on_node must hand check_subsystem the WIRE identity, or a + healthy failed-back volume reports unhealthy forever.""" + from simplyblock_core.controllers import health_controller as hc + + lvol = _failed_back_lvol() + lvol.bdev_stack = [] + + db = MagicMock() + db.get_lvol_by_id.return_value = lvol + db.get_storage_node_by_id.return_value = MagicMock() + + captured = {} + + def _fake_check_subsystem(nqn, *, rpc_client=None, nqns=None, ns_uuid=None): + captured["ns_uuid"] = ns_uuid + return True + + with patch.object(hc, "DBController", lambda: db), \ + patch.object(hc, "check_subsystem", _fake_check_subsystem): + assert hc.check_lvol_on_node("REC", "N1") is True + assert captured["ns_uuid"] == "WIRE" + + +def test_delete_removes_the_namespace_by_its_wire_identity(monkeypatch): + """_remove_lvol_subsys_from_node matched ns['uuid'] against the record + uuid; for a failed-back volume that never matches, and the namespace is + left behind on the shared subsystem forever.""" + from simplyblock_core.controllers import lvol_controller as lc + + lvol = _failed_back_lvol() + + removed = [] + + class _Rpc: + def __init__(self): + self.namespaces = [ + {"nsid": 1, "uuid": "WIRE", "bdev_name": "LVS_1/LVOL_C"}, + {"nsid": 2, "uuid": "SIBLING", "bdev_name": "LVS_1/LVOL_S"}, + ] + + def subsystem_get(self, nqn): + return {"nqn": nqn, "namespaces": list(self.namespaces)} + + def nvmf_subsystem_remove_ns(self, nqn, nsid): + removed.append(nsid) + self.namespaces = [n for n in self.namespaces if n["nsid"] != nsid] + return True + + monkeypatch.setattr(lc.time, "sleep", lambda s: None) + assert lc._remove_lvol_subsys_from_node(lvol, _Rpc()) is True + assert removed == [1] + + +def test_registration_sites_use_the_wire_identity(): + """Every path that (re)registers an lvol namespace must pass + get_ns_uuid(): the monitor's self-heal, node-restart recreation, and + intra-cluster migration all re-run against records whose namespace may + carry a borrowed identity.""" + from simplyblock_core import storage_node_ops + from simplyblock_core.controllers import lvol_controller, migration_controller + + for func in (storage_node_ops.add_lvol_thread, + lvol_controller.recreate_lvol_on_node, + migration_controller.create_migration): + src = inspect.getsource(func) + assert "get_ns_uuid()" in src, \ + f"{func.__name__} must register/verify by the wire identity" + assert "lvol.uuid, lvol.guid" not in src, \ + f"{func.__name__} still registers by the record uuid" diff --git a/simplyblock_core/xfer_timing.py b/simplyblock_core/xfer_timing.py new file mode 100644 index 0000000000..a6f1196e20 --- /dev/null +++ b/simplyblock_core/xfer_timing.py @@ -0,0 +1,123 @@ +"""Phase timing for the replication/cutover pipeline. + +Soak analysis 2026-08-27 could not answer a basic question -- of the 588s a +convergence round took, how much was the DATA TRANSFER and how much was +orchestration? Round duration was the only number available, and it spans the +landing-volume create, the hub attach, the transfer, the detach, add_clone and +convert on two nodes, several DB writes, and up to TASK_EXEC_INTERVAL_SEC of +task-runner latency per state change. Every hardware-level explanation we +tested came out an order of magnitude off because the number being explained +was not a throughput. + +So: emit one line per phase, parseable without guessing. + + XFER-TIMING t=1787868791.244 phase=transfer lvol=1c8874f3 snap=a0f48bf5 \ + round=2 ms=1843.2 bytes=33554432 mbps=18.2 ok=1 + +Every line carries its own epoch timestamp because the container clock is +skewed from the host's, so `docker service logs -t` ordering cannot be trusted +across services. Grep for XFER-TIMING and feed it to +scripts/xfer_timing_report.py. + +Instrumentation only -- no behaviour change, and every helper is safe to call +from any thread and cheap enough for the hot path. +""" +import logging +import time +from contextlib import contextmanager + +logger = logging.getLogger() + +_PREFIX = "XFER-TIMING" + + +def _fmt(**fields): + parts = [] + for k, v in fields.items(): + if v is None: + continue + if isinstance(v, float): + parts.append("%s=%.3f" % (k, v)) + else: + parts.append("%s=%s" % (k, v)) + return " ".join(parts) + + +def _short(value): + """Ids are long and the interesting part is the head.""" + if value is None: + return None + text = str(value) + return text[:8] if len(text) > 8 else text + + +def now(): + """Clock for callers that need a start marker to pair with gap(). + + Exposed here so an instrumented module does not have to import time just + to be measured -- a missing `import time` in replication_final_step.py + would have raised NameError mid-cutover. + """ + return time.time() + + +def stamp(phase, lvol=None, snap=None, round=None, **extra): + """A point event: something happened now, with no duration.""" + logger.info("%s %s", _PREFIX, _fmt( + t=time.time(), phase=phase, lvol=_short(lvol), snap=_short(snap), + round=round, **extra)) + + +@contextmanager +def phase(name, lvol=None, snap=None, round=None, **extra): + """Time a block and emit its duration, whether it succeeds or raises. + + Usage: + with xfer_timing.phase("hub_attach", lvol=lvol_id) as ph: + ... + ph["bytes"] = n # optional, folded into the line + """ + started = time.time() + box: dict = {} + ok = 1 + try: + yield box + except BaseException: + ok = 0 + raise + finally: + elapsed_ms = (time.time() - started) * 1000.0 + fields = dict(extra) + fields.update(box) + nbytes = fields.pop("bytes", None) + if nbytes: + try: + fields["bytes"] = int(nbytes) + fields["mbps"] = (int(nbytes) / 1e6) / max(elapsed_ms / 1000.0, 1e-9) + except (TypeError, ValueError): + fields["bytes"] = nbytes + logger.info("%s %s", _PREFIX, _fmt( + t=time.time(), phase=name, lvol=_short(lvol), snap=_short(snap), + round=round, ms=elapsed_ms, ok=ok, **fields)) + + +def gap(name, since, lvol=None, snap=None, round=None, **extra): + """Time from an earlier epoch to now -- for waits nobody is inside of. + + The dead time between one round completing and the next snapshot being + taken is exactly this shape: no call to wrap, just two moments. + """ + if not since: + return + elapsed_ms = (time.time() - float(since)) * 1000.0 + fields = dict(extra) + nbytes = fields.pop("bytes", None) + if nbytes: + try: + fields["bytes"] = int(nbytes) + fields["mbps"] = (int(nbytes) / 1e6) / max(elapsed_ms / 1000.0, 1e-9) + except (TypeError, ValueError): + fields["bytes"] = nbytes + logger.info("%s %s", _PREFIX, _fmt( + t=time.time(), phase=name, lvol=_short(lvol), snap=_short(snap), + round=round, ms=elapsed_ms, ok=1, **fields)) diff --git a/simplyblock_web/api/v2/cluster/storage_pool/volume/replication.py b/simplyblock_web/api/v2/cluster/storage_pool/volume/replication.py index e99d6720e1..a508b54773 100644 --- a/simplyblock_web/api/v2/cluster/storage_pool/volume/replication.py +++ b/simplyblock_web/api/v2/cluster/storage_pool/volume/replication.py @@ -2,6 +2,7 @@ from uuid import UUID from fastapi import APIRouter, HTTPException, Request, Response +from fastapi.responses import JSONResponse from pydantic import BaseModel from simplyblock_core.controllers import lvol_controller, replication_policy_controller @@ -107,18 +108,35 @@ def trigger(cluster: Cluster, pool: StoragePool, volume: Volume) -> Response: @api.post('/failover', name='clusters:storage-pools:volumes:replication:failover', status_code=204, responses={204: {"content": None}}) -def failover(cluster: Cluster, pool: StoragePool, volume: Volume) -> Response: +def failover(cluster: Cluster, pool: StoragePool, volume: Volume, + generation: int = 0) -> Response: """Bring the volume up on the target cluster. The counterpart's id is read back from this volume's replication relationship, its connection paths from the target volume's `connect`. + + ``generation`` selects WHICH retained point-in-time to come up on: 0 (the + default) is the newest, 1 the one before it, and so on through the + history a retention schedule keeps. Failing over to an older generation + is the recovery path for a logical corruption, which the newest copy has + faithfully replicated. """ - result = lvol_controller.replicate_lvol_on_target_cluster(volume.get_id()) + if generation < 0: + raise HTTPException(400, 'generation cannot be negative') + result = lvol_controller.replicate_lvol_on_target_cluster( + volume.get_id(), generation=generation) if isinstance(result, tuple): # (False, error) raise HTTPException(500, str(result[1])) if not result: raise HTTPException(500, 'Failed to fail the volume over to the target cluster') + # Consistency groups: an older generation may not match current + # membership; the operator must SEE that, so warnings turn the empty 204 + # into a 200 with a body (requirement: API response, not only a log). + if isinstance(result, dict) and result.get("warnings"): + return JSONResponse(status_code=200, + content={"warnings": result["warnings"]}) + return Response(status_code=204) diff --git a/tests/AGENTS.md b/tests/AGENTS.md index b39e8077bb..0d8220dd3f 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -33,6 +33,21 @@ Pure-logic tests. Pick this tier when the test: `tests/unit/conftest.py` stubs out `fdb` (returning `None` from `fdb.open`), so unit tests never touch a real database. Run with `tox run -e unit` — no Docker, no infra. +**Never stand in for the database in a unit test.** The tier split is *about* the database, and there are only two positions: + +- the test does not interact with the database API **at all** → `tests/unit/`; +- the test interacts with the **real** database API → `tests/integration/`. + +A `_FakeDB`, a patched `DBController`, a module's `db` / `db_controller` singleton swapped for a mock, an assigned `kv_store` — each of these invents a third position that does not exist. It neither avoids the database nor exercises it: it asserts against a second, undeclared copy of the `DBController` interface that nothing keeps in sync with the real one. Such a test stays green until production code calls an accessor the fake never implemented, and then fails on an `AttributeError` that says nothing about the behaviour under test. + +That is not hypothetical. The snapshot-replication retention suite had a `_FakeDB` implementing the four accessors retention used when it was written; the moment retention began consulting `get_replication_policy_for_lvol`, twelve tests failed on a missing attribute rather than on anything about retention. They are now `tests/integration/test_snapshot_replication_retention.py`, seeding real models and reading them back through a live `DBController`. + +So, when a unit test seems to need a fake DB: **the need is the signal that it is not a unit test.** Move it to `tests/integration/`, seed real models with `write_to_db(db.kv_store)`, and mock only what sits above the database. If the code under test does not actually need the database, delete the stand-in rather than feeding it a fake. + +`.agents/hooks/guard-fake-db-tests.py` enforces this (see the root `AGENTS.md` § Guard hooks). It is a **ratchet**: the suite carries ~400 pre-existing stand-ins, so the guard compares each file before and after an edit and refuses only what that edit *introduces*. Editing or cleaning up a grandfathered file is unaffected. `python3 .agents/hooks/guard-fake-db-tests.py --scan` lists what is left. + +> **Migration in progress, here too.** `simplyblock_core/test/` is a second unit-tier directory (also collected by `tox run -e unit`, with its own `fdb` stub) and holds the densest concentration of these fakes. Treat it as legacy: don't add files there, and convert what you touch. + ### `tests/integration/` Controller-flow tests, **all of which run against a real FoundationDB**. `tests/integration/conftest.py` provisions FDB once for the whole tier from `pytest_configure` — *before* test collection — reusing `$FDB_CLUSTER_FILE` if set, otherwise starting a `testcontainers` container and binding its cluster file into `simplyblock_core.constants`. Provisioning at `pytest_configure` (rather than in a session fixture) means the real `fdb` client and a live `DBController()` are available at **collection / module-import time**, so test modules may touch the DB at import scope. A separate autouse fixture wipes the user keyspace before every test for isolation. The FDB-backed subdirs (`ftt2/`, `migration/`, `expansion_sim/`) add their own per-suite topology/bootstrap fixtures on top of that same cluster. diff --git a/tests/integration/migration/mock_rpc_server.py b/tests/integration/migration/mock_rpc_server.py index 5bd9df500f..fe86338529 100644 --- a/tests/integration/migration/mock_rpc_server.py +++ b/tests/integration/migration/mock_rpc_server.py @@ -657,21 +657,34 @@ def _bdev_lvol_transfer_stat(s: NodeState, p: dict): def _bdev_lvol_transfer_final_step(s: NodeState, p: dict): - """Start async final-migration (source lvol → target blobstore). - - Wire name for ``RPCClient.bdev_lvol_final_migration`` (a deprecated alias - for ``bdev_lvol_transfer_final_step``). + """Final migration (source lvol → target blobstore) — SYNCHRONOUS. + + The real RPC blocks through the IO drain and delta copy and returns a + stat dict; the runner requires ``transfer_state == 'Done'`` on the return + itself (tasks_runner_lvol_migration, strict check added 2026-08 after the + batch-migration "Failed slipped through" bug). This mock used to model it + as async (register op, return True) — a stale contract that made every + completed migration read as ``transfer_state=None`` and failed 26 tests + while the code worked on real clusters. + + Failure injection still applies: the dispatch layer times out or errors + BEFORE this handler runs, which exercises the runner's not-ret / + exception paths, including the crash-recovery stat poll — served by the + 'Done' op recorded here. """ lvol_name = _req(p, 'lvol_name') composite = lvol_name if lvol_name in s.lvols else s.composite(lvol_name) if composite not in s.lvols: raise _RpcError(-2, f"source lvol {composite} not found") + # Block briefly like the real drain does, but cap the exponential tail — + # a synchronous multi-second stall would serialize the whole suite. + time.sleep(min(_async_delay(s.rng), 0.5)) s.transfer_ops[composite] = { - 'complete_at': time.time() + _async_delay(s.rng), - 'state': 'In progress', + 'complete_at': time.time(), + 'state': 'Done', } - logger.debug("mock bdev_lvol_transfer_final_step started for %s", composite) - return True + logger.debug("mock bdev_lvol_transfer_final_step completed for %s", composite) + return {'transfer_state': 'Done', 'offset': 0} # ---- NVMe-oF subsystems ---- diff --git a/tests/integration/test_clone_namespace_race.py b/tests/integration/test_clone_namespace_race.py index a9bd313d05..f8e8e9475e 100644 --- a/tests/integration/test_clone_namespace_race.py +++ b/tests/integration/test_clone_namespace_race.py @@ -65,6 +65,12 @@ def _lvol_for_add(uuid, namespace="", nqn=None): lv.ha_type = "single" lv.nqn = nqn or ("nqn.test:cluster-1:lvol:" + uuid) lv.namespace = namespace + # Create-flow invariant (see add_lvol): ns_id is 0 = "not assigned yet" + # until the PRIMARY namespace add persists SPDK's assignment. The model + # default is 1 — a legitimate nsid — and a non-zero ns_id reads as + # "dictated by a migration/fail-over caller", which forbids the -32602 + # re-claim these tests exercise. + lv.ns_id = 0 lv.allowed_hosts = [] lv.fabric = "tcp" lv.max_namespace_per_subsys = 32 diff --git a/tests/integration/test_lvol_ns_removal_confirmation.py b/tests/integration/test_lvol_ns_removal_confirmation.py new file mode 100644 index 0000000000..0c643533bf --- /dev/null +++ b/tests/integration/test_lvol_ns_removal_confirmation.py @@ -0,0 +1,78 @@ +# coding=utf-8 +"""Empty-subsystem deletion in ``_remove_lvol_subsys_from_node``. + +Moved from tests/unit/test_lvol_ns_removal_confirmation.py: since 37751bfe4 +the empty-subsystem branch calls DBController().get_lvols_by_node_id to check +for other live volumes still claiming the NQN, so it needs a real DB. +""" + +import unittest +import uuid as uuid_mod +from unittest.mock import MagicMock, patch + +from simplyblock_core.controllers import lvol_controller +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.lvol_model import LVol + + +NQN = "nqn.2023-02.io.simplyblock:cl:lvol:shared" + + +def _make_lvol(node_id="node-1"): + lvol = LVol() + lvol.uuid = str(uuid_mod.uuid4()) + lvol.nqn = NQN + lvol.lvs_name = "LVS_1" + lvol.lvol_bdev = "LVOL_9" + lvol.node_id = node_id + lvol.pool_uuid = "pool-1" + lvol.bdev_stack = [] + return lvol + + +class TestRemoveSubsysConfirmationEmptySubsystem(unittest.TestCase): + + def setUp(self): + self.db = DBController() + if self.db.kv_store is None: + self.skipTest("FoundationDB is not available") + self.db.kv_store.clear_range(b"\x00", b"\xff") + + self.lvol = _make_lvol() + self.lvol.write_to_db(self.db.kv_store) + + self.rpc = MagicMock(name="rpc") + self.rpc.nvmf_subsystem_remove_ns.return_value = True + self.rpc.subsystem_delete.return_value = True + p = patch.object(lvol_controller.time, "sleep") + p.start() + self.addCleanup(p.stop) + + def test_confirmed_removal_then_empty_subsystem_is_deleted(self): + self.rpc.subsystem_get.side_effect = [ + {"namespaces": [{"nsid": 2, "uuid": self.lvol.uuid}]}, + {"namespaces": []}, # confirmation poll: ns gone + ] + ok = lvol_controller._remove_lvol_subsys_from_node(self.lvol, self.rpc) + self.assertTrue(ok) + self.rpc.nvmf_subsystem_remove_ns.assert_called_once_with(NQN, 2) + self.rpc.subsystem_delete.assert_called_once_with(NQN) + + def test_other_live_volume_on_same_nqn_keeps_subsystem(self): + """A second lvol on the same node still sharing the NQN must block + the delete, even though the subsystem is transiently empty.""" + other = _make_lvol(node_id=self.lvol.node_id) + other.status = LVol.STATUS_ONLINE + other.write_to_db(self.db.kv_store) + + self.rpc.subsystem_get.side_effect = [ + {"namespaces": [{"nsid": 2, "uuid": self.lvol.uuid}]}, + {"namespaces": []}, + ] + ok = lvol_controller._remove_lvol_subsys_from_node(self.lvol, self.rpc) + self.assertTrue(ok) + self.rpc.subsystem_delete.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/test_snapshot_replication_retention.py b/tests/integration/test_snapshot_replication_retention.py new file mode 100644 index 0000000000..e2b17382bc --- /dev/null +++ b/tests/integration/test_snapshot_replication_retention.py @@ -0,0 +1,444 @@ +# coding=utf-8 +"""Retention for replication-driven internal snapshots (D2), against real FDB. + +``snapshot_replication._prune_internal_snapshots`` decides which internal +snapshots may be deleted by *reading model state*: the source snapshot chain of +a volume (``get_snapshots_by_node_id``), each snapshot's replicated copy on the +target cluster (``get_snapshot_by_id``), that copy's chain link +(``prev_snap_uuid``), the volumes cloned from it (``get_mini_lvols``), the +owning storage node, and the volume's replication policy +(``get_replication_policy_for_lvol``). Every one of those is a DBController +accessor, so the tests belong to the FDB-backed tier: the state under test IS +database state. + +These cases previously ran against a hand-written ``_FakeDB`` that implemented +the four accessors the code used at the time. They broke the moment retention +started consulting the replication policy — the fake had no +``get_replication_policy_for_lvol``, so twelve tests failed on an +``AttributeError`` rather than on anything about retention. A real +``DBController`` cannot drift out of sync with the code it serves. + +Mocked here — everything *above* the database, per the tier's rule: + +- ``snapshot_controller.delete``, the data-plane delete (cluster-wide object + lock, ``bdev_lvol_delete`` over JSON-RPC, reaped by snapshot_monitor). It is + replaced by a recorder that applies the DB effect the real delete eventually + has — the record is removed — because that is what the prune loop reads back. +- ``StorageNode.rpc_client``, the SPDK chain query on the target node. The + integration tier never talks to a storage node. +""" + +import pytest + +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.snapshot import SnapShot +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import snapshot_replication as sr + +CLUSTER_ID = "cluster-1" +POOL_ID = "pool-1" +SOURCE_NODE_ID = "N1" +TARGET_NODE_ID = "TN1" +TARGET_LVOL_ID = "T_LV1" + + +@pytest.fixture +def db(): + db = DBController() + if db.kv_store is None: + pytest.skip("FoundationDB is not available") + return db + + +def _write_lvol(db, uuid, node_id, cloned_from_snap="", status=LVol.STATUS_ONLINE): + lvol = LVol() + lvol.uuid = uuid + lvol.cluster_id = CLUSTER_ID + lvol.pool_uuid = POOL_ID + lvol.node_id = node_id + lvol.lvol_name = f"VOL_{uuid}" + lvol.lvol_bdev = f"LVOL_{uuid}" + lvol.lvs_name = "LVS_S" if node_id == SOURCE_NODE_ID else "LVS_T" + lvol.top_bdev = f"{lvol.lvs_name}/{lvol.lvol_bdev}" + lvol.size = 1024 ** 3 + lvol.status = status + lvol.cloned_from_snap = cloned_from_snap + lvol.write_to_db(db.kv_store) + return lvol + + +def _write_node(db, uuid, status): + node = StorageNode() + node.uuid = uuid + node.cluster_id = CLUSTER_ID + node.hostname = uuid + node.status = status + node.lvstore = "LVS_T" + node.lvstore_status = "ready" + node.write_to_db(db.kv_store) + return node + + +def _write_snapshot(db, uuid, created_at, snap_type, lvol, target="", + status=SnapShot.STATUS_ONLINE): + """A snapshot on the SOURCE cluster, optionally already replicated.""" + snap = SnapShot() + snap.uuid = uuid + snap.cluster_id = CLUSTER_ID + snap.pool_uuid = POOL_ID + snap.created_at = created_at + snap.snap_type = snap_type + snap.status = status + snap.target_replicated_snap_uuid = target + snap.snap_name = f"SNAP_{uuid}" + snap.snap_bdev = f"LVS_S/{uuid}" + snap.snap_uuid = f"uuid-{uuid}" + snap.size = lvol.size + snap.lvol = lvol + snap.write_to_db(db.kv_store) + return snap + + +def _write_target_copy(db, uuid, target_lvol, prev_snap_uuid=""): + """A replicated snapshot as it exists on the remote cluster. + + ``prev_snap_uuid`` is the chain link retention checks before it deletes a + predecessor: it is only written once bdev_lvol_add_clone + convert + succeeded. + """ + copy = SnapShot() + copy.uuid = uuid + copy.cluster_id = CLUSTER_ID + copy.pool_uuid = POOL_ID + copy.status = SnapShot.STATUS_ONLINE + copy.snap_type = SnapShot.TYPE_INTERNAL + copy.snap_name = f"SNAP_{uuid}" + copy.snap_bdev = f"LVS_T/{uuid}" + copy.snap_uuid = f"uuid-{uuid}" + copy.prev_snap_uuid = prev_snap_uuid + copy.size = target_lvol.size + copy.lvol = target_lvol + copy.write_to_db(db.kv_store) + return copy + + +def _healthy_chain(source_snaps): + """Link each replicated internal target copy onto its predecessor's. + + This is the state a converged replication leaves behind, so it is the + default for the existing cases: they assert on retention, not on chaining. + """ + chain: dict = {} + per_lvol: dict = {} + for s in source_snaps: + if s.snap_type != SnapShot.TYPE_INTERNAL or not s.target_replicated_snap_uuid: + continue + per_lvol.setdefault(s.lvol.get_id(), []).append(s) + for snaps in per_lvol.values(): + snaps.sort(key=lambda s: s.created_at) + for prev, nxt in zip(snaps, snaps[1:]): + chain[nxt.target_replicated_snap_uuid] = prev.target_replicated_snap_uuid + return chain + + +class _RecordingSnapshotController: + """Records what retention decided to delete, and applies it to the real DB. + + The real ``snapshot_controller.delete`` is a data-plane path; what matters + to the prune loop is the state it leaves behind, since the loop reads each + target copy back before acting on it. + """ + + def __init__(self, db): + self._db = db + self.deleted = [] + + def delete(self, uuid, force_delete=False): + self.deleted.append(uuid) + try: + snap = self._db.get_snapshot_by_id(uuid) + except KeyError: + return True + snap.remove(self._db.kv_store) + return True + + +def _seed(db, monkeypatch, source_snaps, existing_targets, chain=None, + target_node_status=StorageNode.STATUS_OFFLINE, target_bdevs=()): + """Materialize the target-side state and stub the layers above the DB. + + The target node is offline by default so the SPDK chain fallback stays out + of the way unless a test explicitly opts into it. + """ + if chain is None: + chain = _healthy_chain(source_snaps) + + target_lvol = _write_lvol(db, TARGET_LVOL_ID, TARGET_NODE_ID) + for uuid in existing_targets: + _write_target_copy(db, uuid, target_lvol, chain.get(uuid, "")) + _write_node(db, TARGET_NODE_ID, target_node_status) + + class _RPC: + def get_bdevs(self, name=None): + return [b for b in target_bdevs if b.get("name") == name] + + monkeypatch.setattr(StorageNode, "rpc_client", lambda self, **kwargs: _RPC()) + + snapctl = _RecordingSnapshotController(db) + monkeypatch.setattr(sr, "snapshot_controller", snapctl) + return snapctl + + +def test_prunes_older_internal_keeps_newest_and_users(db, monkeypatch): + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_old"), + _write_snapshot(db, "user_mid", 150, SnapShot.TYPE_USER, source_lvol, target="T_user"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + snapctl = _seed(db, monkeypatch, snaps, {"T_old", "T_user", "T_mid", "T_new"}) + + sr._prune_internal_snapshots(source_lvol) + + # Target copy deleted before the source snapshot; the newest PAIR is kept + # so an arriving snapshot always has a predecessor to chain onto. + assert snapctl.deleted == ["T_old", "int_old"] + for kept in ("int_mid", "T_mid", "int_new", "T_new", "user_mid", "T_user"): + assert kept not in snapctl.deleted + + +def test_single_internal_not_pruned(db, monkeypatch): + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [_write_snapshot(db, "int_only", 100, SnapShot.TYPE_INTERNAL, + source_lvol, target="T_only")] + snapctl = _seed(db, monkeypatch, snaps, {"T_only"}) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == [] + + +def test_unreplicated_internal_ignored(db, monkeypatch): + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + # Newest internal not yet replicated (no target) -> excluded; the only + # replicated internal is the single oldest, so nothing is pruned. + snaps = [ + _write_snapshot(db, "int_repl", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_repl"), + _write_snapshot(db, "int_pending", 200, SnapShot.TYPE_INTERNAL, source_lvol, target=""), + ] + snapctl = _seed(db, monkeypatch, snaps, {"T_repl"}) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == [] + + +def test_missing_target_still_cleans_source(db, monkeypatch): + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_gone"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + # T_gone already deleted on target -> only source snapshot is cleaned up. + snapctl = _seed(db, monkeypatch, snaps, {"T_mid", "T_new"}) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == ["int_old"] + + +def test_other_lvol_snapshots_untouched(db, monkeypatch): + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + other_lvol = _write_lvol(db, "LV2", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_old"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + # Different lvol on same node — must never be pruned. + _write_snapshot(db, "other_old", 50, SnapShot.TYPE_INTERNAL, other_lvol, target="TO_old"), + _write_snapshot(db, "other_new", 250, SnapShot.TYPE_INTERNAL, other_lvol, target="TO_new"), + ] + snapctl = _seed(db, monkeypatch, snaps, + {"T_old", "T_mid", "T_new", "TO_old", "TO_new"}) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == ["T_old", "int_old"] + + +def test_never_prunes_snapshot_a_failed_over_volume_is_cloned_from(db, monkeypatch): + """Root cause of the all-zeros DR fail-over (labs 2026-08-10/11). + + Fail-over clones the volume from the last replicated TARGET snapshot; the + prune, keyed only on the SOURCE snapshot age, then deleted that target copy. + The delete reaches SPDK as bdev_lvol_delete(sync=False) and frees the blocks + immediately, so no DB-level guard downstream can save the clone. Retention + must skip a target snapshot with a live dependent clone. + + Three snapshots, not two: with only a pair, the newest-``keep`` rule returns + before the clone guard is ever consulted, and the case passes no matter what + that guard does. ``int_old`` has to be a genuine prune candidate — chained, + past the cushion — so that the dependent clone is the only thing saving it. + """ + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_old"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + snapctl = _seed(db, monkeypatch, snaps, {"T_old", "T_mid", "T_new"}) + # A failed-over volume lives on the OLD target snapshot. + _write_lvol(db, "FO_VOL", TARGET_NODE_ID, cloned_from_snap="T_old") + + sr._prune_internal_snapshots(source_lvol) + + assert "T_old" not in snapctl.deleted, ( + "pruned the target snapshot a failed-over volume is cloned from — " + "its blocks are freed by SPDK immediately (sync=False), the volume " + "reads zeros from then on") + # The source-side copy must survive too (it pairs with the kept target). + assert "int_old" not in snapctl.deleted + + +def test_in_deletion_clone_does_not_pin_the_snapshot(db, monkeypatch): + """A clone that is itself going away must not block retention forever.""" + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_old"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + snapctl = _seed(db, monkeypatch, snaps, {"T_old", "T_mid", "T_new"}) + _write_lvol(db, "DYING", TARGET_NODE_ID, cloned_from_snap="T_old", + status=LVol.STATUS_IN_DELETION) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == ["T_old", "int_old"] + + +def test_newest_pair_is_kept_so_arrivals_have_a_chain_parent(db, monkeypatch): + """Chain continuity: a replicated snapshot holds only its own clusters, and + deleting one swap-merges its segments into the successor CHAINED to it. + Keeping just the newest pruned the predecessor the instant a replication + finished, so the next arrival had nothing to chain onto and kept only its + delta — the target then held the last delta over holes (labs run 15 vs 19, + same case passing then failing on timing alone).""" + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_prev", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_prev"), + _write_snapshot(db, "int_new", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + snapctl = _seed(db, monkeypatch, snaps, {"T_prev", "T_new"}) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == [], ( + "pruned the predecessor the next arrival must chain onto") + + +def test_defers_prune_until_the_successor_is_actually_chained(db, monkeypatch): + """The count cushion is not the precondition — the chain link is. + + Keeping the newest N only widens the window in which chaining is expected to + have happened. If it lagged or failed for one snapshot while newer ones kept + arriving, the predecessor was still pruned, and because the delete reaches + SPDK as bdev_lvol_delete(sync=False) its segments were freed instead of + swap-merged into the successor. The target then holds the newest delta over + holes and a fail-over clone reads zeros (labs 2026-08-10..17). Retention must + verify the link and defer while it is absent. + """ + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_old"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + # T_mid arrived but was never chained onto T_old. + snapctl = _seed(db, monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, chain={}) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == [], ( + "pruned a predecessor whose successor is not chained onto it — SPDK frees " + "the blocks immediately, so those segments are lost rather than merged") + + +def test_prunes_once_the_chain_is_established(db, monkeypatch): + """The deferral must release as soon as chaining catches up (no livelock).""" + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_old"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + snapctl = _seed(db, monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, + chain={"T_mid": "T_old", "T_new": "T_mid"}) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == ["T_old", "int_old"] + + +def test_spdk_verdict_releases_a_missing_db_link(db, monkeypatch): + """A missing link must not pin the pair for ever. + + The link write is best-effort, and snapshots replicated before chaining was + implemented have none at all. SPDK is the real authority, so when the DB has + no link we ask the target node before giving up — otherwise retention would + never release those snapshots and both chains would grow without bound. + """ + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_old"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + # SPDK reports T_mid as a clone whose base is T_old, while the DB link is absent. + snapctl = _seed(db, monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, chain={}, + target_node_status=StorageNode.STATUS_ONLINE, + target_bdevs=[{ + "name": "LVS_T/T_mid", + "driver_specific": { + "lvol": {"clone": True, "base_snapshot": "LVS_T/T_old"}}, + }]) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == ["T_old", "int_old"] + + +def test_unchained_in_spdk_is_not_pruned_even_when_node_is_reachable(db, monkeypatch): + """An online target that reports a standalone blob must still defer.""" + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_old"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + snapctl = _seed(db, monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, chain={}, + target_node_status=StorageNode.STATUS_ONLINE, + target_bdevs=[{ + "name": "LVS_T/T_mid", + "driver_specific": { + "lvol": {"clone": False, "base_snapshot": None}}, + }]) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == [] diff --git a/tests/unit/tasks/test_retry_ceiling.py b/tests/unit/tasks/test_retry_ceiling.py index 0b4a429b70..83648dd8f2 100644 --- a/tests/unit/tasks/test_retry_ceiling.py +++ b/tests/unit/tasks/test_retry_ceiling.py @@ -325,13 +325,24 @@ def _spec_node_add(runner, monkeypatch): def _spec_replication_final(runner, monkeypatch): + # The endgame params are pre-set so each pass skips the shrink machinery + # and goes straight to run_cutover, which is mocked to fail every cycle. + # The target node stays ONLINE on purpose: an offline target deliberately + # does NOT burn task.retry (waiting out an outage is transient), so the + # ceiling must be driven by the work itself failing. Each failure walks + # the hub-attempt ladder (REPL_CUTOVER_MAX_HUB_ATTEMPTS cooldown attempts + # per burned retry); the fake clock's giant steps make every cooldown + # already elapsed by the next poll. task = _make_task( JobSchedule.FN_REPLICATION_FINAL, - lvol_id="lv-1", tgt_node_id="tgt-1", src_node_id="src-1") - db, _cluster, node = _wire_base(runner, monkeypatch, task) - # Target node never comes online -> cutover cannot proceed, retry each poll. - node.status = StorageNode.STATUS_OFFLINE + lvol_id="lv-1", tgt_node_id="tgt-1", src_node_id="src-1", + tgt_lvol_composite="lvs_tgt/LVOL_1", tgt_map_id=1, + tgt_snap_composite="lvs_tgt/SNAP_1", + shrink_snap_id="S_endgame", shrink_round=0) + db, _cluster, _node = _wire_base(runner, monkeypatch, task) db.get_lvol_by_id.return_value = MagicMock() + monkeypatch.setattr(runner.replication_final_step, "run_cutover", + lambda *a, **k: (False, "boom")) return task diff --git a/tests/unit/test_jm_rep_wait.py b/tests/unit/test_jm_rep_wait.py index 35b013a6f3..098652000c 100644 --- a/tests/unit/test_jm_rep_wait.py +++ b/tests/unit/test_jm_rep_wait.py @@ -49,6 +49,8 @@ def test_dead_peer_abandons_the_wait_immediately(self): rpc.bdev_lvol_get_lvstores.side_effect = RuntimeError("connection refused") node = _node(rpc=rpc) with self._patch_db(StorageNode.STATUS_OFFLINE), \ + patch("simplyblock_core.models.storage_node.time.time", + return_value=1000.0), \ patch("simplyblock_core.models.storage_node.time.sleep") as sleep: self.assertFalse(node.wait_for_jm_rep_tasks_to_finish(10)) sleep.assert_not_called() @@ -64,6 +66,8 @@ def test_unreachable_rpc_on_a_live_peer_is_bounded(self): rpc.bdev_lvol_get_lvstores.side_effect = RuntimeError("timeout") node = _node(rpc=rpc) with self._patch_db(StorageNode.STATUS_ONLINE), \ + patch("simplyblock_core.models.storage_node.time.time", + return_value=1000.0), \ patch("simplyblock_core.models.storage_node.time.sleep") as sleep: self.assertFalse( node.wait_for_jm_rep_tasks_to_finish(10, retry=4, delay=5)) @@ -77,6 +81,8 @@ def test_pre_check_failure_no_longer_escapes(self): rpc.bdev_lvol_get_lvstores.side_effect = RuntimeError("connection refused") node = _node(rpc=rpc) with self._patch_db(StorageNode.STATUS_ONLINE), \ + patch("simplyblock_core.models.storage_node.time.time", + return_value=1000.0), \ patch("simplyblock_core.models.storage_node.time.sleep"): result = node.wait_for_jm_rep_tasks_to_finish(10, retry=2, delay=1) self.assertFalse(result) # returned, did not raise @@ -85,7 +91,9 @@ def test_no_lvstore_returns_immediately(self): rpc = MagicMock() rpc.bdev_lvol_get_lvstores.return_value = [] node = _node(rpc=rpc) - with patch("simplyblock_core.models.storage_node.time.sleep") as sleep: + with patch("simplyblock_core.models.storage_node.time.time", + return_value=1000.0), \ + patch("simplyblock_core.models.storage_node.time.sleep") as sleep: self.assertTrue(node.wait_for_jm_rep_tasks_to_finish(10)) sleep.assert_not_called() rpc.jc_get_jm_status.assert_not_called() @@ -98,7 +106,9 @@ def test_busy_then_free_returns_true(self): {"jm_a": True, "jm_b": True}, # free ] node = _node(rpc=rpc) - with patch("simplyblock_core.models.storage_node.time.sleep") as sleep: + with patch("simplyblock_core.models.storage_node.time.time", + return_value=1000.0), \ + patch("simplyblock_core.models.storage_node.time.sleep") as sleep: self.assertTrue(node.wait_for_jm_rep_tasks_to_finish(10, delay=7)) self.assertEqual(sleep.call_count, 1) sleep.assert_called_once_with(7) @@ -108,7 +118,9 @@ def test_persistently_busy_peer_exhausts_the_budget(self): rpc.bdev_lvol_get_lvstores.return_value = [{"name": "LVS_10"}] rpc.jc_get_jm_status.return_value = {"jm_a": False} node = _node(rpc=rpc) - with patch("simplyblock_core.models.storage_node.time.sleep") as sleep: + with patch("simplyblock_core.models.storage_node.time.time", + return_value=1000.0), \ + patch("simplyblock_core.models.storage_node.time.sleep") as sleep: self.assertFalse( node.wait_for_jm_rep_tasks_to_finish(10, retry=3, delay=2)) self.assertEqual(rpc.jc_get_jm_status.call_count, 3) @@ -122,6 +134,8 @@ def test_db_lookup_failure_keeps_the_bounded_retries(self): node = _node(rpc=rpc) with patch("simplyblock_core.db_controller.DBController", side_effect=RuntimeError("fdb down")), \ + patch("simplyblock_core.models.storage_node.time.time", + return_value=1000.0), \ patch("simplyblock_core.models.storage_node.time.sleep") as sleep: self.assertFalse( node.wait_for_jm_rep_tasks_to_finish(10, retry=3, delay=1)) diff --git a/tests/unit/test_lvol_ns_removal_confirmation.py b/tests/unit/test_lvol_ns_removal_confirmation.py index 4533d25c45..ca02f18c91 100644 --- a/tests/unit/test_lvol_ns_removal_confirmation.py +++ b/tests/unit/test_lvol_ns_removal_confirmation.py @@ -51,16 +51,6 @@ def setUp(self): p.start() self.addCleanup(p.stop) - def test_confirmed_removal_then_empty_subsystem_is_deleted(self): - self.rpc.subsystem_get.side_effect = [ - {"namespaces": [{"nsid": 2, "uuid": "lvol-1"}]}, - {"namespaces": []}, # confirmation poll: ns gone - ] - ok = lvol_controller._remove_lvol_subsys_from_node(self.lvol, self.rpc) - self.assertTrue(ok) - self.rpc.nvmf_subsystem_remove_ns.assert_called_once_with(NQN, 2) - self.rpc.subsystem_delete.assert_called_once_with(NQN) - def test_confirmed_removal_with_surviving_namespaces_keeps_subsystem(self): self.rpc.subsystem_get.side_effect = [ {"namespaces": [{"nsid": 2, "uuid": "lvol-1"}, diff --git a/tests/unit/test_lvol_nsid_consistency.py b/tests/unit/test_lvol_nsid_consistency.py index f825bfec01..3ac931c510 100644 --- a/tests/unit/test_lvol_nsid_consistency.py +++ b/tests/unit/test_lvol_nsid_consistency.py @@ -29,8 +29,14 @@ class _Base(unittest.TestCase): def setUp(self): self.rpc = MagicMock(name="rpc") - self.rpc.nvmf_subsystem_add_ns2.return_value = ("7", None) - self.rpc.nvmf_subsystem_add_ns.return_value = "7" + # SPDK echoes the nsid it actually used: the requested one when the + # caller asked for a specific number, its own pick otherwise. A fake + # that always answers "7" hides a mismatch between the claim and the + # persisted value. + self.rpc.nvmf_subsystem_add_ns2.side_effect = ( + lambda *a, **kw: (str(kw.get("nsid") or 7), None)) + self.rpc.nvmf_subsystem_add_ns.side_effect = ( + lambda *a, **kw: str(kw.get("nsid") or 7)) self.rpc.get_bdevs.return_value = [ {"uuid": "lvol-bdev-uuid", "driver_specific": {"lvol": {"blobid": 33}}}] @@ -63,7 +69,7 @@ def setUp(self): patch.object(lvol_controller, "DBController"), patch.object( lvol_controller, "_fail_after_bdev", - side_effect=lambda lvol, rpc, msg: (False, msg)), + side_effect=lambda lvol, rpc, msg, is_primary=True: (False, msg)), patch( "simplyblock_core.controllers.migration_controller" ".get_active_migration_for_nqn", @@ -87,6 +93,19 @@ def test_primary_autoassigns_and_persists(self): self.assertEqual(self.lvol.ns_id, 7, "the assigned nsid must be persisted for the replicas") + def test_primary_honours_a_control_plane_claim(self): + """A fail-over copy arrives with an nsid claimed across the target HA + set; the primary must add it at exactly that number instead of + auto-assigning from its own partial view (soak case 7).""" + self.lvol.ns_id = 8 + ret, err = lvol_controller.add_lvol_on_node(self.lvol, self.snode) + self.assertIsNone(err) + kwargs = self.rpc.nvmf_subsystem_add_ns2.call_args.kwargs + self.assertEqual(kwargs.get("nsid"), 8, + "the primary must use the control-plane claim") + self.assertEqual(self.lvol.ns_id, 8, + "the claim must survive the add for the replicas") + def test_replica_reuses_primary_assigned_nsid(self): self.lvol.ns_id = 7 ret, err = lvol_controller.add_lvol_on_node( diff --git a/tests/unit/test_node_online_device_readmit.py b/tests/unit/test_node_online_device_readmit.py index cc99e84870..790b699eaa 100644 --- a/tests/unit/test_node_online_device_readmit.py +++ b/tests/unit/test_node_online_device_readmit.py @@ -59,7 +59,10 @@ def _node(uuid, status): # Distinct per-node mgmt_ip: get_next_cluster_status dedups affected # nodes by physical host (mgmt_ip); leaving the model default ("") on # every node collapses all affected nodes into one and undercounts. - n.mgmt_ip = f"10.99.0.{abs(hash(uuid)) % 250 + 1}" + # Use sum-of-ordinals rather than hash() — Python randomizes hash() per + # process (PYTHONHASHSEED), so two UUIDs can collide under some seeds and + # make the dedup return a count of 1 instead of 2, flipping the verdict. + n.mgmt_ip = f"10.99.0.{(sum(ord(c) for c in uuid) % 250) + 1}" return n diff --git a/tests/unit/test_snapshot_replication_leader_gate.py b/tests/unit/test_snapshot_replication_leader_gate.py new file mode 100644 index 0000000000..2a623a4394 --- /dev/null +++ b/tests/unit/test_snapshot_replication_leader_gate.py @@ -0,0 +1,26 @@ +# coding=utf-8 +"""The LVS-leader gate in the snapshot-replication service. + +Pure logic over a mocked ``is_node_leader`` — no model state, no DB — so it +stays in the unit tier. The retention tests it used to share a file with are +DB-driven and live in ``tests/integration/test_snapshot_replication_retention.py``. +""" + +from simplyblock_core.services import snapshot_replication as sr + + +def test_require_lvs_leader_gate(monkeypatch): + """Convert on a non-leader returns success WITHOUT persisting (silent + conversion error) — leadership must be checked BEFORE the operation and a + non-leader must fail-and-retry, never proceed.""" + import simplyblock_core.controllers.lvol_controller as lc + + class _N: + def get_id(self): + return "N1" + + monkeypatch.setattr(lc, "is_node_leader", lambda node, lvs: False) + assert sr._require_lvs_leader(_N(), "LVS_1", "convert") is False + + monkeypatch.setattr(lc, "is_node_leader", lambda node, lvs: True) + assert sr._require_lvs_leader(_N(), "LVS_1", "convert") is True diff --git a/tests/unit/web/api/v2/test_volume_replication_endpoints.py b/tests/unit/web/api/v2/test_volume_replication_endpoints.py index 4ddf88912a..7813935f3b 100644 --- a/tests/unit/web/api/v2/test_volume_replication_endpoints.py +++ b/tests/unit/web/api/v2/test_volume_replication_endpoints.py @@ -203,7 +203,33 @@ def test_failover(self, client, db, volume, lvol_controller): assert response.status_code == 204 assert response.content == b'' - lvol_controller.replicate_lvol_on_target_cluster.assert_called_once_with(VOLUME_ID) + lvol_controller.replicate_lvol_on_target_cluster.assert_called_once_with( + VOLUME_ID, generation=0) + + def test_failover_forwards_the_requested_generation(self, client, db, volume, + lvol_controller): + """``generation`` selects which retained point-in-time to come up on. + + It has to reach the controller: dropping it silently fails the volume + over to the NEWEST copy, which in the case this parameter exists for — + recovering from a logical corruption — is the copy that faithfully + replicated the corruption. + """ + lvol_controller.replicate_lvol_on_target_cluster.return_value = { + 'lvol_id': TARGET_VOLUME_ID, 'nqn': 'nqn.x', 'ns_id': 1, 'connection_strings': [], + } + + response = client.post(REPLICATION_URL + 'failover?generation=2') + + assert response.status_code == 204 + lvol_controller.replicate_lvol_on_target_cluster.assert_called_once_with( + VOLUME_ID, generation=2) + + def test_negative_generation_rejected(self, client, db, volume, lvol_controller): + response = client.post(REPLICATION_URL + 'failover?generation=-1') + + assert response.status_code == 400 + lvol_controller.replicate_lvol_on_target_cluster.assert_not_called() def test_failed_failover_is_an_error(self, client, db, volume, lvol_controller): lvol_controller.replicate_lvol_on_target_cluster.return_value = (False, 'node is not online') diff --git a/xfer_ut.c b/xfer_ut.c new file mode 100644 index 0000000000..ee993540c7 --- /dev/null +++ b/xfer_ut.c @@ -0,0 +1,239 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (C) 2026 Simplyblock. + * + * Unit tests for the transfer dispatch path (xfer_replication + + * helper_xfer_poller). Written for the fix that removed the one-item-per- + * poller-tick serialization: the dispatcher must fill the whole in-flight + * window in one pass, and the helper must drain its ready ring. + * + * Reuses lvol_ut.c wholesale (its stub set is what makes lvol.c link); + * its main() is renamed away and only the xfer suite is registered here, + * so this binary is independent of the state of the legacy suites. + */ +#define main lvol_ut_main_disabled +#include "../lvol.c/lvol_ut.c" +#undef main + +static struct spdk_lvol g_xfer_lvol; +static struct spdk_lvol_store g_xfer_lvs; + +static struct spdk_lvs_xfer * +make_xfer(int cluster_batch, uint32_t num_clusters, const int *allocated) +{ + struct spdk_lvs_xfer *xfer = calloc(1, sizeof(*xfer)); + SPDK_CU_ASSERT_FATAL(xfer != NULL); + + memset(&g_xfer_lvol, 0, sizeof(g_xfer_lvol)); + memset(&g_xfer_lvs, 0, sizeof(g_xfer_lvs)); + g_xfer_lvol.lvol_store = &g_xfer_lvs; + snprintf(g_xfer_lvol.name, sizeof(g_xfer_lvol.name), "xfer_ut_lvol"); + + xfer->lvol = &g_xfer_lvol; + xfer->type = XFER_REPLICATE_SNAPSHOT; + xfer->state = XFER_STATE_TRANSFER_CLUSTERS; + xfer->final_step = false; + xfer->cluster_batch = cluster_batch; + xfer->page_size = 16; + xfer->page_per_cluster = 4; + xfer->num_clusters = num_clusters; + xfer->clusters = calloc(num_clusters, sizeof(uint64_t)); + SPDK_CU_ASSERT_FATAL(xfer->clusters != NULL); + for (uint32_t i = 0; i < num_clusters; i++) { + xfer->clusters[i] = allocated[i] ? 0xABCD0000 + i : 0; + } + + xfer->free_ring = spdk_ring_create(SPDK_RING_TYPE_MP_MC, cluster_batch, + SPDK_ENV_NUMA_ID_ANY); + xfer->ready_ring = spdk_ring_create(SPDK_RING_TYPE_MP_MC, cluster_batch, + SPDK_ENV_NUMA_ID_ANY); + SPDK_CU_ASSERT_FATAL(xfer->free_ring != NULL && xfer->ready_ring != NULL); + + xfer->reqs = calloc(cluster_batch, sizeof(*xfer->reqs)); + SPDK_CU_ASSERT_FATAL(xfer->reqs != NULL); + for (int i = 0; i < cluster_batch; i++) { + xfer->reqs[i].payload = calloc(1, xfer->page_size * xfer->page_per_cluster); + SPDK_CU_ASSERT_FATAL(xfer->reqs[i].payload != NULL); + xfer->reqs[i].xfer = xfer; + xfer->reqs[i].type = xfer->type; + } + xfer_fill_queue(xfer, cluster_batch); + xfer->timeout = spdk_get_ticks(); + return xfer; +} + +static void +free_xfer(struct spdk_lvs_xfer *xfer) +{ + for (int i = 0; i < xfer->cluster_batch; i++) { + free(xfer->reqs[i].payload); + } + free(xfer->reqs); + spdk_ring_free(xfer->free_ring); + spdk_ring_free(xfer->ready_ring); + free(xfer->clusters); + free(xfer); +} + +static uint32_t +drain_ready(struct spdk_lvs_xfer *xfer, uint64_t *offsets, uint32_t max) +{ + struct spdk_lvs_xfer_req *req; + uint32_t n = 0; + + while (n < max && spdk_ring_dequeue(xfer->ready_ring, (void **)&req, 1) == 1) { + if (offsets != NULL) { + offsets[n] = req->offset; + } + n++; + } + return n; +} + +/* The fix itself: one call must fill the whole window, not one cluster. */ +static void +xfer_fills_whole_window(void) +{ + const int alloc[20] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + struct spdk_lvs_xfer *xfer = make_xfer(8, 20, alloc); + + int count = xfer_replication(xfer); + + CU_ASSERT(count == 8); + CU_ASSERT(xfer->outstanding_io == 8); + CU_ASSERT(xfer->idx == 8); + CU_ASSERT(xfer->hold_idx == 8); + CU_ASSERT(drain_ready(xfer, NULL, 16) == 8); + + free_xfer(xfer); +} + +/* A window larger than the remaining work stops at the work, not the window. */ +static void +xfer_stops_at_last_cluster(void) +{ + const int alloc[5] = {1, 1, 1, 0, 0}; + struct spdk_lvs_xfer *xfer = make_xfer(8, 5, alloc); + + int count = xfer_replication(xfer); + + CU_ASSERT(count == 3); + CU_ASSERT(xfer->outstanding_io == 3); + CU_ASSERT(drain_ready(xfer, NULL, 16) == 3); + + free_xfer(xfer); +} + +/* Sparse maps: unallocated clusters are skipped, offsets match the map. */ +static void +xfer_skips_unallocated_clusters(void) +{ + const int alloc[10] = {1, 0, 0, 1, 0, 1, 0, 0, 0, 1}; + struct spdk_lvs_xfer *xfer = make_xfer(8, 10, alloc); + uint64_t offsets[8] = {0}; + + int count = xfer_replication(xfer); + + CU_ASSERT(count == 4); + CU_ASSERT(drain_ready(xfer, offsets, 8) == 4); + CU_ASSERT(offsets[0] == 0 * xfer->page_per_cluster); + CU_ASSERT(offsets[1] == 3 * xfer->page_per_cluster); + CU_ASSERT(offsets[2] == 5 * xfer->page_per_cluster); + CU_ASSERT(offsets[3] == 9 * xfer->page_per_cluster); + + free_xfer(xfer); +} + +/* Completions recycle requests: the NEXT pass continues where the map left + * off, and the batch transition still fires when everything succeeded. */ +static void +xfer_second_pass_continues_and_completes(void) +{ + const int alloc[6] = {1, 1, 1, 1, 1, 1}; + struct spdk_lvs_xfer *xfer = make_xfer(4, 6, alloc); + struct spdk_lvs_xfer_req *req; + + CU_ASSERT(xfer_replication(xfer) == 4); + CU_ASSERT(xfer->hold_idx == 4); + + /* complete the four in-flight requests the way local/remote completion + * does: status DONE, back on the free ring */ + for (int i = 0; i < 4; i++) { + CU_ASSERT(spdk_ring_dequeue(xfer->ready_ring, (void **)&req, 1) == 1); + req->status = XFER_REQ_STATUS_DONE; + xfer->success_cnt++; + CU_ASSERT(spdk_ring_enqueue(xfer->free_ring, (void **)&req, 1, NULL) == 1); + } + + CU_ASSERT(xfer_replication(xfer) == 2); + CU_ASSERT(xfer->idx == 6); + CU_ASSERT(xfer->hold_idx == 6); + + /* complete the tail, then the state machine must leave the transfer */ + for (int i = 0; i < 2; i++) { + CU_ASSERT(spdk_ring_dequeue(xfer->ready_ring, (void **)&req, 1) == 1); + req->status = XFER_REQ_STATUS_DONE; + xfer->success_cnt++; + CU_ASSERT(spdk_ring_enqueue(xfer->free_ring, (void **)&req, 1, NULL) == 1); + } + xfer_replication(xfer); + CU_ASSERT(xfer->state == XFER_STATE_DONE); + + free_xfer(xfer); +} + +/* The helper must drain its ready ring, not take one request per tick. */ +static void +helper_drains_ready_ring(void) +{ + const int alloc[8] = {1, 1, 1, 1, 1, 1, 1, 1}; + struct spdk_lvs_xfer *xfer = make_xfer(8, 8, alloc); + struct spdk_lvs_poll_group lpg; + struct remote_lvol_info rmt; + + CU_ASSERT(xfer_replication(xfer) == 8); + + memset(&lpg, 0, sizeof(lpg)); + TAILQ_INIT(&lpg.rmt_lvols); + memset(&rmt, 0, sizeof(rmt)); + rmt.status = true; + rmt.type = XFER_REPLICATE_SNAPSHOT; + rmt.desc = (struct spdk_bdev_desc *)0x1; /* only null-checked */ + rmt.channel = (struct spdk_io_channel *)0x1; /* only null-checked */ + rmt.md_channel = (struct spdk_io_channel *)0x1; + rmt.ready_ring = xfer->ready_ring; + rmt.free_ring = xfer->free_ring; + TAILQ_INSERT_TAIL(&lpg.rmt_lvols, &rmt, entry); + + helper_xfer_poller(&lpg); + + CU_ASSERT(rmt.outstanding_io == 8); + CU_ASSERT(drain_ready(xfer, NULL, 8) == 0); /* ring fully drained */ + + free_xfer(xfer); +} + +int +main(int argc, char **argv) +{ + CU_pSuite suite = NULL; + unsigned int num_failures; + + CU_initialize_registry(); + + suite = CU_add_suite("lvol_xfer", NULL, NULL); + CU_ADD_TEST(suite, xfer_fills_whole_window); + CU_ADD_TEST(suite, xfer_stops_at_last_cluster); + CU_ADD_TEST(suite, xfer_skips_unallocated_clusters); + CU_ADD_TEST(suite, xfer_second_pass_continues_and_completes); + CU_ADD_TEST(suite, helper_drains_ready_ring); + + allocate_threads(1); + set_thread(0); + + num_failures = spdk_ut_run_tests(argc, argv, NULL); + + free_threads(); + CU_cleanup_registry(); + return num_failures; +}