fix: preserve original source lvolID through failback cutover by swap…

img_hashes.txt

@@ -0,0 +1,172 @@
+d41d8cd98f00b204e9800998ecf8427e  simplyblock_core/__init__.py
+921aad174688b985519e0c72d1d6d0e7  simplyblock_core/cluster_ops.py
+eb941f1f1c0fbc49bf6c569fef7d31f8  simplyblock_core/constants.py
+d41d8cd98f00b204e9800998ecf8427e  simplyblock_core/controllers/__init__.py
+a440b45b21c4f14f2e40240dbf2d9bc5  simplyblock_core/controllers/backup_controller.py
+eb0275929e5d1dd522763c44b3b40fda  simplyblock_core/controllers/backup_events.py
+43f023313267a2434a2029deda471f61  simplyblock_core/controllers/cluster_events.py
+d41d8cd98f00b204e9800998ecf8427e  simplyblock_core/controllers/cluster_expansion/__init__.py
+38cba0e6dd4e9374b37c3fe2cd0c6b8f  simplyblock_core/controllers/cluster_expansion/executor.py
+7e3ed535a62cdcd5e087044502437269  simplyblock_core/controllers/cluster_expansion/orchestrator.py
+5388e317669ef6ed861b882f75522e27  simplyblock_core/controllers/cluster_expansion/planner.py
+cb317fde8a270cda8bba528d0e4bf286  simplyblock_core/controllers/cluster_expansion/preconditions.py
+1dd883da6a0e99bad07ea12cb6ed2727  simplyblock_core/controllers/device_controller.py
+49318a96a77b96cd2438b1389a927600  simplyblock_core/controllers/device_events.py
+9563d5d608189dc3ee1cc7dc6575d8c8  simplyblock_core/controllers/events_controller.py
+96466463b95d74b79cce2a398949956a  simplyblock_core/controllers/fdb_backup_controller.py
+ef01b95df3f3c501bbd0df35753eeea0  simplyblock_core/controllers/health_controller.py
+583833a31731ee88d6c5e4ed9ed8b2f9  simplyblock_core/controllers/host_auth.py
+122238a7f90215505a85c9fb4e1d06d7  simplyblock_core/controllers/lvol_controller.py
+8ed20e8ada22c39ad2b40dc1aeb91436  simplyblock_core/controllers/lvol_events.py
+d7261581a5308b13e1fc830910b4202d  simplyblock_core/controllers/mgmt_events.py
+4495ba0571fed37306e9d6684e993e87  simplyblock_core/controllers/migration_bdev_ops.py
+7a7a80de2a192802cfab2a4d52a619c4  simplyblock_core/controllers/migration_controller.py
+faf2454ca99acce7a0b8b12f2a0a1c2e  simplyblock_core/controllers/migration_events.py
+3329fdc76d009ab22043a4adfdd31162  simplyblock_core/controllers/ops_gate.py
+16b973f2a2a44a20ae1d76329572ed33  simplyblock_core/controllers/pool_controller.py
+1c79832885997693b642ea125465c224  simplyblock_core/controllers/pool_events.py
+10fbc91df11af77ad23a83c9d0dbe66b  simplyblock_core/controllers/qos_controller.py
+8ef27ba9a004a7f365f7fdc810796cfa  simplyblock_core/controllers/replication_policy_controller.py
+a4a06dc366b026908f02fc21502792d2  simplyblock_core/controllers/snapshot_controller.py
+99f7f70f3a3c0e909aac03802f5006b3  simplyblock_core/controllers/snapshot_events.py
+9c08682f202955ed566ed96d9535c829  simplyblock_core/controllers/storage_events.py
+441da86657062e418f150da66efa5390  simplyblock_core/controllers/tasks_controller.py
+c5cc66cb98bde4768092cdb1a0131ae4  simplyblock_core/controllers/tasks_events.py
+b8d69479e568944c6d26da7e9b61bb56  simplyblock_core/controllers/tcp_ports_events.py
+91613b33cc8eecfd8e36488c42b1efcb  simplyblock_core/db_controller.py
+84a808ea97b86654e5547b13472faf48  simplyblock_core/distr_controller.py
+212bb9c4ba9198b35be212d5dfd89601  simplyblock_core/exceptions.py
+c577aec17bc3aa0428af060e5cbbd8f0  simplyblock_core/fw_api_client.py
+3fc3f4fc5f60993d7b138fa9b4adbcad  simplyblock_core/jm_raid.py
+74e7b278ad56e4adbfc6f5105f23fa70  simplyblock_core/kms/__init__.py
+b21dfac624800ae4695edb6dead53a7c  simplyblock_core/kms/_base.py
+eaa1c679331e65af0a8533856173c486  simplyblock_core/kms/_exceptions.py
+707d37b161f048797f4be19654345709  simplyblock_core/kms/_fdb.py
+7f917ea95665ee16c0ffd1c08ddbd9c2  simplyblock_core/kms/_hcp.py
+fcd90d3a9b7af72c76369eb6fd67e955  simplyblock_core/mgmt_node_ops.py
+d41d8cd98f00b204e9800998ecf8427e  simplyblock_core/models/__init__.py
+f4a6dff300d400be9d9983ea6bcd02b0  simplyblock_core/models/backup.py
+1991ef4a4e6f043d90f08232ccd6998a  simplyblock_core/models/base_model.py
+b7a0b089043fcd1bbc4114cf866f2a98  simplyblock_core/models/cluster.py
+41c4c62321b77e7381709d39345be74c  simplyblock_core/models/events.py
+c1d95ed3eb8531a3736a74ba9d522a16  simplyblock_core/models/hub_cooldown.py
+84a5e67573439d80fc8217bf6956e86d  simplyblock_core/models/hublvol.py
+aaa0f08f3724359abfccac3a6f912651  simplyblock_core/models/iface.py
+bad6cffda3d269eac4528a9fbe3bceca  simplyblock_core/models/job_schedule.py
+fbf8c48e05b78ad0cfadd0d0b6952e3e  simplyblock_core/models/lvol_migration.py
+93e027ba3737c5d3a66a76c4bef1eb99  simplyblock_core/models/lvol_migration_group.py
+7233db81712cd3c02277cf88b2c5509d  simplyblock_core/models/lvol_model.py
+79a31818d478ecce83c63050ece62065  simplyblock_core/models/lvstore_lock.py
+5e8aeda97b87058699292ed0fd3603b9  simplyblock_core/models/mgmt_node.py
+b1076bf14319f93df9906a1ff77e9920  simplyblock_core/models/nvme_device.py
+16e6498b741406a6ca52e531a2a34611  simplyblock_core/models/pool.py
+d350fce28254cf53da607facfdab1cce  simplyblock_core/models/port_stat.py
+76b86dcb5850954d0b78928a772e4616  simplyblock_core/models/qos.py
+82eb6d6570452eaa7052d84dd5a2b47e  simplyblock_core/models/replication.py
+7e34fc1f2577c045667418687e82796d  simplyblock_core/models/restart_lock.py
+b17735bfa9fb6099d948ca1ab954a907  simplyblock_core/models/snapshot.py
+930d082cbb3d3f2f9076837e38a5283d  simplyblock_core/models/stats.py
+afadba47b1b755674ecfb2edbae2dafd  simplyblock_core/models/storage_node.py
+f820ca63541b36fc34dc938a1419da53  simplyblock_core/prom_client.py
+9ef4b502175a7d5cc32f6ea82d110dfa  simplyblock_core/release_upgrades/__init__.py
+cba45f129303eca7295214719b56df4b  simplyblock_core/release_upgrades/jc_compression_upgrade.py
+ae7b5ff638b16236f39f38ecd52563e5  simplyblock_core/rpc_client.py
+fb8278ed7174bd9ca12b44a545b108e6  simplyblock_core/scripts/__init__.py
+649b3072b7625d6bbb732a9e66566dc2  simplyblock_core/scripts/collect_logs.py
+d41d8cd98f00b204e9800998ecf8427e  simplyblock_core/scripts/helpers/__init__.py
+455c553a6598cd0178ecb36433f07d10  simplyblock_core/scripts/helpers/nvme_disconnect_by_ip.py
+a371a8ebc56236f463fa49c6674faa51  simplyblock_core/scripts/restore_fdb_from_kvfiledump.py
+4c20bb18b5f6e04234455dfe8112b3d8  simplyblock_core/services/__init__.py
+89ea5203a611d9eb7ba04d1879aa840c  simplyblock_core/services/cap_monitor.py
+c74e062d9cd028e98503b07fd3a7ee04  simplyblock_core/services/capacity_and_stats_collector.py
+9310b22ccdc7d6f5d850e932a6d6098d  simplyblock_core/services/device_monitor.py
+3b982e4e58d489fc2cc861ee9f5d9e88  simplyblock_core/services/health_check_service.py
+8773b45be91a706f4dd50bf6ba14ff87  simplyblock_core/services/hub_controller_manager.py
+f22a5aa9946183ec33061a253166b9f0  simplyblock_core/services/lvol_monitor.py
+5af944ba3a09848e24984cc560012c00  simplyblock_core/services/lvol_stat_collector.py
+e286e508f7f34ea28b6894df29b6819e  simplyblock_core/services/main_distr_event_collector.py
+db8fd72f53396244adcd18f1e0348655  simplyblock_core/services/mgmt_node_monitor.py
+66a30a4a6386a08eb720530341b3ee0e  simplyblock_core/services/new_device_discovery.py
+1da9113a14a2e5c72438f8174d4d1301  simplyblock_core/services/replication_final_step.py
+f49cf6719f484f43c6f543f09a59356d  simplyblock_core/services/snapshot_monitor.py
+0103fd4d054878e22b450350e72cfd04  simplyblock_core/services/snapshot_replication.py
+a8e8f2ac6baaec3a2781cbc774886036  simplyblock_core/services/spdk_http_proxy_server.py
+d21aa89347671199e4c4f93db11067e6  simplyblock_core/services/storage_node_monitor.py
+f5a0397d343cfc2148eb86192c9e8ebd  simplyblock_core/services/tasks_cluster_status.py
+659ee98597f9f0f8e9607209a3abebd1  simplyblock_core/services/tasks_runner_backup.py
+10a97c1171915bbaa645c649cff88da2  simplyblock_core/services/tasks_runner_backup_merge.py
+ed954f1ff465008efb0222c09bb68deb  simplyblock_core/services/tasks_runner_batch_migration.py
+b4048ca32a99789e6ed76ae16c834985  simplyblock_core/services/tasks_runner_cluster_expand.py
+13faa65654c71c55bd8f3a19e34cba15  simplyblock_core/services/tasks_runner_failed_migration.py
+4894a2715a481f116ee2f092fe963c8b  simplyblock_core/services/tasks_runner_fdb_backup.py
+d7f6eb76fb513c62e8a3d5abd5192fa1  simplyblock_core/services/tasks_runner_jc_comp.py
+1c1ac93f66e096d52addb6373db50de2  simplyblock_core/services/tasks_runner_lvol_migration.py
+9763fc859165deda77a7433f48dc3825  simplyblock_core/services/tasks_runner_migration.py
+a45d0d3c7e952a7598ff15c91e90c6af  simplyblock_core/services/tasks_runner_new_dev_migration.py
+2cd5d1df8cfda73e0eaf1b9e1d96b8b9  simplyblock_core/services/tasks_runner_node_add.py
+2b57c422d936551428c28546ce9d7a02  simplyblock_core/services/tasks_runner_node_removal.py
+5015f0ed7eb5d465a2e61f0d414a8f19  simplyblock_core/services/tasks_runner_port_allow.py
+75434f8a8951e5dabee51f20bd8d8c3a  simplyblock_core/services/tasks_runner_replication_final.py
+dfafaedfafff3b69802d3f2c9f680390  simplyblock_core/services/tasks_runner_restart.py
+6f7c8d54b32d8a3cb60f18eff49a1217  simplyblock_core/services/tasks_runner_sync_lvol_del.py
+a3c56c87ab44a01ba71bcb12d778d23e  simplyblock_core/settings.py
+c0fb4c93ff5b5d32005d63bcdb813d0c  simplyblock_core/shell_utils.py
+03cda4cb8275cd62cb084da29d8a09af  simplyblock_core/snapshot_retention.py
+f1bc395dbd5223ae85a7efe1fee9b9cb  simplyblock_core/snode_client.py
+b12f37e2116565a4801be071c6a2fc10  simplyblock_core/storage_node_ops.py
+cc67f35ed45f89fa61d44fc74069898b  simplyblock_core/test/conftest.py
+4e36ab773d8359225f213cecaf429aab  simplyblock_core/test/test_ana_namespace_scope.py
+a77e777d1924e7964788ab3e1624b9b3  simplyblock_core/test/test_async_delete_poll.py
+93ee14f68659902bb5eb33efabdb7039  simplyblock_core/test/test_backpressure_safety_valve.py
+a99d0eb13088fa96da90263687413d29  simplyblock_core/test/test_chain_lock_scope.py
+4ebd88bc0d2f073317bebe6398a92023  simplyblock_core/test/test_connect_path_resolution.py
+0fdd9f09d496833cbf4450237868243f  simplyblock_core/test/test_del_sync_gate_release.py
+5b6b39afa4d99b0f3185f7d071178f28  simplyblock_core/test/test_delete_absent_is_done.py
+1739de1f19e94c6af8368e0d169be4bf  simplyblock_core/test/test_failback_chain_parent.py
+b2f945c83409c635074eb2179036e043  simplyblock_core/test/test_failover_clone_race.py
+6cd40edbbf823f2d5e8482c3cd39f720  simplyblock_core/test/test_failover_snapshot_selection.py
+f202c98493ee5e1e182dd902b40eccb2  simplyblock_core/test/test_failover_target.py
+cef1f86b3ef92e3de9a181555edbd587  simplyblock_core/test/test_internal_snapshot_backpressure.py
+d394ebf8b8c7e5451817fa4ec07d000d  simplyblock_core/test/test_internal_snapshot_scheduler.py
+51ab8b315c56ddebb6524922d1f05c03  simplyblock_core/test/test_internal_subsystem_exemption.py
+cbc0dd39dd808c076c3b6a0de6e97eea  simplyblock_core/test/test_jc_dual_node.py
+98a25a8b439113f8730b070e32a68151  simplyblock_core/test/test_lvstore_mutation_lock.py
+494ffa54376289a7a3746d003291a785  simplyblock_core/test/test_models.py
+468f1f2ee94262aea2a960e1ba79a519  simplyblock_core/test/test_pool_id_or_name.py
+8eee2578bf5bffcd5b682212499e8b61  simplyblock_core/test/test_replication_backlog_chain.py
+4df442ad73560df92de19c3f02a8499a  simplyblock_core/test/test_replication_backlog_destination.py
+858ad1c5544632989bf9b3a71ced0019  simplyblock_core/test/test_replication_backlog_reporting.py
+87fc1665b2d89826c316e5b9adc6dc06  simplyblock_core/test/test_replication_chain_completeness.py
+d6b2259d2fd45d8b672c58dfdcf333d4  simplyblock_core/test/test_replication_chain_predecessor.py
+142c27321f4107cb8dcd6f80cebc6d71  simplyblock_core/test/test_replication_commit.py
+27c3901c3b17efe712cfeb0441b750ed  simplyblock_core/test/test_replication_destination_resolution.py
+2aef47c5d33aa862031042362abb4552  simplyblock_core/test/test_replication_failback.py
+cbcedc215d05ce9d13e5aa1297da65c4  simplyblock_core/test/test_replication_failback_pool.py
+efdcf72be12304a4d6ea97570052b564  simplyblock_core/test/test_replication_final_step.py
+ec169938b233d87ed30f5356867b1a02  simplyblock_core/test/test_replication_health.py
+428cce0e1f68056e361ec7e7affa4861  simplyblock_core/test/test_replication_leader_follow.py
+7fd80e9a3b36f0cbe9f8204d67fccbb7  simplyblock_core/test/test_replication_models.py
+674100457e09a93981f8d1bbeca581b4  simplyblock_core/test/test_replication_policies.py
+142f851c9f1a973a3b69ad6587ab084f  simplyblock_core/test/test_replication_progress.py
+c5d56733f8e64b4934a83dfa773dd9e8  simplyblock_core/test/test_replication_relationship_and_delete_source.py
+9deb75aea6214d0dc6ad81527332acb2  simplyblock_core/test/test_replication_runner_resilience.py
+722b01fc4a5a39f1165d01d365fdccfa  simplyblock_core/test/test_replication_source_leader.py
+d060d8d46b38d9fca593db7206f0d33a  simplyblock_core/test/test_shutdown_task_guard.py
+78de0df795baaf32863f4e6916a34967  simplyblock_core/test/test_snapshot_chain_target.py
+d6e30188e4eb37f5f54e72c031ca1ccd  simplyblock_core/test/test_snapshot_instance_handoff.py
+98c1f729b8a91aadcfe5bd15ac5466b3  simplyblock_core/test/test_snapshot_monitor_live_clone.py
+7088fd00c8673421cb9d100ec7612b16  simplyblock_core/test/test_snapshot_retention.py
+9a95119047fe465944e67c499cd925b3  simplyblock_core/test/test_sync_delete_peer.py
+33b8ddf70012155e20620d95b495dfcd  simplyblock_core/test/test_tasks_runner_replication_final.py
+bc53f292fae277b435d6b1952cdc6daf  simplyblock_core/test/test_transfer_hub_heal.py
+0c9a3e9dc14315ea372a1f631802c354  simplyblock_core/test/test_utils.py
+9e4fa1baf48be1c833f34da643b1a81f  simplyblock_core/utils/__init__.py
+fc87695c06fb33ca74293dae12f6bb15  simplyblock_core/utils/dial_backoff.py
+4d255423494ff9d96f38b2cae3175195  simplyblock_core/utils/helpers.py
+70c1bf1532768cc2ff86d061b4b8d17b  simplyblock_core/utils/hublvol_reconnect.py
+7dcbc009ce235b4df483d38d5b34c13b  simplyblock_core/utils/nvme.py
+b0b3ac01438e9bbea6b7eeb678f4f52f  simplyblock_core/utils/pci.py
+9d2effb926da5d0e36be1e86778bbdde  simplyblock_core/utils/port_block.py
+b45bc54582bfbe1bea24580cf20e8010  simplyblock_core/utils/secrets.py
+57af18bc39674a7a16cbb7bc9a8684b5  simplyblock_core/utils/ttl_cache.py
+dbf1184940d69c0d5bf7a1f22f000b63  simplyblock_core/workers/cleanup_foundationdb.py

scripts/aws_dual_node_outage_soak_multipath.py

@@ -130,6 +130,18 @@
     "fio: pid=",
     "Killed",
     "Terminated",
+    # fio prints verify failures WITHOUT the "fio: " prefix -- the line reads
+    # "verify: bad magic header a8a4, wanted acca at file ...". Neither
+    # "fio: verify" nor "verify failed" matches that, so run 20260825_155730
+    # corrupted vol6 at 18:53 and vol2 at 19:43 and the soak kept applying
+    # outages for another two hours, reporting PASS each time, until vol4's
+    # fio *process* died at 20:49 and the rc-file branch finally caught it.
+    # A data verification failure is the single most important thing this
+    # harness can find; it must never again depend on fio also crashing.
+    "verify: bad",
+    "bad magic header",
+    "bad header offset",
+    "verify: got",
 )
 #: fio stderr markers for a --max_latency violation. Fatal in phase 1 (a
 #: single-NIC outage must be transparent), counted in phase 2 (promotion
@@ -206,6 +218,14 @@ def parse_args():
                      help="Seconds the NIC stays down on all nodes (default 30).")
     nic.add_argument("--nic-phase-settle", type=int, default=30,
                      help="Seconds to wait after NIC restore before verifying (default 30).")
+    nic.add_argument("--no-nic-phase", action="store_true",
+                     help="Disable phase 1 (the all-nodes single-NIC outage) "
+                          "entirely. REQUIRED on a single-data-NIC, non-multipath "
+                          "cluster: with one data NIC per node, phase 1 takes that "
+                          "NIC down on every node at once and isolates the whole "
+                          "cluster instead of exercising path redundancy. Note that "
+                          "--nic-phase-every 0 does NOT disable phase 1 -- it means "
+                          "'once, on iteration 1'.")
     nic.add_argument("--nic-phase-every", type=int, default=1,
                      help="Run the NIC phase every N iterations. 0 = once, before the "
                           "first pair only (default 1).")
@@ -1220,8 +1240,15 @@ def start_fio(self, volumes):
             if args.fio_max_latency > 0:
                 fio_cmd += f"--max_latency={args.fio_max_latency}s "
             if args.fio_verify:
+                # verify_dump writes <file>.<offset>.received / .expected on a
+                # mismatch. Without it a verify failure gives only fio's one
+                # line, and every corruption so far has died with the returned
+                # bytes unidentified -- we could not tell stale data from
+                # parity noise from a neighbouring block, and the volumes are
+                # on instance store so they vanish when the fleet is stopped.
+                # The dumps are 4 KiB each and only appear on failure.
                 fio_cmd += (f"--verify={args.fio_verify} --verify_fatal=1 "
-                            f"--verify_backlog=1024 ")
+                            f"--verify_backlog=1024 --verify_dump=1 ")
             fio_cmd += f"--output={shlex.quote(volume['fio_log'])}"
 
             start_script = (
@@ -1602,6 +1629,12 @@ def _rpc_json(subcmd, what):
         # node passive. active_passive means one NIC carries all hub IO.
         to_check = [(b, "hublvol") for b in hublvol_bdevs]
         to_check += [(b, "remote") for b in remote_bdevs[:max(0, self.args.policy_sample)]]
+        # Only meaningful with more than one path: on a single-data-NIC
+        # (non-multipath) cluster a bdev legitimately reports active_passive,
+        # and asserting active_active there would make every bdev a "problem",
+        # so the heal gate could never converge.
+        if len(self.args.data_nics) < 2:
+            to_check = []
         for bdev_name, kind in to_check:
             try:
                 bdevs = _rpc_json(f"bdev_get_bdevs -b {shlex.quote(bdev_name)}",
@@ -2152,8 +2185,13 @@ def run(self):
                     + ", ".join(f"{n['uuid'][:12]}:{n['status']}" for n in current))
             uuids = [n["uuid"] for n in current]
 
-            nic_due = (iteration == 1 if args.nic_phase_every == 0
-                       else (iteration - 1) % args.nic_phase_every == 0)
+            # --nic-phase-every 0 means "once, on iteration 1"; only
+            # --no-nic-phase disables phase 1 outright, which is required on a
+            # single-data-NIC cluster where taking "one" NIC down on every node
+            # isolates the whole cluster instead of testing path redundancy.
+            nic_due = ((not args.no_nic_phase)
+                       and (iteration == 1 if args.nic_phase_every == 0
+                            else (iteration - 1) % args.nic_phase_every == 0))
             if nic_due:
                 nic = args.data_nics[(iteration - 1) % len(args.data_nics)]
                 if self.run_nic_phase(iteration, nic, uuids):