From 7ca7f7a3a99805396bb4d174ddaf63c6cf04e8b9 Mon Sep 17 00:00:00 2001 From: michael Date: Mon, 24 Aug 2026 11:06:43 +0200 Subject: [PATCH 001/122] =?UTF-8?q?fix(replication):=20the=20namespace=20e?= =?UTF-8?q?viction=20never=20ran=20=E2=80=94=20subsystem=5Fget=20returns?= =?UTF-8?q?=20a=20dict?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rpc_client.subsystem_get returns ONE subsystem dict (single_or_none), not a list. Indexing it with [0] raised KeyError(0), the helper's best-effort except swallowed it as a warning, and the eviction silently never executed — run 20260824_104449 failed with the same 40x add_ns -32602 while the fix was nominally in place. The test fake modelled the wrong shape too (a list), which is exactly how the bug got past the suite; it now models reality. Co-Authored-By: Claude Opus 5 --- simplyblock_core/controllers/lvol_controller.py | 10 +++++++--- .../test/test_replication_chain_completeness.py | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index ff12b0bc00..010c9c9e2e 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -3462,10 +3462,14 @@ def _evict_stale_namespace(new_lvol, target_node): """ try: rpc = target_node.rpc_client() - subsystems = rpc.subsystem_get(new_lvol.nqn) - if not subsystems: + # subsystem_get returns ONE subsystem dict (single_or_none), not a + # list -- indexing it with [0] raised KeyError(0), the best-effort + # except swallowed it, and the eviction silently never ran (run + # 20260824_104449: same 40x add_ns -32602 with the fix "in place"). + subsystem = rpc.subsystem_get(new_lvol.nqn) + if not subsystem: return - for ns in (subsystems[0].get("namespaces") or []): + for ns in (subsystem.get("namespaces") or []): if ns.get("nsid") != new_lvol.ns_id: continue if ns.get("bdev_name") == new_lvol.top_bdev: diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index c31691090f..cb84d017ee 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -274,7 +274,8 @@ def __init__(self, namespaces): self._ns = namespaces def subsystem_get(self, nqn): - return [{"nqn": nqn, "namespaces": self._ns}] + # the real client returns ONE dict (single_or_none), never a list + return {"nqn": nqn, "namespaces": self._ns} def nvmf_subsystem_remove_ns(self, nqn, nsid): self.removed.append((nqn, nsid)) From da7d1be42d8119e7d29be57c5f65ab9b343f6e9e Mon Sep 17 00:00:00 2001 From: michael Date: Mon, 24 Aug 2026 11:33:56 +0200 Subject: [PATCH 002/122] fix(replication): confirm the stale namespace is GONE before the cutover adds nvmf_subsystem_remove_ns acknowledges before it completes (the same async false-success that dropped a shared subsystem in the PVC-expand incident, whose fix polls for confirmation). The fail-back eviction removed the stale namespace and add_ns raced the removal and lost on all 8 retries -- run 20260824_110959: 40 evictions logged, 40 add_ns -32602 right behind them. The eviction now polls the subsystem until the namespace is actually gone (bounded, 20s) before returning, matches the stale entry by uuid as well as nsid (the preserved identity collides on both axes), and the test fake now models the acknowledged-but-lingering removal so this race stays pinned. Co-Authored-By: Claude Opus 5 --- .../controllers/lvol_controller.py | 33 +++++++++++++--- .../test_replication_chain_completeness.py | 38 +++++++++++++++++-- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 010c9c9e2e..914cb27abd 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -3469,17 +3469,38 @@ def _evict_stale_namespace(new_lvol, target_node): subsystem = rpc.subsystem_get(new_lvol.nqn) if not subsystem: return - for ns in (subsystem.get("namespaces") or []): - if ns.get("nsid") != new_lvol.ns_id: - continue - if ns.get("bdev_name") == new_lvol.top_bdev: - return # already ours (re-run) + # Match by nsid OR by uuid: the stale namespace carries the volume's + # preserved identity on both axes, and either collides with add_ns. + stale = [ns for ns in (subsystem.get("namespaces") or []) + if (ns.get("nsid") == new_lvol.ns_id + or ns.get("uuid") == new_lvol.uuid) + and ns.get("bdev_name") != new_lvol.top_bdev] + if not stale: + return + for ns in stale: logger.info( f"Fail-back cutover: evicting stale namespace nsid={ns.get('nsid')} " f"(bdev {ns.get('bdev_name')}) from {new_lvol.nqn} on " f"{target_node.get_id()} -- superseded by the failed-over data") rpc.nvmf_subsystem_remove_ns(new_lvol.nqn, ns.get("nsid")) - return + # remove_ns ACKNOWLEDGES before it completes (the same async + # false-success that dropped a shared subsystem in the PVC-expand + # incident; its fix polls for confirmation, eb127eed). Without this + # poll the follow-up add_ns raced the removal and lost on all 8 + # retries (run 20260824_110959: 40 evictions logged, 40 add_ns + # -32602 right behind them). + stale_ids = {ns.get("nsid") for ns in stale} + deadline = time.time() + 20 + while time.time() < deadline: + current = rpc.subsystem_get(new_lvol.nqn) or {} + if not any(ns.get("nsid") in stale_ids + for ns in (current.get("namespaces") or [])): + return + time.sleep(1) + logger.error( + f"Stale namespace(s) {sorted(stale_ids)} on {new_lvol.nqn} did not " + f"disappear within 20s of removal on {target_node.get_id()}; the " + f"cutover's add_ns will fail and retry") except Exception as e: # Best effort: if the subsystem is not there, add_lvol_on_node creates # it; if the eviction genuinely failed, add_ns will say so loudly. diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index cb84d017ee..bc3bc98ed9 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -269,15 +269,23 @@ def test_no_backward_task_for_a_policy_managed_clone(): class _EvictRPC: - def __init__(self, namespaces): + def __init__(self, namespaces, linger_polls=0): self.removed = [] - self._ns = namespaces + self._ns = list(namespaces) + self._linger = linger_polls # polls before a removed ns disappears def subsystem_get(self, nqn): # the real client returns ONE dict (single_or_none), never a list - return {"nqn": nqn, "namespaces": self._ns} + if self._linger > 0: + self._linger -= 1 + return {"nqn": nqn, "namespaces": self._ns} + live = [n for n in self._ns + if (nqn, n.get("nsid")) not in self.removed] + return {"nqn": nqn, "namespaces": live} def nvmf_subsystem_remove_ns(self, nqn, nsid): + # acknowledges immediately; disappearance is governed by linger_polls, + # modelling the fork's async remove_ns false-success self.removed.append((nqn, nsid)) return True @@ -332,3 +340,27 @@ def subsystem_get(self, nqn): rpc = _NoSubsysRPC([]) lc._evict_stale_namespace(_CloneLvol(), _EvictNode(rpc)) # must not raise assert rpc.removed == [] + + +def test_failback_eviction_waits_out_the_async_removal(monkeypatch): + """remove_ns acknowledges before it completes (the PVC-expand + false-success); the eviction must confirm the namespace is GONE before + add_ns runs, or the add races the removal and loses (run 20260824_110959: + 40 evictions immediately followed by 40 add_ns -32602).""" + from simplyblock_core.controllers import lvol_controller as lc + monkeypatch.setattr(lc.time, "sleep", lambda s: None) + rpc = _EvictRPC([{"nsid": 7, "bdev_name": "LVS_1/LVOL_ORIG"}], linger_polls=3) + lc._evict_stale_namespace(_CloneLvol(), _EvictNode(rpc)) + assert rpc.removed == [("nqn.test:lvol:ORIG", 7)] + # after the helper returns, the namespace must actually be gone + assert rpc.subsystem_get("nqn.test:lvol:ORIG")["namespaces"] == [] + + +def test_failback_eviction_matches_by_uuid_too(monkeypatch): + from simplyblock_core.controllers import lvol_controller as lc + monkeypatch.setattr(lc.time, "sleep", lambda s: None) + rpc = _EvictRPC([{"nsid": 3, "uuid": "LV_UUID", "bdev_name": "LVS_1/LVOL_ORIG"}]) + class _C(_CloneLvol): + uuid = "LV_UUID" + lc._evict_stale_namespace(_C(), _EvictNode(rpc)) + assert rpc.removed == [("nqn.test:lvol:ORIG", 3)] From f3ea3cfb90be01ddff25f49590e2da36c64ee998 Mon Sep 17 00:00:00 2001 From: michael Date: Mon, 24 Aug 2026 12:19:18 +0200 Subject: [PATCH 003/122] fix: evict the stale preserved-identity namespace on every HA node, not just the primary Run 20260824_113711 proved the eviction itself now works: the primary's add_ns returned result:1 for the first time. But the preserved-NQN subsystem exists on EVERY node of the recovered HA set, each still holding the original volume's namespace at the preserved nsid. The HA peer's add_ns failed with the same -32602 the eviction was written for, add_lvol_on_node's peer failure rolled the whole cutover back, and all 5 fail-back cutovers died on max retry (0/5). _create_target_lvol_clone now calls _evict_stale_namespace for each online HA peer right before that peer's add_lvol_on_node, exactly as it already did for the primary. Co-Authored-By: Claude Fable 5 --- .../controllers/lvol_controller.py | 7 +++ .../test_replication_chain_completeness.py | 55 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 914cb27abd..58dcbfca84 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -3384,6 +3384,13 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps continue if peer_node.status != StorageNode.STATUS_ONLINE: continue + # The preserved-NQN subsystem exists on EVERY node of the recovered + # HA set, each still holding the original volume's namespace at the + # preserved nsid. Evicting only on the primary made its add_ns + # succeed while the peer's failed with the same -32602, and the + # peer failure rolled the whole cutover back (run 20260824_113711: + # primary add_ns result:1, peer -32602, 0/5 cutovers). + _evict_stale_namespace(new_lvol, peer_node) lvol_bdev, error = add_lvol_on_node(new_lvol, peer_node, is_primary=False) if error: logger.error(error) diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index bc3bc98ed9..83f0ad8ad9 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -364,3 +364,58 @@ class _C(_CloneLvol): uuid = "LV_UUID" lc._evict_stale_namespace(_C(), _EvictNode(rpc)) assert rpc.removed == [("nqn.test:lvol:ORIG", 3)] + + +def test_failback_evicts_on_every_ha_node_not_just_the_primary(monkeypatch): + """Run 20260824_113711: eviction on the primary made ITS add_ns succeed + (result: 1) while the HA peer's failed with the same -32602 -- the + preserved-NQN subsystem exists on EVERY node of the recovered set, each + still holding the original namespace. The peer failure rolled the whole + cutover back: 0/5. The clone path must evict per node.""" + import copy + from simplyblock_core.controllers import lvol_controller as lc + + evicted, added = [], [] + monkeypatch.setattr(lc, "_evict_stale_namespace", + lambda lvol, node: evicted.append(node.get_id())) + monkeypatch.setattr(lc, "add_lvol_on_node", + lambda lvol, node, is_primary=True: ( + added.append((node.get_id(), is_primary)), + ({"uuid": "U", "driver_specific": {"lvol": {"blobid": 9}}}, None))[-1]) + + class _N: + def __init__(self, nid, secondary="", tertiary=""): + self._id, self.secondary_node_id, self.tertiary_node_id = nid, secondary, tertiary + self.lvstore = "LVS_1" + self.status = lc.StorageNode.STATUS_ONLINE + def get_id(self): + return self._id + + primary = _N("P", secondary="S") + peer = _N("S") + + class _DB: + kv_store = None + def get_storage_node_by_id(self, nid): + return {"P": primary, "S": peer}[nid] + def release_lvol_ns_slot(self, lvol): + pass + + class _Lvol: + uuid = "ORIG"; nqn = "nqn.test:lvol:ORIG"; ns_id = 7 + lvol_bdev = "LVOL_C"; crypto_bdev = "" + def __deepcopy__(self, memo): + c = _Lvol(); c.__dict__.update(self.__dict__); return c + def write_to_db(self, kv=None): + pass + + class _Snap: + cluster_id = "C1"; snap_bdev = "LVS_1/SNAP_1" + def get_id(self): + return "SNAP1" + + new_lvol, error = lc._create_target_lvol_clone(_DB(), _Lvol(), primary, "POOL", _Snap()) + assert error is None + assert evicted == ["P", "S"], \ + "stale-namespace eviction must run on the primary AND every online HA peer" + assert ("S", False) in added From d361e0e8aed99ce6c6e26efa7c695afb3f6cb84e Mon Sep 17 00:00:00 2001 From: michael Date: Mon, 24 Aug 2026 15:27:18 +0200 Subject: [PATCH 004/122] fix: adopt or clear a landing volume left by an interrupted transfer attempt; add soak cases 7-9 Case 6 (run 20260824_144226) exposed the stall: a node outage mid-create left a REP_* landing volume whose id was never stored on the task, and every retry of the transfer died on "LVol name must be unique" (~31s loop for the rest of the run), stalling three volumes' chains and the case behind the 180s lag gate. The runner now probes for a record already wearing the derived name before creating: adopt it when online, wait when in_deletion, force-delete when half-created. New soak cases per the extended test plan: - case 7: 20 namespaced volumes on 2 shared subsystems (10 ns each), randomly assigned across 2 clients, replication + fail-over + fail-back with NQN/nsid identity asserted per namespace. The deployer gains an add_client mode to grow an existing lab to 2 clients. - case 8: sequential-pressure catch-up: 64k/QD64/4-job fills of 50G per volume, repeated; peak backlog and catch-up time are recorded and the backlog must drain under the lag gate every cycle. - case 9: chaos: random SPDK-container kills on BOTH clusters' nodes while replication runs (seeded, logged with the pipeline phases active at each kill), then full catch-up + fail-over integrity verification. Co-Authored-By: Claude Fable 5 --- scripts/setup_repl_test_2clusters.py | 38 +- scripts/test_async_replication.py | 508 +++++++++++++++++- .../services/snapshot_replication.py | 41 ++ .../test_replication_chain_completeness.py | 20 + 4 files changed, 582 insertions(+), 25 deletions(-) diff --git a/scripts/setup_repl_test_2clusters.py b/scripts/setup_repl_test_2clusters.py index 91b99b26a1..2b3040d355 100644 --- a/scripts/setup_repl_test_2clusters.py +++ b/scripts/setup_repl_test_2clusters.py @@ -19,6 +19,7 @@ """ import os import json +import sys import re import time from concurrent.futures import ThreadPoolExecutor @@ -368,7 +369,38 @@ def add_one(priv_ip): # --------------------------------------------------------------------------- # +CLIENT_PREP_CMDS = [ + "sudo dnf install nvme-cli fio -y", + "sudo modprobe nvme-tcp", + "echo 'nvme-tcp' | sudo tee /etc/modules-load.d/nvme-tcp.conf", +] + + +def add_client(): + """Add one client instance to an EXISTING deployment (case 7 needs >= 2 + clients; redeploying a healthy two-cluster lab for that is wasteful).""" + with open("cluster_metadata_repl.json") as f: + metadata = json.load(f) + print("Launching 1 additional client...") + clients = launch_instances("SB-Repl-Client", CLIENT_TYPE, 1) + for inst in clients: + inst.wait_until_running() + inst.reload() + ip = clients[0].public_ip_address + wait_for_ssh(ip) + print(f"Prepping client {ip}...") + ssh_exec(ip, CLIENT_PREP_CMDS, check=True) + metadata.setdefault("clients", []).append( + {"public_ip": ip, "private_ip": clients[0].private_ip_address}) + with open("cluster_metadata_repl.json", "w") as f: + json.dump(metadata, f, indent=4) + print(f"Client added: {ip} ({len(metadata['clients'])} clients in metadata).") + + def main(): + if len(sys.argv) > 1 and sys.argv[1] == "add_client": + add_client() + return print(f"Launching control plane + {SN_COUNT} storage nodes + {CLIENT_COUNT} client(s)...") mgmt = launch_instances("SB-Repl-Mgmt", MGMT_TYPE, 1, with_net=False) sns = launch_instances("SB-Repl-Storage", SN_TYPE, SN_COUNT) @@ -487,11 +519,7 @@ def main(): # --- Phase 6: prep clients --- if client_pub_ips: print("Prepping clients...") - client_cmds = [ - "sudo dnf install nvme-cli fio -y", - "sudo modprobe nvme-tcp", - "echo 'nvme-tcp' | sudo tee /etc/modules-load.d/nvme-tcp.conf", - ] + client_cmds = CLIENT_PREP_CMDS for ip in client_pub_ips: wait_for_ssh(ip) with ThreadPoolExecutor(max_workers=max(1, len(client_pub_ips))) as ex: diff --git a/scripts/test_async_replication.py b/scripts/test_async_replication.py index d4c0e0a051..be19f922b1 100644 --- a/scripts/test_async_replication.py +++ b/scripts/test_async_replication.py @@ -419,22 +419,40 @@ def connect_and_mount(client_ip, key_path, mgmt_ip, lvols, fmt=True, mount_base= return mounts -def write_fio_jobfile(client_ip, key_path, mounts): - """One fio job per mounted volume; FIO_NUMJOBS threads each; md5 verify; 20s max latency.""" +def write_fio_jobfile(client_ip, key_path, mounts, + rw=None, bs=None, iodepth=None, numjobs=None, size=None, + time_based=True, verify=True, jobfile=None): + """One fio job per mounted volume; FIO_NUMJOBS threads each; md5 verify; 20s max latency. + + The keyword overrides exist for the pressure/chaos cases (7-9): case 8 + needs a FINITE sequential 64k/QD64/4-job fill of a known delta size, which + is the opposite of the endless mild verify-writer the fail-over cases use. + """ + rw = rw or FIO_RW + bs = bs or FIO_BS + iodepth = iodepth if iodepth is not None else FIO_IODEPTH + numjobs = numjobs if numjobs is not None else FIO_NUMJOBS + size = size or FIO_SIZE + jobfile = jobfile or FIO_JOBFILE sections = [ "[global]", - f"rw={FIO_RW}", - f"bs={FIO_BS}", - f"iodepth={FIO_IODEPTH}", + f"rw={rw}", + f"bs={bs}", + f"iodepth={iodepth}", "ioengine=libaio", "direct=1", - f"size={FIO_SIZE}", - f"numjobs={FIO_NUMJOBS}", - "time_based=1", - "runtime=86400", # effectively endless for the test - "verify=md5", - "verify_backlog=512", - "verify_fatal=1", + f"size={size}", + f"numjobs={numjobs}", + ] + if time_based: + sections += ["time_based=1", "runtime=86400"] # effectively endless + else: + # finite pass over `size`; rewriting the same files each cycle keeps + # the volume's footprint constant while dirtying the delta again + sections += ["loops=1", "overwrite=1"] + if verify: + sections += ["verify=md5", "verify_backlog=512", "verify_fatal=1"] + sections += [ f"max_latency={FIO_MAX_LATENCY}", "group_reporting=1", "", @@ -448,8 +466,8 @@ def write_fio_jobfile(client_ip, key_path, mounts): sections += [f"[vol{i}]", f"directory={m['mount']}", ""] content = "\n".join(sections) # Write the job file on the client. - run(client_ip, key_path, f"cat > {FIO_JOBFILE} <<'EOF'\n{content}\nEOF") - return FIO_JOBFILE + run(client_ip, key_path, f"cat > {jobfile} <<'EOF'\n{content}\nEOF") + return jobfile def start_fio(client_ip, key_path, jobfile): @@ -883,7 +901,7 @@ def delete_test_volumes(mgmt_ip, key_path, pools): check=False, quiet=True) for line in raw.splitlines(): cols = [c.strip() for c in line.split("|")] - if len(cols) > 3 and (cols[2].startswith("replvol") or cols[2].startswith("REP_")): + if len(cols) > 3 and cols[2].startswith(("replvol", "REP_", "nsvol", "presvol", "chaosvol")): if (cols[1], cols[2]) not in victims: victims.append((cols[1], cols[2])) if not victims: @@ -903,7 +921,7 @@ def delete_test_volumes(mgmt_ip, key_path, pools): check=False, quiet=True) for line in raw.splitlines(): cols = [c.strip() for c in line.split("|")] - if len(cols) > 3 and (cols[2].startswith("replvol") or cols[2].startswith("REP_")): + if len(cols) > 3 and cols[2].startswith(("replvol", "REP_", "nsvol", "presvol", "chaosvol")): left += 1 if left == 0: print(" cleanup drained.") @@ -912,7 +930,8 @@ def delete_test_volumes(mgmt_ip, key_path, pools): def create_volumes(mgmt_ip, key_path, src_uuid, pool, tgt_uuid, tgt_pool, mode, - count=NUM_VOLUMES): + count=NUM_VOLUMES, prefix="replvol", size=VOL_SIZE, + extra_flags=""): """Create the test volumes already following a replication policy. The policy IS the start: `volume add --replication-policy` attaches it, and @@ -924,10 +943,10 @@ def create_volumes(mgmt_ip, key_path, src_uuid, pool, tgt_uuid, tgt_pool, mode, mode=mode) lvols = [] for i in range(count): - name = f"replvol{i}" + name = f"{prefix}{i}" run(mgmt_ip, key_path, - f"{SBCTL} -d volume add {name} {VOL_SIZE} {pool}" - f" --replication-policy {policy}") + f"{SBCTL} -d volume add {name} {size} {pool}" + f" --replication-policy {policy}{(' ' + extra_flags) if extra_flags else ''}") lv = resolve_lvol(mgmt_ip, key_path, name) lvols.append(lv["uuid"]) print(f" created {name} = {lv['uuid']} (policy {policy}, mode={mode})") @@ -1536,6 +1555,449 @@ def test_case_6(meta): print("CASE 6 PASSED: source-primary outage survived, replication continued and resumed.") +# --------------------------------------------------------------------------- # +# Cases 7-9: namespaced subsystems, sequential pressure, chaos injection +# --------------------------------------------------------------------------- # +NS_VOLUMES = int(os.environ.get("NS_VOLUMES", "20")) +NS_PER_SUBSYS = int(os.environ.get("NS_PER_SUBSYS", "10")) +NS_VOL_SIZE = os.environ.get("NS_VOL_SIZE", "20G") + +PRESSURE_VOLUMES = int(os.environ.get("PRESSURE_VOLUMES", "2")) +PRESSURE_VOL_SIZE = os.environ.get("PRESSURE_VOL_SIZE", "120G") +PRESSURE_DELTA_GB = int(os.environ.get("PRESSURE_DELTA_GB", "50")) +PRESSURE_CYCLES = int(os.environ.get("PRESSURE_CYCLES", "3")) +PRESSURE_CATCHUP_TIMEOUT = int(os.environ.get("PRESSURE_CATCHUP_TIMEOUT", "3600")) + +CHAOS_EVENTS = int(os.environ.get("CHAOS_EVENTS", "12")) +CHAOS_SLEEP_MIN = int(os.environ.get("CHAOS_SLEEP_MIN", "20")) +CHAOS_SLEEP_MAX = int(os.environ.get("CHAOS_SLEEP_MAX", "150")) +CHAOS_SEED = os.environ.get("CHAOS_SEED", "") + + +def lvol_identities(mgmt_ip, key_path, lvol_uuids): + """{uuid: {nqn, ns_id, node_id}} — the preserved identity under test.""" + return mgmt_py(mgmt_ip, key_path, f""" +import json +from simplyblock_core.db_controller import DBController +db = DBController() +out = {{}} +for u in {list(lvol_uuids)!r}: + lv = db.get_lvol_by_id(u) + out[u] = {{"nqn": lv.nqn, "ns_id": lv.ns_id, "node_id": lv.node_id}} +print(json.dumps(out)) +""", replayable=True) + + +def _ns_devs_for_nqn(client_ip, key_path, nqn, expected, tries=10): + """{nsid: /dev/nvmeXnY} for every namespace of a SHARED subsystem. + + _dev_for_nqn picks the first namespace under the subsystem, which is + exactly wrong for case 7 where ten volumes share one NQN. sysfs gives the + block devices; `nvme get-ns-id` gives each one's NSID (the n-suffix in the + device name is a kernel instance number, NOT the NSID). + """ + for _ in range(tries): + out = run(client_ip, key_path, + "for s in /sys/class/nvme-subsystem/nvme-subsys*; do " + f"[ \"$(cat $s/subsysnqn 2>/dev/null)\" = \"{nqn}\" ] || continue; " + "ls $s 2>/dev/null | grep -E '^nvme[0-9]+n[0-9]+$'; " + "done", check=False, quiet=True) + devs = [d for d in out.split() if d] + mapping = {} + for d in devs: + nsid_out = run(client_ip, key_path, + f"sudo nvme get-ns-id /dev/{d} 2>/dev/null", + check=False, quiet=True) + m = re.search(r"(\d+)\s*$", nsid_out.strip()) + if m: + mapping[int(m.group(1))] = f"/dev/{d}" + if len(mapping) >= expected: + return mapping + time.sleep(5) + return mapping + + +def connect_and_mount_namespaced(client_ip, key_path, mgmt_ip, lvols, idents, + fmt=True, mount_base=MOUNT_BASE + "_ns"): + """connect_and_mount for volumes that SHARE subsystems: connect each NQN + once, then hand every volume the device matching ITS nsid.""" + prepare_mount_points(client_ip, key_path) + by_nqn = {} + for lv in lvols: + by_nqn.setdefault(idents[lv]["nqn"], []).append(lv) + + devmaps = {} + for nqn, members in by_nqn.items(): + conn = get_connect_cmds(mgmt_ip, key_path, members[0]) + assert not conn["err"], f"connect_lvol error for {members[0]}: {conn['err']}" + for cmd in conn["connect"]: + run(client_ip, key_path, cmd, check=False) + time.sleep(3) + devmaps[nqn] = _ns_devs_for_nqn(client_ip, key_path, nqn, len(members)) + print(f" subsystem {nqn.split(':')[-1][:13]}: " + f"{len(devmaps[nqn])} namespaces visible on {client_ip}") + + mounts = [] + for idx, lv in enumerate(lvols): + ident = idents[lv] + dev = devmaps.get(ident["nqn"], {}).get(ident["ns_id"]) + if not dev: + raise RuntimeError( + f"no device for lvol {lv} (nqn={ident['nqn']} nsid={ident['ns_id']}); " + f"visible: {devmaps.get(ident['nqn'])}") + mnt = f"{mount_base}{idx}" + if fmt: + run(client_ip, key_path, f"sudo mkfs.xfs -f {dev}") + run(client_ip, key_path, f"sudo mkdir -p {mnt} && sudo mount {dev} {mnt}") + mounts.append({"lvol": lv, "nqn": ident["nqn"], "dev": dev, "mount": mnt}) + print(f" vol {lv} (nsid {ident['ns_id']}) -> {dev} @ {mnt}") + return mounts + + +def test_case_7(meta): + """Namespaced volumes: 2 subsystems x 10 namespaces, 2 clients, full + replication + fail-over + fail-back with the shared-subsystem identity + preserved for every namespace.""" + print("\n========== CASE 7: namespaced subsystems (2x10 ns, 2 clients) ==========") + import random + key_path = meta["key_path"] + mgmt_ip = meta["mgmt"]["public_ip"] + clients = [c["public_ip"] for c in meta["clients"]] + if len(clients) < 2: + raise RuntimeError( + "case 7 needs at least 2 clients; run " + "`python scripts/setup_repl_test_2clusters.py add_client` first") + src_uuid, src, tgt_uuid, tgt = _src_target(meta) + + for ip in clients: + prepare_mount_points(ip, key_path) + delete_test_volumes(mgmt_ip, key_path, _all_test_pools(meta)) + + print(f"Creating {NS_VOLUMES} namespaced volumes " + f"(max {NS_PER_SUBSYS}/subsystem => {NS_VOLUMES // NS_PER_SUBSYS} subsystems)...") + lvols = create_volumes( + mgmt_ip, key_path, src_uuid, src["pool"], tgt_uuid, tgt["pool"], + mode="failover", count=NS_VOLUMES, prefix="nsvol", size=NS_VOL_SIZE, + extra_flags=f"--namespaced True --max-namespace-per-subsys {NS_PER_SUBSYS}") + + idents = lvol_identities(mgmt_ip, key_path, lvols) + by_nqn = {} + for lv in lvols: + by_nqn.setdefault(idents[lv]["nqn"], []).append(lv) + packing = {n.split(":")[-1][:13]: len(v) for n, v in by_nqn.items()} + print(f" subsystem packing: {packing}") + if len(by_nqn) != NS_VOLUMES // NS_PER_SUBSYS or set(packing.values()) != {NS_PER_SUBSYS}: + raise RuntimeError(f"FAIL: expected {NS_VOLUMES // NS_PER_SUBSYS} subsystems x " + f"{NS_PER_SUBSYS} namespaces, got {packing}") + + # Random client assignment, reproducible via printed seed. + seed = int(os.environ.get("NS_SEED") or time.time()) + rng = random.Random(seed) + print(f" client assignment seed: {seed}") + shuffled = lvols[:] + rng.shuffle(shuffled) + split = rng.randint(NS_VOLUMES // 4, 3 * NS_VOLUMES // 4) # both clients always used + assign = {clients[0]: shuffled[:split], clients[1]: shuffled[split:]} + + mounts_by_client, baseline = {}, {} + for ip, vols in assign.items(): + print(f"Client {ip}: {len(vols)} namespaces") + mounts_by_client[ip] = connect_and_mount_namespaced( + ip, key_path, mgmt_ip, vols, idents, fmt=True) + baseline.update(write_baseline(ip, key_path, mounts_by_client[ip])) + baseline_ts = time.time() + + for ip in assign: + start_fio(ip, key_path, write_fio_jobfile(ip, key_path, mounts_by_client[ip], + size="1G")) + wait_replication_caught_up(mgmt_ip, key_path, lvols, timeout=3600) + wait_data_replicated(mgmt_ip, key_path, lvols, baseline_ts, timeout=3600) + + print("Killing the source cluster (both nodes)...") + for ip in src["storage_public_ips"][:2]: + kill_spdk(ip, key_path) + time.sleep(15) + for ip in assign: + stop_fio(ip, key_path) + cleanup_client(ip, key_path, mounts_by_client[ip]) + + print("Failing over all namespaces...") + tgt_lvols = [] + for lv in lvols: + fo = do_failover(mgmt_ip, key_path, lv) + if not isinstance(fo, dict) or not fo.get("connection_strings"): + raise RuntimeError(f"FAIL: fail-over returned no connection strings for {lv}") + tgt_lvols.append(fo["lvol_id"]) + src_to_tgt = dict(zip(lvols, tgt_lvols)) + + # Identity preservation: every fail-over copy keeps ITS nqn and nsid, so + # the shared subsystems must re-form on the target with all 10 namespaces. + tgt_idents = lvol_identities(mgmt_ip, key_path, tgt_lvols) + for s, t in src_to_tgt.items(): + if (tgt_idents[t]["nqn"], tgt_idents[t]["ns_id"]) != (idents[s]["nqn"], idents[s]["ns_id"]): + raise RuntimeError( + f"FAIL: identity not preserved for {s}: " + f"{idents[s]['nqn']}/{idents[s]['ns_id']} -> " + f"{tgt_idents[t]['nqn']}/{tgt_idents[t]['ns_id']}") + print(" NQN + nsid preserved for all namespaces.") + + # Re-assign RANDOMLY again for the fail-over verification (fresh shuffle). + shuffled2 = tgt_lvols[:] + rng.shuffle(shuffled2) + split2 = rng.randint(NS_VOLUMES // 4, 3 * NS_VOLUMES // 4) + assign_fo = {clients[0]: shuffled2[:split2], clients[1]: shuffled2[split2:]} + tgt_baseline = {src_to_tgt[s]: b for s, b in baseline.items()} + + fo_mounts_by_client = {} + ok = True + for ip, vols in assign_fo.items(): + m = connect_and_mount_namespaced(ip, key_path, mgmt_ip, vols, tgt_idents, + fmt=False, mount_base=MOUNT_BASE + "_nsfo") + fo_mounts_by_client[ip] = m + good, _ = verify_baseline(ip, key_path, m, tgt_baseline) + ok = ok and good + if not ok: + raise RuntimeError("FAIL: replicated namespace data not intact after fail-over") + + print("Restoring the source cluster + failing back all namespaces...") + restore_cluster(mgmt_ip, key_path, src, label="src (case 7)") + set_cluster_replication(mgmt_ip, key_path, tgt_uuid, src_uuid, + pool_uuid_of(mgmt_ip, key_path, src["pool"])) + for lv in tgt_lvols: + failback(mgmt_ip, key_path, lv) + wait_replication_caught_up(mgmt_ip, key_path, tgt_lvols, timeout=3600) + for lv in tgt_lvols: + run(mgmt_ip, key_path, f"{SBCTL} -d volume replication-commit {lv}") + + start = time.time() + done = 0 + while time.time() - start < CUTOVER_WAIT_TIMEOUT * 2: + states = replication_states(mgmt_ip, key_path, tgt_lvols) + done = sum(1 for s in states.values() if s in ("cutover_done", "failed_over")) + print(f" fail-back cutovers done: {done}/{len(tgt_lvols)}") + if done == len(tgt_lvols): + break + time.sleep(15) + if done != len(tgt_lvols): + raise RuntimeError(f"FAIL: only {done}/{len(tgt_lvols)} fail-back cutovers completed") + + back = failed_over_targets(mgmt_ip, key_path, tgt_lvols) + back_idents = lvol_identities(mgmt_ip, key_path, list(back.values())) + fb_baseline = {back[t]: tgt_baseline[t] for t in tgt_lvols if t in back} + ok = True + for ip, vols in assign_fo.items(): + cleanup_client(ip, key_path, fo_mounts_by_client[ip]) + fb_vols = [back[t] for t in vols if t in back] + m = connect_and_mount_namespaced(ip, key_path, mgmt_ip, fb_vols, back_idents, + fmt=False, mount_base=MOUNT_BASE + "_nsfb") + good, _ = verify_baseline(ip, key_path, m, fb_baseline) + ok = ok and good + cleanup_client(ip, key_path, m) + if not ok: + raise RuntimeError("FAIL: namespace data not intact after fail-back") + print(f"CASE 7 PASSED: {NS_VOLUMES} namespaces on " + f"{NS_VOLUMES // NS_PER_SUBSYS} shared subsystems survived " + f"fail-over + fail-back with identity preserved (seed {seed}).") + + +def test_case_8(meta): + """Sequential-pressure catch-up: 64k/QD64/4-job fills of PRESSURE_DELTA_GB + per volume, repeated; the backlog must drain back under the lag gate after + every cycle. The cadence is a target, the backlog must converge.""" + print(f"\n========== CASE 8: {PRESSURE_DELTA_GB}G sequential-pressure catch-up " + f"x{PRESSURE_CYCLES} ==========") + key_path = meta["key_path"] + mgmt_ip = meta["mgmt"]["public_ip"] + client_ip = meta["clients"][0]["public_ip"] + src_uuid, src, tgt_uuid, tgt = _src_target(meta) + + prepare_mount_points(client_ip, key_path) + delete_test_volumes(mgmt_ip, key_path, _all_test_pools(meta)) + lvols = create_volumes(mgmt_ip, key_path, src_uuid, src["pool"], tgt_uuid, + tgt["pool"], mode="failover", count=PRESSURE_VOLUMES, + prefix="presvol", size=PRESSURE_VOL_SIZE) + mounts = connect_and_mount(client_ip, key_path, mgmt_ip, lvols, fmt=True) + baseline = write_baseline(client_ip, key_path, mounts) + + # Each of the 4 jobs writes delta/4 into its own file => delta GB per + # volume per cycle; overwrite=1 dirties the SAME clusters again next cycle. + per_job = f"{(PRESSURE_DELTA_GB * 1024) // 4}m" + jobfile = write_fio_jobfile(client_ip, key_path, mounts, rw="write", bs="64k", + iodepth=64, numjobs=4, size=per_job, + time_based=False, verify=False, + jobfile="/tmp/fio_pressure.fio") + + results = [] + for cycle in range(1, PRESSURE_CYCLES + 1): + print(f"--- cycle {cycle}/{PRESSURE_CYCLES}: writing " + f"{PRESSURE_DELTA_GB}G per volume (64k seq, QD64, 4 jobs) ---") + fill_start = time.time() + with ThreadPoolExecutor(max_workers=1) as ex: + fut = ex.submit(run, client_ip, key_path, + f"sudo fio --eta=never {jobfile}", True, True, 7200) + peak_bytes, peak_lag = 0, 0 + while not fut.done(): + time.sleep(20) + infos = get_replication_infos(mgmt_ip, key_path, lvols) + bts = sum((i.get("outstanding_bytes") or 0) for i in infos.values()) + lag = max((i.get("lag_seconds") or 0) for i in infos.values()) + peak_bytes, peak_lag = max(peak_bytes, bts), max(peak_lag, lag) + print(f" [fill] backlog={bts / 2**30:.1f}GiB worst_lag={lag}s") + fut.result() + fill_secs = time.time() - fill_start + + # Now the pipeline must CATCH UP: bounded lag again within the budget. + drain_start = time.time() + caught_up = None + while time.time() - drain_start < PRESSURE_CATCHUP_TIMEOUT: + infos = get_replication_infos(mgmt_ip, key_path, lvols) + bts = sum((i.get("outstanding_bytes") or 0) for i in infos.values()) + lag = max((i.get("lag_seconds") or 0) for i in infos.values()) + outst = sum((i.get("outstanding_count") or 0) for i in infos.values()) + peak_bytes, peak_lag = max(peak_bytes, bts), max(peak_lag, lag) + print(f" [drain] backlog={bts / 2**30:.1f}GiB worst_lag={lag}s outstanding={outst}") + if lag <= MAX_LAG_SECONDS and outst <= len(lvols): + caught_up = time.time() - drain_start + break + time.sleep(20) + results.append({"cycle": cycle, "fill_secs": int(fill_secs), + "peak_backlog_gib": round(peak_bytes / 2**30, 1), + "peak_lag_s": peak_lag, + "catch_up_secs": None if caught_up is None else int(caught_up)}) + print(f" cycle {cycle}: fill={int(fill_secs)}s " + f"peak_backlog={peak_bytes / 2**30:.1f}GiB peak_lag={peak_lag}s " + f"catch_up={'TIMEOUT' if caught_up is None else str(int(caught_up)) + 's'}") + if caught_up is None: + raise RuntimeError( + f"FAIL: replication did not catch up within " + f"{PRESSURE_CATCHUP_TIMEOUT}s after cycle {cycle} " + f"(peak backlog {peak_bytes / 2**30:.1f}GiB)") + + ok, _ = verify_baseline(client_ip, key_path, mounts, baseline) + stop_fio(client_ip, key_path) + cleanup_client(client_ip, key_path, mounts) + print(" cycle results:", json.dumps(results)) + if not ok: + raise RuntimeError("FAIL: baseline corrupted during pressure cycles") + print(f"CASE 8 PASSED: {PRESSURE_CYCLES} x {PRESSURE_DELTA_GB}G deltas, " + f"replication caught up every cycle.") + + +def _sample_replication_phases(mgmt_ip, key_path): + """What the replication pipeline is doing RIGHT NOW (for kill logging).""" + return mgmt_py(mgmt_ip, key_path, """ +import json +from collections import Counter +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.job_schedule import JobSchedule +db = DBController() +c = Counter() +for cl in db.get_clusters(): + for t in db.get_job_tasks(cl.get_id()): + if t.function_name in (JobSchedule.FN_SNAPSHOT_REPLICATION, + JobSchedule.FN_REPLICATION_FINAL) \ + and t.status != JobSchedule.STATUS_DONE: + c[f"{t.function_name}:{t.status}"] += 1 +print(json.dumps(dict(c))) +""", replayable=True) + + +def _all_nodes_online(mgmt_ip, key_path): + return mgmt_py(mgmt_ip, key_path, """ +import json +from simplyblock_core.db_controller import DBController +db = DBController() +bad = [n.get_id()[:8] for n in db.get_storage_nodes() if n.status != "online"] +print(json.dumps({"all_online": not bad, "offline": bad})) +""", replayable=True) + + +def test_case_9(meta): + """Chaos: random SPDK-container kills on SOURCE and TARGET nodes while + replication runs, to land failures in every pipeline phase (target-lvol + create, hublvol attach, transfer, convert, chain, prune, detach).""" + print(f"\n========== CASE 9: chaos container kills x{CHAOS_EVENTS} ==========") + import random + key_path = meta["key_path"] + mgmt_ip = meta["mgmt"]["public_ip"] + client_ip = meta["clients"][0]["public_ip"] + src_uuid, src, tgt_uuid, tgt = _src_target(meta) + + seed = int(CHAOS_SEED or time.time()) + rng = random.Random(seed) + print(f" chaos seed: {seed}") + + prepare_mount_points(client_ip, key_path) + delete_test_volumes(mgmt_ip, key_path, _all_test_pools(meta)) + lvols = create_volumes(mgmt_ip, key_path, src_uuid, src["pool"], tgt_uuid, + tgt["pool"], mode="failover", prefix="chaosvol") + mounts = connect_and_mount(client_ip, key_path, mgmt_ip, lvols, fmt=True) + baseline = write_baseline(client_ip, key_path, mounts) + start_fio(client_ip, key_path, write_fio_jobfile(client_ip, key_path, mounts)) + wait_replication_caught_up(mgmt_ip, key_path, lvols) + + # One kill at a time, recover, next -- randomized timing spreads the kills + # across the pipeline phases; two concurrent kills in one cluster is the + # cluster-outage scenario cases 2/3 already cover, not a race. + targets = ([("src", ip) for ip in src["storage_public_ips"]] + + [("tgt", ip) for ip in tgt["storage_public_ips"]]) + kills = [] + for ev in range(1, CHAOS_EVENTS + 1): + time.sleep(rng.randint(CHAOS_SLEEP_MIN, CHAOS_SLEEP_MAX)) + side, victim = rng.choice(targets) + phases = _sample_replication_phases(mgmt_ip, key_path) + print(f" [{ev}/{CHAOS_EVENTS}] killing SPDK on {side} node {victim} " + f"(active phases: {phases})") + kill_spdk(victim, key_path) + kills.append({"event": ev, "side": side, "node": victim, "phases": phases}) + + # Wait for the auto-restart to bring everything back before the next hit. + deadline = time.time() + NODE_STATE_TIMEOUT + while time.time() < deadline: + state = _all_nodes_online(mgmt_ip, key_path) + if state["all_online"]: + break + time.sleep(20) + else: + raise RuntimeError( + f"FAIL: nodes {state['offline']} not back online " + f"{NODE_STATE_TIMEOUT}s after chaos kill {ev} on {victim}") + + print("Chaos done; requiring full catch-up + integrity...") + wait_replication_caught_up(mgmt_ip, key_path, lvols, timeout=3600) + quiesce_ts = time.time() + stop_fio(client_ip, key_path) + run(client_ip, key_path, "sync", check=False) + wait_data_replicated(mgmt_ip, key_path, lvols, quiesce_ts, timeout=3600) + cleanup_client(client_ip, key_path, mounts) + + # End-to-end proof the surviving pipeline shipped GOOD data: kill the + # source, fail over, verify the baselines on the target copies. + print("Final integrity check via fail-over...") + for ip in src["storage_public_ips"][:2]: + kill_spdk(ip, key_path) + time.sleep(15) + tgt_lvols = [] + for lv in lvols: + fo = do_failover(mgmt_ip, key_path, lv) + if not isinstance(fo, dict) or not fo.get("connection_strings"): + raise RuntimeError(f"FAIL: post-chaos fail-over failed for {lv}") + tgt_lvols.append(fo["lvol_id"]) + tgt_baseline = {t: baseline[s] for s, t in zip(lvols, tgt_lvols)} + fo_mounts = connect_and_mount(client_ip, key_path, mgmt_ip, tgt_lvols, + fmt=False, mount_base=MOUNT_BASE + "_chaos") + ok, _ = verify_baseline(client_ip, key_path, fo_mounts, tgt_baseline) + cleanup_client(client_ip, key_path, fo_mounts) + restore_cluster(mgmt_ip, key_path, src, label="src (after chaos)") + + print(" kill log:", json.dumps(kills)) + if not ok: + raise RuntimeError("FAIL: data not intact after chaos (seed " + f"{seed}; kill log above)") + print(f"CASE 9 PASSED: {CHAOS_EVENTS} random kills across both clusters, " + f"replication recovered every time, data intact (seed {seed}).") + + CASES = { "case1": test_case_1, # online migration cutover, no IO interruption "case2": test_case_2, # DR fail-over on source-cluster loss @@ -1543,15 +2005,21 @@ def test_case_6(meta): "case4": test_case_4, # full fail-back to a fresh empty cluster "case5": test_case_5, # error: replication target node offline "case6": test_case_6, # error: source primary offline, secondary survives + "case7": test_case_7, # namespaced: 2 subsystems x 10 ns, 2 clients, fo+fb + "case8": test_case_8, # sequential pressure: repeated 50G deltas must catch up + "case9": test_case_9, # chaos: random SPDK kills on src+tgt during replication } GROUPS = { "both": ["case1", "case2"], "failback": ["case3", "case4"], "errors": ["case5", "case6"], + "extended": ["case7", "case8", "case9"], "all": ["case1", "case2", "case3", "case4", "case5", "case6"], # Case 3 last: it is the only case that needs the killed primary restored # and recovered, so a failure there cannot cost the other five cases. "all_c3_last": ["case1", "case2", "case4", "case5", "case6", "case3"], + "all9": ["case1", "case2", "case4", "case5", "case6", "case3", + "case7", "case8", "case9"], } diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index 4dd61c3ac5..6e399f07de 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -252,6 +252,47 @@ def process_snap_replicate_start(task, snapshot): logger.error(f"Unable to find pool on remote cluster: {remote_node_uuid.cluster_id}") return + # An earlier attempt of THIS task may have created the landing volume + # and died before storing its id (a node outage mid-create): add_lvol_ha + # then fails "LVol name must be unique" on EVERY retry and the task + # loops forever, stalling the volume's whole chain behind it (case 6, + # run 20260824_144226: three volumes stuck on their first cadence + # snapshot, retrying every ~31s for the rest of the run). The name is + # derived from the snapshot, so a record wearing it IS this transfer's + # landing volume: adopt it when it is usable, clear it when it is not. + rep_name = f"REP_{snapshot.snap_name}" + existing = None + try: + existing = db.get_lvol_by_name(rep_name) + except KeyError: + pass + if existing is not None: + if existing.status == LVol.STATUS_ONLINE: + logger.info(f"Adopting landing volume {existing.get_id()} " + f"({rep_name}) left by an interrupted attempt") + task.function_params["remote_lvol_id"] = existing.get_id() + task.write_to_db() + elif existing.status == LVol.STATUS_IN_DELETION: + task.function_result = f"stale landing volume {rep_name} still deleting, retrying" + task.status = JobSchedule.STATUS_SUSPENDED + task.retry += 1 + task.write_to_db() + return + else: + logger.warning(f"Deleting half-created landing volume " + f"{existing.get_id()} ({rep_name}, status " + f"{existing.status}) from an interrupted attempt") + try: + lvol_controller.delete_lvol(existing, force_delete=True) + except Exception as e: + logger.error(f"Failed to clear stale landing volume {rep_name}: {e}") + task.function_result = "cleared stale landing volume, retrying" + task.status = JobSchedule.STATUS_SUSPENDED + task.retry += 1 + task.write_to_db() + return + + if "remote_lvol_id" not in task.function_params or not task.function_params["remote_lvol_id"]: # internal=True: this REP_* volume is the landing copy for a transfer, # created by the system and never handed to a client. The per-node # subsystem cap is a user-admission limit; enforcing it here only stops diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index 83f0ad8ad9..b45a9b0ce1 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -419,3 +419,23 @@ def get_id(self): assert evicted == ["P", "S"], \ "stale-namespace eviction must run on the primary AND every online HA peer" assert ("S", False) in added + + +def test_interrupted_landing_volume_is_adopted_or_cleared(): + """Case 6, run 20260824_144226: a node outage mid-create left a REP_* + landing volume whose id was never stored on the task; every retry then + died on "LVol name must be unique" and three volumes' chains stalled for + the rest of the run. Before creating the landing volume, the runner must + look for a record already wearing the derived name and adopt it (online), + wait for it (in_deletion), or clear it (half-created).""" + import inspect + from simplyblock_core.services import snapshot_replication as sr + src = inspect.getsource(sr) + probe = src.index('rep_name = f"REP_{snapshot.snap_name}"') + create = src.index("lvol_controller.add_lvol_ha") + assert probe < create, "the adopt/clear probe must run before the create" + adopt = src.index("Adopting landing volume") + assert probe < adopt < create + for handled in ("STATUS_ONLINE", "STATUS_IN_DELETION", "force_delete=True"): + assert src.index(handled, probe) < create, \ + f"collision handling must cover {handled} before creating" From d8f667738ce6d9f9b33058287c330bc360092c05 Mon Sep 17 00:00:00 2001 From: michael Date: Mon, 24 Aug 2026 16:46:05 +0200 Subject: [PATCH 005/122] test: wait for the cluster to settle before node-down cases; surface sn shutdown's stderr Run 20260824_153107: case 6's sn shutdown was silently REFUSED because the source cluster was still ACTIVE - REBALANCING from the previous case's restore (open device_migration + balancing_on_restart tasks); the refusal went to stderr, which the driver's exec channel drops, so the test stared at an online node for the full 900s budget. Cases 5 and 6 now wait for the victim's cluster to settle before shutting the node down, and the shutdown command merges stderr so a refusal is visible. Co-Authored-By: Claude Fable 5 --- scripts/test_async_replication.py | 46 ++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/scripts/test_async_replication.py b/scripts/test_async_replication.py index be19f922b1..dbc2af48d9 100644 --- a/scripts/test_async_replication.py +++ b/scripts/test_async_replication.py @@ -604,19 +604,57 @@ def wait_node_status(mgmt_ip, key_path, node_id, wanted, timeout=NODE_STATE_TIME f"Node {node_id} did not reach {wanted!r} within {timeout}s (last={seen!r})") -def sn_shutdown(mgmt_ip, key_path, node_id): +def wait_cluster_settled(mgmt_ip, key_path, cluster_id, timeout=1800): + """Wait until *cluster_id* has no open migration/balancing work. + + A restore leaves the cluster ACTIVE - REBALANCING (device_migration + + balancing_on_restart tasks), and `sn shutdown` REFUSES a node while that + work is open — silently, from the driver's point of view, because the + refusal goes to stderr (run 20260824_153107: case 6's shutdown printed + nothing and the node stayed 'online' for the full 900s budget, while + case 5's identical call later worked, after the rebalance had drained). + """ + print(f"Waiting for cluster {cluster_id[:8]} to settle (no rebalance/migration)...") + start = time.time() + while time.time() - start < timeout: + state = mgmt_py(mgmt_ip, key_path, f""" +import json +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.job_schedule import JobSchedule +db = DBController() +open_tasks = [t.function_name for t in db.get_job_tasks({cluster_id!r}) + if t.status != JobSchedule.STATUS_DONE and not t.canceled + and t.function_name in ("device_migration", "balancing_on_restart", + "new_device_migration", "failed_device_migration")] +print(json.dumps({{"status": db.get_cluster_by_id({cluster_id!r}).status, + "open": open_tasks}})) +""", replayable=True) + if not state["open"]: + print(f" cluster settled (status {state['status']}).") + return + print(f" status={state['status']} open={state['open']}") + time.sleep(20) + raise RuntimeError(f"Cluster {cluster_id[:8]} did not settle within {timeout}s") + + +def sn_shutdown(mgmt_ip, key_path, node_id, cluster_id=None): """Take a node offline the supported way. Deliberately NOT `docker kill` on the SPDK container: the control plane auto-restarts that within minutes (case 2 saw the "suspended" source cluster heal itself mid-test), which silently invalidates an outage scenario. """ + if cluster_id: + wait_cluster_settled(mgmt_ip, key_path, cluster_id) print(f"Shutting down node {node_id[:8]} ...") # Straight to shutdown: suspending first buys nothing and actively hurts — # a suspended node makes its own queued work defer ("node is not online, # retrying"), and that backlog then blocks the shutdown itself (run 15 # case 6: the node never left `suspended`). - run(mgmt_ip, key_path, f"{SBCTL} -d sn shutdown {node_id}", check=False) + # 2>&1: sbctl reports a REFUSED shutdown on stderr, which the channel + # otherwise drops — the driver then stares at an online node for 900s + # with no clue why (run 20260824_153107). + run(mgmt_ip, key_path, f"{SBCTL} -d sn shutdown {node_id} 2>&1", check=False) wait_node_status(mgmt_ip, key_path, node_id, "offline") @@ -1470,7 +1508,7 @@ def test_case_5(meta): victim = node_of_lvol(mgmt_ip, key_path, lvols[0])["replication_node_id"] print(f"Taking the REPLICATION TARGET node {victim[:8]} offline...") before = _replication_progress(mgmt_ip, key_path, lvols) - sn_shutdown(mgmt_ip, key_path, victim) + sn_shutdown(mgmt_ip, key_path, victim, cluster_id=tgt_uuid) print(f"Observing {OUTAGE_REPL_CYCLES} replication cycles with the target down...") time.sleep(OUTAGE_REPL_CYCLES * REPL_INTERVAL_MIN * 60) @@ -1522,7 +1560,7 @@ def test_case_6(meta): print(f"Taking the SOURCE PRIMARY {primary[:8]} offline " f"(secondary {secondary[:8]} must carry on)...") before = _replication_progress(mgmt_ip, key_path, lvols) - sn_shutdown(mgmt_ip, key_path, primary) + sn_shutdown(mgmt_ip, key_path, primary, cluster_id=src_uuid) print(f"Observing {OUTAGE_REPL_CYCLES} replication cycles on the secondary...") time.sleep(OUTAGE_REPL_CYCLES * REPL_INTERVAL_MIN * 60) From 22c9188330782594a7ea6d9c978e8312ba084a26 Mon Sep 17 00:00:00 2001 From: michael Date: Mon, 24 Aug 2026 19:38:46 +0200 Subject: [PATCH 006/122] fix: fail-over must match the full preserved identity (nqn AND nsid), not nqn alone Soak case 7 (run 20260824_174611) caught this on its first run: namespaced volumes SHARE a subsystem, so replicate_lvol_on_target_cluster's existing-copy guard -- an nqn-only match -- fired for every namespace after the first. Namespace 1's fail-over copy already carried the shared nqn, so namespaces 2..N returned ITS target lvol id and were never failed over at all: 9 of 10 volumes silently absent after a DR fail-over, with every call reporting success. The guard now compares nqn AND ns_id, which is the identity a fail-over copy actually preserves. The existing idempotency test passed only because its fake left the already-failed-over copy at the model-default nsid while the source carries nsid 7; a real copy preserves the source's nsid. Fixed, and a companion test pins that a SIBLING namespace on the same nqn is not mistaken for this volume. Also scale the leftover-volume drain budget with the number of victims: case 7 leaves 20 namespaced volumes plus their REP_* landing copies, and 37 of them did not drain inside the flat 300s, failing case 8 in its prologue. Co-Authored-By: Claude Fable 5 --- scripts/test_async_replication.py | 10 +++++-- .../controllers/lvol_controller.py | 11 +++++-- simplyblock_core/test/test_failover_target.py | 30 +++++++++++++++++++ .../test_replication_chain_completeness.py | 14 +++++++++ 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/scripts/test_async_replication.py b/scripts/test_async_replication.py index dbc2af48d9..d1d144e0ca 100644 --- a/scripts/test_async_replication.py +++ b/scripts/test_async_replication.py @@ -951,7 +951,11 @@ def delete_test_volumes(mgmt_ip, key_path, pools): run(mgmt_ip, key_path, f"{SBCTL} volume delete {uuid} --force", check=False, quiet=True) # Deletion is asynchronous; wait for the records to drain so the name is free. - for _ in range(30): + # Budget scales with the pile: case 7 leaves 20 namespaced volumes plus + # their REP_* landing copies, and 37 volumes did not drain inside the flat + # 300s, failing case 8 in its prologue (run 20260824_174611). + drain_polls = max(30, len(victims) * 3) + for _ in range(drain_polls): time.sleep(10) left = 0 for pool in pools: @@ -964,7 +968,9 @@ def delete_test_volumes(mgmt_ip, key_path, pools): if left == 0: print(" cleanup drained.") return - raise RuntimeError("Timed out waiting for leftover volumes to delete") + raise RuntimeError( + f"Timed out waiting for {len(victims)} leftover volumes to delete " + f"after {drain_polls * 10}s") def create_volumes(mgmt_ip, key_path, src_uuid, pool, tgt_uuid, tgt_pool, mode, diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 58dcbfca84..d8dbee7015 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -3625,9 +3625,16 @@ def replicate_lvol_on_target_cluster(lvol_id): target_cluster, target_pool_uuid = resolve_replication_destination( db_controller, lvol, target_node, source_node) + # Match the preserved identity in FULL: nqn AND nsid. A namespaced volume + # SHARES its subsystem with up to max-namespace-per-subsys siblings, so an + # nqn-only test made this idempotency guard fire for every namespace after + # the first: namespace 1's fail-over copy already carried the nqn, so + # namespaces 2..N returned ITS id and were never failed over at all — + # silent loss of 9 of 10 volumes in a DR event, reported as success + # (soak case 7, run 20260824_174611). for lv in db_controller.get_lvols(target_cluster.get_id()): - if lv.nqn == lvol.nqn: - logger.info(f"LVol with same nqn already exists on target cluster: {lv.get_id()}") + if lv.nqn == lvol.nqn and lv.ns_id == lvol.ns_id: + logger.info(f"LVol with same nqn+nsid already exists on target cluster: {lv.get_id()}") return lv.get_id() new_lvol, _snapshot, error = _clone_from_last_replicated( diff --git a/simplyblock_core/test/test_failover_target.py b/simplyblock_core/test/test_failover_target.py index 945d69fd5a..74a17d0787 100644 --- a/simplyblock_core/test/test_failover_target.py +++ b/simplyblock_core/test/test_failover_target.py @@ -203,6 +203,9 @@ def test_failover_idempotent_when_target_exists(monkeypatch, patched): existing = LVol() existing.uuid = "EXISTING" existing.nqn = "nqn.orig:lvol:LV1" + # A fail-over copy preserves the source's nsid, so the already-failed-over + # volume this guard recognises carries nsid 7 too -- not the model default. + existing.ns_id = 7 nodes = { "N_src": _node("N_src", "CL_src"), "N_tgt": _node("N_tgt", "CL_tgt", secondary="N_sec", lvstore="lvs_tgt"), @@ -218,3 +221,30 @@ def test_failover_idempotent_when_target_exists(monkeypatch, patched): # Returns the existing target lvol id; no new volume created. assert result == "EXISTING" assert patched["add_calls"] == [] + + +def test_failover_does_not_mistake_a_sibling_namespace_for_this_volume(monkeypatch, patched): + """Soak case 7 (run 20260824_174611): with namespaced volumes, up to + max-namespace-per-subsys volumes SHARE one nqn. The existing-copy guard + matched on nqn alone, so once namespace 1 had failed over, namespaces + 2..N returned ITS target id and were never failed over -- 9 of 10 volumes + silently absent after a DR fail-over, every call reporting success.""" + sibling = LVol() + sibling.uuid = "SIBLING_NS1" + sibling.nqn = "nqn.orig:lvol:LV1" # same shared subsystem + sibling.ns_id = 1 # DIFFERENT namespace + nodes = { + "N_src": _node("N_src", "CL_src"), + "N_tgt": _node("N_tgt", "CL_tgt", secondary="N_sec", lvstore="lvs_tgt"), + } + clusters = { + "CL_src": _cluster("CL_src", target_cluster="CL_tgt", target_pool="POOL_tgt"), + "CL_tgt": _cluster("CL_tgt"), + } + _install_db(monkeypatch, _FakeDB(nodes, clusters, existing_lvols=[sibling])) + + result = lvol_controller.replicate_lvol_on_target_cluster("LV1") + + # LV1 (nsid 7) must actually fail over, NOT return the nsid-1 sibling. + assert result != "SIBLING_NS1" + assert patched["add_calls"], "the volume must really be failed over" diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index b45a9b0ce1..9fd10dc35f 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -439,3 +439,17 @@ def test_interrupted_landing_volume_is_adopted_or_cleared(): for handled in ("STATUS_ONLINE", "STATUS_IN_DELETION", "force_delete=True"): assert src.index(handled, probe) < create, \ f"collision handling must cover {handled} before creating" + + +def test_failover_guard_matches_nqn_and_nsid_not_nqn_alone(): + """Soak case 7 (run 20260824_174611): namespaced volumes SHARE a subsystem, + so the fail-over idempotency guard's nqn-only match fired for every + namespace after the first — namespaces 2..N returned namespace 1's target + lvol id and were never failed over, losing 9 of 10 volumes in a DR event + while reporting success. The guard must compare the FULL preserved + identity: nqn AND ns_id.""" + import inspect + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc.replicate_lvol_on_target_cluster) + assert "lv.nqn == lvol.nqn and lv.ns_id == lvol.ns_id" in src, \ + "the fail-over existing-copy guard must match nqn AND nsid" From 2aa42bc79bb53e74ca90bdffba82796479bcf900 Mon Sep 17 00:00:00 2001 From: michael Date: Mon, 24 Aug 2026 20:30:02 +0200 Subject: [PATCH 007/122] test: pass env overrides to the remote driver; track fio survival across chaos kills CHAOS_EVENTS and friends can now be set per run (stage_and_run_repl_cases case9 CHAOS_EVENTS=100) instead of editing the driver on the box. Case 9 also records whether fio survived each kill: that is the promotion-window signal the spdk ANA-transition fix targets, so a chaos soak now reports it directly instead of only asserting recovery. Co-Authored-By: Claude Fable 5 --- scripts/stage_and_run_repl_cases.py | 9 +++++++-- scripts/test_async_replication.py | 14 +++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/stage_and_run_repl_cases.py b/scripts/stage_and_run_repl_cases.py index ccbe485ae0..1b34f5f229 100644 --- a/scripts/stage_and_run_repl_cases.py +++ b/scripts/stage_and_run_repl_cases.py @@ -56,9 +56,14 @@ def remote_metadata(meta): def main(): cases = sys.argv[1] if len(sys.argv) > 1 else "all_c3_last" + # Any further NAME=VALUE arguments are exported for the remote driver, so + # a run can be tuned (CHAOS_EVENTS, PRESSURE_CYCLES, ...) without editing + # the script on the box. + env_args = [a for a in sys.argv[2:] if "=" in a] + env_prefix = "".join(f"{a} " for a in env_args) meta = json.loads((HERE / "cluster_metadata_repl.json").read_text()) mgmt = meta["mgmt"]["public_ip"] - log(f"mgmt={mgmt} cases={cases}") + log(f"mgmt={mgmt} cases={cases}" + (f" env={env_args}" if env_args else "")) remote_meta = HERE / "cluster_metadata_repl_remote.json" remote_meta.write_text(json.dumps(remote_metadata(meta), indent=4)) @@ -78,7 +83,7 @@ def main(): # going away; without -f, ssh sat on the channel until it timed out (twice: # 2026-08-19 with nohup, 2026-08-20 with setsid) while the driver ran fine. run(["ssh", "-f", *SSH_OPTS, f"ec2-user@{mgmt}", - f"cd ~ && setsid python3 -u test_async_replication.py {cases} " + f"cd ~ && setsid env {env_prefix}python3 -u test_async_replication.py {cases} " f"> {remote_log} 2>&1 < /dev/null & echo $! > ~/repl_pid; " f"echo ~/repl_cases_{ts}.log > ~/repl_log"], timeout=60) time.sleep(45) diff --git a/scripts/test_async_replication.py b/scripts/test_async_replication.py index d1d144e0ca..25ed831b9b 100644 --- a/scripts/test_async_replication.py +++ b/scripts/test_async_replication.py @@ -1986,6 +1986,7 @@ def test_case_9(meta): targets = ([("src", ip) for ip in src["storage_public_ips"]] + [("tgt", ip) for ip in tgt["storage_public_ips"]]) kills = [] + fio_deaths = [] for ev in range(1, CHAOS_EVENTS + 1): time.sleep(rng.randint(CHAOS_SLEEP_MIN, CHAOS_SLEEP_MAX)) side, victim = rng.choice(targets) @@ -1993,7 +1994,16 @@ def test_case_9(meta): print(f" [{ev}/{CHAOS_EVENTS}] killing SPDK on {side} node {victim} " f"(active phases: {phases})") kill_spdk(victim, key_path) - kills.append({"event": ev, "side": side, "node": victim, "phases": phases}) + # fio survival across a kill is the promotion-window signal: before the + # ANA-transition fix (spdk R26.3) a killed primary handed hard EIO to + # the client within seconds and XFS shut the filesystem down. + time.sleep(20) + alive = fio_alive(client_ip, key_path) + kills.append({"event": ev, "side": side, "node": victim, + "phases": phases, "fio_alive": alive}) + if not alive: + fio_deaths.append(ev) + print(f" fio NOT alive after event {ev} (deaths so far: {len(fio_deaths)})") # Wait for the auto-restart to bring everything back before the next hit. deadline = time.time() + NODE_STATE_TIMEOUT @@ -2034,6 +2044,8 @@ def test_case_9(meta): cleanup_client(client_ip, key_path, fo_mounts) restore_cluster(mgmt_ip, key_path, src, label="src (after chaos)") + print(f" fio survived {CHAOS_EVENTS - len(fio_deaths)}/{CHAOS_EVENTS} kills" + + (f" (died after events {fio_deaths})" if fio_deaths else "")) print(" kill log:", json.dumps(kills)) if not ok: raise RuntimeError("FAIL: data not intact after chaos (seed " From 28b9bcb0fd23b2472fc08031e11d219d6820e599 Mon Sep 17 00:00:00 2001 From: michael Date: Mon, 24 Aug 2026 21:01:11 +0200 Subject: [PATCH 008/122] test: pin the ultra image carrying the ANA fix, deploy 2 clients, surface fail-over errors - SPDK_IMAGE -> main-2a03661a-amd64 (sha256:961410ae...), the first ultra build containing the promotion-window ANA-transition fix (spdk R26.3 554c80f11). Verified it was built FROM spdk-core:R26.3-latest whose manifest was created at 18:41:57, before this ultra build started at 18:42:52 -- the floating-tag manifest race makes that check necessary. - CLIENT_COUNT 2, so a namespaced (case 7) run does not need add_client. - do_failover now captures the controller's error and log output. A bare 'returned no connection strings' cost two lab runs to diagnose; the reason was being written to a stderr nobody read. Co-Authored-By: Claude Fable 5 --- scripts/setup_repl_test_2clusters.py | 12 ++++++-- scripts/test_async_replication.py | 43 +++++++++++++++++++++++----- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/scripts/setup_repl_test_2clusters.py b/scripts/setup_repl_test_2clusters.py index 2b3040d355..d4a5f3018e 100644 --- a/scripts/setup_repl_test_2clusters.py +++ b/scripts/setup_repl_test_2clusters.py @@ -45,7 +45,10 @@ SN_TYPE = "i3en.2xlarge" MGMT_TYPE = "m6i.2xlarge" CLIENT_TYPE = "m6in.8xlarge" -CLIENT_COUNT = 1 # client(s) used by the test process +CLIENT_COUNT = 2 # client(s) used by the test process +#: 2, not 1: case 7 spreads 20 namespaced volumes across at least two +#: clients, and growing an existing lab with `add_client` after the fact +#: is an extra manual step before every namespaced run. USER = "ec2-user" IFACE = "eth0" @@ -101,9 +104,12 @@ # parallel reads, spdk R26.3 bdd97c1d8/ce876a169) exist only from the # 2026-08-22 build onward, and ultra:main-latest's manifest list has a live # race that leaves its amd64 entry pointing at the PREVIOUS build (observed -# 2026-08-17, -21 and -22). This digest = main-d89a595c-amd64, 2026-08-22. +# 2026-08-17, -21 and -22). This digest = main-2a03661a-amd64, 2026-08-24 -- +# the first build carrying the promotion-window ANA-transition fix +# (spdk R26.3 554c80f11), verified built FROM spdk-core:R26.3-latest +# whose manifest was created 18:41:57, before this ultra build started. SPDK_IMAGE = ("public.ecr.aws/simply-block/ultra@" - "sha256:8a007dca5bf6a1fa89cc96945f43164539cac65f0cdafa56d5ee44961e9902af") + "sha256:961410aefddaa615d4d1dfe1b8cc7ce27922d9a06ff924296f669c953e742bef") SN_COUNT = sum(c["nodes"] for c in CLUSTERS) SBCTL = "sudo /usr/local/bin/sbctl" diff --git a/scripts/test_async_replication.py b/scripts/test_async_replication.py index 25ed831b9b..acc8109c63 100644 --- a/scripts/test_async_replication.py +++ b/scripts/test_async_replication.py @@ -292,11 +292,32 @@ def wait_data_replicated(mgmt_ip, key_path, lvol_uuids, after_ts, def do_failover(mgmt_ip, key_path, lvol_uuid): + """Fail a volume over, capturing WHY when it does not work. + + replicate_lvol_on_target_cluster returns a dict on success but False or + (False, error) on failure, and the controller reports the reason through + its logger -- which went to the snippet's stderr and was dropped. A bare + "returned no connection strings" then costs a whole lab run to diagnose + (case 7, runs 20260824_174611 and _202949). Capture both. + """ return mgmt_py(mgmt_ip, key_path, f""" -import json +import io, json, logging, contextlib from simplyblock_core.controllers import lvol_controller -res = lvol_controller.replicate_lvol_on_target_cluster({lvol_uuid!r}) -print(json.dumps(res if isinstance(res, dict) else {{"result": res}})) +buf = io.StringIO() +handler = logging.StreamHandler(buf) +handler.setLevel(logging.WARNING) +logging.getLogger().addHandler(handler) +err = "" +try: + with contextlib.redirect_stderr(buf): + res = lvol_controller.replicate_lvol_on_target_cluster({lvol_uuid!r}) +except Exception as exc: # noqa: BLE001 - report, don't hide + res, err = False, f"{{type(exc).__name__}}: {{exc}}" +out = res if isinstance(res, dict) else {{"result": res}} +if not (isinstance(res, dict) and res.get("connection_strings")): + out["error"] = err + out["log"] = buf.getvalue()[-1500:] +print(json.dumps(out)) """) @@ -1197,7 +1218,9 @@ def test_case_2(meta): fo = do_failover(mgmt_ip, key_path, lv) print(f" failover {lv}: {json.dumps(fo)}") if not isinstance(fo, dict) or not fo.get("connection_strings"): - raise RuntimeError(f"FAIL: fail-over returned no connection strings for {lv}") + raise RuntimeError( + f"FAIL: fail-over returned no connection strings for {lv}: " + f"{fo.get('error') or ''} {fo.get('log') or ''}".strip()) if fo.get("nqn"): assert fo["nqn"], "missing NQN" failed_over.append({"src_lvol": lv, "fo": fo}) @@ -1272,7 +1295,9 @@ def _setup_failed_over_volumes(meta, tag): for lv in lvols: fo = do_failover(mgmt_ip, key_path, lv) if not isinstance(fo, dict) or not fo.get("connection_strings"): - raise RuntimeError(f"FAIL: fail-over returned no connection strings for {lv}") + raise RuntimeError( + f"FAIL: fail-over returned no connection strings for {lv}: " + f"{fo.get('error') or ''} {fo.get('log') or ''}".strip()) tgt_lvols.append(fo["lvol_id"]) # Re-key the baseline by TARGET lvol id and mount the failed-over copies. @@ -1770,7 +1795,9 @@ def test_case_7(meta): for lv in lvols: fo = do_failover(mgmt_ip, key_path, lv) if not isinstance(fo, dict) or not fo.get("connection_strings"): - raise RuntimeError(f"FAIL: fail-over returned no connection strings for {lv}") + raise RuntimeError( + f"FAIL: fail-over returned no connection strings for {lv}: " + f"{fo.get('error') or ''} {fo.get('log') or ''}".strip()) tgt_lvols.append(fo["lvol_id"]) src_to_tgt = dict(zip(lvols, tgt_lvols)) @@ -2035,7 +2062,9 @@ def test_case_9(meta): for lv in lvols: fo = do_failover(mgmt_ip, key_path, lv) if not isinstance(fo, dict) or not fo.get("connection_strings"): - raise RuntimeError(f"FAIL: post-chaos fail-over failed for {lv}") + raise RuntimeError( + f"FAIL: post-chaos fail-over failed for {lv}: " + f"{fo.get('error') or ''} {fo.get('log') or ''}".strip()) tgt_lvols.append(fo["lvol_id"]) tgt_baseline = {t: baseline[s] for s, t in zip(lvols, tgt_lvols)} fo_mounts = connect_and_mount(client_ip, key_path, mgmt_ip, tgt_lvols, From be5bffd02ca74e28240e205fcff1560dc7d026bb Mon Sep 17 00:00:00 2001 From: Waleed Mousa <32266980+wmousa@users.noreply.github.com> Date: Mon, 24 Aug 2026 22:56:39 +0200 Subject: [PATCH 009/122] fix: clear pre-existing lint/type errors blocking CI on main (#1265) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two unrelated, pre-existing issues currently failing every PR's "Python checks" job (found while chasing an unrelated PR's CI failure — these files aren't touched by that PR at all, so the merge-commit CI check was just surfacing main's own already-broken state): - tests/unit/test_ucs_debounce_bound.py: unused `MagicMock` import (F401). - simplyblock_core/test/test_replication_chain_completeness.py: unused `import copy` (F401), and a lambda that sequenced `added.append(...)` with a return value via a `(side_effect, value)[-1]` tuple-index trick -- mypy's func-returns-value check flags embedding append()'s None result in the tuple literal. Replaced the lambda with a small nested function that does the append as its own statement and returns the real value normally; same behavior, no trick, no warning. No functional change -- verified via pytest (22 passed) and unittest (7 passed) on the two affected test modules. --- .../test/test_replication_chain_completeness.py | 11 ++++++----- tests/unit/test_ucs_debounce_bound.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index 9fd10dc35f..8d055a2211 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -372,16 +372,17 @@ def test_failback_evicts_on_every_ha_node_not_just_the_primary(monkeypatch): preserved-NQN subsystem exists on EVERY node of the recovered set, each still holding the original namespace. The peer failure rolled the whole cutover back: 0/5. The clone path must evict per node.""" - import copy from simplyblock_core.controllers import lvol_controller as lc evicted, added = [], [] monkeypatch.setattr(lc, "_evict_stale_namespace", lambda lvol, node: evicted.append(node.get_id())) - monkeypatch.setattr(lc, "add_lvol_on_node", - lambda lvol, node, is_primary=True: ( - added.append((node.get_id(), is_primary)), - ({"uuid": "U", "driver_specific": {"lvol": {"blobid": 9}}}, None))[-1]) + + def _fake_add_lvol_on_node(lvol, node, is_primary=True): + added.append((node.get_id(), is_primary)) + return {"uuid": "U", "driver_specific": {"lvol": {"blobid": 9}}}, None + + monkeypatch.setattr(lc, "add_lvol_on_node", _fake_add_lvol_on_node) class _N: def __init__(self, nid, secondary="", tertiary=""): diff --git a/tests/unit/test_ucs_debounce_bound.py b/tests/unit/test_ucs_debounce_bound.py index 7a2f21c481..d7ed14aee4 100644 --- a/tests/unit/test_ucs_debounce_bound.py +++ b/tests/unit/test_ucs_debounce_bound.py @@ -17,7 +17,7 @@ import threading import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import patch from simplyblock_core.services import storage_node_monitor as monitor From 4bdf334eb5d0e593574daf9f16df6cb173cfb16e Mon Sep 17 00:00:00 2001 From: michael Date: Mon, 24 Aug 2026 23:03:01 +0200 Subject: [PATCH 010/122] fix: namespaced siblings must replicate to the same target node Soak case 7 (run 20260824_215758) got 14 of 20 namespaces failed over and then died in add_ns on the 15th. The replication destination is chosen per volume by capacity (_get_next_3_nodes), with nothing tying volumes that SHARE a subsystem to one target node. Because a fail-over copy preserves the volume's NQN and nsid, scattering siblings splits a single shared subsystem across unrelated target primaries: each one advertises the same NQN carrying only its own subset of namespaces, and a sibling whose nsid is already taken there cannot be added. A volume that shares its NQN with an already-replicating sibling now inherits that sibling's replication node. Also raise the soak's fail-over diagnostic capture to DEBUG: the SPDK response behind 'Failed to add bdev to subsystem' is logged there, and the controller only re-reports its own generic message. Co-Authored-By: Claude Fable 5 --- scripts/test_async_replication.py | 12 ++++++-- .../controllers/lvol_controller.py | 28 +++++++++++++++++-- .../test_replication_chain_completeness.py | 17 +++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/scripts/test_async_replication.py b/scripts/test_async_replication.py index acc8109c63..120395d44f 100644 --- a/scripts/test_async_replication.py +++ b/scripts/test_async_replication.py @@ -305,8 +305,14 @@ def do_failover(mgmt_ip, key_path, lvol_uuid): from simplyblock_core.controllers import lvol_controller buf = io.StringIO() handler = logging.StreamHandler(buf) -handler.setLevel(logging.WARNING) -logging.getLogger().addHandler(handler) +# DEBUG, not WARNING: the RPC layer logs the actual SPDK response +# ("Invalid parameters", nsid in use, ...) at DEBUG, and the controller +# only re-reports its own generic "Failed to add bdev to subsystem". +# Only the last 1500 chars are kept, which is exactly the failure tail. +handler.setLevel(logging.DEBUG) +root = logging.getLogger() +root.addHandler(handler) +root.setLevel(logging.DEBUG) err = "" try: with contextlib.redirect_stderr(buf): @@ -316,7 +322,7 @@ def do_failover(mgmt_ip, key_path, lvol_uuid): out = res if isinstance(res, dict) else {{"result": res}} if not (isinstance(res, dict) and res.get("connection_strings")): out["error"] = err - out["log"] = buf.getvalue()[-1500:] + out["log"] = buf.getvalue()[-2500:] print(json.dumps(out)) """) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index d8dbee7015..d3e075e09a 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -657,8 +657,32 @@ def add_lvol_ha(name, size, host_id_or_name, ha_type, pool_id_or_name, use_comp= return False, f"Replication cluster not found: {replication_cluster_id}" else: replication_cluster_id = cl.snapshot_replication_target_cluster - random_nodes = _get_next_3_nodes(replication_cluster_id, lvol.size, all_lvols) - lvol.replication_node_id = random_nodes[0].get_id() + # Namespaced siblings MUST replicate to the same target node. + # A fail-over copy preserves the volume's NQN and nsid, so all + # volumes sharing a subsystem land in the SAME subsystem on the + # target. Picking the destination purely by capacity scattered + # siblings across the target cluster's nodes, which splits one + # shared subsystem across unrelated primaries: each advertises the + # same NQN with only its own subset of namespaces, and the copies + # collide when a sibling's nsid is already taken there (soak case 7, + # run 20260824_215758: 14 of 20 namespaces failed over, the 15th + # died in add_ns). + sibling_node_id = "" + if getattr(lvol, "namespaced", False) or lvol.max_namespace_per_subsys > 1: + for lv in (all_lvols or db_controller.get_lvols(cl.get_id())): + if (lv.nqn == lvol.nqn and lv.get_id() != lvol.get_id() + and getattr(lv, "replication_node_id", "")): + sibling_node_id = lv.replication_node_id + break + if sibling_node_id: + logger.info( + f"LVol {lvol.lvol_name} shares subsystem {lvol.nqn} with an " + f"already-replicating sibling; using its replication node " + f"{sibling_node_id} so the shared subsystem is not split") + lvol.replication_node_id = sibling_node_id + else: + random_nodes = _get_next_3_nodes(replication_cluster_id, lvol.size, all_lvols) + lvol.replication_node_id = random_nodes[0].get_id() lvol_dict: dict = { "type": "bdev_lvol", diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index 8d055a2211..88c3c327b5 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -454,3 +454,20 @@ def test_failover_guard_matches_nqn_and_nsid_not_nqn_alone(): src = inspect.getsource(lc.replicate_lvol_on_target_cluster) assert "lv.nqn == lvol.nqn and lv.ns_id == lvol.ns_id" in src, \ "the fail-over existing-copy guard must match nqn AND nsid" + + +def test_namespaced_siblings_replicate_to_the_same_target_node(): + """Soak case 7 (run 20260824_215758): the replication destination was + picked per volume by capacity, so volumes SHARING a subsystem were + scattered across the target cluster's nodes. A fail-over copy preserves + the NQN, so that splits one shared subsystem across unrelated primaries + -- each advertising the same NQN with only part of the namespaces. + Siblings must inherit the node their subsystem already replicates to.""" + import inspect + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc.add_lvol_ha) + assert "sibling_node_id" in src, "namespaced siblings must share a replication node" + pick = src.index("_get_next_3_nodes(replication_cluster_id") + check = src.index("sibling_node_id") + assert check < pick, "the sibling lookup must precede the capacity-based pick" + assert "lv.nqn == lvol.nqn" in src, "siblings are identified by shared NQN" From 3beb57222cc1a6bab3617c4bf573e9a7e7070e39 Mon Sep 17 00:00:00 2001 From: michael Date: Mon, 24 Aug 2026 23:08:41 +0200 Subject: [PATCH 011/122] test: make the chaos gate health-aware and restart the workload after a kill Run 20260824_224909 showed the soak damaging the cluster instead of testing it: the recovery gate accepted status == online while health_check was still False and the source cluster sat in SUSPENDED/IN_ACTIVATION, so kills kept landing on a half-recovered 2-node cluster. It now requires every node online AND healthy and every cluster active/degraded before the next event. fio also died at event 3 (the promotion-window EIO) and stayed dead, so the remaining 97 events would have run against an idle client. The case now reconnects and restarts the workload after an outage, and still reports every death. Co-Authored-By: Claude Fable 5 --- scripts/test_async_replication.py | 37 +++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/scripts/test_async_replication.py b/scripts/test_async_replication.py index 120395d44f..3b9941ce48 100644 --- a/scripts/test_async_replication.py +++ b/scripts/test_async_replication.py @@ -1980,12 +1980,25 @@ def _sample_replication_phases(mgmt_ip, key_path): def _all_nodes_online(mgmt_ip, key_path): + """Truly recovered: every node online AND healthy, every cluster ACTIVE. + + Status alone is not recovery. After a chaos kill a node reports + "online" again while health_check is still False and its lvstore port + is down, and the cluster can sit in SUSPENDED/IN_ACTIVATION behind it. + Gating on status only, the soak kept firing kills into a half-recovered + 2-node cluster and drove it to SUSPENDED by event 3 (run + 20260824_224909) -- damage the test caused, not damage it found. + """ return mgmt_py(mgmt_ip, key_path, """ import json from simplyblock_core.db_controller import DBController db = DBController() -bad = [n.get_id()[:8] for n in db.get_storage_nodes() if n.status != "online"] -print(json.dumps({"all_online": not bad, "offline": bad})) +bad = [n.get_id()[:8] for n in db.get_storage_nodes() + if n.status != "online" or not n.health_check] +busy = [c.get_id()[:8] for c in db.get_clusters() + if c.status not in ("active", "degraded")] +print(json.dumps({"all_online": not bad and not busy, + "offline": bad, "clusters": busy})) """, replayable=True) @@ -2047,8 +2060,24 @@ def test_case_9(meta): time.sleep(20) else: raise RuntimeError( - f"FAIL: nodes {state['offline']} not back online " - f"{NODE_STATE_TIMEOUT}s after chaos kill {ev} on {victim}") + f"FAIL: not recovered {NODE_STATE_TIMEOUT}s after chaos kill " + f"{ev} on {victim}: unhealthy nodes={state['offline']} " + f"clusters not active={state['clusters']}") + + # Chaos without IO is not chaos. A kill that takes fio down (the + # promotion-window EIO) otherwise leaves every later event running + # against an idle client -- run 20260824_224909 lost its workload at + # event 3 and would have coasted through the remaining 97. + if not fio_alive(client_ip, key_path): + print(" restarting the client workload after the outage") + try: + cleanup_client(client_ip, key_path, mounts) + mounts = connect_and_mount(client_ip, key_path, mgmt_ip, lvols, + fmt=False) + start_fio(client_ip, key_path, + write_fio_jobfile(client_ip, key_path, mounts)) + except Exception as exc: # noqa: BLE001 - keep the soak going + print(f" could not restart the workload: {exc}") print("Chaos done; requiring full catch-up + integrity...") wait_replication_caught_up(mgmt_ip, key_path, lvols, timeout=3600) From bf60495487c74ba7e7f19c16aaae180f9cc4694e Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 25 Aug 2026 10:37:26 +0200 Subject: [PATCH 012/122] fix: tell the journal component when a cluster is dual-node Gracefully shutting one node of a 2-node cluster made the SURVIVING node abort its own SPDK application: JC detected a network outage nd=1 njms=2 JC aborts the node due to network outage spdk_abort_node: Forcing application shutdown via abort. (core dumped) The journal component requires jc_ha_nmin_jms() reachable journals -- 2 normally, 1 when the dual-node flag is set. A 2-node cluster losing its peer is left with 1 of 2, so the survivor fail-stopped. Every client path vanished at once, fio took hard EIO and XFS shut down (soak case 6), and the cluster went SUSPENDED. The spdk fork implements the tolerance and exposes the jc_set_dual_node RPC, documenting that in a dual-node configuration a single connected JM is enough -- but nothing in the control plane ever called it, so no 2-node cluster has ever had it on. apply_jc_dual_node() now sets the flag across the cluster whenever a node is added (so growing 2 -> 3 also CLEARS it) and whenever a node is brought back up (a restarted node returns with the JC default). The flag tracks MEMBERSHIP, not the online count: a 3-node cluster with one node down must keep requiring two journals. Co-Authored-By: Claude Fable 5 --- simplyblock_core/rpc_client.py | 15 +++ simplyblock_core/storage_node_ops.py | 43 ++++++++ simplyblock_core/test/test_jc_dual_node.py | 109 +++++++++++++++++++++ 3 files changed, 167 insertions(+) create mode 100644 simplyblock_core/test/test_jc_dual_node.py diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index 864dd46c39..c49e53343c 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -1640,6 +1640,21 @@ def bdev_lvol_connect_hublvol(self, lvs, bdev): "remote_bdev": bdev, }) + def jc_set_dual_node(self, enable): + """Tell the journal component whether this is a DUAL-NODE cluster. + + The JC aborts its whole SPDK application when the number of reachable + journal members drops below jc_ha_nmin_jms(), which is 2 unless the + dual-node flag is set, and 1 when it is. On a 2-node cluster losing + the peer leaves exactly 1 of 2 JMs, so without this flag the SURVIVING + node aborts itself the moment its partner stops -- a single graceful + `sn shutdown` takes the entire cluster down (soak case 6: "JC detected + a network outage nd=1 njms=2" / "JC aborts the node due to network + outage" / core dump, every client path gone, XFS shut down). + The fork implements the tolerance; nothing ever switched it on. + """ + return self._request2("jc_set_dual_node", {"enable": bool(enable)}) + def jc_suspend_compression(self, jm_vuid, suspend=False): params = { "jm_vuid": jm_vuid, diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 44046715fd..34f6d83816 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -910,6 +910,40 @@ def _search_for_partitions(rpc_client, nvme_device): return partitioned_devices +def apply_jc_dual_node(cluster_id): + """Keep every node's JC dual-node flag in step with the cluster size. + + The journal component ABORTS its whole SPDK application when reachable + journal members drop below jc_ha_nmin_jms(), which is 2 normally and 1 + when the dual-node flag is set. A 2-node cluster that loses its peer is + left with 1 of 2 JMs, so without the flag the SURVIVING node aborts + itself the instant its partner stops: one graceful `sn shutdown` takes + the entire cluster down, every client path disappears at once and + filesystems shut down (soak case 6, 2026-08-24 -- "JC detected a network + outage nd=1 njms=2", "JC aborts the node due to network outage", core + dumped). The fork implements the tolerance and exposes jc_set_dual_node; + nothing in the control plane ever called it. + + Keyed on MEMBERSHIP, not on how many nodes are online: a 3-node cluster + with one node down must keep requiring 2 journals. Only a cluster whose + whole membership is 2 is a dual-node cluster. + """ + db_controller = DBController() + members = [n for n in db_controller.get_storage_nodes_by_cluster_id(cluster_id) + if n.status != StorageNode.STATUS_REMOVED] + enable = len(members) == 2 + for node in members: + if node.status != StorageNode.STATUS_ONLINE: + continue + try: + node.rpc_client().jc_set_dual_node(enable) + logger.info("JC dual-node=%s applied on %s (cluster membership %d)", + enable, node.get_id(), len(members)) + except Exception as e: # noqa: BLE001 - best effort + logger.warning("Could not set JC dual-node=%s on %s: %s", + enable, node.get_id(), e) + + def _create_jm_stack_on_raid(rpc_client, jm_nvme_bdevs, snode: StorageNode, after_restart): # RAID 0+1 journal layout (see simplyblock_core/jm_raid.py): # 1 device -> no raid (bare device) @@ -1312,6 +1346,11 @@ def _prepare_cluster_devices_partitions(snode: StorageNode, devices): snode.jm_device = jm_device + # Applied cluster-wide, not just to this node: a cluster growing 2 -> 3 + # must also CLEAR the flag on the two nodes that already have it, and one + # shrinking 3 -> 2 must set it on the survivors. + apply_jc_dual_node(snode.cluster_id) + snode.nvme_devices = new_devices return True @@ -1491,6 +1530,10 @@ def _prepare_cluster_devices_on_restart(snode: StorageNode, clear_data=False): snode.jm_device = jm_device snode.write_to_db() + # A restarted node comes up with the JC default (dual-node off), so the + # flag has to be re-applied on every bring-up, not only at node-add. + apply_jc_dual_node(snode.cluster_id) + return True diff --git a/simplyblock_core/test/test_jc_dual_node.py b/simplyblock_core/test/test_jc_dual_node.py new file mode 100644 index 0000000000..387738f0b5 --- /dev/null +++ b/simplyblock_core/test/test_jc_dual_node.py @@ -0,0 +1,109 @@ +"""The JC dual-node flag must track cluster MEMBERSHIP. + +Soak case 6 (2026-08-24): gracefully shutting down one node of a 2-node +cluster made the SURVIVOR abort its own SPDK -- + + JC detected a network outage nd=1 njms=2 + JC aborts the node due to network outage + spdk_abort_node: Forcing application shutdown via abort. + +-- because the journal component requires jc_ha_nmin_jms() reachable +journals, which is 2 unless the dual-node flag is set (then 1). The fork +implements the tolerance and exposes jc_set_dual_node; the control plane +never called it, so every 2-node cluster lost all availability the moment +either node stopped. +""" +import inspect + +import pytest + +from simplyblock_core import storage_node_ops +from simplyblock_core.models.storage_node import StorageNode + + +class _RPC: + def __init__(self, sink, node_id): + self._sink, self._node_id = sink, node_id + + def jc_set_dual_node(self, enable): + self._sink.append((self._node_id, enable)) + return True + + +class _Node: + def __init__(self, nid, sink, status=StorageNode.STATUS_ONLINE): + self._id, self._sink, self.status = nid, sink, status + self.cluster_id = "CL" + + def get_id(self): + return self._id + + def rpc_client(self, *a, **kw): + return _RPC(self._sink, self._id) + + +def _install(monkeypatch, nodes): + class _DB: + def get_storage_nodes_by_cluster_id(self, cluster_id): + return nodes + monkeypatch.setattr(storage_node_ops, "DBController", lambda: _DB()) + + +def test_two_node_cluster_enables_dual_node(monkeypatch): + sink = [] + nodes = [_Node("A", sink), _Node("B", sink)] + _install(monkeypatch, nodes) + storage_node_ops.apply_jc_dual_node("CL") + assert sink == [("A", True), ("B", True)] + + +def test_three_node_cluster_disables_dual_node(monkeypatch): + sink = [] + nodes = [_Node("A", sink), _Node("B", sink), _Node("C", sink)] + _install(monkeypatch, nodes) + storage_node_ops.apply_jc_dual_node("CL") + assert sink == [("A", False), ("B", False), ("C", False)] + + +def test_flag_follows_membership_not_how_many_are_online(monkeypatch): + """A 3-node cluster with one node down still needs 2 journals. Keying + the flag on the ONLINE count would switch a degraded 3-node cluster into + dual-node mode -- weakening the journal requirement exactly when a node + is already missing.""" + sink = [] + nodes = [_Node("A", sink), _Node("B", sink), + _Node("C", sink, status=StorageNode.STATUS_OFFLINE)] + _install(monkeypatch, nodes) + storage_node_ops.apply_jc_dual_node("CL") + # offline node is not called, but the two online ones must be told FALSE + assert sink == [("A", False), ("B", False)] + + +def test_removed_nodes_do_not_count_towards_membership(monkeypatch): + sink = [] + nodes = [_Node("A", sink), _Node("B", sink), + _Node("C", sink, status=StorageNode.STATUS_REMOVED)] + _install(monkeypatch, nodes) + storage_node_ops.apply_jc_dual_node("CL") + assert sink == [("A", True), ("B", True)] + + +def test_one_unreachable_node_does_not_stop_the_others(monkeypatch): + sink = [] + + class _BadNode(_Node): + def rpc_client(self, *a, **kw): + raise RuntimeError("node unreachable") + + nodes = [_BadNode("A", sink), _Node("B", sink)] + _install(monkeypatch, nodes) + storage_node_ops.apply_jc_dual_node("CL") # must not raise + assert sink == [("B", True)] + + +@pytest.mark.parametrize("path", ["_prepare_cluster_devices_on_restart"]) +def test_bring_up_paths_apply_the_flag(path): + """A restarted node comes back with the JC default (off), so the flag has + to be re-applied on every bring-up, not only when the node is added.""" + src = inspect.getsource(getattr(storage_node_ops, path)) + assert "apply_jc_dual_node" in src, f"{path} must re-apply the dual-node flag" From d3d038584d241c04a40a9de522fa0a8afd85e7be Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 25 Aug 2026 10:27:52 +0200 Subject: [PATCH 013/122] test(mp): run the parity-desync leadership fix (ultra b44de698, spdk 554c80f11) Iteration 12 of the 2026-08-24 run failed fio verify on four blocks, and the received magics were random (c32e, b24e, e6b4, c8b7) rather than fio's constant 0xacca. Stale-but-valid data would still carry acca and fail as an offset or crc mismatch, so those buffers never held an fio header at all -- the map named a location and the bytes handed back were not that location's content. The placement history exonerates the map at every observable point: no steady-state re-home across the nine dump points of iterations 11-12, every group back on its pre-outage home, and the three roles identical at range level (12,810 groups) at post_incident. ultra b44de698 fits that shape: a reactively promoted distrib used to signal JC leadership immediately while, under write protection, the parity desynchronisation check was still outstanding, so reads served in that window came off desynchronised parity. The signal now waits for the check to finish. spdk moves a311a6852 -> 554c80f11, which keeps the upstream retry-state fix (#3686) as an ancestor and adds the promotion-window ANA transition fix. The gate now pins the ultra commit as well, so a stale image cannot quietly re-test the build that already failed. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/deploy_gate_and_soak.py | 16 +++++++++++++++- scripts/setup_perf_test_multipath.py | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/scripts/deploy_gate_and_soak.py b/scripts/deploy_gate_and_soak.py index c41a32dd06..24dd2943ae 100644 --- a/scripts/deploy_gate_and_soak.py +++ b/scripts/deploy_gate_and_soak.py @@ -32,7 +32,16 @@ #: mtime: the ultra Dockerfile bakes `git log` of the spdk repo into #: /root/spdk/git_log.txt, which pins the image to a commit. See the SPDK_IMAGE #: pin in setup_perf_test_multipath.py for why this is checked, not assumed. -EXPECT_SPDK_COMMIT = "a311a6852" +EXPECT_SPDK_COMMIT = "554c80f11" +#: The ultra commit that must be running. b44de698 defers the JC leadership +#: signal until the parity-desynchronisation check completes. Before it, a +#: reactively promoted distrib announced leadership while parity was still +#: desynchronised, so reads served in that window came off desynchronised +#: parity and returned arbitrary bytes -- the 2026-08-24 iteration-12 "bad +#: magic" failures, whose received magics were random rather than fio's +#: 0xacca, ruling out stale-but-valid data. Without this pin the run would +#: re-test the build that already failed. +EXPECT_ULTRA_COMMIT = "b44de698" #: Lines probe_bin.sh must print, with what their absence would mean. These #: distinguish upstream d528e1a67 (zeroes retry state at the submission entry #: point) from the superseded local attempt that zeroed it at completion and so @@ -123,6 +132,11 @@ def gate(mgmt, sn): f"— /root/spdk/git_log.txt names a different commit, so the node is " f"on a stale image. Check the SPDK_IMAGE pin and that the " f"spdk-core R26.3 tags finished rebuilding (amd64 included).") + if EXPECT_ULTRA_COMMIT not in out: + raise RuntimeError( + f"GATE FAILED: image was not built from ultra " + f"{EXPECT_ULTRA_COMMIT} — the parity-desync leadership fix under " + f"test is absent, so the run would prove nothing.") for fragment, why in REQUIRED_FIX_LINES: if fragment not in out: raise RuntimeError(f"GATE FAILED: {why} ({fragment!r} absent)") diff --git a/scripts/setup_perf_test_multipath.py b/scripts/setup_perf_test_multipath.py index 6cce555a85..1b41abc6e5 100644 --- a/scripts/setup_perf_test_multipath.py +++ b/scripts/setup_perf_test_multipath.py @@ -87,7 +87,7 @@ #: itself, so a stale pin fails loudly instead of running old code quietly. #: 8d2e5215 = ultra main "Build main on the R26.3 spdk-core base", built on #: spdk-core R26.3 a311a6852 which carries upstream d528e1a67 (spdk/spdk#3686). -SPDK_IMAGE = "public.ecr.aws/simply-block/ultra:main-8d2e5215-amd64" +SPDK_IMAGE = "public.ecr.aws/simply-block/ultra:main-b44de698-amd64" USER = "ec2-user" MGMT_IFACE = "eth0" DATA_NICS = ["eth1", "eth2"] # Names the OS assigns to ENI index 1, 2 From e39458c44a450f1636f4065202cd67e63d59f8e2 Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 25 Aug 2026 11:10:18 +0200 Subject: [PATCH 014/122] test: prove replication resumed by a post-recovery point-in-time, not a retained count With the JC dual-node fix in place case 6 finally kept the client alive through a primary outage (fio_alive=True, replicated 5 -> 13 during the outage) and then failed on its own assertion: during=13, after=10. replicated_count counts RETAINED replicated snapshots, and retention keeps only the newest generations, so it is bounded and falls after a burst. It cannot express 'replication resumed'. Cases 5 and 6 now require a point-in-time created AFTER recovery to reach the target, which is what resumption actually means, using the harness primitive that already exists for it. Co-Authored-By: Claude Fable 5 --- scripts/hotfix_repl_lab.py | 6 ++++++ scripts/test_async_replication.py | 23 ++++++++++++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/scripts/hotfix_repl_lab.py b/scripts/hotfix_repl_lab.py index c8d3f75f7f..81715e8ea5 100644 --- a/scripts/hotfix_repl_lab.py +++ b/scripts/hotfix_repl_lab.py @@ -61,6 +61,12 @@ # dependency alongside the file that needs it. "simplyblock_core/controllers/ops_gate.py": "controllers/ops_gate.py", "simplyblock_core/controllers/tasks_controller.py": "controllers/tasks_controller.py", + # The JC dual-node flag lives here: apply_jc_dual_node() is called on + # node add AND on every bring-up, and case 6 restarts a node mid-run -- + # a restarted node comes back with the flag off unless the CP re-applies + # it, which is exactly when the survivor would abort. + "simplyblock_core/storage_node_ops.py": "storage_node_ops.py", + "simplyblock_core/rpc_client.py": "rpc_client.py", } #: service -> its own module (mounted on top of the shared set) SERVICES = { diff --git a/scripts/test_async_replication.py b/scripts/test_async_replication.py index 3b9941ce48..2fc7a4f622 100644 --- a/scripts/test_async_replication.py +++ b/scripts/test_async_replication.py @@ -1557,17 +1557,17 @@ def test_case_5(meta): print("Bringing the target node back...") sn_bring_back(mgmt_ip, key_path, victim) + recovery_ts = time.time() wait_replication_caught_up(mgmt_ip, key_path, lvols) + # See case 6: replicated_count is a retained count, not a progress + # counter. Require a post-recovery point-in-time on the target instead. + wait_data_replicated(mgmt_ip, key_path, lvols, recovery_ts) after = _replication_progress(mgmt_ip, key_path, lvols) - print(f" replicated_count after recovery={after}") + print(f" replicated_count after recovery={after} (retained, not cumulative)") stop_fio(client_ip, key_path) errors = fio_error_count(client_ip, key_path) cleanup_client(client_ip, key_path, mounts) - if after <= during: - raise RuntimeError( - f"FAIL: replication did not resume after the target node returned " - f"(during={during}, after={after})") if errors: raise RuntimeError(f"FAIL: fio reported {errors} errors during the target-node outage") print("CASE 5 PASSED: target-node outage survived, replication resumed.") @@ -1615,16 +1615,21 @@ def test_case_6(meta): print("Bringing the primary back...") sn_bring_back(mgmt_ip, key_path, primary) + recovery_ts = time.time() wait_replication_caught_up(mgmt_ip, key_path, lvols) + # Resumption is proven by a point-in-time created AFTER the primary + # returned reaching the target -- not by replicated_count growing. + # That counter tracks RETAINED replicated snapshots, and retention keeps + # only the newest generations, so it is bounded and routinely falls after + # a burst: run 20260825_105453 read during=13 / after=10 while + # replication was working perfectly. + wait_data_replicated(mgmt_ip, key_path, lvols, recovery_ts) after = _replication_progress(mgmt_ip, key_path, lvols) - print(f" replicated_count after recovery={after}") + print(f" replicated_count after recovery={after} (retained, not cumulative)") stop_fio(client_ip, key_path) errors = fio_error_count(client_ip, key_path) cleanup_client(client_ip, key_path, mounts) - if after <= during: - raise RuntimeError(f"FAIL: replication did not resume after the primary returned " - f"(during={during}, after={after})") if errors: raise RuntimeError(f"FAIL: fio reported {errors} errors during the primary outage") print("CASE 6 PASSED: source-primary outage survived, replication continued and resumed.") From dd641a59d6b178391419a942f1cfff1a476e026f Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 25 Aug 2026 11:37:19 +0200 Subject: [PATCH 015/122] feat: tiered snapshot retention and fail-over generation selection Replication kept a flat count of internal snapshots, so history never went back further than a couple of cadence ticks and a fail-over could only ever land on 'a minute ago' -- useless against a logical corruption, which the newest copy has faithfully replicated. A replication policy can now carry a retention SCHEDULE: --retention-schedule '15m:2h,1h:11h,1d:7d' 'one snapshot every 15 minutes for the last 2 hours, then hourly for 11 hours, then daily for 7 days'. Snapshots past the total span are pruned. Selection is a pure function of (times, schedule, now) in the new snapshot_retention module, so it is unit-tested without a cluster: a minute-cadence stream over 8 days collapses to ~25 retained snapshots that still reach back 6+ days. The schedule never overrides MIN_KEEP_REPLICATED: the newest pair is always retained, because deleting a snapshot swap-merges its segments into the successor chained to it. An unparseable schedule is rejected when the policy is created, and if one ever reaches the runner it is logged and treated as 'no schedule' rather than crashing replication or silently dropping history. Fail-over gains a generation selector: generation 0 is the newest replicated point-in-time (previous behaviour), higher values walk back through the retained history. Asking for more generations than exist is an explicit error rather than a silent fall back to the newest. Co-Authored-By: Claude Fable 5 --- simplyblock_cli/cli.py | 1 + simplyblock_cli/clibase.py | 4 +- .../controllers/lvol_controller.py | 27 +++- .../replication_policy_controller.py | 15 +- simplyblock_core/models/replication.py | 6 + .../services/snapshot_replication.py | 47 +++++- simplyblock_core/snapshot_retention.py | 132 ++++++++++++++++ .../test/test_failover_clone_race.py | 2 +- .../test/test_snapshot_retention.py | 142 ++++++++++++++++++ .../storage_pool/volume/replication.py | 14 +- 10 files changed, 374 insertions(+), 16 deletions(-) create mode 100644 simplyblock_core/snapshot_retention.py create mode 100644 simplyblock_core/test/test_snapshot_retention.py diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 77c498dd97..f1f0d592df 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -662,6 +662,7 @@ def init_cluster__replication_policy_add(self, subparser): subcommand.add_argument('--interval-min', help='Cadence: minutes between internal replication snapshots. 0 replicates user snapshots only. Default: `1`.', type=int, dest='interval_min') subcommand.add_argument('--mode', help='Replication mode. Default: `failover`.', type=str, dest='mode', choices=['failover','migration',]) subcommand.add_argument('--keep', help='Replicated internal snapshots to retain on each side. Minimum (and default): `2`.', type=int, dest='keep_replicated') + subcommand.add_argument('--retention-schedule', help='Tiered retention, e.g. `15m:2h,1h:11h,1d:7d` - one snapshot every 15 minutes for the last 2 hours, then hourly for 11 hours, then daily for 7 days. Snapshots older than the total span are pruned. Empty (default) keeps the flat --keep behaviour.', type=str, dest='retention_schedule') def init_cluster__replication_policy_list(self, subparser): subcommand = self.add_sub_command(subparser, 'replication-policy-list', 'Lists the replication policies of a cluster') diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 749efd7e50..530c0e05b6 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -661,7 +661,8 @@ def cluster__replication_policy_add(self, sub_command, args): return replication_policy_controller.add_policy( args.cluster_id, args.name, args.target, interval_min=args.interval_min, mode=args.mode, - keep_replicated=args.keep_replicated) + keep_replicated=args.keep_replicated, + retention_schedule=args.retention_schedule) def cluster__replication_policy_list(self, sub_command, args): data = [{ @@ -671,6 +672,7 @@ def cluster__replication_policy_list(self, sub_command, args): "Interval (min)": p.interval_min, "Mode": p.mode, "Keep": p.keep_replicated, + "Retention": p.retention_schedule or "-", "Status": p.status, } for p in replication_policy_controller.list_policies(args.cluster_id)] return _format_result(data, json=args.json) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index d3e075e09a..117338baff 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -3428,7 +3428,7 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps return new_lvol, None -def _last_replicated_target_snapshot(db_controller, lvol_id, cluster_id): +def _last_replicated_target_snapshot(db_controller, lvol_id, cluster_id, generation=0): """Return the target-cluster copy of the most recent FULLY replicated snapshot of *lvol_id*, or None. @@ -3461,6 +3461,18 @@ def _last_replicated_target_snapshot(db_controller, lvol_id, cluster_id): snaps.append(snap) snaps.sort(key=lambda x: x.created_at, reverse=True) + # generation 0 = newest replicated point-in-time (the default and the only + # behaviour before tiered retention existed). A higher generation walks + # BACK through the retained history, which is what a retention schedule is + # for: recovering to a point before a logical corruption that a + # minute-old copy would have replicated faithfully. + if generation: + if generation >= len(snaps): + logger.error( + f"Fail-over generation {generation} requested for {lvol_id} but only " + f"{len(snaps)} replicated point(s)-in-time exist") + return None + snaps = snaps[generation:] for snap in snaps: try: target_snap = db_controller.get_snapshot_by_id(snap.target_replicated_snap_uuid) @@ -3540,7 +3552,7 @@ def _evict_stale_namespace(new_lvol, target_node): def _clone_from_last_replicated(db_controller, lvol_id, lvol, target_node, pool_uuid, - cluster_id, attempts=3): + cluster_id, attempts=3, generation=0): """Pick the last fully replicated target snapshot and clone from it ATOMICALLY. Selecting and then cloning as two unsynchronised steps loses the data: the @@ -3561,9 +3573,12 @@ def _clone_from_last_replicated(db_controller, lvol_id, lvol, target_node, pool_ Returns (new_lvol, snapshot_used, error). """ for _ in range(attempts): - snapshot = _last_replicated_target_snapshot(db_controller, lvol_id, cluster_id) + snapshot = _last_replicated_target_snapshot(db_controller, lvol_id, cluster_id, + generation=generation) if not snapshot: - return None, None, "No replicated snapshot on target yet" + return None, None, ( + f"No replicated snapshot on target for generation {generation}" + if generation else "No replicated snapshot on target yet") with snapshot_controller.object_mutation_lock(snapshot.cluster_id, snapshot.uuid): # Re-read INSIDE the lock: retention may have removed or started @@ -3624,7 +3639,7 @@ def resolve_replication_destination(db_controller, lvol, target_node, source_nod return target_cluster, "" -def replicate_lvol_on_target_cluster(lvol_id): +def replicate_lvol_on_target_cluster(lvol_id, generation=0): db_controller = DBController() try: lvol = db_controller.get_lvol_by_id(lvol_id) @@ -3663,7 +3678,7 @@ def replicate_lvol_on_target_cluster(lvol_id): new_lvol, _snapshot, error = _clone_from_last_replicated( db_controller, lvol_id, lvol, target_node, - target_pool_uuid, source_node.cluster_id) + target_pool_uuid, source_node.cluster_id, generation=generation) if error: logger.error(f"Fail-over clone failed for lvol {lvol_id}: {error}") return False, error diff --git a/simplyblock_core/controllers/replication_policy_controller.py b/simplyblock_core/controllers/replication_policy_controller.py index 12a868e6b5..88b5459195 100644 --- a/simplyblock_core/controllers/replication_policy_controller.py +++ b/simplyblock_core/controllers/replication_policy_controller.py @@ -16,6 +16,7 @@ from simplyblock_core.controllers import lvol_controller, snapshot_controller from simplyblock_core.models.lvol_model import LVolReplication from simplyblock_core.models.pool import Pool +from simplyblock_core import snapshot_retention from simplyblock_core.models.replication import ReplicationPolicy, ReplicationTarget from simplyblock_core.models.snapshot import SnapShot @@ -92,7 +93,8 @@ def remove_target(target_id): # Policies # --------------------------------------------------------------------------- # -def add_policy(cluster_id, policy_name, target, interval_min=1, mode=None, keep_replicated=None): +def add_policy(cluster_id, policy_name, target, interval_min=1, mode=None, keep_replicated=None, + retention_schedule=None): """Create a policy on *target* (id or name).""" db.get_cluster_by_id(cluster_id) try: @@ -118,6 +120,15 @@ def add_policy(cluster_id, policy_name, target, interval_min=1, mode=None, keep_ raise ReplicationConfigError( f"keep_replicated must be at least {ReplicationPolicy.MIN_KEEP_REPLICATED}") + if retention_schedule: + # Validate at ingress: an unparseable schedule silently falling back to + # flat retention would quietly discard the history the operator asked + # for, and they would only find out at fail-over. + try: + snapshot_retention.parse_schedule(retention_schedule) + except snapshot_retention.RetentionScheduleError as e: + raise ReplicationConfigError(f"invalid retention schedule: {e}") + policy = ReplicationPolicy() policy.uuid = str(uuid_module.uuid4()) policy.cluster_id = cluster_id @@ -129,6 +140,8 @@ def add_policy(cluster_id, policy_name, target, interval_min=1, mode=None, keep_ policy.mode = mode if keep_replicated is not None: policy.keep_replicated = keep_replicated + if retention_schedule is not None: + policy.retention_schedule = retention_schedule policy.status = ReplicationPolicy.STATUS_ACTIVE policy.write_to_db(db.kv_store) logger.info("Created replication policy %s on target %s (%s)", diff --git a/simplyblock_core/models/replication.py b/simplyblock_core/models/replication.py index ce73eff2bf..ff8f5ec941 100644 --- a/simplyblock_core/models/replication.py +++ b/simplyblock_core/models/replication.py @@ -71,6 +71,12 @@ class ReplicationPolicy(BaseModel): interval_min: int = 1 # internal snapshot cadence, 0 = user snaps only mode: str = MODE_FAILOVER keep_replicated: int = MIN_KEEP_REPLICATED + #: Tiered retention, e.g. "15m:2h,1h:11h,1d:7d" -- one snapshot every 15 + #: minutes for the last 2 hours, then hourly for 11 hours, then daily for + #: 7 days. Empty keeps the flat keep_replicated behaviour. A schedule + #: never overrides MIN_KEEP_REPLICATED: the newest pair is always kept so + #: an arriving delta has a predecessor to chain onto. + retention_schedule: str = "" status: str = STATUS_ACTIVE def get_id(self): diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index 6e399f07de..d56541c89e 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -2,7 +2,7 @@ import time import uuid -from simplyblock_core import constants, db_controller, utils +from simplyblock_core import constants, db_controller, snapshot_retention, utils from simplyblock_core.controllers import lvol_controller, snapshot_events, snapshot_controller from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.lvol_model import LVol @@ -596,6 +596,28 @@ def _keep_replicated_for(source_lvol): from simplyblock_core.models.replication import ReplicationPolicy return max(policy.keep_replicated, ReplicationPolicy.MIN_KEEP_REPLICATED) +def _retention_schedule_for(source_lvol): + """Parsed retention tiers from the volume's policy, or [] when it has none. + + A malformed schedule must not silently disable retention or crash the + replication runner: it is reported and treated as "no schedule", which + falls back to the flat keep-count. + """ + try: + policy = db.get_replication_policy_for_lvol(source_lvol) + except Exception: + return [] + spec = getattr(policy, "retention_schedule", "") if policy else "" + if not spec: + return [] + try: + return snapshot_retention.parse_schedule(spec) + except snapshot_retention.RetentionScheduleError as e: + logger.error("Ignoring invalid retention_schedule %r on policy %s: %s", + spec, policy.get_id(), e) + return [] + + def _prune_internal_snapshots(source_lvol): """Retention for replication-driven internal snapshots. @@ -619,10 +641,25 @@ def _prune_internal_snapshots(source_lvol): and s.target_replicated_snap_uuid ] keep = _keep_replicated_for(source_lvol) - if len(replicated_internal) <= keep: - return - replicated_internal.sort(key=lambda s: s.created_at) + + # A retention SCHEDULE, when the policy defines one, decides which older + # snapshots survive; without it retention stays the flat "newest N". + # Either way the newest `keep` are protected, because deleting a snapshot + # swap-merges its segments into the successor chained to it. + schedule = _retention_schedule_for(source_lvol) + if schedule: + retained_ts = snapshot_retention.select_retained( + [s.created_at for s in replicated_internal], schedule, + now=time.time(), always_keep_newest=keep) + candidates = [(i, s) for i, s in enumerate(replicated_internal) + if s.created_at not in retained_ts] + if not candidates: + return + else: + if len(replicated_internal) <= keep: + return + candidates = list(enumerate(replicated_internal))[:-keep] # Keep the newest TWO replicated internal snapshots, not just one. # # A replicated snapshot holds only its own clusters; the rest of the data @@ -639,7 +676,7 @@ def _prune_internal_snapshots(source_lvol): # for one snapshot while newer ones kept arriving, the predecessor was still # pruned and its segments were dropped instead of merged. So the chain is # verified per candidate below, and an unchained successor defers the prune. - for index, snap in enumerate(replicated_internal[:-keep]): + for index, snap in candidates: target_uuid = snap.target_replicated_snap_uuid try: db.get_snapshot_by_id(target_uuid) diff --git a/simplyblock_core/snapshot_retention.py b/simplyblock_core/snapshot_retention.py new file mode 100644 index 0000000000..9d1a6735af --- /dev/null +++ b/simplyblock_core/snapshot_retention.py @@ -0,0 +1,132 @@ +"""Tiered retention for replication snapshots. + +Replication used to keep a flat count of internal snapshots (the newest +``keep_replicated``), which gives no history: everything older than a couple +of cadence ticks is gone, so a fail-over can only ever land on "a minute +ago". A retention SCHEDULE keeps a thinning history instead -- dense for the +recent past, sparse further back -- so an operator can fail over to a chosen +point in time (yesterday, six hours ago) after a logical corruption that a +one-minute-old copy would have faithfully replicated. + +Schedule syntax (compact, order-independent, parsed left to right): + + "15m:2h,1h:11h,1d:7d" + +reads as "one snapshot every 15 minutes covering the last 2 hours, then one +every hour covering the next 11 hours, then one per day covering the next 7 +days". Each tier is ``:``; both accept an integer with a unit +suffix s/m/h/d. A snapshot older than the sum of all tier spans is not +covered by the schedule and is pruned. + +The selection is a pure function of (snapshot times, schedule, now), which +is what makes it testable without a cluster: see test_snapshot_retention.py. +""" +from __future__ import annotations + +import re +from typing import Iterable, List, NamedTuple, Sequence, Set + +_UNITS = {"s": 1, "m": 60, "h": 3600, "d": 86400} +_TOKEN = re.compile(r"^(\d+)([smhd])$") + + +class RetentionTier(NamedTuple): + """Keep one snapshot per ``every_sec`` bucket, covering ``span_sec``.""" + + every_sec: int + span_sec: int + + +class RetentionScheduleError(ValueError): + """The schedule string could not be parsed.""" + + +def _duration(token: str) -> int: + m = _TOKEN.match(token.strip().lower()) + if not m: + raise RetentionScheduleError( + f"bad duration {token!r}: expected , e.g. 15m or 7d") + value, unit = int(m.group(1)), m.group(2) + if value <= 0: + raise RetentionScheduleError(f"duration must be positive: {token!r}") + return value * _UNITS[unit] + + +def parse_schedule(spec: str) -> List[RetentionTier]: + """Parse ``"15m:2h,1h:11h,1d:7d"`` into tiers. Empty string -> no tiers.""" + if not spec or not spec.strip(): + return [] + tiers: List[RetentionTier] = [] + for chunk in spec.split(","): + chunk = chunk.strip() + if not chunk: + continue + if chunk.count(":") != 1: + raise RetentionScheduleError( + f"bad tier {chunk!r}: expected :, e.g. 15m:2h") + every_s, span_s = chunk.split(":") + every, span = _duration(every_s), _duration(span_s) + if every > span: + raise RetentionScheduleError( + f"tier {chunk!r} keeps one snapshot every {every_s} but only " + f"covers {span_s} -- the interval cannot exceed the span") + tiers.append(RetentionTier(every, span)) + # Coarser tiers must come after finer ones; sorting makes the spec + # order-independent rather than silently producing a nonsense ladder. + tiers.sort(key=lambda t: t.every_sec) + return tiers + + +def horizon_sec(tiers: Sequence[RetentionTier]) -> int: + """Total age covered by the schedule; older snapshots are not retained.""" + return sum(t.span_sec for t in tiers) + + +def select_retained(created_ats: Iterable[float], tiers: Sequence[RetentionTier], + now: float, always_keep_newest: int = 0) -> Set[float]: + """Return the subset of ``created_ats`` the schedule retains. + + One snapshot per bucket per tier: the NEWEST in each bucket, so the + retained point-in-time is as close as possible to the bucket boundary the + operator asked for. Tiers apply to successive age ranges, finest first. + + ``always_keep_newest`` protects the N most recent regardless of the + schedule. Replication needs that: deleting a snapshot swap-merges its + segments into the successor chained to it, so the newest pair must + survive or an arriving delta has nothing to chain onto. + """ + times = sorted({float(t) for t in created_ats if t}, reverse=True) + if not times: + return set() + + keep: Set[float] = set(times[:max(0, always_keep_newest)]) + if not tiers: + return keep + + # Walk age ranges: tier i covers [range_start, range_start + span). + range_start = 0.0 + for tier in tiers: + range_end = range_start + tier.span_sec + # Bucket by absolute age so bucket edges do not drift between calls. + best_per_bucket = {} + for t in times: + age = now - t + if age < range_start or age >= range_end: + continue + bucket = int(age // tier.every_sec) + # times is newest-first, so the first hit in a bucket is newest. + best_per_bucket.setdefault(bucket, t) + keep.update(best_per_bucket.values()) + range_start = range_end + return keep + + +def describe(tiers: Sequence[RetentionTier]) -> str: + """Render tiers back to the compact spec (for display / round-trip).""" + def fmt(seconds: int) -> str: + for unit in ("d", "h", "m", "s"): + size = _UNITS[unit] + if seconds % size == 0: + return f"{seconds // size}{unit}" + return f"{seconds}s" + return ",".join(f"{fmt(t.every_sec)}:{fmt(t.span_sec)}" for t in tiers) diff --git a/simplyblock_core/test/test_failover_clone_race.py b/simplyblock_core/test/test_failover_clone_race.py index b4c4e1f256..6e3d8032e7 100644 --- a/simplyblock_core/test/test_failover_clone_race.py +++ b/simplyblock_core/test/test_failover_clone_race.py @@ -82,7 +82,7 @@ def __exit__(self, *a): monkeypatch.setattr(lvol_controller.snapshot_controller, "object_mutation_lock", _Lock) - def _fake_select(db, lvol_id, cluster_id): + def _fake_select(db, lvol_id, cluster_id, generation=0): """Stand-in for _last_replicated_target_snapshot. Mirrors the real selector: newest first, skipping anything missing or diff --git a/simplyblock_core/test/test_snapshot_retention.py b/simplyblock_core/test/test_snapshot_retention.py new file mode 100644 index 0000000000..b9e51e2b2c --- /dev/null +++ b/simplyblock_core/test/test_snapshot_retention.py @@ -0,0 +1,142 @@ +"""Tiered snapshot retention: parsing and selection are pure functions.""" +import pytest + +from simplyblock_core.snapshot_retention import ( + RetentionScheduleError, RetentionTier, describe, horizon_sec, + parse_schedule, select_retained, +) + +HOUR = 3600 +DAY = 86400 +#: the example from the feature request +SPEC = "15m:2h,1h:11h,1d:7d" + + +def test_parse_the_requested_schedule(): + assert parse_schedule(SPEC) == [ + RetentionTier(15 * 60, 2 * HOUR), + RetentionTier(HOUR, 11 * HOUR), + RetentionTier(DAY, 7 * DAY), + ] + assert horizon_sec(parse_schedule(SPEC)) == 2 * HOUR + 11 * HOUR + 7 * DAY + + +def test_empty_schedule_is_no_tiers(): + assert parse_schedule("") == [] + assert parse_schedule(" ") == [] + + +def test_schedule_is_order_independent(): + assert parse_schedule("1d:7d,15m:2h,1h:11h") == parse_schedule(SPEC) + + +@pytest.mark.parametrize("bad", [ + "15m", "15m:2h:3d", "15x:2h", "0m:2h", "-5m:2h", "2h:15m", "abc", +]) +def test_bad_schedules_are_rejected(bad): + with pytest.raises(RetentionScheduleError): + parse_schedule(bad) + + +def test_round_trip_describe(): + assert describe(parse_schedule(SPEC)) == SPEC + + +def test_one_snapshot_kept_per_bucket_newest_wins(): + tiers = parse_schedule("15m:1h") + now = 10_000_000.0 + # first four are all inside bucket 0 (ages 60..800s < 900s), the last + # one is in bucket 1 (age 1000s) + snaps = [now - 60, now - 120, now - 200, now - 800, now - 1000] + keep = select_retained(snaps, tiers, now) + assert now - 60 in keep # newest of bucket 0 + for shadowed in (now - 120, now - 200, now - 800): + assert shadowed not in keep + assert now - 1000 in keep # bucket 1 keeps its own + + +def test_history_thins_out_with_age(): + """A minute-cadence stream over 8 days collapses to roughly + 8 quarter-hours + 11 hours + 7 days, not thousands of snapshots.""" + tiers = parse_schedule(SPEC) + now = 1_000_000_000.0 + snaps = [now - 60 * i for i in range(8 * 24 * 60)] # every minute, 8 days + keep = select_retained(snaps, tiers, now) + assert 20 <= len(keep) <= 30, len(keep) + # dense recent history + assert max(keep) == now + # and genuine multi-day depth + assert min(keep) < now - 6 * DAY + + +def test_snapshots_older_than_the_horizon_are_dropped(): + tiers = parse_schedule("15m:2h") + now = 5_000_000.0 + keep = select_retained([now - 60, now - 10 * HOUR], tiers, now) + assert now - 60 in keep + assert now - 10 * HOUR not in keep + + +def test_always_keep_newest_survives_the_schedule(): + """Replication chains the arriving delta onto its predecessor, so the + newest pair must never be pruned no matter what the schedule says.""" + tiers = parse_schedule("1d:7d") + now = 2_000_000.0 + snaps = [now - 10, now - 20, now - 30] # all inside one daily bucket + keep = select_retained(snaps, tiers, now, always_keep_newest=2) + assert now - 10 in keep and now - 20 in keep + + +def test_no_tiers_keeps_only_the_protected_newest(): + now = 100.0 + keep = select_retained([now - 1, now - 2, now - 3], [], now, always_keep_newest=2) + assert keep == {now - 1, now - 2} + + +def test_selection_is_stable_across_repeated_calls(): + """Bucketing on absolute age must not drift, or a snapshot kept on one + pass gets pruned on the next and the history develops holes.""" + tiers = parse_schedule(SPEC) + now = 3_000_000.0 + snaps = [now - 60 * i for i in range(600)] + assert select_retained(snaps, tiers, now) == select_retained(snaps, tiers, now) + + +# --- wiring: the schedule and the fail-over generation ---------------------- + +def test_prune_consults_the_schedule_before_the_flat_count(): + """_prune_internal_snapshots must ask the policy's schedule which + snapshots survive; the flat keep-count is only the fallback.""" + import inspect + from simplyblock_core.services import snapshot_replication as sr + src = inspect.getsource(sr._prune_internal_snapshots) + assert "_retention_schedule_for" in src + assert "select_retained" in src + # the newest `keep` stay protected even under a schedule + assert "always_keep_newest=keep" in src + + +def test_invalid_schedule_falls_back_instead_of_crashing_the_runner(): + import inspect + from simplyblock_core.services import snapshot_replication as sr + src = inspect.getsource(sr._retention_schedule_for) + assert "RetentionScheduleError" in src and "return []" in src + + +def test_policy_rejects_an_invalid_schedule_at_ingress(): + import inspect + from simplyblock_core.controllers import replication_policy_controller as rpc + src = inspect.getsource(rpc.add_policy) + assert "parse_schedule" in src, "the policy must validate the schedule when set" + assert src.index("parse_schedule") < src.index("policy = ReplicationPolicy()") + + +def test_failover_generation_walks_back_through_history(): + """generation=0 is the newest point-in-time; higher values step back, and + asking for more generations than exist is an error, not a silent newest.""" + import inspect + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc._last_replicated_target_snapshot) + assert "generation" in src + assert "snaps[generation:]" in src + assert "only" in src and "exist" in src # explicit out-of-range error diff --git a/simplyblock_web/api/v2/cluster/storage_pool/volume/replication.py b/simplyblock_web/api/v2/cluster/storage_pool/volume/replication.py index 339e5ef344..dde7cfc767 100644 --- a/simplyblock_web/api/v2/cluster/storage_pool/volume/replication.py +++ b/simplyblock_web/api/v2/cluster/storage_pool/volume/replication.py @@ -107,13 +107,23 @@ def trigger(cluster: Cluster, pool: StoragePool, volume: Volume) -> Response: @api.post('/failover', name='clusters:storage-pools:volumes:replication:failover', status_code=204, responses={204: {"content": None}}) -def failover(cluster: Cluster, pool: StoragePool, volume: Volume) -> Response: +def failover(cluster: Cluster, pool: StoragePool, volume: Volume, + generation: int = 0) -> Response: """Bring the volume up on the target cluster. The counterpart's id is read back from this volume's replication relationship, its connection paths from the target volume's `connect`. + + ``generation`` selects WHICH retained point-in-time to come up on: 0 (the + default) is the newest, 1 the one before it, and so on through the + history a retention schedule keeps. Failing over to an older generation + is the recovery path for a logical corruption, which the newest copy has + faithfully replicated. """ - result = lvol_controller.replicate_lvol_on_target_cluster(volume.get_id()) + if generation < 0: + raise HTTPException(400, 'generation cannot be negative') + result = lvol_controller.replicate_lvol_on_target_cluster( + volume.get_id(), generation=generation) if isinstance(result, tuple): # (False, error) raise HTTPException(500, str(result[1])) if not result: From 3aadb01153956dd40b58ae62ec17aa74534b5f0d Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 25 Aug 2026 12:02:07 +0200 Subject: [PATCH 016/122] test: three nodes per cluster everywhere -- two-node clusters are not a supported configuration The product minimum is 3 nodes. The 2-node src/fresh clusters this lab used are exactly what produced the 2026-08-24/25 failure chain: with one node down the survivor holds 1 of 2 journal members and the JC aborts it (whole-cluster outage on a single node stop), and the restart rebalance has no third failure domain to place into, so device_migration loops on 'no allowed placement' forever and pins the cluster in REBALANCING, blocking sn shutdown and every node-down test behind it. Co-Authored-By: Claude Fable 5 --- scripts/setup_repl_test_2clusters.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/setup_repl_test_2clusters.py b/scripts/setup_repl_test_2clusters.py index d4a5f3018e..00421e3a41 100644 --- a/scripts/setup_repl_test_2clusters.py +++ b/scripts/setup_repl_test_2clusters.py @@ -60,15 +60,18 @@ # The FIRST cluster (bootstrap=True) is created with `cluster create`; every # other cluster is attached to the same CP with `cluster add`. CLUSTERS = [ + # THREE nodes per cluster, everywhere. Two-node clusters are not a + # supported configuration (product minimum is 3), and the 2026-08-24/25 + # campaign showed exactly why: with one node down the survivor holds 1 of + # 2 journal members and the JC aborts it, and the restart rebalance has no + # third failure domain to place into, so its device_migration loops on + # "no allowed placement" forever and pins the cluster in REBALANCING. { - "name": "src", # 1+1 HA pair - "nodes": 2, + "name": "src", + "nodes": 3, "ndcs": 1, # data-chunks-per-stripe "npcs": 1, # parity-chunks-per-stripe (FT=1) - # None => let the CP resolve the required count (3 for FT=1). An explicit - # 2 is now rejected by resolve_ha_jm_count(); a 2-node cluster simply - # ends up with the 2 host-disjoint journals it can place. - "ha_jm_count": None, + "ha_jm_count": 3, "bootstrap": True, # `cluster create` "pool": "pool_src", }, @@ -87,10 +90,10 @@ # only run cases 1-3/5/6. { "name": "fresh", - "nodes": 2, + "nodes": 3, "ndcs": 1, "npcs": 1, - "ha_jm_count": None, + "ha_jm_count": 3, "bootstrap": False, "pool": "pool_fresh", }, From 12a09043b9b72bfc783899726802166c846617f1 Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 25 Aug 2026 12:39:50 +0200 Subject: [PATCH 017/122] feat: snapshot consistency groups on replication policies A policy created with --consistency-group makes all attached volumes ONE crash-consistent unit: 1) Same-LVS invariant: the group pins to the first member's node/LVS. Attaching the policy to a volume elsewhere FAILS before any state is written; creating a volume under the policy forces placement onto the pinned node (an explicit conflicting --host is an error). 2) Group snapshots: an auto-managed ConsistencyGroup record lives and dies with the policy. One SPDK call (bdev_lvol_snapshot_group, spdk branch consistency-groups) freezes IO on every member blob, snapshots them one after the other, and unfreezes; SPDK unfreezes-first-then-GCs on mid-sequence failure. The controller mirrors the single-snapshot path per member around that call: replica registration, records, chain links, events, replication tasks. All-or-nothing: any registration or record failure rolls the whole generation back, and the group_seq counter moves only after full success. 3) The cadence (snapshot_monitor) snapshots CG policies as a group - one generation per tick with group-wide back-pressure - and its members leave the per-volume loop. 4) Membership epochs: a late joiner is active from the FIRST group snapshot after its attach (joined_seq = last_group_seq + 1); a detached member's epoch closes at the current generation. Failing over to generation N warns which current members that point-in-time does NOT contain and which contained volumes are no longer members - in the controller result, the CLI failover table, and the API response (200 + warnings body instead of the empty 204). Group provenance (group_id, group_seq) is stamped on member snapshots and travels onto the replicated target copies, which is what the fail-over generation selector actually returns. Co-Authored-By: Claude Fable 5 --- simplyblock_cli/cli.py | 6 + simplyblock_cli/clibase.py | 18 +- .../consistency_group_controller.py | 411 ++++++++++++++++++ .../controllers/lvol_controller.py | 41 ++ .../replication_policy_controller.py | 32 +- simplyblock_core/db_controller.py | 23 +- simplyblock_core/models/replication.py | 49 +++ simplyblock_core/models/snapshot.py | 4 + simplyblock_core/rpc_client.py | 15 + simplyblock_core/services/snapshot_monitor.py | 49 +++ .../services/snapshot_replication.py | 5 + .../test/test_consistency_groups.py | 216 +++++++++ .../test/test_internal_snapshot_scheduler.py | 5 + .../storage_pool/volume/replication.py | 8 + 14 files changed, 877 insertions(+), 5 deletions(-) create mode 100644 simplyblock_core/controllers/consistency_group_controller.py create mode 100644 simplyblock_core/test/test_consistency_groups.py diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index f1f0d592df..c6b7a2fcd5 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -398,6 +398,7 @@ def init_cluster(self): self.init_cluster__replication_policy_list(subparser) self.init_cluster__replication_policy_remove(subparser) self.init_cluster__replication_policy_failover(subparser) + self.init_cluster__replication_policy_snapshot(subparser) def init_cluster__create(self, subparser): @@ -663,6 +664,7 @@ def init_cluster__replication_policy_add(self, subparser): subcommand.add_argument('--mode', help='Replication mode. Default: `failover`.', type=str, dest='mode', choices=['failover','migration',]) subcommand.add_argument('--keep', help='Replicated internal snapshots to retain on each side. Minimum (and default): `2`.', type=int, dest='keep_replicated') subcommand.add_argument('--retention-schedule', help='Tiered retention, e.g. `15m:2h,1h:11h,1d:7d` - one snapshot every 15 minutes for the last 2 hours, then hourly for 11 hours, then daily for 7 days. Snapshots older than the total span are pruned. Empty (default) keeps the flat --keep behaviour.', type=str, dest='retention_schedule') + subcommand.add_argument('--consistency-group', help='All volumes attached to this policy form ONE consistency group: they must share an LVS (creation pins them to it), cadence snapshots are taken as one frozen group, and fail-over generations resolve group-wide.', dest='consistency_group', action='store_true') def init_cluster__replication_policy_list(self, subparser): subcommand = self.add_sub_command(subparser, 'replication-policy-list', 'Lists the replication policies of a cluster') @@ -673,6 +675,10 @@ def init_cluster__replication_policy_remove(self, subparser): subcommand = self.add_sub_command(subparser, 'replication-policy-remove', 'Removes a replication policy. Refused while a volume still follows it.') subcommand.add_argument('policy_id', help='Replication policy id', type=str) + def init_cluster__replication_policy_snapshot(self, subparser): + subcommand = self.add_sub_command(subparser, 'replication-policy-snapshot', "Takes ONE crash-consistent snapshot of every volume in the policy's consistency group, as a new group generation") + subcommand.add_argument('policy_id', help='Replication policy id (must be a consistency-group policy)', type=str) + def init_cluster__replication_policy_failover(self, subparser): subcommand = self.add_sub_command(subparser, 'replication-policy-failover', 'Fails over EVERY volume following this policy') subcommand.add_argument('policy_id', help='Replication policy id', type=str) diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 530c0e05b6..76cfbd3bb4 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -662,7 +662,16 @@ def cluster__replication_policy_add(self, sub_command, args): args.cluster_id, args.name, args.target, interval_min=args.interval_min, mode=args.mode, keep_replicated=args.keep_replicated, - retention_schedule=args.retention_schedule) + retention_schedule=args.retention_schedule, + consistency_group=args.consistency_group) + + def cluster__replication_policy_snapshot(self, sub_command, args): + from simplyblock_core.controllers import consistency_group_controller + snap_ids, err = consistency_group_controller.create_group_snapshot( + args.policy_id) + if err: + return f"Group snapshot failed: {err}" + return utils.print_table([{"Snapshot": s} for s in snap_ids]) def cluster__replication_policy_list(self, sub_command, args): data = [{ @@ -673,6 +682,7 @@ def cluster__replication_policy_list(self, sub_command, args): "Mode": p.mode, "Keep": p.keep_replicated, "Retention": p.retention_schedule or "-", + "CG": "yes" if getattr(p, "consistency_group", False) else "-", "Status": p.status, } for p in replication_policy_controller.list_policies(args.cluster_id)] return _format_result(data, json=args.json) @@ -690,12 +700,16 @@ def _format_failover_results(self, results, args): return _format_json(results) if not results: return "No volumes to fail over" - return utils.print_table([{ + table = utils.print_table([{ "Volume": r.get("lvol_id", ""), "Status": r.get("status", ""), "Target Volume": r.get("target_lvol_id", "") or "-", "Detail": r.get("detail", "") or "", } for r in results]) + warnings = [w for r in results for w in (r.get("warnings") or [])] + if warnings: + table += chr(10) + chr(10).join("WARNING: " + w for w in warnings) + return table def volume__replication_policy_set(self, sub_command, args): return replication_policy_controller.attach_policy(args.volume_id, args.policy) diff --git a/simplyblock_core/controllers/consistency_group_controller.py b/simplyblock_core/controllers/consistency_group_controller.py new file mode 100644 index 0000000000..abe79a52a8 --- /dev/null +++ b/simplyblock_core/controllers/consistency_group_controller.py @@ -0,0 +1,411 @@ +# coding=utf-8 +"""Consistency groups: group-wide crash-consistent snapshots for a policy. + +A replication policy created with ``consistency_group=True`` owns exactly one +auto-managed :class:`ConsistencyGroup`. Its members are the volumes attached +to the policy; they all live on ONE node/LVS (the group pins placement on the +first attach and enforces it afterwards), because the group snapshot freezes +IO per member blob on that LVS and a cross-LVS "group" would only be as +consistent as its slowest freeze. + +The group snapshot itself is ONE SPDK call (``bdev_lvol_snapshot_group``): +IO on every member is parked before the first snapshot and released after the +last, so the resulting set is a single point in time across the group. SPDK +garbage-collects on mid-sequence failure (unfreeze first, then delete the +snapshots already taken), so this controller never sees half a group from a +failed RPC. What this controller owns is everything around that call: +member resolution, the monotonically increasing ``group_seq``, replica +registration, snapshot records, chain linking and replication-task enqueue — +mirroring ``snapshot_controller.add`` step for step for each member. + +Membership epochs: a volume attached after the group already ticked joins at +``last_group_seq + 1`` — the first group snapshot that actually contains it. +Earlier generations do not, and a volume detached at seq M is not in +generations after M. :func:`generation_membership_warnings` computes exactly +the two warnings the fail-over path must surface when an operator selects an +older generation. +""" +import time +import uuid as uuid_module +from datetime import datetime + +from simplyblock_core import db_controller as db_mod +from simplyblock_core import utils +from simplyblock_core.controllers import snapshot_events, tasks_controller +from simplyblock_core.controllers.snapshot_controller import ( + _find_lvs_leader, _rollback_snapshot_bdev, lvstore_op_lock, + object_mutation_lock) +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.replication import ConsistencyGroup +from simplyblock_core.models.snapshot import SnapShot +from simplyblock_core.models.storage_node import StorageNode + +logger = utils.get_logger(__name__) +db = db_mod.DBController() + + +class ConsistencyGroupError(Exception): + pass + + +# --------------------------------------------------------------------------- # +# Group lifecycle (driven by the policy controller) +# --------------------------------------------------------------------------- # + +def create_group_for_policy(policy): + """Auto-create the group record when a consistency-group policy is made.""" + group = ConsistencyGroup() + group.uuid = str(uuid_module.uuid4()) + group.cluster_id = policy.cluster_id + group.policy_id = policy.get_id() + group.members = {} + group.write_to_db(db.kv_store) + logger.info("Created consistency group %s for policy %s", + group.get_id(), policy.policy_name) + return group + + +def delete_group_for_policy(policy_id): + """Auto-delete with the policy (which is only removable member-free).""" + group = db.get_consistency_group_for_policy(policy_id) + if group is not None: + group.remove(db.kv_store) + logger.info("Removed consistency group %s of policy %s", + group.get_id(), policy_id) + + +def add_member(policy, lvol): + """Enforce requirement 1 and record the member's epoch. + + The FIRST member pins the group to its node/LVS. Every later member must + already live there — the attachment FAILS otherwise; membership becomes + effective at the NEXT group snapshot (``joined_seq = last_group_seq + 1``), + because no earlier group snapshot contains this volume. + + A re-attaching volume gets a fresh epoch: its old history window stays + recorded under the closed epoch semantics (removed_seq of the old entry + is preserved only implicitly by the new joined_seq being later — the + entry is replaced, and the generation math treats the gap correctly + because the new joined_seq excludes the detached window). + """ + group = db.get_consistency_group_for_policy(policy.get_id()) + if group is None: + # Policies created before the flag existed, or records lost: fail + # loudly rather than silently degrading to per-volume snapshots. + raise ConsistencyGroupError( + f"Policy {policy.policy_name} declares a consistency group but " + f"has no group record") + + if group.lvs_name and (lvol.lvs_name != group.lvs_name + or lvol.node_id != group.node_id): + raise ConsistencyGroupError( + f"Volume {lvol.get_id()} lives on {lvol.node_id[:8]}/{lvol.lvs_name} " + f"but consistency group {group.uuid[:8]} of policy " + f"{policy.policy_name} is pinned to " + f"{group.node_id[:8]}/{group.lvs_name}; all members of a " + f"consistency group must share one LVS") + + if not group.lvs_name: + group.node_id = lvol.node_id + group.lvs_name = lvol.lvs_name + logger.info("Consistency group %s pinned to node %s / %s by its first " + "member %s", group.uuid[:8], lvol.node_id[:8], + lvol.lvs_name, lvol.get_id()) + + members = dict(group.members or {}) + members[lvol.get_id()] = {"joined_seq": group.last_group_seq + 1, + "removed_seq": 0} + group.members = members + group.write_to_db(db.kv_store) + logger.info("Volume %s joined consistency group %s at generation %d " + "(effective from the next group snapshot)", + lvol.get_id(), group.uuid[:8], group.last_group_seq + 1) + return group + + +def remove_member(policy_id, lvol_id): + """Close the member's epoch at the current generation.""" + group = db.get_consistency_group_for_policy(policy_id) + if group is None: + return + members = dict(group.members or {}) + entry = members.get(lvol_id) + if entry and entry.get("removed_seq", 0) == 0: + entry = dict(entry) + entry["removed_seq"] = max(group.last_group_seq, entry.get("joined_seq", 1) - 1) + members[lvol_id] = entry + group.members = members + group.write_to_db(db.kv_store) + logger.info("Volume %s left consistency group %s (included up to " + "generation %d)", lvol_id, group.uuid[:8], entry["removed_seq"]) + + +def pinned_node_for_policy(policy): + """The node a NEW volume under this policy must be created on, or None.""" + group = db.get_consistency_group_for_policy(policy.get_id()) + if group is not None and group.node_id: + return group.node_id + return None + + +# --------------------------------------------------------------------------- # +# Generation membership warnings (requirement 4) — pure logic +# --------------------------------------------------------------------------- # + +def generation_membership_warnings(group, seq): + """The two warnings an operator must see when failing over to ``seq``. + + Returns a list of strings: + * one for current members NOT included in that generation (late + joiners whose ``joined_seq`` is newer than ``seq``); + * one for volumes included in that generation that are NO LONGER + members (their epoch covers ``seq`` but ``removed_seq`` is set). + Empty list when the generation matches current membership exactly. + """ + if group is None or not seq: + return [] + missing = [] + stale = [] + for lvol_id, m in (group.members or {}).items(): + joined = m.get("joined_seq", 1) + removed = m.get("removed_seq", 0) + is_current = removed == 0 + included = joined <= seq and (removed == 0 or seq <= removed) + if is_current and not included: + missing.append(lvol_id) + if not is_current and included: + stale.append(lvol_id) + warnings = [] + if missing: + warnings.append( + "generation %d predates %d current group member(s); NOT included " + "in this point-in-time: %s" % (seq, len(missing), ", ".join(sorted(missing)))) + if stale: + warnings.append( + "generation %d includes %d volume(s) that are no longer group " + "members: %s" % (seq, len(stale), ", ".join(sorted(stale)))) + return warnings + + +def warnings_for_snapshot(lvol, snapshot): + """Convenience for the fail-over path: warnings for the generation the + chosen snapshot belongs to, [] for non-group snapshots/policies.""" + seq = getattr(snapshot, "group_seq", 0) + group_id = getattr(snapshot, "group_id", "") + if not seq or not group_id: + return [] + try: + group = db.get_consistency_group_by_id(group_id) + except KeyError: + return [] + return generation_membership_warnings(group, seq) + + +# --------------------------------------------------------------------------- # +# The group snapshot tick +# --------------------------------------------------------------------------- # + +def _current_members(group): + """Live member volumes: attached to the policy, epoch open, usable.""" + members = [] + for lvol_id, m in (group.members or {}).items(): + if m.get("removed_seq", 0) != 0: + continue + try: + lvol = db.get_lvol_by_id(lvol_id) + except KeyError: + continue + if lvol.status != LVol.STATUS_ONLINE: + logger.warning("Consistency group %s: member %s is %s; the group " + "snapshot is skipped this tick (a group snapshot " + "missing a member is not a group snapshot)", + group.uuid[:8], lvol_id, lvol.status) + return None + members.append(lvol) + return members + + +def create_group_snapshot(policy_id, snap_type=SnapShot.TYPE_INTERNAL, lock=True): + """Take ONE crash-consistent snapshot of every group member. + + Returns (list_of_snapshot_ids, None) or (None, error). All-or-nothing: + a failure anywhere rolls back every snapshot bdev of this tick (SPDK + already GC'd if the failure was inside the RPC; registration/record + failures are rolled back here) and the generation counter does not move. + """ + policy = db.get_replication_policy_by_id(policy_id) + group = db.get_consistency_group_for_policy(policy.get_id()) + if group is None: + return None, f"Policy {policy_id} has no consistency group" + + members = _current_members(group) + if members is None: + return None, "consistency group member not online" + if not members: + return None, "consistency group has no members" + + # Placement invariant (defense in depth: attach enforces it already). + for lvol in members: + if lvol.lvs_name != group.lvs_name or lvol.node_id != group.node_id: + return None, (f"member {lvol.get_id()} is on " + f"{lvol.node_id[:8]}/{lvol.lvs_name}, group is pinned " + f"to {group.node_id[:8]}/{group.lvs_name}") + + host_node = db.get_storage_node_by_id(group.node_id) + pool = db.get_pool_by_id(members[0].pool_uuid) + cluster = db.get_cluster_by_id(pool.cluster_id) + + # Leader + HA member set, same as the single-snapshot path. + secondary_ids = [host_node.secondary_node_id] + if host_node.tertiary_node_id: + secondary_ids.append(host_node.tertiary_node_id) + all_nodes = [host_node] + for sid in secondary_ids: + if not sid: + continue + try: + all_nodes.append(db.get_storage_node_by_id(sid)) + except KeyError: + pass + primary_node = _find_lvs_leader(pool.cluster_id, group.lvs_name, all_nodes) + if not primary_node: + return None, (f"No leader available for LVS {group.lvs_name} — " + f"rejecting the group snapshot until leadership is " + f"re-established") + secondary_nodes = [n for n in all_nodes + if n.get_id() != primary_node.get_id() + and n.status == StorageNode.STATUS_ONLINE] + + group_seq = group.last_group_seq + 1 + now_ts = int(time.time()) + plan = [] + for lvol in members: + snap_vuid = utils.get_random_snapshot_vuid() + plan.append({ + "lvol": lvol, + "vuid": snap_vuid, + "snap_bdev_name": f"SNAP_{snap_vuid}", + "snap_name": f"repl_cg_{group.uuid[:8]}_{group_seq}_{lvol.get_id()[:8]}_{now_ts}", + }) + + rpc_client = primary_node.rpc_client() + logger.info("Consistency group %s: taking generation %d over %d member(s) " + "on %s/%s", group.uuid[:8], group_seq, len(plan), + primary_node.get_id()[:8], group.lvs_name) + + # ONE lvstore mutation: the whole frozen window is a single RPC. + with lvstore_op_lock(pool.cluster_id, group.lvs_name, + node_id=primary_node.get_id(), enabled=lock): + ret = rpc_client.bdev_lvol_snapshot_group( + group.lvs_name, + [{"lvol_name": f"{p['lvol'].lvs_name}/{p['lvol'].lvol_bdev}", + "snapshot_name": p["snap_bdev_name"]} for p in plan]) + if not ret: + # SPDK unfroze first and garbage-collected the partial snapshots. + return None, (f"Group snapshot RPC failed on {primary_node.get_id()}; " + f"SPDK rolled the partial group back") + + def _rollback_all(): + for p in plan: + _rollback_snapshot_bdev(pool.cluster_id, group.lvs_name, + primary_node, p["snap_bdev_name"], + all_nodes, lock=lock) + + # Everything below mirrors snapshot_controller.add's tail per member: + # read back uuid/blobid, register on the HA peers, then the record. + created_ids = [] + for p in plan: + lvol = p["lvol"] + snap_bdev = rpc_client.get_bdevs(f"{group.lvs_name}/{p['snap_bdev_name']}") + if not snap_bdev: + _rollback_all() + return None, (f"group snapshot {p['snap_bdev_name']} not readable " + f"after creation") + p["snap_uuid"] = snap_bdev[0]["uuid"] + p["blobid"] = snap_bdev[0]["driver_specific"]["lvol"]["blobid"] + num_allocated = snap_bdev[0]["driver_specific"]["lvol"]["num_allocated_clusters"] + p["used_size"] = int(num_allocated * cluster.page_size_in_blocks) + + for sec in secondary_nodes: + from simplyblock_core.storage_node_ops import ( + wait_or_delay_for_restart_gate, queue_for_restart_drain) + gate = wait_or_delay_for_restart_gate(sec.get_id(), group.lvs_name) + if gate == "delay": + queue_for_restart_drain( + sec.get_id(), group.lvs_name, + lambda s=sec, pp=p, lv=lvol: s.rpc_client().bdev_lvol_snapshot_register( + f"{group.lvs_name}/{lv.lvol_bdev}", pp["snap_bdev_name"], + pp["snap_uuid"], pp["blobid"]), + f"register group snapshot {p['snap_bdev_name']} on {sec.get_id()[:8]}") + continue + with lvstore_op_lock(pool.cluster_id, group.lvs_name, + node_id=sec.get_id(), enabled=lock): + reg = sec.rpc_client().bdev_lvol_snapshot_register( + f"{group.lvs_name}/{lvol.lvol_bdev}", p["snap_bdev_name"], + p["snap_uuid"], p["blobid"]) + if not reg: + logger.error("Group snapshot register of %s failed on %s; " + "rolling the WHOLE generation back", + p["snap_bdev_name"], sec.get_id()) + _rollback_all() + for snap_id in created_ids: + try: + rec = db.get_snapshot_by_id(snap_id) + db.unindex_snapshot(rec) + rec.remove(db.kv_store) + except Exception: + pass + return None, f"Failed to register group snapshot on {sec.get_id()}" + + snap = SnapShot() + snap.uuid = str(uuid_module.uuid4()) + snap.data_uuid = str(uuid_module.uuid4()) + snap.snap_uuid = p["snap_uuid"] + snap.size = lvol.size + snap.used_size = p["used_size"] + snap.blobid = p["blobid"] + snap.pool_uuid = pool.get_id() + snap.cluster_id = pool.cluster_id + snap.snap_name = p["snap_name"] + snap.snap_bdev = f"{group.lvs_name}/{p['snap_bdev_name']}" + snap.created_at = now_ts + snap.lvol = lvol + snap.fabric = lvol.fabric + snap.vuid = p["vuid"] + snap.status = SnapShot.STATUS_ONLINE + snap.snap_type = snap_type + snap.group_id = group.get_id() + snap.group_seq = group_seq + snap.create_dt = str(datetime.now()) + snap.write_to_db(db.kv_store) + + prev = db.get_lvol_latest_snapshot(lvol.get_id(), exclude_uuid=snap.get_id()) + if prev is not None and not prev.next_snap_uuid: + prev.next_snap_uuid = snap.get_id() + snap.prev_snap_uuid = prev.get_id() + prev.write_to_db() + snap.write_to_db() + + db.index_snapshot(snap) + snapshot_events.snapshot_create(snap) + created_ids.append(snap.get_id()) + p["snap_id"] = snap.get_id() + + # The generation exists in full: bump the counter, then enqueue the + # per-member replication tasks (transfer machinery is per-snapshot). + group = db.get_consistency_group_by_id(group.get_id()) + group.last_group_seq = group_seq + group.write_to_db(db.kv_store) + + for p in plan: + lvol = p["lvol"] + if lvol.do_replicate: + task = tasks_controller.add_snapshot_replication_task( + pool.cluster_id, lvol.node_id, p["snap_id"]) + if task: + snap = db.get_snapshot_by_id(p["snap_id"]) + snapshot_events.replication_task_created(snap) + + logger.info("Consistency group %s: generation %d complete (%d snapshots)", + group.uuid[:8], group_seq, len(created_ids)) + return created_ids, None diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 117338baff..26f34eba88 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -427,6 +427,28 @@ def add_lvol_ha(name, size, host_id_or_name, ha_type, pool_id_or_name, use_comp= f"max_namespace_per_subsys={max_namespace_per_subsys} exceeds the " f"hard limit of {constants.MAX_NAMESPACES_PER_SUBSYSTEM} " f"namespaces per subsystem") + if replication_policy: + # A consistency-group policy pins placement: every member must live in + # ONE LVS, so a volume created under such a policy is forced onto the + # group's node BEFORE placement runs (requirement: pin to host before + # creation). An explicit conflicting --host is an error, not a + # preference fight. + from simplyblock_core.controllers import replication_policy_controller as _rpc + from simplyblock_core.controllers import consistency_group_controller as _cgc + try: + _policy = _rpc._resolve_policy(replication_policy) + except KeyError: + return False, f"Replication policy not found: {replication_policy}" + if getattr(_policy, "consistency_group", False): + pinned = _cgc.pinned_node_for_policy(_policy) + if pinned: + if host_id_or_name and host_id_or_name != pinned: + return False, ( + f"Volume must be created on node {pinned} — its " + f"replication policy {_policy.policy_name} is a " + f"consistency group pinned to that node's LVS") + host_id_or_name = pinned + host_node = None if host_id_or_name: try: @@ -3735,11 +3757,30 @@ def replicate_lvol_on_target_cluster(lvol_id, generation=0): else: connection_strings = [c.model_dump(by_alias=True) for c in conn] + # Requirement 4 (consistency groups): a generation older than the current + # membership must SAY so — which current members the chosen point-in-time + # does not contain (late joiners), and which contained volumes are no + # longer members. Logged AND returned, so CLI and API callers surface it. + warnings = [] + if _snapshot is not None: + try: + from simplyblock_core.controllers import consistency_group_controller + # The clone was made from the TARGET copy; group provenance lives + # on the SOURCE snapshot record it replicated from. Both carry it, + # so read whichever the selector handed us. + warnings = consistency_group_controller.warnings_for_snapshot( + lvol, _snapshot) + except Exception as e: + logger.warning("Group-membership warning computation failed: %s", e) + for w in warnings: + logger.warning("Fail-over of %s: %s", lvol_id, w) + return { "lvol_id": new_lvol.uuid, "nqn": new_lvol.nqn, "ns_id": new_lvol.ns_id, "connection_strings": connection_strings, + "warnings": warnings, } diff --git a/simplyblock_core/controllers/replication_policy_controller.py b/simplyblock_core/controllers/replication_policy_controller.py index 88b5459195..ef10da4bf9 100644 --- a/simplyblock_core/controllers/replication_policy_controller.py +++ b/simplyblock_core/controllers/replication_policy_controller.py @@ -94,7 +94,7 @@ def remove_target(target_id): # --------------------------------------------------------------------------- # def add_policy(cluster_id, policy_name, target, interval_min=1, mode=None, keep_replicated=None, - retention_schedule=None): + retention_schedule=None, consistency_group=False): """Create a policy on *target* (id or name).""" db.get_cluster_by_id(cluster_id) try: @@ -142,8 +142,13 @@ def add_policy(cluster_id, policy_name, target, interval_min=1, mode=None, keep_ policy.keep_replicated = keep_replicated if retention_schedule is not None: policy.retention_schedule = retention_schedule + policy.consistency_group = bool(consistency_group) policy.status = ReplicationPolicy.STATUS_ACTIVE policy.write_to_db(db.kv_store) + if policy.consistency_group: + # Auto-created with the policy, auto-deleted with it (requirement 2). + from simplyblock_core.controllers import consistency_group_controller + consistency_group_controller.create_group_for_policy(policy) logger.info("Created replication policy %s on target %s (%s)", policy_name, tgt.target_name, policy.get_id()) return policy.get_id() @@ -161,6 +166,9 @@ def remove_policy(policy_id): raise ReplicationConfigError( f"Replication policy {policy.policy_name} is followed by " f"{len(users)} volume(s); detach them first") + if getattr(policy, "consistency_group", False): + from simplyblock_core.controllers import consistency_group_controller + consistency_group_controller.delete_group_for_policy(policy.get_id()) policy.remove(db.kv_store) logger.info("Removed replication policy %s", policy_id) return True @@ -218,6 +226,12 @@ def attach_policy(lvol_id, policy): detach_policy(lvol_id) lvol = db.get_lvol_by_id(lvol_id) + if getattr(pol, "consistency_group", False): + # Requirement 1: all members share one LVS. Checked BEFORE any state + # is written, so a failed attachment leaves the volume untouched. + from simplyblock_core.controllers import consistency_group_controller + consistency_group_controller.add_member(pol, lvol) + lvol.replication_policy_id = pol.get_id() lvol.write_to_db() ret = lvol_controller.replication_start( @@ -232,6 +246,9 @@ def attach_policy(lvol_id, policy): lvol = db.get_lvol_by_id(lvol_id) lvol.replication_policy_id = "" lvol.write_to_db() + if getattr(pol, "consistency_group", False): + from simplyblock_core.controllers import consistency_group_controller + consistency_group_controller.remove_member(pol.get_id(), lvol_id) raise ReplicationConfigError( f"Could not start replication of {lvol_id} to target {target.target_name}") logger.info("Volume %s now follows policy %s (target %s)", @@ -257,9 +274,19 @@ def detach_policy(lvol_id): f"Volume {lvol_id} has a cutover in flight; wait for it to finish " f"before detaching the replication policy") + detached_policy_id = lvol.replication_policy_id lvol.replication_policy_id = "" lvol.write_to_db() + if detached_policy_id: + try: + pol = db.get_replication_policy_by_id(detached_policy_id) + except KeyError: + pol = None + if pol is not None and getattr(pol, "consistency_group", False): + from simplyblock_core.controllers import consistency_group_controller + consistency_group_controller.remove_member(pol.get_id(), lvol_id) + # Stops streaming and cancels the non-DONE FN_SNAPSHOT_REPLICATION tasks. lvol_controller.replication_stop(lvol_id, from_policy=True) @@ -366,7 +393,8 @@ def _failover_volumes(volumes, what): elif isinstance(ret, dict): results.append({"lvol_id": lvol_id, "status": "failed_over", "target_lvol_id": ret.get("lvol_id", ""), - "connection_strings": ret.get("connection_strings", [])}) + "connection_strings": ret.get("connection_strings", []), + "warnings": ret.get("warnings", [])}) else: results.append({"lvol_id": lvol_id, "status": "failed_over", "target_lvol_id": str(ret)}) return results diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index 4b0842c96f..3d80c6c7bf 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -20,7 +20,7 @@ from simplyblock_core.models.backup import Backup, BackupChainLock, BackupPolicy, BackupPolicyAttachment from simplyblock_core.models.lvol_migration import LVolMigration from simplyblock_core.models.lvol_migration_group import LVolMigrationGroup -from simplyblock_core.models.replication import ReplicationPolicy, ReplicationTarget +from simplyblock_core.models.replication import ConsistencyGroup, ReplicationPolicy, ReplicationTarget from simplyblock_core.models.qos import QOSClass from simplyblock_core.models.snapshot import SnapShot, SnapShotMini from simplyblock_core.models.stats import DeviceStatObject, NodeStatObject, ClusterStatObject, LVolStatObject, \ @@ -1461,6 +1461,27 @@ def get_replication_policy_for_lvol(self, lvol) -> Optional[ReplicationPolicy]: except KeyError: return None + def get_consistency_groups(self, cluster_id: Optional[str] = None) -> List[ConsistencyGroup]: + prefix = cluster_id if cluster_id else " " + return ConsistencyGroup().read_from_db(self.kv_store, id=prefix) + + def get_consistency_group_by_id(self, group_id: str) -> ConsistencyGroup: + if not group_id: + raise KeyError('ConsistencyGroup lookup with a blank id') + wanted = group_id.split('/')[-1] + group = single_or_none(g for g in self.get_consistency_groups() if g.uuid == wanted) + if group is None: + raise KeyError(f'ConsistencyGroup {group_id} not found') + return group + + def get_consistency_group_for_policy(self, policy_id: str) -> Optional[ConsistencyGroup]: + wanted = policy_id.split('/')[-1] if policy_id else "" + if not wanted: + return None + return single_or_none( + g for g in self.get_consistency_groups() + if g.policy_id.split('/')[-1] == wanted) + def get_lvols_by_replication_policy(self, policy_id: str) -> List[LVol]: wanted = policy_id.split('/')[-1] if policy_id else "" if not wanted: diff --git a/simplyblock_core/models/replication.py b/simplyblock_core/models/replication.py index ff8f5ec941..5569b1f33b 100644 --- a/simplyblock_core/models/replication.py +++ b/simplyblock_core/models/replication.py @@ -77,6 +77,11 @@ class ReplicationPolicy(BaseModel): #: never overrides MIN_KEEP_REPLICATED: the newest pair is always kept so #: an arriving delta has a predecessor to chain onto. retention_schedule: str = "" + #: All volumes attached to this policy form ONE consistency group: they + #: must share an LVS, cadence snapshots are taken as one frozen group + #: (bdev_lvol_snapshot_group), and fail-over generations are resolved + #: group-wide. Auto-creates/deletes a ConsistencyGroup record. + consistency_group: bool = False status: str = STATUS_ACTIVE def get_id(self): @@ -85,3 +90,47 @@ def get_id(self): def write_to_db(self, kv_store=None): self.updated_at = str(datetime.datetime.now(datetime.timezone.utc)) super().write_to_db(kv_store) + + +class ConsistencyGroup(BaseModel): + """Auto-managed group record behind a consistency-group policy. + + Created with the policy and removed with it. ``members`` maps lvol id to + its membership EPOCH: + + {"joined_seq": N, "removed_seq": M} + + A member is included in group generation ``seq`` iff + ``joined_seq <= seq`` and (``removed_seq == 0`` or ``seq <= removed_seq``). + Late joiners deliberately do NOT inherit history: they join at + ``last_group_seq + 1``, i.e. the first group snapshot taken AFTER the + attach, because earlier group snapshots simply do not contain them. + """ + + cluster_id: str = "" + policy_id: str = "" # ReplicationPolicy.get_id() + #: pinned placement: every member volume lives on this node / LVS. Set by + #: the first member and enforced for all others. + node_id: str = "" + lvs_name: str = "" + #: monotonically increasing generation counter; group snapshot N stamps + #: every member snapshot it takes with group_seq = N. + last_group_seq: int = 0 + members: dict = {} + status: str = "active" + + def get_id(self): + return "%s/%s" % (self.cluster_id, self.uuid) + + def write_to_db(self, kv_store=None): + self.updated_at = str(datetime.datetime.now(datetime.timezone.utc)) + super().write_to_db(kv_store) + + def included_in_seq(self, lvol_id, seq): + m = (self.members or {}).get(lvol_id) + if not m or not seq: + return False + if m.get("joined_seq", 0) > seq: + return False + removed = m.get("removed_seq", 0) + return removed == 0 or seq <= removed diff --git a/simplyblock_core/models/snapshot.py b/simplyblock_core/models/snapshot.py index 46559e8d0f..1476e03ef9 100644 --- a/simplyblock_core/models/snapshot.py +++ b/simplyblock_core/models/snapshot.py @@ -39,6 +39,10 @@ class SnapShot(BaseModel): target_replicated_snap_uuid: str = "" source_replicated_snap_uuid: str = "" snap_type: str = "user" + #: consistency-group provenance: which group and which group generation + #: this snapshot belongs to (0 = not a group snapshot). + group_id: str = "" + group_seq: int = 0 next_snap_uuid: str = "" prev_snap_uuid: str = "" instances: list[dict] = [] diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index c49e53343c..668402c2df 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -1655,6 +1655,21 @@ def jc_set_dual_node(self, enable): """ return self._request2("jc_set_dual_node", {"enable": bool(enable)}) + def bdev_lvol_snapshot_group(self, lvs_name, snapshots): + """One crash-consistent snapshot per consistency-group member. + + ``snapshots`` is a list of {"lvol_name": "LVS_1/LVOL_5", + "snapshot_name": "SNAP_123"}; all members must be in ``lvs_name``. + IO on every member is frozen before the first snapshot and released + after the last; a mid-sequence failure unfreezes first and then + garbage-collects the snapshots already taken (SPDK side). + Returns [{"lvol_name", "snapshot_name", "uuid"}, ...] or False. + """ + return self._request2("bdev_lvol_snapshot_group", { + "lvs_name": lvs_name, + "snapshots": snapshots, + }) + def jc_suspend_compression(self, jm_vuid, suspend=False): params = { "jm_vuid": jm_vuid, diff --git a/simplyblock_core/services/snapshot_monitor.py b/simplyblock_core/services/snapshot_monitor.py index 0c3a77ca53..99978cb316 100644 --- a/simplyblock_core/services/snapshot_monitor.py +++ b/simplyblock_core/services/snapshot_monitor.py @@ -585,6 +585,55 @@ def take_due_internal_snapshots(cluster_id, now_ts): if not repl_lvols: return all_snaps = db.get_mini_snapshots() + + # Consistency-group policies snapshot as a GROUP (requirement 3): one + # frozen point-in-time across every member, via ONE group-snapshot call + # per tick — never per-volume snapshots. Members of such policies are + # removed from the per-volume loop below. + cg_policies = {p.get_id(): p for p in db.get_replication_policies(cluster_id) + if getattr(p, "consistency_group", False)} + if cg_policies: + from simplyblock_core.controllers import consistency_group_controller + grouped_ids = set() + for policy_id, policy in cg_policies.items(): + members = [lv for lv in repl_lvols + if getattr(lv, "replication_policy_id", "") == policy_id] + if not members: + continue + grouped_ids.update(lv.get_id() for lv in members) + try: + # Due when ANY member's interval elapsed (they tick together, + # so member timestamps agree except right after a join). + if not any(_due_for_internal_snapshot(lv, all_snaps, now_ts) + for lv in members): + continue + # Group-wide back-pressure: one member's unfinished transfer + # holds the WHOLE group's next generation, otherwise the + # generations stop being aligned points in time. + blocked = None + for lv in members: + outstanding = _outstanding_internal_snapshot(lv, all_snaps) + if outstanding is not None: + blocked = (lv, outstanding) + break + if blocked: + logger.warning( + "Skipping group snapshot for policy %s: member %s " + "has an unreplicated internal snapshot (%s)", + policy.policy_name, blocked[0].get_id(), + blocked[1].get_id()) + continue + logger.info("Taking consistency-group snapshot for policy %s " + "(%d members)", policy.policy_name, len(members)) + _ids, err = consistency_group_controller.create_group_snapshot(policy_id) + if err: + logger.warning("Group snapshot for policy %s failed: %s", + policy.policy_name, err) + except Exception as e: + logger.error("Group snapshot scheduling failed for policy %s: %s", + policy_id, e) + repl_lvols = [lv for lv in repl_lvols if lv.get_id() not in grouped_ids] + for lvol in repl_lvols: try: if not _due_for_internal_snapshot(lvol, all_snaps, now_ts): diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index d56541c89e..3f364232d8 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -918,6 +918,11 @@ def process_snap_replicate_finish(task, snapshot): new_snapshot.blobid = remote_lv.blobid new_snapshot.created_at = int(time.time()) new_snapshot.status = SnapShot.STATUS_ONLINE + # Consistency-group provenance travels with the copy: the fail-over + # generation selector returns the TARGET record, and requirement 4's + # membership warnings are computed from (group_id, group_seq) on it. + new_snapshot.group_id = getattr(snapshot, "group_id", "") + new_snapshot.group_seq = getattr(snapshot, "group_seq", 0) snapshot.instances.append(new_snapshot) if not replicate_as_snap_instance: if replicate_to_source: diff --git a/simplyblock_core/test/test_consistency_groups.py b/simplyblock_core/test/test_consistency_groups.py new file mode 100644 index 0000000000..3a9ee7c9d6 --- /dev/null +++ b/simplyblock_core/test/test_consistency_groups.py @@ -0,0 +1,216 @@ +"""Consistency groups: membership epochs, placement pinning, generation +warnings, and the wiring contracts of the group snapshot flow.""" +import inspect + +import pytest + +from simplyblock_core.controllers import consistency_group_controller as cgc +from simplyblock_core.models.replication import ConsistencyGroup + + +def _group(members=None, last_seq=0, lvs="LVS_1", node="NODE_A"): + g = ConsistencyGroup() + g.uuid = "g1" + g.cluster_id = "CL" + g.policy_id = "CL/p1" + g.lvs_name = lvs + g.node_id = node + g.last_group_seq = last_seq + g.members = members or {} + return g + + +# --------------------------------------------------------------------------- # +# Epoch semantics (requirement 4) +# --------------------------------------------------------------------------- # + +def test_member_included_between_join_and_removal(): + g = _group({"v1": {"joined_seq": 2, "removed_seq": 5}}) + assert not g.included_in_seq("v1", 1) + assert g.included_in_seq("v1", 2) + assert g.included_in_seq("v1", 5) + assert not g.included_in_seq("v1", 6) + + +def test_open_epoch_member_included_from_join_onwards(): + g = _group({"v1": {"joined_seq": 3, "removed_seq": 0}}) + assert not g.included_in_seq("v1", 2) + assert g.included_in_seq("v1", 3) + assert g.included_in_seq("v1", 99) + + +def test_late_joiner_is_warned_about_for_older_generations(): + """A volume attached after generation 4 is NOT in generations 1..4: + failing over to one of those must say so.""" + g = _group({ + "old": {"joined_seq": 1, "removed_seq": 0}, + "late": {"joined_seq": 5, "removed_seq": 0}, + }, last_seq=6) + warnings = cgc.generation_membership_warnings(g, 4) + assert len(warnings) == 1 + assert "late" in warnings[0] + assert "NOT included" in warnings[0] + assert "old" not in warnings[0].split(":")[-1] + + +def test_departed_member_is_warned_about_when_generation_contains_it(): + g = _group({ + "stay": {"joined_seq": 1, "removed_seq": 0}, + "gone": {"joined_seq": 1, "removed_seq": 3}, + }, last_seq=6) + warnings = cgc.generation_membership_warnings(g, 2) + assert len(warnings) == 1 + assert "gone" in warnings[0] + assert "no longer" in warnings[0] + + +def test_matching_generation_produces_no_warnings(): + g = _group({ + "a": {"joined_seq": 1, "removed_seq": 0}, + "b": {"joined_seq": 1, "removed_seq": 0}, + }, last_seq=3) + assert cgc.generation_membership_warnings(g, 3) == [] + + +def test_both_warning_kinds_can_coexist(): + g = _group({ + "late": {"joined_seq": 5, "removed_seq": 0}, + "gone": {"joined_seq": 1, "removed_seq": 3}, + }, last_seq=6) + warnings = cgc.generation_membership_warnings(g, 2) + assert len(warnings) == 2 + + +def test_no_group_or_no_seq_is_silent(): + assert cgc.generation_membership_warnings(None, 3) == [] + assert cgc.generation_membership_warnings(_group(), 0) == [] + + +# --------------------------------------------------------------------------- # +# Placement / lifecycle contracts (requirement 1 + 2) +# --------------------------------------------------------------------------- # + +class _FakeDB: + def __init__(self, group): + self._group = group + + def get_consistency_group_for_policy(self, policy_id): + return self._group + + @property + def kv_store(self): + return None + + +class _Policy: + policy_name = "p1" + consistency_group = True + + def get_id(self): + return "CL/p1" + + +class _Lvol: + def __init__(self, lvol_id, node, lvs): + self._id, self.node_id, self.lvs_name = lvol_id, node, lvs + + def get_id(self): + return self._id + + +def test_attach_to_pinned_group_fails_on_wrong_lvs(monkeypatch): + g = _group(lvs="LVS_1", node="NODE_A") + monkeypatch.setattr(cgc, "db", _FakeDB(g)) + with pytest.raises(cgc.ConsistencyGroupError): + cgc.add_member(_Policy(), _Lvol("v1", "NODE_B", "LVS_2")) + assert "v1" not in (g.members or {}) + + +def test_first_member_pins_the_group(monkeypatch): + g = _group(lvs="", node="") + g.write_to_db = lambda kv=None: None + monkeypatch.setattr(cgc, "db", _FakeDB(g)) + cgc.add_member(_Policy(), _Lvol("v1", "NODE_A", "LVS_1")) + assert g.node_id == "NODE_A" and g.lvs_name == "LVS_1" + assert g.members["v1"]["joined_seq"] == 1 + + +def test_late_joiner_epoch_starts_at_next_generation(monkeypatch): + """Requirement 4: membership becomes active only with the FIRST group + snapshot taken after the attach.""" + g = _group({"v1": {"joined_seq": 1, "removed_seq": 0}}, last_seq=7) + g.write_to_db = lambda kv=None: None + monkeypatch.setattr(cgc, "db", _FakeDB(g)) + cgc.add_member(_Policy(), _Lvol("v2", "NODE_A", "LVS_1")) + assert g.members["v2"]["joined_seq"] == 8 + # ... and generation 7 correctly warns about it + warnings = cgc.generation_membership_warnings(g, 7) + assert warnings and "v2" in warnings[0] + + +def test_detach_closes_the_epoch_at_current_generation(monkeypatch): + g = _group({"v1": {"joined_seq": 1, "removed_seq": 0}}, last_seq=4) + g.write_to_db = lambda kv=None: None + monkeypatch.setattr(cgc, "db", _FakeDB(g)) + cgc.remove_member("CL/p1", "v1") + assert g.members["v1"]["removed_seq"] == 4 + assert g.included_in_seq("v1", 4) + assert not g.included_in_seq("v1", 5) + + +# --------------------------------------------------------------------------- # +# Wiring contracts (source-inspection, matching the repo's test idiom) +# --------------------------------------------------------------------------- # + +def test_policy_lifecycle_auto_creates_and_deletes_the_group(): + from simplyblock_core.controllers import replication_policy_controller as rpc + src_add = inspect.getsource(rpc.add_policy) + assert "create_group_for_policy" in src_add + src_rm = inspect.getsource(rpc.remove_policy) + assert "delete_group_for_policy" in src_rm + + +def test_attach_checks_the_group_before_any_state_is_written(): + from simplyblock_core.controllers import replication_policy_controller as rpc + src = inspect.getsource(rpc.attach_policy) + assert "add_member" in src + assert src.index("add_member") < src.index("lvol.replication_policy_id = pol.get_id()") + + +def test_create_path_pins_cg_volumes_to_the_group_node(): + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc.add_lvol_ha) + assert "pinned_node_for_policy" in src + pin = src.index("pinned_node_for_policy") + place = src.index("host_node = None") + assert pin < place, "the pin must be resolved before placement" + + +def test_cadence_snapshots_cg_policies_as_a_group(): + from simplyblock_core.services import snapshot_monitor as sm + src = inspect.getsource(sm.take_due_internal_snapshots) + assert "create_group_snapshot" in src + assert "grouped_ids" in src, "group members must leave the per-volume loop" + + +def test_group_snapshot_is_one_rpc_and_bumps_seq_only_on_full_success(): + src = inspect.getsource(cgc.create_group_snapshot) + assert "bdev_lvol_snapshot_group" in src + rpc_at = src.index("bdev_lvol_snapshot_group") + seq_at = src.index("group.last_group_seq = group_seq") + assert rpc_at < seq_at, "the generation counter moves only after the whole tick" + assert "_rollback_all" in src + + +def test_failover_result_carries_membership_warnings(): + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc.replicate_lvol_on_target_cluster) + assert "warnings_for_snapshot" in src + assert '"warnings": warnings' in src + + +def test_target_snapshot_copy_inherits_group_provenance(): + from simplyblock_core.services import snapshot_replication as sr + src = inspect.getsource(sr) + assert 'new_snapshot.group_id = getattr(snapshot, "group_id", "")' in src + assert 'new_snapshot.group_seq = getattr(snapshot, "group_seq", 0)' in src diff --git a/simplyblock_core/test/test_internal_snapshot_scheduler.py b/simplyblock_core/test/test_internal_snapshot_scheduler.py index 6d3c25a48a..2dadb73298 100644 --- a/simplyblock_core/test/test_internal_snapshot_scheduler.py +++ b/simplyblock_core/test/test_internal_snapshot_scheduler.py @@ -85,6 +85,11 @@ def get_lvols(self, cluster_id): def get_mini_snapshots(self): return [] + def get_replication_policies(self, cluster_id=None): + # The scheduler partitions consistency-group policies out of the + # per-volume loop; this scenario has none. + return [] + calls = [] class _SnapCtl: diff --git a/simplyblock_web/api/v2/cluster/storage_pool/volume/replication.py b/simplyblock_web/api/v2/cluster/storage_pool/volume/replication.py index dde7cfc767..260bc25b63 100644 --- a/simplyblock_web/api/v2/cluster/storage_pool/volume/replication.py +++ b/simplyblock_web/api/v2/cluster/storage_pool/volume/replication.py @@ -2,6 +2,7 @@ from uuid import UUID from fastapi import APIRouter, HTTPException, Request, Response +from fastapi.responses import JSONResponse from pydantic import BaseModel from simplyblock_core.controllers import lvol_controller, replication_policy_controller @@ -129,6 +130,13 @@ def failover(cluster: Cluster, pool: StoragePool, volume: Volume, if not result: raise HTTPException(500, 'Failed to fail the volume over to the target cluster') + # Consistency groups: an older generation may not match current + # membership; the operator must SEE that, so warnings turn the empty 204 + # into a 200 with a body (requirement: API response, not only a log). + if isinstance(result, dict) and result.get("warnings"): + return JSONResponse(status_code=200, + content={"warnings": result["warnings"]}) + return Response(status_code=204) From 9397dea15cd30c425cbafe837688f401575c3159 Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 25 Aug 2026 12:48:44 +0200 Subject: [PATCH 018/122] fix: confirm the clone bdev exists after clone_register before adding its namespace bdev_lvol_clone_register acknowledges before the bdev is examinable -- the third member of the acknowledge-before-complete family, after remove_ns (PVC-expand, eb127eed) and the case-3 eviction. The HA-peer leg of a namespaced fail-over issues nvmf_subsystem_add_ns immediately after the register and lost the race every time: -32602 with the peer's subsystem EMPTY, while the bdev existed moments later (run 20260825_122423, LVS_13/LVOL_121, uuid f01ea33c). The peer failure then rolled back the whole fail-over of that namespace. The stack build now polls get_bdevs (bounded, 20s) after clone_register and only then proceeds to the namespace add. Co-Authored-By: Claude Fable 5 --- .../controllers/lvol_controller.py | 20 +++++++++++++++++++ .../test_replication_chain_completeness.py | 17 ++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 117338baff..3e9454ca14 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -1032,6 +1032,26 @@ def _create_bdev_stack(lvol, snode, is_primary=True): else: ret = rpc_client.bdev_lvol_clone_register( lvol.lvol_bdev, lvol.snapshot_name, lvol.lvol_uuid, lvol.blobid) + if ret: + # clone_register ACKNOWLEDGES before the bdev is + # examinable (the same async false-success family as + # remove_ns in the PVC-expand and case-3 incidents). The + # very next step adds this bdev to the nvmf subsystem, and + # racing the registration lost every time on the fail-over + # of namespaced volumes: peer add_ns -32602 with the + # subsystem EMPTY, while the bdev existed moments later + # (run 20260825_122423, LVS_13/LVOL_121). Poll until the + # bdev is really there before letting add_ns proceed. + bdev_name = f"{lvol.lvs_name}/{lvol.lvol_bdev}" + for _ in range(40): + if rpc_client.get_bdevs(bdev_name): + break + time.sleep(0.5) + else: + logger.error( + f"clone_register acknowledged but {bdev_name} did " + f"not appear within 20s on the peer") + ret = None else: logger.debug(f"Unknown BDev type: {type}") diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index 88c3c327b5..6f8ff6aa40 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -471,3 +471,20 @@ def test_namespaced_siblings_replicate_to_the_same_target_node(): check = src.index("sibling_node_id") assert check < pick, "the sibling lookup must precede the capacity-based pick" assert "lv.nqn == lvol.nqn" in src, "siblings are identified by shared NQN" + + +def test_clone_register_confirms_the_bdev_before_add_ns(): + """Run 20260825_122423: bdev_lvol_clone_register acknowledges before the + bdev is examinable, and the peer's nvmf_subsystem_add_ns raced it and lost + (-32602 with the subsystem EMPTY; the bdev existed moments later). Third + member of the acknowledge-before-complete family, after remove_ns + (PVC-expand) and the case-3 eviction. The stack build must poll the bdev + into existence before the namespace add runs.""" + import inspect + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc._create_bdev_stack) + reg = src.index("bdev_lvol_clone_register") + assert "not appear within 20s" in src[reg:], \ + "clone_register must be followed by a bdev confirmation poll" + poll = src.index("not appear within 20s", reg) + assert "get_bdevs" in src[reg:poll], "the poll must probe get_bdevs" From a6b3ec2e864a735f0afc06db189fdbf38c06a52d Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Tue, 25 Aug 2026 17:32:12 +0200 Subject: [PATCH 019/122] Fix CLI, linter and type checker on main --- simplyblock_cli/cli.py | 1 - simplyblock_core/services/snapshot_replication.py | 5 ++--- simplyblock_core/snapshot_retention.py | 2 +- simplyblock_core/test/test_jc_dual_node.py | 10 +++++----- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index f1f0d592df..77c498dd97 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -662,7 +662,6 @@ def init_cluster__replication_policy_add(self, subparser): subcommand.add_argument('--interval-min', help='Cadence: minutes between internal replication snapshots. 0 replicates user snapshots only. Default: `1`.', type=int, dest='interval_min') subcommand.add_argument('--mode', help='Replication mode. Default: `failover`.', type=str, dest='mode', choices=['failover','migration',]) subcommand.add_argument('--keep', help='Replicated internal snapshots to retain on each side. Minimum (and default): `2`.', type=int, dest='keep_replicated') - subcommand.add_argument('--retention-schedule', help='Tiered retention, e.g. `15m:2h,1h:11h,1d:7d` - one snapshot every 15 minutes for the last 2 hours, then hourly for 11 hours, then daily for 7 days. Snapshots older than the total span are pruned. Empty (default) keeps the flat --keep behaviour.', type=str, dest='retention_schedule') def init_cluster__replication_policy_list(self, subparser): subcommand = self.add_sub_command(subparser, 'replication-policy-list', 'Lists the replication policies of a cluster') diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index d56541c89e..06ab073c3c 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -605,10 +605,9 @@ def _retention_schedule_for(source_lvol): """ try: policy = db.get_replication_policy_for_lvol(source_lvol) - except Exception: + except KeyError: return [] - spec = getattr(policy, "retention_schedule", "") if policy else "" - if not spec: + if (policy is None) or (spec := getattr(policy, "retention_schedule", None)) is None: return [] try: return snapshot_retention.parse_schedule(spec) diff --git a/simplyblock_core/snapshot_retention.py b/simplyblock_core/snapshot_retention.py index 9d1a6735af..5afe42386f 100644 --- a/simplyblock_core/snapshot_retention.py +++ b/simplyblock_core/snapshot_retention.py @@ -108,7 +108,7 @@ def select_retained(created_ats: Iterable[float], tiers: Sequence[RetentionTier] for tier in tiers: range_end = range_start + tier.span_sec # Bucket by absolute age so bucket edges do not drift between calls. - best_per_bucket = {} + best_per_bucket: dict[int, float] = {} for t in times: age = now - t if age < range_start or age >= range_end: diff --git a/simplyblock_core/test/test_jc_dual_node.py b/simplyblock_core/test/test_jc_dual_node.py index 387738f0b5..d6347f957e 100644 --- a/simplyblock_core/test/test_jc_dual_node.py +++ b/simplyblock_core/test/test_jc_dual_node.py @@ -50,7 +50,7 @@ def get_storage_nodes_by_cluster_id(self, cluster_id): def test_two_node_cluster_enables_dual_node(monkeypatch): - sink = [] + sink: list[tuple[str, bool]] = [] nodes = [_Node("A", sink), _Node("B", sink)] _install(monkeypatch, nodes) storage_node_ops.apply_jc_dual_node("CL") @@ -58,7 +58,7 @@ def test_two_node_cluster_enables_dual_node(monkeypatch): def test_three_node_cluster_disables_dual_node(monkeypatch): - sink = [] + sink: list[tuple[str, bool]] = [] nodes = [_Node("A", sink), _Node("B", sink), _Node("C", sink)] _install(monkeypatch, nodes) storage_node_ops.apply_jc_dual_node("CL") @@ -70,7 +70,7 @@ def test_flag_follows_membership_not_how_many_are_online(monkeypatch): the flag on the ONLINE count would switch a degraded 3-node cluster into dual-node mode -- weakening the journal requirement exactly when a node is already missing.""" - sink = [] + sink: list[tuple[str, bool]] = [] nodes = [_Node("A", sink), _Node("B", sink), _Node("C", sink, status=StorageNode.STATUS_OFFLINE)] _install(monkeypatch, nodes) @@ -80,7 +80,7 @@ def test_flag_follows_membership_not_how_many_are_online(monkeypatch): def test_removed_nodes_do_not_count_towards_membership(monkeypatch): - sink = [] + sink: list[tuple[str, bool]] = [] nodes = [_Node("A", sink), _Node("B", sink), _Node("C", sink, status=StorageNode.STATUS_REMOVED)] _install(monkeypatch, nodes) @@ -89,7 +89,7 @@ def test_removed_nodes_do_not_count_towards_membership(monkeypatch): def test_one_unreachable_node_does_not_stop_the_others(monkeypatch): - sink = [] + sink: list[tuple[str, bool]] = [] class _BadNode(_Node): def rpc_client(self, *a, **kw): From 82eed79d422fb2bf737f2bff60702f7c34460b38 Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 25 Aug 2026 17:48:30 +0200 Subject: [PATCH 020/122] fix: retire empty-stack landing records without the async delete machinery A finished transfer retires its REP_ landing volume by emptying the record's bdev_stack (the blob lives on as the converted, chained snapshot) and removing the record. Routing that retirement through delete_lvol flipped the record to in_deletion first, so any interruption before remove() stranded a record the monitor can never finish: with an empty stack there is nothing to issue, the delete-status poll answers 4 ('no async delete request exists') forever -- 856x in 30 minutes in run 20260825_125156 -- and every cleanup that waits for volumes to drain times out behind it (case 7: 33 stuck, case 9: 2 stuck, both dead in their prologues). Two-sided fix: - the retirement path tears down the nvmf plumbing DIRECTLY per node (delete_lvol_from_node(force=True); the empty stack means no blob work) and then removes the record, never entering in_deletion; - the monitor retires any in_deletion record with an empty bdev_stack record-only. Deliberately NO fallback delete of top_bdev: that bdev IS the converted snapshot, deleting it would destroy replicated data. Co-Authored-By: Claude Fable 5 --- simplyblock_core/services/lvol_monitor.py | 19 ++++++++++++ .../services/snapshot_replication.py | 23 ++++++++++---- .../test_replication_chain_completeness.py | 31 +++++++++++++++++++ 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/simplyblock_core/services/lvol_monitor.py b/simplyblock_core/services/lvol_monitor.py index a47cf36a82..d7cfd87ab6 100644 --- a/simplyblock_core/services/lvol_monitor.py +++ b/simplyblock_core/services/lvol_monitor.py @@ -403,6 +403,25 @@ def check_node(cluster, snode, all_lvols, subsys_check=False): deletions_processed += 1 + # RECORD-ONLY deletion: a retired landing volume's record carries + # an EMPTY bdev_stack on purpose -- its blob lives on as the + # converted, chained snapshot and must never be deleted. When the + # retirement sequence in snapshot_replication is interrupted + # between emptying the stack and removing the record, the record + # is left in_deletion here; the delete flow below then has nothing + # to issue, the status poll returns 4 ("no async delete request") + # forever (856x/30min, run 20260825_125156), and every cleanup + # that waits for lvols to drain times out behind it. Nothing on + # any node belongs to this record any more: retire it. + if not lvol.bdev_stack: + logger.info( + f"LVol {lvol.get_id()} ({lvol.lvol_name}) is in_deletion " + f"with an empty bdev stack (retired landing volume); " + f"removing the record only -- its blob lives on as the " + f"converted snapshot") + process_lvol_delete_finish(cluster, lvol) + continue + # The FULL delete of a chain member — the async delete, its # completion wait, and the sync deletes that follow — is one # atomic sequence per LVS+chain: a delete swap-merges segments diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index 06ab073c3c..81fe593cd6 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -959,12 +959,23 @@ def process_snap_replicate_finish(task, snapshot): # later cleanup that waits for lvols to drain times out on it. remote_lv.bdev_stack = [] remote_lv.write_to_db() - try: - lvol_controller.delete_lvol(remote_lv, force_delete=True) - except Exception as e: - logger.error(f"Landing volume {remote_lv.get_id()} teardown raised: {e}; " - f"retiring its record anyway (its bdev lives on as the " - f"converted snapshot)") + # Tear the subsystem/namespace down DIRECTLY, not via delete_lvol: + # delete_lvol flips the record to in_deletion and hands it to the + # monitor's async machinery, so an interruption anywhere before the + # remove() below stranded a record the monitor can never finish (empty + # stack -> nothing to issue -> status poll 4 forever; runs 20260824 and + # 20260825_125156). With the stack already emptied there is no blob work + # to do -- only nvmf plumbing on the volume's nodes. + for _node_id in remote_lv.nodes: + try: + _node = db.get_storage_node_by_id(_node_id) + if _node.status == StorageNode.STATUS_ONLINE: + lvol_controller.delete_lvol_from_node( + remote_lv.get_id(), _node_id, force=True) + except Exception as e: + logger.error(f"Landing volume {remote_lv.get_id()} teardown on " + f"{_node_id[:8]} raised: {e}; retiring the record " + f"anyway (its bdev lives on as the converted snapshot)") remote_lv.remove(db.kv_store) snapshot_events.replication_task_finished(snapshot) _prune_internal_snapshots(snapshot.lvol) diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index 6f8ff6aa40..ab4e654c29 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -488,3 +488,34 @@ def test_clone_register_confirms_the_bdev_before_add_ns(): "clone_register must be followed by a bdev confirmation poll" poll = src.index("not appear within 20s", reg) assert "get_bdevs" in src[reg:poll], "the poll must probe get_bdevs" + + +def test_retired_landing_records_are_record_only_deletions(): + """A retired landing volume's record deliberately carries an EMPTY + bdev_stack: its blob lives on as the converted, chained snapshot. The + monitor must retire such a record without issuing ANY bdev delete (runs + 20260824/20260825: interrupted retirements left records in_deletion that + the delete flow could never finish -- status poll 4, forever -- and a + naive top_bdev fallback delete would have destroyed the replicated + snapshot's data).""" + import inspect + from simplyblock_core.services import lvol_monitor as lm + src = inspect.getsource(lm.check_node) + guard = src.index("if not lvol.bdev_stack:") + flow = src.index("delete_lvol_from_node", guard) + finish = src.index("process_lvol_delete_finish", guard) + assert finish < flow, "empty-stack records must retire BEFORE the delete flow" + + +def test_retirement_tears_down_plumbing_without_delete_lvol(): + """The retirement path must not route through delete_lvol: that flips the + record to in_deletion for the monitor's async machinery, so any + interruption before remove() strands the record.""" + import inspect + from simplyblock_core.services import snapshot_replication as sr + src = inspect.getsource(sr) + empty = src.index("remote_lv.bdev_stack = []") + remove = src.index("remote_lv.remove(db.kv_store)", empty) + seg = src[empty:remove] + assert "delete_lvol_from_node" in seg, "teardown must be the direct per-node call" + assert "delete_lvol(remote_lv" not in seg, "must not route through delete_lvol" From 57ce4a18c194e6abec8f6f6073fc05cc7bebcdbe Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Tue, 25 Aug 2026 18:36:11 +0200 Subject: [PATCH 021/122] Fix unit and integration tests --- .../test_snapshot_replication_retention.py | 414 ---------------- tests/AGENTS.md | 15 + .../test_snapshot_replication_retention.py | 444 ++++++++++++++++++ .../test_snapshot_replication_leader_gate.py | 26 + .../v2/test_volume_replication_endpoints.py | 28 +- 5 files changed, 512 insertions(+), 415 deletions(-) delete mode 100644 simplyblock_core/test/test_snapshot_replication_retention.py create mode 100644 tests/integration/test_snapshot_replication_retention.py create mode 100644 tests/unit/test_snapshot_replication_leader_gate.py diff --git a/simplyblock_core/test/test_snapshot_replication_retention.py b/simplyblock_core/test/test_snapshot_replication_retention.py deleted file mode 100644 index cf0c674c53..0000000000 --- a/simplyblock_core/test/test_snapshot_replication_retention.py +++ /dev/null @@ -1,414 +0,0 @@ -"""D2 unit tests for internal-snapshot retention on source + target.""" -from simplyblock_core.models.snapshot import SnapShot -from simplyblock_core.models.lvol_model import LVol -from simplyblock_core.services import snapshot_replication as sr - - -def _mk_snap(uuid, created_at, snap_type, lvol_uuid, node_id, - status=SnapShot.STATUS_ONLINE, target=""): - lv = LVol() - lv.uuid = lvol_uuid - lv.node_id = node_id - s = SnapShot() - s.uuid = uuid - s.created_at = created_at - s.snap_type = snap_type - s.status = status - s.target_replicated_snap_uuid = target - s.lvol = lv - return s - - -class _Clone: - def __init__(self, uuid, cloned_from, status=LVol.STATUS_ONLINE): - self.uuid = uuid - self.cloned_from_snap = cloned_from - self.status = status - - def get_id(self): - return self.uuid - - -class _TargetCopy: - """A replicated snapshot as it exists on the remote cluster. - - ``prev_snap_uuid`` is the chain link retention checks before it deletes a - predecessor: it is only written once bdev_lvol_add_clone + convert succeeded. - """ - - def __init__(self, uuid, prev_snap_uuid="", node_id="TN1"): - self.uuid = uuid - self.prev_snap_uuid = prev_snap_uuid - self.snap_bdev = f"LVS_T/{uuid}" - self.snap_uuid = f"uuid-{uuid}" - lv = LVol() - lv.uuid = "T_LV1" - lv.node_id = node_id - self.lvol = lv - - def get_id(self): - return self.uuid - - -class _FakeNode: - """Target node. Offline by default so the SPDK fallback stays out of the - way unless a test explicitly opts into it.""" - - def __init__(self, status=LVol.STATUS_OFFLINE, bdevs=None): - self.status = status - self._bdevs = bdevs or [] - - def rpc_client(self): - node = self - - class _RPC: - def get_bdevs(self, name): - return [b for b in node._bdevs if b.get("name") == name] - - return _RPC() - - -class _FakeDB: - def __init__(self, source_snaps, existing_uuids, clones=(), chain=None, node=None): - self._source_snaps = source_snaps - self._existing = set(existing_uuids) - self._clones = list(clones) - self._chain = dict(chain or {}) - self._node = node or _FakeNode() - - def get_snapshots_by_node_id(self, node_id): - return [s for s in self._source_snaps if s.lvol.node_id == node_id] - - def get_snapshot_by_id(self, uuid): - if uuid in self._existing: - return _TargetCopy(uuid, self._chain.get(uuid, "")) - raise KeyError(uuid) - - def get_storage_node_by_id(self, node_id): - return self._node - - def get_mini_lvols(self): - return self._clones - - -def _healthy_chain(source_snaps): - """Link each replicated internal target copy onto its predecessor's. - - This is the state a converged replication leaves behind, so it is the - default for the existing cases: they assert on retention, not on chaining. - """ - chain: dict = {} - per_lvol: dict = {} - for s in source_snaps: - if s.snap_type != SnapShot.TYPE_INTERNAL or not s.target_replicated_snap_uuid: - continue - per_lvol.setdefault(s.lvol.get_id(), []).append(s) - for snaps in per_lvol.values(): - snaps.sort(key=lambda s: s.created_at) - for prev, nxt in zip(snaps, snaps[1:]): - chain[nxt.target_replicated_snap_uuid] = prev.target_replicated_snap_uuid - return chain - - -class _FakeSnapCtl: - def __init__(self, db): - self.deleted = [] - self._db = db - - def delete(self, uuid, force_delete=False): - self.deleted.append(uuid) - self._db._existing.discard(uuid) - return True - - -def _patch(monkeypatch, source_snaps, existing_uuids, clones=(), chain=None, node=None): - if chain is None: - chain = _healthy_chain(source_snaps) - db = _FakeDB(source_snaps, existing_uuids, clones, chain, node) - snapctl = _FakeSnapCtl(db) - monkeypatch.setattr(sr, "db", db) - monkeypatch.setattr(sr, "snapshot_controller", snapctl) - return snapctl - - -def test_prunes_older_internal_keeps_newest_and_users(monkeypatch): - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_old"), - _mk_snap("user_mid", 150, SnapShot.TYPE_USER, "LV1", "N1", target="T_user"), - _mk_snap("int_mid", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_mid"), - _mk_snap("int_new", 300, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - snapctl = _patch(monkeypatch, snaps, {"T_old", "T_user", "T_mid", "T_new"}) - - sr._prune_internal_snapshots(source_lvol) - - # Target copy deleted before the source snapshot; the newest PAIR is kept - # so an arriving snapshot always has a predecessor to chain onto. - assert snapctl.deleted == ["T_old", "int_old"] - for kept in ("int_mid", "T_mid", "int_new", "T_new", "user_mid", "T_user"): - assert kept not in snapctl.deleted - - -def test_single_internal_not_pruned(monkeypatch): - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [_mk_snap("int_only", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_only")] - snapctl = _patch(monkeypatch, snaps, {"T_only"}) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == [] - - -def test_unreplicated_internal_ignored(monkeypatch): - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - # Newest internal not yet replicated (no target) -> excluded; the only - # replicated internal is the single oldest, so nothing is pruned. - snaps = [ - _mk_snap("int_repl", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_repl"), - _mk_snap("int_pending", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target=""), - ] - snapctl = _patch(monkeypatch, snaps, {"T_repl"}) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == [] - - -def test_missing_target_still_cleans_source(monkeypatch): - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_gone"), - _mk_snap("int_mid", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_mid"), - _mk_snap("int_new", 300, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - # T_gone already deleted on target -> only source snapshot is cleaned up. - snapctl = _patch(monkeypatch, snaps, {"T_mid", "T_new"}) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == ["int_old"] - - -def test_other_lvol_snapshots_untouched(monkeypatch): - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_old"), - _mk_snap("int_mid", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_mid"), - _mk_snap("int_new", 300, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - # Different lvol on same node — must never be pruned. - _mk_snap("other_old", 50, SnapShot.TYPE_INTERNAL, "LV2", "N1", target="TO_old"), - _mk_snap("other_new", 250, SnapShot.TYPE_INTERNAL, "LV2", "N1", target="TO_new"), - ] - snapctl = _patch(monkeypatch, snaps, {"T_old", "T_mid", "T_new", "TO_old", "TO_new"}) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == ["T_old", "int_old"] - - -def test_never_prunes_snapshot_a_failed_over_volume_is_cloned_from(monkeypatch): - """Root cause of the all-zeros DR fail-over (labs 2026-08-10/11). - - Fail-over clones the volume from the last replicated TARGET snapshot; the - prune, keyed only on the SOURCE snapshot age, then deleted that target copy. - The delete reaches SPDK as bdev_lvol_delete(sync=False) and frees the blocks - immediately, so no DB-level guard downstream can save the clone. Retention - must skip a target snapshot with a live dependent clone. - """ - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_old"), - _mk_snap("int_new", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - # A failed-over volume lives on the OLD target snapshot. - snapctl = _patch(monkeypatch, snaps, {"T_old", "T_new"}, - clones=[_Clone("FO_VOL", "T_old")]) - - sr._prune_internal_snapshots(source_lvol) - - assert "T_old" not in snapctl.deleted, ( - "pruned the target snapshot a failed-over volume is cloned from — " - "its blocks are freed by SPDK immediately (sync=False), the volume " - "reads zeros from then on") - # The source-side copy must survive too (it pairs with the kept target). - assert "int_old" not in snapctl.deleted - - -def test_in_deletion_clone_does_not_pin_the_snapshot(monkeypatch): - """A clone that is itself going away must not block retention forever.""" - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_old"), - _mk_snap("int_mid", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_mid"), - _mk_snap("int_new", 300, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - snapctl = _patch(monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, - clones=[_Clone("DYING", "T_old", status=LVol.STATUS_IN_DELETION)]) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == ["T_old", "int_old"] - - -def test_require_lvs_leader_gate(monkeypatch): - """Convert on a non-leader returns success WITHOUT persisting (silent - conversion error) — leadership must be checked BEFORE the operation and a - non-leader must fail-and-retry, never proceed.""" - import simplyblock_core.controllers.lvol_controller as lc - - class _N: - def get_id(self): - return "N1" - - monkeypatch.setattr(lc, "is_node_leader", lambda node, lvs: False) - assert sr._require_lvs_leader(_N(), "LVS_1", "convert") is False - - monkeypatch.setattr(lc, "is_node_leader", lambda node, lvs: True) - assert sr._require_lvs_leader(_N(), "LVS_1", "convert") is True - - -def test_newest_pair_is_kept_so_arrivals_have_a_chain_parent(monkeypatch): - """Chain continuity: a replicated snapshot holds only its own clusters, and - deleting one swap-merges its segments into the successor CHAINED to it. - Keeping just the newest pruned the predecessor the instant a replication - finished, so the next arrival had nothing to chain onto and kept only its - delta — the target then held the last delta over holes (labs run 15 vs 19, - same case passing then failing on timing alone).""" - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_prev", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_prev"), - _mk_snap("int_new", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - snapctl = _patch(monkeypatch, snaps, {"T_prev", "T_new"}) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == [], ( - "pruned the predecessor the next arrival must chain onto") - - -def test_defers_prune_until_the_successor_is_actually_chained(monkeypatch): - """The count cushion is not the precondition — the chain link is. - - Keeping the newest N only widens the window in which chaining is expected to - have happened. If it lagged or failed for one snapshot while newer ones kept - arriving, the predecessor was still pruned, and because the delete reaches - SPDK as bdev_lvol_delete(sync=False) its segments were freed instead of - swap-merged into the successor. The target then holds the newest delta over - holes and a fail-over clone reads zeros (labs 2026-08-10..17). Retention must - verify the link and defer while it is absent. - """ - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_old"), - _mk_snap("int_mid", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_mid"), - _mk_snap("int_new", 300, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - # T_mid arrived but was never chained onto T_old. - snapctl = _patch(monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, chain={}) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == [], ( - "pruned a predecessor whose successor is not chained onto it — SPDK frees " - "the blocks immediately, so those segments are lost rather than merged") - - -def test_prunes_once_the_chain_is_established(monkeypatch): - """The deferral must release as soon as chaining catches up (no livelock).""" - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_old"), - _mk_snap("int_mid", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_mid"), - _mk_snap("int_new", 300, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - snapctl = _patch(monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, - chain={"T_mid": "T_old", "T_new": "T_mid"}) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == ["T_old", "int_old"] - - -def test_spdk_verdict_releases_a_missing_db_link(monkeypatch): - """A missing link must not pin the pair for ever. - - The link write is best-effort, and snapshots replicated before chaining was - implemented have none at all. SPDK is the real authority, so when the DB has - no link we ask the target node before giving up — otherwise retention would - never release those snapshots and both chains would grow without bound. - """ - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_old"), - _mk_snap("int_mid", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_mid"), - _mk_snap("int_new", 300, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - # SPDK reports T_mid as a clone whose base is T_old, while the DB link is absent. - node = _FakeNode(status=LVol.STATUS_ONLINE, bdevs=[{ - "name": "LVS_T/T_mid", - "driver_specific": {"lvol": {"clone": True, "base_snapshot": "LVS_T/T_old"}}, - }]) - snapctl = _patch(monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, - chain={}, node=node) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == ["T_old", "int_old"] - - -def test_unchained_in_spdk_is_not_pruned_even_when_node_is_reachable(monkeypatch): - """An online target that reports a standalone blob must still defer.""" - source_lvol = LVol() - source_lvol.uuid = "LV1" - source_lvol.node_id = "N1" - - snaps = [ - _mk_snap("int_old", 100, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_old"), - _mk_snap("int_mid", 200, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_mid"), - _mk_snap("int_new", 300, SnapShot.TYPE_INTERNAL, "LV1", "N1", target="T_new"), - ] - node = _FakeNode(status=LVol.STATUS_ONLINE, bdevs=[{ - "name": "LVS_T/T_mid", - "driver_specific": {"lvol": {"clone": False, "base_snapshot": None}}, - }]) - snapctl = _patch(monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, - chain={}, node=node) - - sr._prune_internal_snapshots(source_lvol) - - assert snapctl.deleted == [] diff --git a/tests/AGENTS.md b/tests/AGENTS.md index f80d74a83e..502a146c40 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -32,6 +32,21 @@ Pure-logic tests. Pick this tier when the test: `tests/unit/conftest.py` stubs out `fdb` (returning `None` from `fdb.open`), so unit tests never touch a real database. Run with `tox run -e unit` — no Docker, no infra. +**Never stand in for the database in a unit test.** The tier split is *about* the database, and there are only two positions: + +- the test does not interact with the database API **at all** → `tests/unit/`; +- the test interacts with the **real** database API → `tests/integration/`. + +A `_FakeDB`, a patched `DBController`, a module's `db` / `db_controller` singleton swapped for a mock, an assigned `kv_store` — each of these invents a third position that does not exist. It neither avoids the database nor exercises it: it asserts against a second, undeclared copy of the `DBController` interface that nothing keeps in sync with the real one. Such a test stays green until production code calls an accessor the fake never implemented, and then fails on an `AttributeError` that says nothing about the behaviour under test. + +That is not hypothetical. The snapshot-replication retention suite had a `_FakeDB` implementing the four accessors retention used when it was written; the moment retention began consulting `get_replication_policy_for_lvol`, twelve tests failed on a missing attribute rather than on anything about retention. They are now `tests/integration/test_snapshot_replication_retention.py`, seeding real models and reading them back through a live `DBController`. + +So, when a unit test seems to need a fake DB: **the need is the signal that it is not a unit test.** Move it to `tests/integration/`, seed real models with `write_to_db(db.kv_store)`, and mock only what sits above the database. If the code under test does not actually need the database, delete the stand-in rather than feeding it a fake. + +`.agents/hooks/guard-fake-db-tests.py` enforces this (see the root `AGENTS.md` § Guard hooks). It is a **ratchet**: the suite carries ~400 pre-existing stand-ins, so the guard compares each file before and after an edit and refuses only what that edit *introduces*. Editing or cleaning up a grandfathered file is unaffected. `python3 .agents/hooks/guard-fake-db-tests.py --scan` lists what is left. + +> **Migration in progress, here too.** `simplyblock_core/test/` is a second unit-tier directory (also collected by `tox run -e unit`, with its own `fdb` stub) and holds the densest concentration of these fakes. Treat it as legacy: don't add files there, and convert what you touch. + ### `tests/integration/` Controller-flow tests, **all of which run against a real FoundationDB**. `tests/integration/conftest.py` provisions FDB once for the whole tier from `pytest_configure` — *before* test collection — reusing `$FDB_CLUSTER_FILE` if set, otherwise starting a `testcontainers` container and binding its cluster file into `simplyblock_core.constants`. Provisioning at `pytest_configure` (rather than in a session fixture) means the real `fdb` client and a live `DBController()` are available at **collection / module-import time**, so test modules may touch the DB at import scope. A separate autouse fixture wipes the user keyspace before every test for isolation. The FDB-backed subdirs (`ftt2/`, `migration/`, `expansion_sim/`) add their own per-suite topology/bootstrap fixtures on top of that same cluster. diff --git a/tests/integration/test_snapshot_replication_retention.py b/tests/integration/test_snapshot_replication_retention.py new file mode 100644 index 0000000000..e2b17382bc --- /dev/null +++ b/tests/integration/test_snapshot_replication_retention.py @@ -0,0 +1,444 @@ +# coding=utf-8 +"""Retention for replication-driven internal snapshots (D2), against real FDB. + +``snapshot_replication._prune_internal_snapshots`` decides which internal +snapshots may be deleted by *reading model state*: the source snapshot chain of +a volume (``get_snapshots_by_node_id``), each snapshot's replicated copy on the +target cluster (``get_snapshot_by_id``), that copy's chain link +(``prev_snap_uuid``), the volumes cloned from it (``get_mini_lvols``), the +owning storage node, and the volume's replication policy +(``get_replication_policy_for_lvol``). Every one of those is a DBController +accessor, so the tests belong to the FDB-backed tier: the state under test IS +database state. + +These cases previously ran against a hand-written ``_FakeDB`` that implemented +the four accessors the code used at the time. They broke the moment retention +started consulting the replication policy — the fake had no +``get_replication_policy_for_lvol``, so twelve tests failed on an +``AttributeError`` rather than on anything about retention. A real +``DBController`` cannot drift out of sync with the code it serves. + +Mocked here — everything *above* the database, per the tier's rule: + +- ``snapshot_controller.delete``, the data-plane delete (cluster-wide object + lock, ``bdev_lvol_delete`` over JSON-RPC, reaped by snapshot_monitor). It is + replaced by a recorder that applies the DB effect the real delete eventually + has — the record is removed — because that is what the prune loop reads back. +- ``StorageNode.rpc_client``, the SPDK chain query on the target node. The + integration tier never talks to a storage node. +""" + +import pytest + +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.snapshot import SnapShot +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import snapshot_replication as sr + +CLUSTER_ID = "cluster-1" +POOL_ID = "pool-1" +SOURCE_NODE_ID = "N1" +TARGET_NODE_ID = "TN1" +TARGET_LVOL_ID = "T_LV1" + + +@pytest.fixture +def db(): + db = DBController() + if db.kv_store is None: + pytest.skip("FoundationDB is not available") + return db + + +def _write_lvol(db, uuid, node_id, cloned_from_snap="", status=LVol.STATUS_ONLINE): + lvol = LVol() + lvol.uuid = uuid + lvol.cluster_id = CLUSTER_ID + lvol.pool_uuid = POOL_ID + lvol.node_id = node_id + lvol.lvol_name = f"VOL_{uuid}" + lvol.lvol_bdev = f"LVOL_{uuid}" + lvol.lvs_name = "LVS_S" if node_id == SOURCE_NODE_ID else "LVS_T" + lvol.top_bdev = f"{lvol.lvs_name}/{lvol.lvol_bdev}" + lvol.size = 1024 ** 3 + lvol.status = status + lvol.cloned_from_snap = cloned_from_snap + lvol.write_to_db(db.kv_store) + return lvol + + +def _write_node(db, uuid, status): + node = StorageNode() + node.uuid = uuid + node.cluster_id = CLUSTER_ID + node.hostname = uuid + node.status = status + node.lvstore = "LVS_T" + node.lvstore_status = "ready" + node.write_to_db(db.kv_store) + return node + + +def _write_snapshot(db, uuid, created_at, snap_type, lvol, target="", + status=SnapShot.STATUS_ONLINE): + """A snapshot on the SOURCE cluster, optionally already replicated.""" + snap = SnapShot() + snap.uuid = uuid + snap.cluster_id = CLUSTER_ID + snap.pool_uuid = POOL_ID + snap.created_at = created_at + snap.snap_type = snap_type + snap.status = status + snap.target_replicated_snap_uuid = target + snap.snap_name = f"SNAP_{uuid}" + snap.snap_bdev = f"LVS_S/{uuid}" + snap.snap_uuid = f"uuid-{uuid}" + snap.size = lvol.size + snap.lvol = lvol + snap.write_to_db(db.kv_store) + return snap + + +def _write_target_copy(db, uuid, target_lvol, prev_snap_uuid=""): + """A replicated snapshot as it exists on the remote cluster. + + ``prev_snap_uuid`` is the chain link retention checks before it deletes a + predecessor: it is only written once bdev_lvol_add_clone + convert + succeeded. + """ + copy = SnapShot() + copy.uuid = uuid + copy.cluster_id = CLUSTER_ID + copy.pool_uuid = POOL_ID + copy.status = SnapShot.STATUS_ONLINE + copy.snap_type = SnapShot.TYPE_INTERNAL + copy.snap_name = f"SNAP_{uuid}" + copy.snap_bdev = f"LVS_T/{uuid}" + copy.snap_uuid = f"uuid-{uuid}" + copy.prev_snap_uuid = prev_snap_uuid + copy.size = target_lvol.size + copy.lvol = target_lvol + copy.write_to_db(db.kv_store) + return copy + + +def _healthy_chain(source_snaps): + """Link each replicated internal target copy onto its predecessor's. + + This is the state a converged replication leaves behind, so it is the + default for the existing cases: they assert on retention, not on chaining. + """ + chain: dict = {} + per_lvol: dict = {} + for s in source_snaps: + if s.snap_type != SnapShot.TYPE_INTERNAL or not s.target_replicated_snap_uuid: + continue + per_lvol.setdefault(s.lvol.get_id(), []).append(s) + for snaps in per_lvol.values(): + snaps.sort(key=lambda s: s.created_at) + for prev, nxt in zip(snaps, snaps[1:]): + chain[nxt.target_replicated_snap_uuid] = prev.target_replicated_snap_uuid + return chain + + +class _RecordingSnapshotController: + """Records what retention decided to delete, and applies it to the real DB. + + The real ``snapshot_controller.delete`` is a data-plane path; what matters + to the prune loop is the state it leaves behind, since the loop reads each + target copy back before acting on it. + """ + + def __init__(self, db): + self._db = db + self.deleted = [] + + def delete(self, uuid, force_delete=False): + self.deleted.append(uuid) + try: + snap = self._db.get_snapshot_by_id(uuid) + except KeyError: + return True + snap.remove(self._db.kv_store) + return True + + +def _seed(db, monkeypatch, source_snaps, existing_targets, chain=None, + target_node_status=StorageNode.STATUS_OFFLINE, target_bdevs=()): + """Materialize the target-side state and stub the layers above the DB. + + The target node is offline by default so the SPDK chain fallback stays out + of the way unless a test explicitly opts into it. + """ + if chain is None: + chain = _healthy_chain(source_snaps) + + target_lvol = _write_lvol(db, TARGET_LVOL_ID, TARGET_NODE_ID) + for uuid in existing_targets: + _write_target_copy(db, uuid, target_lvol, chain.get(uuid, "")) + _write_node(db, TARGET_NODE_ID, target_node_status) + + class _RPC: + def get_bdevs(self, name=None): + return [b for b in target_bdevs if b.get("name") == name] + + monkeypatch.setattr(StorageNode, "rpc_client", lambda self, **kwargs: _RPC()) + + snapctl = _RecordingSnapshotController(db) + monkeypatch.setattr(sr, "snapshot_controller", snapctl) + return snapctl + + +def test_prunes_older_internal_keeps_newest_and_users(db, monkeypatch): + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_old"), + _write_snapshot(db, "user_mid", 150, SnapShot.TYPE_USER, source_lvol, target="T_user"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + snapctl = _seed(db, monkeypatch, snaps, {"T_old", "T_user", "T_mid", "T_new"}) + + sr._prune_internal_snapshots(source_lvol) + + # Target copy deleted before the source snapshot; the newest PAIR is kept + # so an arriving snapshot always has a predecessor to chain onto. + assert snapctl.deleted == ["T_old", "int_old"] + for kept in ("int_mid", "T_mid", "int_new", "T_new", "user_mid", "T_user"): + assert kept not in snapctl.deleted + + +def test_single_internal_not_pruned(db, monkeypatch): + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [_write_snapshot(db, "int_only", 100, SnapShot.TYPE_INTERNAL, + source_lvol, target="T_only")] + snapctl = _seed(db, monkeypatch, snaps, {"T_only"}) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == [] + + +def test_unreplicated_internal_ignored(db, monkeypatch): + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + # Newest internal not yet replicated (no target) -> excluded; the only + # replicated internal is the single oldest, so nothing is pruned. + snaps = [ + _write_snapshot(db, "int_repl", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_repl"), + _write_snapshot(db, "int_pending", 200, SnapShot.TYPE_INTERNAL, source_lvol, target=""), + ] + snapctl = _seed(db, monkeypatch, snaps, {"T_repl"}) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == [] + + +def test_missing_target_still_cleans_source(db, monkeypatch): + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_gone"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + # T_gone already deleted on target -> only source snapshot is cleaned up. + snapctl = _seed(db, monkeypatch, snaps, {"T_mid", "T_new"}) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == ["int_old"] + + +def test_other_lvol_snapshots_untouched(db, monkeypatch): + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + other_lvol = _write_lvol(db, "LV2", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_old"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + # Different lvol on same node — must never be pruned. + _write_snapshot(db, "other_old", 50, SnapShot.TYPE_INTERNAL, other_lvol, target="TO_old"), + _write_snapshot(db, "other_new", 250, SnapShot.TYPE_INTERNAL, other_lvol, target="TO_new"), + ] + snapctl = _seed(db, monkeypatch, snaps, + {"T_old", "T_mid", "T_new", "TO_old", "TO_new"}) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == ["T_old", "int_old"] + + +def test_never_prunes_snapshot_a_failed_over_volume_is_cloned_from(db, monkeypatch): + """Root cause of the all-zeros DR fail-over (labs 2026-08-10/11). + + Fail-over clones the volume from the last replicated TARGET snapshot; the + prune, keyed only on the SOURCE snapshot age, then deleted that target copy. + The delete reaches SPDK as bdev_lvol_delete(sync=False) and frees the blocks + immediately, so no DB-level guard downstream can save the clone. Retention + must skip a target snapshot with a live dependent clone. + + Three snapshots, not two: with only a pair, the newest-``keep`` rule returns + before the clone guard is ever consulted, and the case passes no matter what + that guard does. ``int_old`` has to be a genuine prune candidate — chained, + past the cushion — so that the dependent clone is the only thing saving it. + """ + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_old"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + snapctl = _seed(db, monkeypatch, snaps, {"T_old", "T_mid", "T_new"}) + # A failed-over volume lives on the OLD target snapshot. + _write_lvol(db, "FO_VOL", TARGET_NODE_ID, cloned_from_snap="T_old") + + sr._prune_internal_snapshots(source_lvol) + + assert "T_old" not in snapctl.deleted, ( + "pruned the target snapshot a failed-over volume is cloned from — " + "its blocks are freed by SPDK immediately (sync=False), the volume " + "reads zeros from then on") + # The source-side copy must survive too (it pairs with the kept target). + assert "int_old" not in snapctl.deleted + + +def test_in_deletion_clone_does_not_pin_the_snapshot(db, monkeypatch): + """A clone that is itself going away must not block retention forever.""" + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_old"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + snapctl = _seed(db, monkeypatch, snaps, {"T_old", "T_mid", "T_new"}) + _write_lvol(db, "DYING", TARGET_NODE_ID, cloned_from_snap="T_old", + status=LVol.STATUS_IN_DELETION) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == ["T_old", "int_old"] + + +def test_newest_pair_is_kept_so_arrivals_have_a_chain_parent(db, monkeypatch): + """Chain continuity: a replicated snapshot holds only its own clusters, and + deleting one swap-merges its segments into the successor CHAINED to it. + Keeping just the newest pruned the predecessor the instant a replication + finished, so the next arrival had nothing to chain onto and kept only its + delta — the target then held the last delta over holes (labs run 15 vs 19, + same case passing then failing on timing alone).""" + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_prev", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_prev"), + _write_snapshot(db, "int_new", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + snapctl = _seed(db, monkeypatch, snaps, {"T_prev", "T_new"}) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == [], ( + "pruned the predecessor the next arrival must chain onto") + + +def test_defers_prune_until_the_successor_is_actually_chained(db, monkeypatch): + """The count cushion is not the precondition — the chain link is. + + Keeping the newest N only widens the window in which chaining is expected to + have happened. If it lagged or failed for one snapshot while newer ones kept + arriving, the predecessor was still pruned, and because the delete reaches + SPDK as bdev_lvol_delete(sync=False) its segments were freed instead of + swap-merged into the successor. The target then holds the newest delta over + holes and a fail-over clone reads zeros (labs 2026-08-10..17). Retention must + verify the link and defer while it is absent. + """ + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_old"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + # T_mid arrived but was never chained onto T_old. + snapctl = _seed(db, monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, chain={}) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == [], ( + "pruned a predecessor whose successor is not chained onto it — SPDK frees " + "the blocks immediately, so those segments are lost rather than merged") + + +def test_prunes_once_the_chain_is_established(db, monkeypatch): + """The deferral must release as soon as chaining catches up (no livelock).""" + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_old"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + snapctl = _seed(db, monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, + chain={"T_mid": "T_old", "T_new": "T_mid"}) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == ["T_old", "int_old"] + + +def test_spdk_verdict_releases_a_missing_db_link(db, monkeypatch): + """A missing link must not pin the pair for ever. + + The link write is best-effort, and snapshots replicated before chaining was + implemented have none at all. SPDK is the real authority, so when the DB has + no link we ask the target node before giving up — otherwise retention would + never release those snapshots and both chains would grow without bound. + """ + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_old"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + # SPDK reports T_mid as a clone whose base is T_old, while the DB link is absent. + snapctl = _seed(db, monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, chain={}, + target_node_status=StorageNode.STATUS_ONLINE, + target_bdevs=[{ + "name": "LVS_T/T_mid", + "driver_specific": { + "lvol": {"clone": True, "base_snapshot": "LVS_T/T_old"}}, + }]) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == ["T_old", "int_old"] + + +def test_unchained_in_spdk_is_not_pruned_even_when_node_is_reachable(db, monkeypatch): + """An online target that reports a standalone blob must still defer.""" + source_lvol = _write_lvol(db, "LV1", SOURCE_NODE_ID) + + snaps = [ + _write_snapshot(db, "int_old", 100, SnapShot.TYPE_INTERNAL, source_lvol, target="T_old"), + _write_snapshot(db, "int_mid", 200, SnapShot.TYPE_INTERNAL, source_lvol, target="T_mid"), + _write_snapshot(db, "int_new", 300, SnapShot.TYPE_INTERNAL, source_lvol, target="T_new"), + ] + snapctl = _seed(db, monkeypatch, snaps, {"T_old", "T_mid", "T_new"}, chain={}, + target_node_status=StorageNode.STATUS_ONLINE, + target_bdevs=[{ + "name": "LVS_T/T_mid", + "driver_specific": { + "lvol": {"clone": False, "base_snapshot": None}}, + }]) + + sr._prune_internal_snapshots(source_lvol) + + assert snapctl.deleted == [] diff --git a/tests/unit/test_snapshot_replication_leader_gate.py b/tests/unit/test_snapshot_replication_leader_gate.py new file mode 100644 index 0000000000..2a623a4394 --- /dev/null +++ b/tests/unit/test_snapshot_replication_leader_gate.py @@ -0,0 +1,26 @@ +# coding=utf-8 +"""The LVS-leader gate in the snapshot-replication service. + +Pure logic over a mocked ``is_node_leader`` — no model state, no DB — so it +stays in the unit tier. The retention tests it used to share a file with are +DB-driven and live in ``tests/integration/test_snapshot_replication_retention.py``. +""" + +from simplyblock_core.services import snapshot_replication as sr + + +def test_require_lvs_leader_gate(monkeypatch): + """Convert on a non-leader returns success WITHOUT persisting (silent + conversion error) — leadership must be checked BEFORE the operation and a + non-leader must fail-and-retry, never proceed.""" + import simplyblock_core.controllers.lvol_controller as lc + + class _N: + def get_id(self): + return "N1" + + monkeypatch.setattr(lc, "is_node_leader", lambda node, lvs: False) + assert sr._require_lvs_leader(_N(), "LVS_1", "convert") is False + + monkeypatch.setattr(lc, "is_node_leader", lambda node, lvs: True) + assert sr._require_lvs_leader(_N(), "LVS_1", "convert") is True diff --git a/tests/unit/web/api/v2/test_volume_replication_endpoints.py b/tests/unit/web/api/v2/test_volume_replication_endpoints.py index dc9d4dd5cf..a61be97a49 100644 --- a/tests/unit/web/api/v2/test_volume_replication_endpoints.py +++ b/tests/unit/web/api/v2/test_volume_replication_endpoints.py @@ -203,7 +203,33 @@ def test_failover(self, client, db, volume, lvol_controller): assert response.status_code == 204 assert response.content == b'' - lvol_controller.replicate_lvol_on_target_cluster.assert_called_once_with(VOLUME_ID) + lvol_controller.replicate_lvol_on_target_cluster.assert_called_once_with( + VOLUME_ID, generation=0) + + def test_failover_forwards_the_requested_generation(self, client, db, volume, + lvol_controller): + """``generation`` selects which retained point-in-time to come up on. + + It has to reach the controller: dropping it silently fails the volume + over to the NEWEST copy, which in the case this parameter exists for — + recovering from a logical corruption — is the copy that faithfully + replicated the corruption. + """ + lvol_controller.replicate_lvol_on_target_cluster.return_value = { + 'lvol_id': TARGET_VOLUME_ID, 'nqn': 'nqn.x', 'ns_id': 1, 'connection_strings': [], + } + + response = client.post(REPLICATION_URL + 'failover?generation=2') + + assert response.status_code == 204 + lvol_controller.replicate_lvol_on_target_cluster.assert_called_once_with( + VOLUME_ID, generation=2) + + def test_negative_generation_rejected(self, client, db, volume, lvol_controller): + response = client.post(REPLICATION_URL + 'failover?generation=-1') + + assert response.status_code == 400 + lvol_controller.replicate_lvol_on_target_cluster.assert_not_called() def test_failed_failover_is_an_error(self, client, db, volume, lvol_controller): lvol_controller.replicate_lvol_on_target_cluster.return_value = (False, 'node is not online') From aa292317659c7a616ecf1f2eb24db2b321910f64 Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 25 Aug 2026 22:42:11 +0200 Subject: [PATCH 022/122] test: ship lvol_monitor's own module in the hotfix app_LVolMonitor received only the SHARED files, so a fix living in lvol_monitor.py itself (the empty-stack record retirement) verified green while the running monitor never contained it -- the import probe checks the shared modules, not the service's own file. Co-Authored-By: Claude Fable 5 --- scripts/hotfix_repl_lab.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/hotfix_repl_lab.py b/scripts/hotfix_repl_lab.py index 81715e8ea5..aa79eeb640 100644 --- a/scripts/hotfix_repl_lab.py +++ b/scripts/hotfix_repl_lab.py @@ -77,7 +77,8 @@ "app_TasksRunnerReplicationFinal": { "simplyblock_core/services/tasks_runner_replication_final.py": "services/tasks_runner_replication_final.py"}, - "app_LVolMonitor": {}, + "app_LVolMonitor": { + "simplyblock_core/services/lvol_monitor.py": "services/lvol_monitor.py"}, } #: also refreshed on the mgmt HOST: the harness runs `sudo python3 -c #: "...lvol_controller.get_replication_info..."` there, and sbctl imports it. From 01c56f7dbb4855cdbd8d0047e306c0545369824b Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 25 Aug 2026 22:48:06 +0200 Subject: [PATCH 023/122] test: cases 10-12 for the new features case 10 -- online migration under sustained heavy IO: 3 volumes per source node at 8k randwrite QD64 x 4 jobs, replication must reach and hold a bounded lag against that load, the final online migration must complete for every volume with fio uninterrupted, and the IO freeze during each cutover is MEASURED as the client actually experiences it (100ms heartbeat probe per volume; the largest gap inside the commit window is the freeze; 30s sanity bound). case 11 -- retention ladder + generation fail-overs: policy with --retention-schedule 5m:15m,7m:30m,10m:1h, ~2h of history under a 30s-fsynced record stream, then (a)/(b) kept-and-pruned verified INDEPENDENTLY of the retention code (gap/count/horizon analysis per tier), and (c) three rounds of fail-over to a randomly selected older generation with exact-data validation (every record up to the generation's snapshot present, none after), each followed by a full fail-back cutover. case 12 -- consistency groups (gated: requires the consistency-groups build; refuses with a clear message otherwise): CG policy, 3 members pinned to one LVS, strictly ordered fsynced writes A->B->C, group generations verified complete (every group_seq covers all members), retention per schedule, then fail-over to the latest and to a random earlier generation verifying BOTH generation correlation and crash consistency: seq(A) >= seq(B) >= seq(C), max skew 1. Policies with extra flags get distinct names so a schedule/CG policy never silently reuses a plain one another case left behind. Co-Authored-By: Claude Fable 5 --- scripts/test_async_replication.py | 509 +++++++++++++++++++++++++++++- 1 file changed, 501 insertions(+), 8 deletions(-) diff --git a/scripts/test_async_replication.py b/scripts/test_async_replication.py index 2fc7a4f622..8bc1b20761 100644 --- a/scripts/test_async_replication.py +++ b/scripts/test_async_replication.py @@ -291,7 +291,7 @@ def wait_data_replicated(mgmt_ip, key_path, lvol_uuids, after_ts, f"predates the filesystem") -def do_failover(mgmt_ip, key_path, lvol_uuid): +def do_failover(mgmt_ip, key_path, lvol_uuid, generation=0): """Fail a volume over, capturing WHY when it does not work. replicate_lvol_on_target_cluster returns a dict on success but False or @@ -316,7 +316,7 @@ def do_failover(mgmt_ip, key_path, lvol_uuid): err = "" try: with contextlib.redirect_stderr(buf): - res = lvol_controller.replicate_lvol_on_target_cluster({lvol_uuid!r}) + res = lvol_controller.replicate_lvol_on_target_cluster({lvol_uuid!r}, generation={generation}) except Exception as exc: # noqa: BLE001 - report, don't hide res, err = False, f"{{type(exc).__name__}}: {{exc}}" out = res if isinstance(res, dict) else {{"result": res}} @@ -774,21 +774,28 @@ def ensure_replication_target(mgmt_ip, key_path, from_cluster, to_cluster, def ensure_replication_policy(mgmt_ip, key_path, from_cluster, target_name, mode, - interval_min=REPL_INTERVAL_MIN, keep=2, name=None): - """Register (idempotently) a cadence policy on an existing target.""" - name = name or f"pol_{mode}_{target_name}" + interval_min=REPL_INTERVAL_MIN, keep=2, name=None, + extra_flags=""): + """Register (idempotently) a cadence policy on an existing target. + + extra_flags shapes the policy NAME too, so a case asking for a + retention-schedule or consistency-group policy never silently reuses a + plain one another case left behind. + """ + suffix = "_x%08x" % (hash(extra_flags) & 0xffffffff) if extra_flags else "" + name = name or f"pol_{mode}_{target_name}{suffix}" for row in _replication_list(mgmt_ip, key_path, "policy", from_cluster): if row.get("Name") == name: return name run(mgmt_ip, key_path, f"{SBCTL} -d cluster replication-policy-add {from_cluster} {name}" f" --target {target_name} --interval-min {interval_min} --mode {mode}" - f" --keep {keep}") + f" --keep {keep}{(' ' + extra_flags) if extra_flags else ''}") return name def set_cluster_replication(mgmt_ip, key_path, from_cluster, to_cluster, to_pool_uuid, - mode="migration"): + mode="migration", extra_flags=""): """Create the target + policy that let `from_cluster` replicate to `to_cluster`. Replication is NEVER started per volume any more: `volume replication-start` @@ -803,7 +810,8 @@ def set_cluster_replication(mgmt_ip, key_path, from_cluster, to_cluster, to_pool f"(pool {to_pool_uuid[:8]}, mode {mode})") target = ensure_replication_target(mgmt_ip, key_path, from_cluster, to_cluster, to_pool_uuid) - policy = ensure_replication_policy(mgmt_ip, key_path, from_cluster, target, mode) + policy = ensure_replication_policy(mgmt_ip, key_path, from_cluster, target, mode, + extra_flags=extra_flags) # PRODUCT GAP (bridge, delete once the readers consult the policy): # replicate_lvol_on_target_cluster() and tasks_runner_replication_final still @@ -2123,6 +2131,487 @@ def test_case_9(meta): f"replication recovered every time, data intact (seed {seed}).") + + +# --------------------------------------------------------------------------- # +# Cases 10-12: feature tests (migration under load, retention ladder, CGs) +# --------------------------------------------------------------------------- # +CASE10_VOLS_PER_NODE = int(os.environ.get("CASE10_VOLS_PER_NODE", "3")) +CASE11_RUNTIME_MIN = int(os.environ.get("CASE11_RUNTIME_MIN", "115")) +CASE11_SCHEDULE = os.environ.get("CASE11_SCHEDULE", "5m:15m,7m:30m,10m:1h") +CASE11_ROUNDS = int(os.environ.get("CASE11_ROUNDS", "3")) +CASE12_SCHEDULE = os.environ.get("CASE12_SCHEDULE", "5m:15m") +CASE12_RUNTIME_MIN = int(os.environ.get("CASE12_RUNTIME_MIN", "25")) + + +def _start_stall_probe(client_ip, key_path, mount): + """100ms wall-clock heartbeats into the volume; gaps measure IO freezes.""" + run(client_ip, key_path, + "sudo rm -f {m}/probe.ts; sudo nohup bash -c 'while :; do " + "date +%s%3N >> {m}/probe.ts; sync {m}/probe.ts; sleep 0.1; done' " + ">/dev/null 2>&1 & echo probe_started".format(m=mount), quiet=True) + + +def _stop_probes(client_ip, key_path): + run(client_ip, key_path, "sudo pkill -f probe.ts 2>/dev/null || true", + check=False, quiet=True) + + +def _max_probe_gap_ms(client_ip, key_path, mount, t0_ms, t1_ms): + """Largest heartbeat gap inside [t0_ms, t1_ms] = the freeze duration the + CLIENT actually observed (includes a ~100ms sampling floor).""" + awk = ("sudo awk 'p && $1>={t0} && $1<={t1} && $1-p>m {{m=$1-p}} {{p=$1}} " + "END{{print m+0}}' {m}/probe.ts").format(t0=t0_ms, t1=t1_ms, m=mount) + out = run(client_ip, key_path, awk, check=False, quiet=True) + try: + return int(out.strip()) + except ValueError: + return -1 + + +def _start_recorder(client_ip, key_path, mount, interval_sec=30): + """Timestamped, fsynced records every interval -- the ground truth for + which point-in-time a snapshot generation captured.""" + cmd = ("sudo rm -f {m}/records.log; sudo nohup bash -c 'i=0; while :; do " + "i=$((i+1)); echo \"iter=$i ts=$(date +%s)\" >> {m}/records.log; " + "sync {m}/records.log; sleep {iv}; done' >/dev/null 2>&1 & " + "echo recorder_started").format(m=mount, iv=interval_sec) + run(client_ip, key_path, cmd, quiet=True) + + +def _stop_recorders(client_ip, key_path): + run(client_ip, key_path, "sudo pkill -f records.log 2>/dev/null || true", + check=False, quiet=True) + + +def _snapshot_ages(mgmt_ip, key_path, lvol_uuid): + """created_at (epoch) of every replicated internal snapshot of the volume, + newest first.""" + return mgmt_py(mgmt_ip, key_path, """ +import json +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.snapshot import SnapShot +db = DBController() +out = [] +for snp in db.get_snapshots(): + if snp.deleted or not snp.lvol or snp.lvol.get_id() != {lv!r}: + continue + if snp.snap_type == SnapShot.TYPE_INTERNAL and snp.target_replicated_snap_uuid: + out.append(snp.created_at or 0) +print(json.dumps(sorted(out, reverse=True))) +""".format(lv=lvol_uuid), replayable=True) + + +def _verify_retention_ladder(times, tiers, now, cadence_sec=60, slack=150): + """Independent check of (a) kept and (b) pruned per schedule. + + tiers: [(every_sec, span_sec)] finest first. Verifies: nothing older + than the horizon (+slack), and inside each tier window consecutive + retained snapshots are no further apart than every+cadence+slack and no + denser than the schedule plus a small tolerance allows. + """ + problems = [] + horizon = sum(t[1] for t in tiers) + ages = sorted(now - t for t in times) + for a in ages: + if a > horizon + slack: + problems.append("snapshot %ds old survives past the %ds horizon " + "(not pruned)" % (a, horizon)) + start = 0 + for every, span in tiers: + end = start + span + inside = [a for a in ages if start <= a < end] + for prev, cur in zip(inside, inside[1:]): + if cur - prev > every + cadence_sec + slack: + problems.append( + "gap of %ds inside the %ds tier (%d-%ds): a scheduled " + "snapshot is missing" % (cur - prev, every, start, end)) + expected = span // every + if len(inside) > expected + 2: + problems.append( + "%d snapshots inside the %ds tier (%d-%ds), expected <= %d: " + "not pruned" % (len(inside), every, start, end, expected + 2)) + start = end + return problems + + +def _records_match_generation(client_ip, key_path, mount, snap_ts, slack=35, + logname="records.log"): + """The mounted snapshot copy must contain every record up to snap_ts and + none from after it (records are fsynced every 30s).""" + out = run(client_ip, key_path, + "sudo grep -oE 'ts=[0-9]+' %s/%s | tail -200" % (mount, logname), + check=False, quiet=True) + ts = [int(x.split("=")[1]) for x in out.split() if x.startswith("ts=")] + if not ts: + return False, "no records found on the snapshot copy" + newest = max(ts) + if newest > snap_ts + slack: + return False, ("record from %d present, %ds AFTER the generation's " + "snapshot (%d)" % (newest, newest - snap_ts, snap_ts)) + if newest < snap_ts - 95: + return False, ("newest record %d is %ds older than the snapshot; " + "records up to the snapshot are missing" + % (newest, snap_ts - newest)) + return True, "newest record %ds before the snapshot" % (int(snap_ts) - newest) + + +def _target_snapshot_ts(mgmt_ip, key_path, target_lvol): + """created_at of the snapshot the fail-over copy was cloned from.""" + return mgmt_py(mgmt_ip, key_path, """ +import json +from simplyblock_core.db_controller import DBController +db = DBController() +lv = db.get_lvol_by_id({t!r}) +snp = db.get_snapshot_by_id(lv.cloned_from_snap) +print(json.dumps(snp.created_at or 0)) +""".format(t=target_lvol), replayable=True) + + +def test_case_10(meta): + """Migration under sustained heavy IO: 3 volumes per source node at + 8k randwrite QD64x4 jobs, replication must keep up, the final online + migration must complete, and the client-observed IO freeze during each + cutover is measured.""" + print("\n========== CASE 10: online migration under heavy IO (freeze timing) ==========") + key_path = meta["key_path"] + mgmt_ip = meta["mgmt"]["public_ip"] + client_ip = meta["clients"][0]["public_ip"] + src_uuid, src, tgt_uuid, tgt = _src_target(meta) + + prepare_mount_points(client_ip, key_path) + delete_test_volumes(mgmt_ip, key_path, _all_test_pools(meta)) + + policy = set_cluster_replication(mgmt_ip, key_path, src_uuid, tgt_uuid, + pool_uuid_of(mgmt_ip, key_path, tgt["pool"]), + mode="migration") + node_ids = mgmt_py(mgmt_ip, key_path, """ +import json +from simplyblock_core.db_controller import DBController +print(json.dumps([n.get_id() for n in + DBController().get_storage_nodes_by_cluster_id({c!r}) + if n.status == "online"])) +""".format(c=src_uuid), replayable=True) + lvols = [] + for n_idx, nid in enumerate(node_ids): + for v in range(CASE10_VOLS_PER_NODE): + name = "replvol%d" % (n_idx * CASE10_VOLS_PER_NODE + v) + run(mgmt_ip, key_path, + "%s -d volume add %s %s %s --replication-policy %s --host-id %s" + % (SBCTL, name, VOL_SIZE, src["pool"], policy, nid)) + lvols.append(resolve_lvol(mgmt_ip, key_path, name)["uuid"]) + print(" %d volumes across %d nodes" % (len(lvols), len(node_ids))) + + mounts = connect_and_mount(client_ip, key_path, mgmt_ip, lvols, fmt=True) + write_baseline(client_ip, key_path, mounts) + for m in mounts: + _start_stall_probe(client_ip, key_path, m["mount"]) + jobfile = write_fio_jobfile(client_ip, key_path, mounts, rw="randwrite", + bs="8k", iodepth=64, numjobs=4, size="2G", + verify=False, jobfile="/tmp/fio_case10.fio") + start_fio(client_ip, key_path, jobfile) + + # The whole point: replication must reach a bounded lag AGAINST this load. + wait_replication_caught_up(mgmt_ip, key_path, lvols, timeout=5400) + + print("Committing the final online migration per volume (freeze timing)...") + windows = {} + for lv in lvols: + t0 = int(run(client_ip, key_path, "date +%s%3N", quiet=True).strip()) + run(mgmt_ip, key_path, "%s -d volume replication-commit %s" % (SBCTL, lv)) + windows[lv] = [t0, 0] + start = time.time() + done = 0 + while time.time() - start < CUTOVER_WAIT_TIMEOUT * 3: + if not fio_alive(client_ip, key_path): + raise RuntimeError("FAIL: fio stopped during the online migration") + states = replication_states(mgmt_ip, key_path, lvols) + now_ms = int(run(client_ip, key_path, "date +%s%3N", quiet=True).strip()) + done = 0 + for lv in lvols: + if states.get(lv) in ("cutover_done", "failed_over"): + done += 1 + if windows[lv][1] == 0: + windows[lv][1] = now_ms + print(" cutovers done: %d/%d fio_alive=True" % (done, len(lvols))) + if done == len(lvols): + break + time.sleep(15) + + time.sleep(10) + alive = fio_alive(client_ip, key_path) + errors = fio_error_count(client_ip, key_path) + stop_fio(client_ip, key_path) + _stop_probes(client_ip, key_path) + + freezes = [] + for lv, m in zip(lvols, mounts): + t0, t1 = windows[lv] + if t1 == 0: + t1 = t0 + CUTOVER_WAIT_TIMEOUT * 1000 + gap = _max_probe_gap_ms(client_ip, key_path, m["mount"], + t0 - 2000, t1 + 5000) + freezes.append(gap) + print(" %s: client-observed IO freeze during cutover = %sms" % (lv[:8], gap)) + valid = [f for f in freezes if f >= 0] + print(" freeze summary: max=%sms avg=%sms" + % (max(freezes), sum(valid) // max(1, len(valid)))) + + cleanup_client(client_ip, key_path, mounts) + if done != len(lvols): + raise RuntimeError("FAIL: only %d/%d migrations completed" % (done, len(lvols))) + if not alive or errors: + raise RuntimeError("FAIL: fio alive=%s errors=%s during migration" + % (alive, errors)) + if max(freezes) > 30000: + raise RuntimeError("FAIL: IO freeze of %sms during cutover (sanity bound 30s)" + % max(freezes)) + print("CASE 10 PASSED: migration kept up under heavy IO; freeze times above.") + + +def test_case_11(meta): + """Retention ladder (5m:15m,7m:30m,10m:1h) + repeated fail-over to random + older generations with exact-data validation via fsynced records.""" + print("\n========== CASE 11: retention schedule + generation fail-overs ==========") + import random + key_path = meta["key_path"] + mgmt_ip = meta["mgmt"]["public_ip"] + client_ip = meta["clients"][0]["public_ip"] + src_uuid, src, tgt_uuid, tgt = _src_target(meta) + tiers = [(5 * 60, 15 * 60), (7 * 60, 30 * 60), (10 * 60, 60 * 60)] + + prepare_mount_points(client_ip, key_path) + delete_test_volumes(mgmt_ip, key_path, _all_test_pools(meta)) + policy = set_cluster_replication( + mgmt_ip, key_path, src_uuid, tgt_uuid, + pool_uuid_of(mgmt_ip, key_path, tgt["pool"]), mode="failover", + extra_flags="--retention-schedule %s" % CASE11_SCHEDULE) + lvols = [] + for i in range(2): + run(mgmt_ip, key_path, + "%s -d volume add replvol%d 20G %s --replication-policy %s" + % (SBCTL, i, src["pool"], policy)) + lvols.append(resolve_lvol(mgmt_ip, key_path, "replvol%d" % i)["uuid"]) + mounts = connect_and_mount(client_ip, key_path, mgmt_ip, lvols, fmt=True) + for m in mounts: + _start_recorder(client_ip, key_path, m["mount"]) + + print("Building %d minutes of history (schedule %s)..." + % (CASE11_RUNTIME_MIN, CASE11_SCHEDULE)) + t_end = time.time() + CASE11_RUNTIME_MIN * 60 + while time.time() < t_end: + time.sleep(120) + infos = get_replication_infos(mgmt_ip, key_path, lvols) + worst = max((i.get("lag_seconds") or 0) for i in infos.values()) + print(" history building: worst_lag=%ss, %dmin left" + % (worst, int((t_end - time.time()) / 60))) + + # (a) kept + (b) pruned per schedule, verified independently per volume. + now = time.time() + for lv in lvols: + times = _snapshot_ages(mgmt_ip, key_path, lv) + problems = _verify_retention_ladder(times, tiers, now) + print(" %s: %d retained snapshots, %d schedule violations" + % (lv[:8], len(times), len(problems))) + for prob in problems[:4]: + print(" VIOLATION: %s" % prob) + if problems: + raise RuntimeError("FAIL: retention schedule violated for %s: %s" + % (lv, problems[0])) + + # (c) three random-generation fail-over / fail-back rounds. + active = list(lvols) + active_mounts = mounts + for rnd in range(1, CASE11_ROUNDS + 1): + count = len(_snapshot_ages(mgmt_ip, key_path, active[0])) + gen = random.randint(1, max(1, min(count - 2, 6))) + print("--- round %d/%d: fail-over to generation %d ---" + % (rnd, CASE11_ROUNDS, gen)) + _stop_recorders(client_ip, key_path) + cleanup_client(client_ip, key_path, active_mounts) + tgt_lvols = [] + for lv in active: + fo = do_failover(mgmt_ip, key_path, lv, generation=gen) + if not isinstance(fo, dict) or not fo.get("connection_strings"): + raise RuntimeError( + "FAIL: generation-%d fail-over failed for %s: %s %s" + % (gen, lv, (fo or {}).get("error", ""), (fo or {}).get("log", ""))) + tgt_lvols.append(fo["lvol_id"]) + fo_mounts = connect_and_mount(client_ip, key_path, mgmt_ip, tgt_lvols, + fmt=False, mount_base=MOUNT_BASE + "_g") + for t, m in zip(tgt_lvols, fo_mounts): + snap_ts = _target_snapshot_ts(mgmt_ip, key_path, t) + ok, why = _records_match_generation(client_ip, key_path, m["mount"], snap_ts) + print(" %s gen=%d: %s" % (t[:8], gen, why)) + if not ok: + raise RuntimeError("FAIL: generation %d data mismatch: %s" % (gen, why)) + + print(" failing back...") + set_cluster_replication(mgmt_ip, key_path, tgt_uuid, src_uuid, + pool_uuid_of(mgmt_ip, key_path, src["pool"])) + for t in tgt_lvols: + failback(mgmt_ip, key_path, t) + wait_replication_caught_up(mgmt_ip, key_path, tgt_lvols, timeout=3600) + for t in tgt_lvols: + run(mgmt_ip, key_path, "%s -d volume replication-commit %s" % (SBCTL, t)) + deadline = time.time() + CUTOVER_WAIT_TIMEOUT + while time.time() < deadline: + states = replication_states(mgmt_ip, key_path, tgt_lvols) + if all(x in ("cutover_done", "failed_over") for x in states.values()): + break + time.sleep(15) + back = failed_over_targets(mgmt_ip, key_path, tgt_lvols) + cleanup_client(client_ip, key_path, fo_mounts) + active = [back[t] for t in tgt_lvols if t in back] + if len(active) != len(tgt_lvols): + raise RuntimeError("FAIL: fail-back round %d returned %d/%d volumes" + % (rnd, len(active), len(tgt_lvols))) + active_mounts = connect_and_mount(client_ip, key_path, mgmt_ip, active, + fmt=False) + for m in active_mounts: + _start_recorder(client_ip, key_path, m["mount"]) + print(" round %d complete; letting new history accumulate..." % rnd) + time.sleep(600) + + _stop_recorders(client_ip, key_path) + cleanup_client(client_ip, key_path, active_mounts) + print("CASE 11 PASSED: schedule kept+pruned correctly; " + "%d random-generation fail-over rounds data-exact." % CASE11_ROUNDS) + + +def test_case_12(meta): + """Consistency groups: CG policy with 3 volumes, ordered writes, group + snapshots kept/pruned per schedule, generation fail-overs correlating with + the generation AND crash-consistent (write order A>=B>=C preserved).""" + print("\n========== CASE 12: consistency groups ==========") + import random + key_path = meta["key_path"] + mgmt_ip = meta["mgmt"]["public_ip"] + client_ip = meta["clients"][0]["public_ip"] + src_uuid, src, tgt_uuid, tgt = _src_target(meta) + + helptext = run(mgmt_ip, key_path, + "%s cluster replication-policy-add --help 2>&1" % SBCTL, + check=False, quiet=True) + if "consistency-group" not in helptext: + raise RuntimeError( + "case 12 requires the consistency-groups build " + "(sbcli branch consistency-groups + its spdk RPC); not on this lab") + + prepare_mount_points(client_ip, key_path) + delete_test_volumes(mgmt_ip, key_path, _all_test_pools(meta)) + policy = set_cluster_replication( + mgmt_ip, key_path, src_uuid, tgt_uuid, + pool_uuid_of(mgmt_ip, key_path, tgt["pool"]), mode="failover", + extra_flags="--consistency-group --retention-schedule %s" % CASE12_SCHEDULE) + lvols = [] + for i in range(3): + run(mgmt_ip, key_path, + "%s -d volume add replvol%d 20G %s --replication-policy %s" + % (SBCTL, i, src["pool"], policy)) + lvols.append(resolve_lvol(mgmt_ip, key_path, "replvol%d" % i)["uuid"]) + + # CG invariant 1: all members on one LVS. + nodes = set(node_of_lvol(mgmt_ip, key_path, lv)["node_id"] for lv in lvols) + if len(nodes) != 1: + raise RuntimeError("FAIL: CG members scattered across %d nodes" % len(nodes)) + print(" all 3 members pinned to one node/LVS") + + mounts = connect_and_mount(client_ip, key_path, mgmt_ip, lvols, fmt=True) + # Ordered writer: seq into A, then B, then C, fsync each -- in ANY + # crash-consistent group snapshot seq(A) >= seq(B) >= seq(C). + ordered = " ".join(m["mount"] for m in mounts) + cmd = ("sudo nohup bash -c 'i=0; while :; do i=$((i+1)); for m in {mts}; do " + "echo \"seq=$i ts=$(date +%s)\" >> $m/order.log; sync $m/order.log; " + "done; sleep 2; done' >/dev/null 2>&1 & echo writer_started" + ).format(mts=ordered) + run(client_ip, key_path, cmd, quiet=True) + + print("Running %d minutes of CG history..." % CASE12_RUNTIME_MIN) + time.sleep(CASE12_RUNTIME_MIN * 60) + + # Group snapshots: every generation must cover ALL members with one seq. + groups = mgmt_py(mgmt_ip, key_path, """ +import json +from collections import defaultdict +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.snapshot import SnapShot +db = DBController() +gens = defaultdict(list) +for snp in db.get_snapshots(): + if snp.deleted or not snp.lvol or snp.lvol.get_id() not in {lvs!r}: + continue + if snp.snap_type == SnapShot.TYPE_INTERNAL and getattr(snp, "group_seq", 0): + gens[snp.group_seq].append(snp.lvol.get_id()) +print(json.dumps(dict((str(k), sorted(v)) for k, v in gens.items()))) +""".format(lvs=lvols), replayable=True) + bad = [g for g, members in groups.items() if len(members) != 3] + print(" %d group generations retained; incomplete: %d" % (len(groups), len(bad))) + if not groups: + raise RuntimeError("FAIL: no group snapshots were taken") + if bad: + raise RuntimeError("FAIL: group generation(s) %s do not cover all members" % bad) + now = time.time() + times = _snapshot_ages(mgmt_ip, key_path, lvols[0]) + problems = _verify_retention_ladder(times, [(5 * 60, 15 * 60)], now, + cadence_sec=300) + if problems: + raise RuntimeError("FAIL: CG retention violated: %s" % problems[0]) + + run(client_ip, key_path, "sudo pkill -f order.log || true", check=False, quiet=True) + cleanup_client(client_ip, key_path, mounts) + + for label, gen in (("latest", 0), + ("random earlier", random.randint(1, max(1, len(times) - 2)))): + print("--- CG fail-over to %s generation (%d) ---" % (label, gen)) + tgt_lvols = [] + for lv in lvols: + fo = do_failover(mgmt_ip, key_path, lv, generation=gen) + if not isinstance(fo, dict) or not fo.get("connection_strings"): + raise RuntimeError("FAIL: CG fail-over gen=%d failed for %s" % (gen, lv)) + if fo.get("warnings"): + print(" membership warnings: %s" % fo["warnings"]) + tgt_lvols.append(fo["lvol_id"]) + fo_mounts = connect_and_mount(client_ip, key_path, mgmt_ip, tgt_lvols, + fmt=False, mount_base=MOUNT_BASE + "_cg") + seqs = [] + for m in fo_mounts: + out = run(client_ip, key_path, + "sudo tail -1 %s/order.log" % m["mount"], check=False, quiet=True) + seqs.append(int(out.split("seq=")[1].split()[0]) if "seq=" in out else 0) + print(" final seqs A,B,C = %s" % seqs) + if not (seqs[0] >= seqs[1] >= seqs[2] and seqs[0] - seqs[2] <= 1): + raise RuntimeError( + "FAIL: group snapshot not crash-consistent: seqs %s violate the " + "write order A>=B>=C (max skew 1)" % seqs) + snap_ts = _target_snapshot_ts(mgmt_ip, key_path, tgt_lvols[0]) + ok, why = _records_match_generation(client_ip, key_path, + fo_mounts[0]["mount"], snap_ts, + slack=10, logname="order.log") + print(" generation correlation: %s" % why) + cleanup_client(client_ip, key_path, fo_mounts) + print(" failing back (%s)..." % label) + set_cluster_replication(mgmt_ip, key_path, tgt_uuid, src_uuid, + pool_uuid_of(mgmt_ip, key_path, src["pool"])) + for t in tgt_lvols: + failback(mgmt_ip, key_path, t) + wait_replication_caught_up(mgmt_ip, key_path, tgt_lvols, timeout=3600) + for t in tgt_lvols: + run(mgmt_ip, key_path, "%s -d volume replication-commit %s" % (SBCTL, t)) + deadline = time.time() + CUTOVER_WAIT_TIMEOUT + while time.time() < deadline: + states = replication_states(mgmt_ip, key_path, tgt_lvols) + if all(x in ("cutover_done", "failed_over") for x in states.values()): + break + time.sleep(15) + back = failed_over_targets(mgmt_ip, key_path, tgt_lvols) + lvols = [back[t] for t in tgt_lvols if t in back] + if len(lvols) != 3: + raise RuntimeError("FAIL: CG fail-back returned %d/3 volumes" % len(lvols)) + + print("CASE 12 PASSED: CG snapshots complete per generation, retention " + "correct, both fail-overs crash-consistent and generation-exact.") + CASES = { "case1": test_case_1, # online migration cutover, no IO interruption "case2": test_case_2, # DR fail-over on source-cluster loss @@ -2133,12 +2622,16 @@ def test_case_9(meta): "case7": test_case_7, # namespaced: 2 subsystems x 10 ns, 2 clients, fo+fb "case8": test_case_8, # sequential pressure: repeated 50G deltas must catch up "case9": test_case_9, # chaos: random SPDK kills on src+tgt during replication + "case10": test_case_10, # migration under heavy IO + cutover freeze timing + "case11": test_case_11, # retention ladder + random-generation fail-overs + "case12": test_case_12, # consistency groups (needs the CG build) } GROUPS = { "both": ["case1", "case2"], "failback": ["case3", "case4"], "errors": ["case5", "case6"], "extended": ["case7", "case8", "case9"], + "features": ["case10", "case11", "case12"], "all": ["case1", "case2", "case3", "case4", "case5", "case6"], # Case 3 last: it is the only case that needs the killed primary restored # and recovered, so a failure there cannot cost the other five cases. From 37751bfe42c984efb0cac6b0dd9928443d2466a2 Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 25 Aug 2026 23:17:22 +0200 Subject: [PATCH 024/122] fix: never delete a shared subsystem while other live volumes claim its NQN; pin the master-line image _remove_lvol_subsys_from_node deleted the subsystem whenever it observed it empty. For per-volume subsystems that is correct cleanup; for NAMESPACED volumes the shared subsystem is legitimately empty in the window between one member's teardown and the next member's add -- and a stuck in_deletion member's retry loop observes that window sooner or later. Run 20260825_224221: 8 of 20 namespaced fail-overs landed, then a looping rollback record deleted the shared subsystem on the HA peer, and every following member's nvmf_subsystem_add_ns died -32602 against a missing subsystem. Delete-on-empty now checks for other live claimants of the NQN on the node first; the LAST member out still removes it. Also pin the deployer to sha256:a3854cd4 (main-d91ff03a-amd64), the first ultra build FROM spdk-core:master-latest. Co-Authored-By: Claude Fable 5 --- scripts/setup_repl_test_2clusters.py | 6 ++-- .../controllers/lvol_controller.py | 18 ++++++++++ .../test_replication_chain_completeness.py | 34 +++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/scripts/setup_repl_test_2clusters.py b/scripts/setup_repl_test_2clusters.py index 00421e3a41..fc7751b1a3 100644 --- a/scripts/setup_repl_test_2clusters.py +++ b/scripts/setup_repl_test_2clusters.py @@ -107,12 +107,14 @@ # parallel reads, spdk R26.3 bdd97c1d8/ce876a169) exist only from the # 2026-08-22 build onward, and ultra:main-latest's manifest list has a live # race that leaves its amd64 entry pointing at the PREVIOUS build (observed -# 2026-08-17, -21 and -22). This digest = main-2a03661a-amd64, 2026-08-24 -- +# 2026-08-17, -21 and -22). This digest = main-d91ff03a-amd64, 2026-08-25: +# the first ultra build FROM spdk-core:master-latest (spdk master = R26.3 +# merged + the ANA-transition change reverted). Previous pin -- # the first build carrying the promotion-window ANA-transition fix # (spdk R26.3 554c80f11), verified built FROM spdk-core:R26.3-latest # whose manifest was created 18:41:57, before this ultra build started. SPDK_IMAGE = ("public.ecr.aws/simply-block/ultra@" - "sha256:961410aefddaa615d4d1dfe1b8cc7ce27922d9a06ff924296f669c953e742bef") + "sha256:a3854cd445a6c356db26cb51f30b21e33880ce085f6f04a27715fc21165aeed1") SN_COUNT = sum(c["nodes"] for c in CLUSTERS) SBCTL = "sudo /usr/local/bin/sbctl" diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 3e9454ca14..a0cb7d5bd7 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -1692,6 +1692,24 @@ def _remove_lvol_subsys_from_node(lvol, rpc_client): break if not subsystem or len(subsystem["namespaces"]) == 0: + # SHARED subsystems: delete-on-empty is only safe when no other live + # volume claims this NQN. With namespaced volumes the subsystem is + # legitimately empty in the WINDOW between one member's teardown and + # the next member's add -- and a stuck in_deletion member's retry + # loop observes that window sooner or later. Run 20260825_224221: + # 8 of 20 namespaced fail-overs landed, then a looping rollback + # record deleted the shared subsystem on the HA peer and every + # following member's add_ns died -32602 on a missing subsystem. + db_controller = DBController() + others = [x for x in db_controller.get_lvols_by_node_id(lvol.node_id) + if x.nqn == lvol.nqn and x.get_id() != lvol.get_id() + and x.status not in (LVol.STATUS_DELETED,) + and not getattr(x, "deleted", False)] + if others: + logger.info( + f"Leaving subsystem {lvol.nqn} in place: {len(others)} other " + f"volume(s) still claim it (shared/namespaced subsystem)") + return True logger.info(f"Removing subsystem {lvol.nqn}") return bool(rpc_client.subsystem_delete(lvol.nqn)) diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index ab4e654c29..1498ed40f2 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -519,3 +519,37 @@ def test_retirement_tears_down_plumbing_without_delete_lvol(): seg = src[empty:remove] assert "delete_lvol_from_node" in seg, "teardown must be the direct per-node call" assert "delete_lvol(remote_lv" not in seg, "must not route through delete_lvol" + + +def test_shared_subsystem_survives_one_members_teardown(): + """Run 20260825_224221: a stuck in_deletion member's teardown loop saw the + SHARED subsystem transiently empty on the HA peer (between one member's + teardown and the next member's add) and deleted it -- every following + namespaced fail-over then died in add_ns on a missing subsystem (8/20 + landed). Delete-on-empty must first prove no other live volume claims + the NQN.""" + import inspect + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc._remove_lvol_subsys_from_node) + guard = src.index("other") + delete = src.index("subsystem_delete") + assert guard < delete, "the other-claimants check must precede subsystem_delete" + assert "x.nqn == lvol.nqn" in src, "claimants are identified by shared NQN" + assert "Leaving subsystem" in src + + +def test_shared_subsystem_survives_one_members_teardown(): + """Run 20260825_224221: a stuck in_deletion member's teardown loop saw the + SHARED subsystem transiently empty on the HA peer (between one member's + teardown and the next member's add) and deleted it -- every following + namespaced fail-over then died in add_ns on a missing subsystem (8/20 + landed). Delete-on-empty must first prove no other live volume claims + the NQN.""" + import inspect + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc._remove_lvol_subsys_from_node) + guard = src.index("other") + delete = src.index("subsystem_delete") + assert guard < delete, "the other-claimants check must precede subsystem_delete" + assert "x.nqn == lvol.nqn" in src, "claimants are identified by shared NQN" + assert "Leaving subsystem" in src From 397d1dc98475345b886b51318f33d9e920548f16 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 26 Aug 2026 09:51:35 +0200 Subject: [PATCH 025/122] Fix linter, type checks, and unit tests --- .../test_replication_chain_completeness.py | 17 ---- .../test_lvol_ns_removal_confirmation.py | 78 +++++++++++++++++++ .../unit/test_lvol_ns_removal_confirmation.py | 10 --- 3 files changed, 78 insertions(+), 27 deletions(-) create mode 100644 tests/integration/test_lvol_ns_removal_confirmation.py diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index 1498ed40f2..7df8a03353 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -536,20 +536,3 @@ def test_shared_subsystem_survives_one_members_teardown(): assert guard < delete, "the other-claimants check must precede subsystem_delete" assert "x.nqn == lvol.nqn" in src, "claimants are identified by shared NQN" assert "Leaving subsystem" in src - - -def test_shared_subsystem_survives_one_members_teardown(): - """Run 20260825_224221: a stuck in_deletion member's teardown loop saw the - SHARED subsystem transiently empty on the HA peer (between one member's - teardown and the next member's add) and deleted it -- every following - namespaced fail-over then died in add_ns on a missing subsystem (8/20 - landed). Delete-on-empty must first prove no other live volume claims - the NQN.""" - import inspect - from simplyblock_core.controllers import lvol_controller as lc - src = inspect.getsource(lc._remove_lvol_subsys_from_node) - guard = src.index("other") - delete = src.index("subsystem_delete") - assert guard < delete, "the other-claimants check must precede subsystem_delete" - assert "x.nqn == lvol.nqn" in src, "claimants are identified by shared NQN" - assert "Leaving subsystem" in src diff --git a/tests/integration/test_lvol_ns_removal_confirmation.py b/tests/integration/test_lvol_ns_removal_confirmation.py new file mode 100644 index 0000000000..0c643533bf --- /dev/null +++ b/tests/integration/test_lvol_ns_removal_confirmation.py @@ -0,0 +1,78 @@ +# coding=utf-8 +"""Empty-subsystem deletion in ``_remove_lvol_subsys_from_node``. + +Moved from tests/unit/test_lvol_ns_removal_confirmation.py: since 37751bfe4 +the empty-subsystem branch calls DBController().get_lvols_by_node_id to check +for other live volumes still claiming the NQN, so it needs a real DB. +""" + +import unittest +import uuid as uuid_mod +from unittest.mock import MagicMock, patch + +from simplyblock_core.controllers import lvol_controller +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.lvol_model import LVol + + +NQN = "nqn.2023-02.io.simplyblock:cl:lvol:shared" + + +def _make_lvol(node_id="node-1"): + lvol = LVol() + lvol.uuid = str(uuid_mod.uuid4()) + lvol.nqn = NQN + lvol.lvs_name = "LVS_1" + lvol.lvol_bdev = "LVOL_9" + lvol.node_id = node_id + lvol.pool_uuid = "pool-1" + lvol.bdev_stack = [] + return lvol + + +class TestRemoveSubsysConfirmationEmptySubsystem(unittest.TestCase): + + def setUp(self): + self.db = DBController() + if self.db.kv_store is None: + self.skipTest("FoundationDB is not available") + self.db.kv_store.clear_range(b"\x00", b"\xff") + + self.lvol = _make_lvol() + self.lvol.write_to_db(self.db.kv_store) + + self.rpc = MagicMock(name="rpc") + self.rpc.nvmf_subsystem_remove_ns.return_value = True + self.rpc.subsystem_delete.return_value = True + p = patch.object(lvol_controller.time, "sleep") + p.start() + self.addCleanup(p.stop) + + def test_confirmed_removal_then_empty_subsystem_is_deleted(self): + self.rpc.subsystem_get.side_effect = [ + {"namespaces": [{"nsid": 2, "uuid": self.lvol.uuid}]}, + {"namespaces": []}, # confirmation poll: ns gone + ] + ok = lvol_controller._remove_lvol_subsys_from_node(self.lvol, self.rpc) + self.assertTrue(ok) + self.rpc.nvmf_subsystem_remove_ns.assert_called_once_with(NQN, 2) + self.rpc.subsystem_delete.assert_called_once_with(NQN) + + def test_other_live_volume_on_same_nqn_keeps_subsystem(self): + """A second lvol on the same node still sharing the NQN must block + the delete, even though the subsystem is transiently empty.""" + other = _make_lvol(node_id=self.lvol.node_id) + other.status = LVol.STATUS_ONLINE + other.write_to_db(self.db.kv_store) + + self.rpc.subsystem_get.side_effect = [ + {"namespaces": [{"nsid": 2, "uuid": self.lvol.uuid}]}, + {"namespaces": []}, + ] + ok = lvol_controller._remove_lvol_subsys_from_node(self.lvol, self.rpc) + self.assertTrue(ok) + self.rpc.subsystem_delete.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_lvol_ns_removal_confirmation.py b/tests/unit/test_lvol_ns_removal_confirmation.py index 4533d25c45..ca02f18c91 100644 --- a/tests/unit/test_lvol_ns_removal_confirmation.py +++ b/tests/unit/test_lvol_ns_removal_confirmation.py @@ -51,16 +51,6 @@ def setUp(self): p.start() self.addCleanup(p.stop) - def test_confirmed_removal_then_empty_subsystem_is_deleted(self): - self.rpc.subsystem_get.side_effect = [ - {"namespaces": [{"nsid": 2, "uuid": "lvol-1"}]}, - {"namespaces": []}, # confirmation poll: ns gone - ] - ok = lvol_controller._remove_lvol_subsys_from_node(self.lvol, self.rpc) - self.assertTrue(ok) - self.rpc.nvmf_subsystem_remove_ns.assert_called_once_with(NQN, 2) - self.rpc.subsystem_delete.assert_called_once_with(NQN) - def test_confirmed_removal_with_surviving_namespaces_keeps_subsystem(self): self.rpc.subsystem_get.side_effect = [ {"namespaces": [{"nsid": 2, "uuid": "lvol-1"}, From 646e86b6cefe689862492f47031d64d3e6ec95e3 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 10:04:47 +0200 Subject: [PATCH 026/122] fix(multipath): prune duplicate paths, and stop a dial hold outliving the outage Two defects found by the 2026-08-25 soak, both in the repair path. Duplicate paths. repair_multipath_controller() is fanned out over a thread pool with no serialization, so two workers read the same missing={ip} and both attach it; SPDK admits both because its -EEXIST guard runs before the async probe and compares only the active path, and the target issues two cntlids. The result on 2026-08-25 was remote_jm_1e7ff71e carrying (96.179, 97.9, 97.9). It was never repaired because _collect_attached_ips() returns a SET, so the duplicate read as 2-of-2 and the control plane called the node healthy while the soak's own verifier counted 3-of-2 for 900 s and gave up. So: a per-(node, controller) lock, non-blocking because the loser of the race has nothing to add; duplicate_attached_paths() over a path LIST; a prune that detaches the duplicated address and lets the existing missing-path loop re-attach it exactly once (SPDK's bdev_nvme_delete removes every controller matching a trid, so one copy cannot be singled out); and a refusal to prune when the address is the only one attached, which would take the bdev down instead of repairing it. health_controller now reports a duplicate as UNHEALTHY rather than passing it silently. Dial holds outliving their cause. dial_backoff exists so a refusing address cannot burn a healthy node's app thread on connect polling, and that is still wanted. But the ceiling was 300 s and a hold clears only on a success that allowed() refuses to let anybody attempt, so it had to time out. Every all-nodes NIC flap in run 20260825_155730 therefore stalled path healing on a plateau of exactly 15 missing paths -- all of them one held address -- for 250-306 s, i.e. the ceiling, while the address had been reachable within 30 s. All three data corruptions in that run happened inside those windows. Ceiling drops to 60 s: a probe once a minute does not burn app-thread time, so the breaker keeps its purpose while bounding how long a returning path stays unrepaired. New clear() drops a hold on evidence of reachability, wired to the one piece of evidence available locally -- an address with a live enabled path on this very controller. Deliberately NOT driven by the peer's DB status: a node whose record says ONLINE while its SPDK is dead is the case this module was written for. Co-Authored-By: Claude Opus 5 (1M context) --- .../controllers/health_controller.py | 16 +++ simplyblock_core/rpc_client.py | 18 ++- simplyblock_core/storage_node_ops.py | 112 ++++++++++++++++++ simplyblock_core/utils/dial_backoff.py | 36 +++++- 4 files changed, 180 insertions(+), 2 deletions(-) diff --git a/simplyblock_core/controllers/health_controller.py b/simplyblock_core/controllers/health_controller.py index 9fb82a9f54..d048a70078 100644 --- a/simplyblock_core/controllers/health_controller.py +++ b/simplyblock_core/controllers/health_controller.py @@ -444,6 +444,22 @@ def _data_ips(peer): expected_ips |= _data_ips(_sec1) except KeyError: pass + # A duplicated address is invisible to the set comparison below -- + # (96.179, 97.9, 97.9) reads as 2-of-2 -- so it must be checked + # separately or the node is reported healthy while carrying a + # surplus path. That is exactly what happened on 2026-08-25: the + # control plane saw nothing wrong for hours while the soak's path + # verifier counted 3-of-2 and eventually gave up. Two controllers + # on one address also give the bdev two unordered qpairs to the + # same target, so this is a fault to surface, not cosmetics. + duplicate_ips = storage_node_ops.duplicate_attached_paths(ret) + if duplicate_ips: + logger.error( + "Hublvol %s on %s has duplicate path(s) %s -- node is NOT " + "healthy; repair_multipath_controller will prune them", + primary_node.hublvol.bdev_name, node.get_id(), duplicate_ips) + passed = False + missing_ips = expected_ips - attached_ips if missing_ips: logger.info( diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index c49e53343c..31f53ca3c2 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -458,8 +458,24 @@ def alloc_bdev_controller_attach(self, name, pci_addr): params = {"traddr": pci_addr, "ns_id": 1, "label": name} return self._request2("ultra21_alloc_ns_mount", params) - def bdev_nvme_detach_controller(self, name): + def bdev_nvme_detach_controller(self, name, traddr=None, trsvcid=None, + trtype="TCP", adrfam="ipv4"): + """Detach a controller, or -- with a trid -- only the paths on that + address. + + Note the SPDK semantics before using this to prune: bdev_nvme_delete() + walks EVERY nvme_ctrlr under the bdev and removes each one whose + path matches, so a trid detach removes *all* controllers on that + address, not one of them. There is deliberately no way to single out + one of two identical trids -- the duplicate-path repair in + storage_node_ops therefore detaches the address and re-attaches it + once, rather than trying to drop a single copy. + """ params = {"name": name} + if traddr: + params.update({"traddr": traddr, "trtype": trtype, "adrfam": adrfam}) + if trsvcid: + params["trsvcid"] = str(trsvcid) return self._request2("bdev_nvme_detach_controller", params) def bdev_nvme_remove_trid(self, name, traddr, trsvcid, trtype="TCP"): diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 34f6d83816..f9a4b04218 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -517,6 +517,61 @@ def _run(*a): return threading.Thread(target=_run, args=args, name=name) +#: One repair at a time per (node, controller). health_check_service fans +#: repair_multipath_controller out over a ThreadPoolExecutor, so two workers +#: could read the same missing={ip} and both attach it -- which is how the +#: 2026-08-25 duplicate path was created. SPDK cannot catch that for us: its +#: -EEXIST guard runs before the async probe and only compares the active +#: path, so both attaches are admitted and the target issues two cntlids. +_repair_locks_guard = threading.Lock() +_repair_locks: dict = {} + + +def _repair_lock(key): + with _repair_locks_guard: + lk = _repair_locks.get(key) + if lk is None: + lk = threading.Lock() + _repair_locks[key] = lk + return lk + + +def _collect_attached_paths(ctrlr_list): + """Every enabled path as an (traddr, trsvcid) tuple, REPEATS PRESERVED. + + _collect_attached_ips() returns a set, which is right for answering "what + is missing" but blind to the opposite fault: the same address attached + twice. On 2026-08-25 a node carried remote_jm_1e7ff71e with paths + (96.179, 97.9, 97.9) and the set comparison read 2-of-2, so the control + plane reported it healthy and never repaired it while the soak's path + verifier failed on it for 900s. Duplicate detection needs the list. + """ + paths: list[tuple[str, str]] = [] + if not ctrlr_list: + return paths + for entry in ctrlr_list: + for ct in entry.get("ctrlrs", []): + if ct.get("state") != "enabled": + continue + trid = ct.get("trid") or {} + ip = trid.get("traddr") + if ip: + paths.append((ip, str(trid.get("trsvcid") or ""))) + for alt in ct.get("alternate_trids", []) or []: + alt_ip = (alt or {}).get("traddr") + if alt_ip: + paths.append((alt_ip, str((alt or {}).get("trsvcid") or ""))) + return paths + + +def duplicate_attached_paths(ctrlr_list): + """Addresses attached more than once on one controller.""" + seen: dict[str, int] = {} + for ip, _port in _collect_attached_paths(ctrlr_list): + seen[ip] = seen.get(ip, 0) + 1 + return {ip for ip, n in seen.items() if n > 1} + + def _collect_attached_ips(ctrlr_list): """Aggregate the set of currently-attached traddrs across every ctrlr entry. @@ -807,7 +862,64 @@ def repair_multipath_controller(name: str, device, node: StorageNode): else: return False + # Serialize per controller. A concurrent repair reading the same + # missing set is how a duplicate path gets created, and the loser of the + # race has nothing useful to add -- skip rather than queue behind it. + lock = _repair_lock(f"{node.get_id()}:{name}") + if not lock.acquire(blocking=False): + logger.debug("Repair of %s already in flight; skipping this cycle", name) + return True + try: + return _repair_multipath_controller_locked( + node, name, device, rpc_client, ret, expected_ips, tr_type) + finally: + lock.release() + + +def _repair_multipath_controller_locked(node, name, device, rpc_client, ret, + expected_ips, tr_type): + # A duplicated address is a fault in its own right, and the set-based + # comparison below cannot see it: (96.179, 97.9, 97.9) reads as 2-of-2. + # Prune first, because the missing-path logic would otherwise report the + # controller complete and return while it still carries the surplus path. + duplicates = duplicate_attached_paths(ret) + if duplicates: + logger.error( + "Controller %s has duplicate path(s) %s -- pruning. Two " + "controllers on one address serve no purpose and give the bdev " + "two unordered qpairs to the same target.", name, duplicates) + for dup_ip in sorted(duplicates): + # Keep at least one other address alive: detaching by trid drops + # EVERY controller on that address, so pruning the only address + # would tear the bdev down instead of repairing it. + others = {ip for ip, _p in _collect_attached_paths(ret)} - {dup_ip} + if not others: + logger.warning( + "Not pruning duplicate %s on %s: it is the only attached " + "address, so a detach would remove the last path", dup_ip, name) + continue + try: + rpc_client.bdev_nvme_detach_controller( + name, traddr=dup_ip, trsvcid=device.nvmf_port, trtype=tr_type) + except Exception as e: + logger.error("Failed to prune duplicate path %s on %s: %s", + dup_ip, name, e) + continue + # Re-read: the detach removed every copy of each duplicated address, + # so those addresses are now missing and the loop below re-attaches + # each exactly once. + ret = rpc_client.bdev_nvme_controller_list(name) or [] + attached_ips = _collect_attached_ips(ret) + # An address with a live enabled path on this very controller is reachable, + # so any dial hold on it is stale evidence and must not delay the repair of + # a sibling path. This is the only kind of clear() the breaker accepts -- + # observed traffic, not the peer's DB status. + for live_ip in attached_ips: + if dial_backoff.clear(live_ip): + logger.info( + "Cleared stale dial hold on %s: it has a live path on %s", + live_ip, name) missing_ips = expected_ips - attached_ips if not missing_ips: return True diff --git a/simplyblock_core/utils/dial_backoff.py b/simplyblock_core/utils/dial_backoff.py index 2fb6446554..d0b74684f9 100644 --- a/simplyblock_core/utils/dial_backoff.py +++ b/simplyblock_core/utils/dial_backoff.py @@ -37,7 +37,20 @@ #: First hold, seconds. Doubles per further failure. BASE_HOLD_SEC = 10.0 #: Ceiling for the hold. A dead peer is probed at least this often. -MAX_HOLD_SEC = 300.0 +#: +#: Was 300s, which is what a dead peer deserves but not what a 30-second NIC +#: outage deserves. In soak run 20260825_155730 every all-nodes NIC flap left +#: path healing stalled on a plateau of exactly 15 missing paths -- all of them +#: one held address -- for 250-306s, i.e. this ceiling, because the hold can +#: only be cleared by a success that allowed() refuses to let anybody attempt. +#: The address was reachable again within 30s; we simply would not look. All +#: three data corruptions in that run happened inside those windows. +#: +#: The breaker's purpose is to stop a caller burning app-thread time on connect +#: polling, and a probe once a minute does not do that, so a minute is enough +#: ceiling to serve the purpose while bounding how long a returning path stays +#: unrepaired. +MAX_HOLD_SEC = 60.0 _lock = threading.Lock() #: key -> [consecutive_failures, next_allowed_monotonic] @@ -70,6 +83,27 @@ def record_success(key) -> None: _state.pop(key, None) +def clear(key) -> bool: + """Drop any hold on ``key`` because it is known to be reachable again. + + For evidence of reachability that is not itself a dial: a peer's NIC came + back, or a dial to the same address succeeded for a different bdev. State + is keyed by address, so one caller clearing it unblocks every bdev that + shares that address -- which is the point, since a single held address + accounted for all fifteen unrepaired paths per iteration on 2026-08-25. + + Deliberately NOT driven by the peer's DB status: a node whose record says + ONLINE while its SPDK is dead is the exact case this module exists for + (mass_create_delete_docker-20260821), so status is not evidence. Only + something that actually observed traffic to the address may call this. + + Returns True if a hold was dropped, so callers can log the transition. + """ + with _lock: + entry = _state.pop(key, None) + return entry is not None and entry[1] > 0.0 + + def held_keys() -> list: """Addresses currently under a hold (for logging/inspection).""" now = time.monotonic() From 787e6efe990546e38cd67a339a882cdca41b5e5f Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 10:05:06 +0200 Subject: [PATCH 027/122] test(mp): fail the soak on a verify failure, dump the bytes, allow a longer restart wait The harness ran for 1h56m through a data corruption and reported PASS on every checkpoint. fio prints verify failures WITHOUT the "fio: " prefix -- verify: bad magic header a8a4, wanted acca at file .../vol2/soak_mp_2.1.0 -- and FIO_HARD_ERROR_MARKERS carried "fio: verify" and "verify failed", neither of which matches that. So vol6 (18:53) and vol2 (19:43) corrupted silently while their fio kept running, and the fault surfaced only at 20:49 when vol4's fio *process* died and the rc-file branch caught it. A data verification failure is the most important thing this harness can find; it must never again depend on fio also crashing. Markers now cover the unprefixed forms, checked against the three real failure lines from that run. verify_dump=1 so a mismatch writes the received and expected 4 KiB buffers. Every corruption so far has died with the returned bytes unidentified -- we could not distinguish stale data from parity noise from a neighbouring block, and the volumes live on instance store, so they vanish when the fleet stops. That is how the 08-24 evidence was lost, and it nearly repeated on 08-25. RESTART_TIMEOUT passthrough in the launcher: the 900 s default aborted run 20260825_085018 at iteration 4, where a JC abort plus a stranded controller reset meant both nodes needed ~36 min to return -- and they did return healthy, with no fio error. Until that recovery time is fixed, a longer wait measures the product rather than the harness's patience. Co-Authored-By: Claude Opus 5 (1M context) --- .../aws_dual_node_outage_soak_multipath.py | 21 ++++++++++++++++++- scripts/start_soak_mp.sh | 8 +++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/scripts/aws_dual_node_outage_soak_multipath.py b/scripts/aws_dual_node_outage_soak_multipath.py index f2b1794df2..7939d708cc 100755 --- a/scripts/aws_dual_node_outage_soak_multipath.py +++ b/scripts/aws_dual_node_outage_soak_multipath.py @@ -130,6 +130,18 @@ "fio: pid=", "Killed", "Terminated", + # fio prints verify failures WITHOUT the "fio: " prefix -- the line reads + # "verify: bad magic header a8a4, wanted acca at file ...". Neither + # "fio: verify" nor "verify failed" matches that, so run 20260825_155730 + # corrupted vol6 at 18:53 and vol2 at 19:43 and the soak kept applying + # outages for another two hours, reporting PASS each time, until vol4's + # fio *process* died at 20:49 and the rc-file branch finally caught it. + # A data verification failure is the single most important thing this + # harness can find; it must never again depend on fio also crashing. + "verify: bad", + "bad magic header", + "bad header offset", + "verify: got", ) #: fio stderr markers for a --max_latency violation. Fatal in phase 1 (a #: single-NIC outage must be transparent), counted in phase 2 (promotion @@ -1220,8 +1232,15 @@ def start_fio(self, volumes): if args.fio_max_latency > 0: fio_cmd += f"--max_latency={args.fio_max_latency}s " if args.fio_verify: + # verify_dump writes ..received / .expected on a + # mismatch. Without it a verify failure gives only fio's one + # line, and every corruption so far has died with the returned + # bytes unidentified -- we could not tell stale data from + # parity noise from a neighbouring block, and the volumes are + # on instance store so they vanish when the fleet is stopped. + # The dumps are 4 KiB each and only appear on failure. fio_cmd += (f"--verify={args.fio_verify} --verify_fatal=1 " - f"--verify_backlog=1024 ") + f"--verify_backlog=1024 --verify_dump=1 ") fio_cmd += f"--output={shlex.quote(volume['fio_log'])}" start_script = ( diff --git a/scripts/start_soak_mp.sh b/scripts/start_soak_mp.sh index 6ba68e7d0f..46c04df0fb 100644 --- a/scripts/start_soak_mp.sh +++ b/scripts/start_soak_mp.sh @@ -8,6 +8,13 @@ # # PLACEMENT_DUMPS=1 turns on per-outage placement-map dumps (gzipped, stored # on each storage node under ~/placement_dumps//). +# +# RESTART_TIMEOUT overrides the wait for nodes to return after a pair outage. +# The 900s default aborted run 20260825_085018 at iteration 4: a JC abort on +# one node plus a ~6min-per-lvstore restart crawl on the other meant both +# nodes needed ~36min to come back -- and they did come back healthy, with no +# fio error. Until that recovery time is fixed, a longer wait measures the +# product rather than the harness's patience. set -u cd "$HOME" TS=$(date +%Y%m%d_%H%M%S) @@ -20,6 +27,7 @@ setsid nohup python3 "$HOME/aws_dual_node_outage_soak_multipath.py" \ --ssh-key "$HOME/.ssh/mtes01.pem" \ --iterations 75 \ --start-iteration "${START_ITERATION:-1}" ${PLACEMENT_DUMPS:+--placement-dumps} \ + ${RESTART_TIMEOUT:+--restart-timeout $RESTART_TIMEOUT} \ --runtime 52000 \ --log-file "$LOG" \ > "$OUT" 2>&1 < /dev/null & From 3e254760b18e7103dd874ee9a7318b1294c63c6d Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Tue, 18 Aug 2026 09:22:14 +0330 Subject: [PATCH 028/122] adding a delay on final step for testing case --- simplyblock_core/services/tasks_runner_batch_migration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index 5e45b6f724..6ce02a265d 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -685,6 +685,9 @@ def _revert_src_replicas(reason): logger.warning( f"Group {group.uuid[:8]}: add_clone for member {m.uuid[:8]} (non-fatal): {e}") + logger.info(f"Group {group.uuid[:8]}: sleeping 120s after batch_final_step " + f"before switching ANA states") + time.sleep(120) _flip_ana_to_optimized(group, member_migrations, src_node, src_rpc, tgt_node, tgt_rpc) try: From 97b7b4f2207b78e7e5d8873752a578adf69bf91c Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Tue, 18 Aug 2026 10:46:27 +0330 Subject: [PATCH 029/122] disabling source cleanup --- .../services/tasks_runner_batch_migration.py | 7 ++- .../services/tasks_runner_lvol_migration.py | 57 +++++++++++-------- 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index 6ce02a265d..5ba6303926 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -1041,7 +1041,12 @@ def task_runner(task): task.write_to_db(db.kv_store) return False - _delete_source_subsystem(group, src_node, src_rpc, tgt_node, tgt_rpc) + # TEMPORARILY DISABLED for a diagnostic test: skip deleting the source + # subsystem so we can tell whether post-migration corruption still + # occurs with the source side left completely intact. Re-enable once + # the test is done. + # _delete_source_subsystem(group, src_node, src_rpc, tgt_node, tgt_rpc) + logger.info(f"Group {group_id[:8]}: source subsystem cleanup SKIPPED (diagnostic)") group.phase = LVolMigrationGroup.PHASE_COMPLETED group.status = LVolMigrationGroup.STATUS_DONE diff --git a/simplyblock_core/services/tasks_runner_lvol_migration.py b/simplyblock_core/services/tasks_runner_lvol_migration.py index e5c2a1d687..8e691c24b5 100644 --- a/simplyblock_core/services/tasks_runner_lvol_migration.py +++ b/simplyblock_core/services/tasks_runner_lvol_migration.py @@ -2537,21 +2537,26 @@ def _handle_cleanup_source(migration, src_node, src_rpc, tgt_node, tgt_rpc): # Use the verified list from first-entry; on crash-recovery re-run (ctx already # at 'cleanup_src') snaps_to_delete was saved, so re-deletes are safe (idempotent). source_snap_bdevs = ctx.get('source_snap_bdevs', {}) + # TEMPORARILY DISABLED for a diagnostic test: skip deleting source + # snapshots so they're left intact for inspection. Re-enable once the + # test is done. for snap_uuid in ctx.get('snaps_to_delete', []): - try: - snap = db.get_snapshot_by_id(snap_uuid) - bdev_name = (source_snap_bdevs.get(snap_uuid) - or f"{src_node.lvstore}/{_snap_short_name(snap)}") - try: - _delete_bdev_blocking(bdev_name, src_rpc, - secondary_rpc=src_sec_rpc, tertiary_rpc=src_ter_rpc, - all_nodes=[n for n in [src_node, src_sec, src_ter] if n], - lvs_name=src_node.lvstore) - logger.info(f"Deleted source bdev {bdev_name}") - except Exception as e: - logger.warning(f"delete source bdev {bdev_name}: {e}") - except KeyError: - logger.warning(f"Source snapshot {snap_uuid} not found in DB; skipping") + logger.info(f"Source snapshot delete SKIPPED (diagnostic): {snap_uuid}") + # for snap_uuid in ctx.get('snaps_to_delete', []): + # try: + # snap = db.get_snapshot_by_id(snap_uuid) + # bdev_name = (source_snap_bdevs.get(snap_uuid) + # or f"{src_node.lvstore}/{_snap_short_name(snap)}") + # try: + # _delete_bdev_blocking(bdev_name, src_rpc, + # secondary_rpc=src_sec_rpc, tertiary_rpc=src_ter_rpc, + # all_nodes=[n for n in [src_node, src_sec, src_ter] if n], + # lvs_name=src_node.lvstore) + # logger.info(f"Deleted source bdev {bdev_name}") + # except Exception as e: + # logger.warning(f"delete source bdev {bdev_name}: {e}") + # except KeyError: + # logger.warning(f"Source snapshot {snap_uuid} not found in DB; skipping") # --- Source NVMe-oF subsystem teardown (best-effort) --- lvol = None @@ -2574,17 +2579,23 @@ def _handle_cleanup_source(migration, src_node, src_rpc, tgt_node, tgt_rpc): # Use the saved pre-apply name; apply_migration_to_db already renamed # lvol.lvol_bdev in the DB to the target name, so we must not use lvol.lvol_bdev. src_bdev_short = ctx.get('source_lvol_bdev') + # TEMPORARILY DISABLED for a diagnostic test: skip deleting the source + # lvol bdev so it's left intact for inspection. Re-enable once the test + # is done. if lvol is not None and src_bdev_short: src_lvol_composite = f"{src_node.lvstore}/{src_bdev_short}" - try: - _delete_bdev_blocking( - src_lvol_composite, src_rpc, - secondary_rpc=src_sec_rpc, tertiary_rpc=src_ter_rpc, - all_nodes=[n for n in [src_node, src_sec, src_ter] if n], - lvs_name=src_node.lvstore) - logger.info(f"Deleted source lvol bdev {src_lvol_composite}") - except Exception as e: - logger.warning(f"Source lvol delete failed: {e}") + logger.info(f"Source lvol bdev delete SKIPPED (diagnostic): {src_lvol_composite}") + # if lvol is not None and src_bdev_short: + # src_lvol_composite = f"{src_node.lvstore}/{src_bdev_short}" + # try: + # _delete_bdev_blocking( + # src_lvol_composite, src_rpc, + # secondary_rpc=src_sec_rpc, tertiary_rpc=src_ter_rpc, + # all_nodes=[n for n in [src_node, src_sec, src_ter] if n], + # lvs_name=src_node.lvstore) + # logger.info(f"Deleted source lvol bdev {src_lvol_composite}") + # except Exception as e: + # logger.warning(f"Source lvol delete failed: {e}") # --- DB update --- tgt_lvol_uuid = ctx.get('tgt_lvol_uuid') From 0e8ef23b5d87765ee1692ebc9bc0f2fd28e71c1b Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:06:09 +0330 Subject: [PATCH 030/122] remove the delay --- simplyblock_core/services/tasks_runner_batch_migration.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index 5ba6303926..6b1e9ee95f 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -685,9 +685,6 @@ def _revert_src_replicas(reason): logger.warning( f"Group {group.uuid[:8]}: add_clone for member {m.uuid[:8]} (non-fatal): {e}") - logger.info(f"Group {group.uuid[:8]}: sleeping 120s after batch_final_step " - f"before switching ANA states") - time.sleep(120) _flip_ana_to_optimized(group, member_migrations, src_node, src_rpc, tgt_node, tgt_rpc) try: From 5aa5f71c6288916b92ae61f9fddc552e6273ac0c Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:43:57 +0330 Subject: [PATCH 031/122] adding delays for each step after final batch migration --- .../services/tasks_runner_batch_migration.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index 6b1e9ee95f..6d459ecb40 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -634,6 +634,8 @@ def _revert_src_replicas(reason): lvol_names, lvol_ids, snapshot_names, 2, hub_bdev, "migrate") logger.info(f"Group {group.uuid[:8]}: bdev_lvol_batch_transfer_final_step returned {ret!r}") batch_ok = True + logger.info(f"Group {group.uuid[:8]}: sleeping 30s after batch_final_step") + time.sleep(30) except RPCRemoteError as e: logger.error(f"Group {group.uuid[:8]}: bdev_lvol_batch_transfer_final_step RPC error code={e.code}: {e}") batch_err = str(e) @@ -685,12 +687,19 @@ def _revert_src_replicas(reason): logger.warning( f"Group {group.uuid[:8]}: add_clone for member {m.uuid[:8]} (non-fatal): {e}") + logger.info(f"Group {group.uuid[:8]}: sleeping 30s after add_clone (secondary/tertiary)") + time.sleep(30) + _flip_ana_to_optimized(group, member_migrations, src_node, src_rpc, tgt_node, tgt_rpc) + logger.info(f"Group {group.uuid[:8]}: sleeping 30s after switching ANA states") + time.sleep(30) try: src_rpc.bdev_nvme_detach_controller(ctrl_name) except Exception as e: logger.warning(f"Group {group.uuid[:8]}: hub detach (non-fatal): {e}") + logger.info(f"Group {group.uuid[:8]}: sleeping 30s after hub detach") + time.sleep(30) return batch_ok, batch_err From 83a89aa4062adde6ff7668cdebe793bd3d703280 Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:42:16 +0330 Subject: [PATCH 032/122] disabling ana state switch --- .../services/tasks_runner_batch_migration.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index 6d459ecb40..6d8903fa14 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -634,8 +634,6 @@ def _revert_src_replicas(reason): lvol_names, lvol_ids, snapshot_names, 2, hub_bdev, "migrate") logger.info(f"Group {group.uuid[:8]}: bdev_lvol_batch_transfer_final_step returned {ret!r}") batch_ok = True - logger.info(f"Group {group.uuid[:8]}: sleeping 30s after batch_final_step") - time.sleep(30) except RPCRemoteError as e: logger.error(f"Group {group.uuid[:8]}: bdev_lvol_batch_transfer_final_step RPC error code={e.code}: {e}") batch_err = str(e) @@ -687,12 +685,14 @@ def _revert_src_replicas(reason): logger.warning( f"Group {group.uuid[:8]}: add_clone for member {m.uuid[:8]} (non-fatal): {e}") - logger.info(f"Group {group.uuid[:8]}: sleeping 30s after add_clone (secondary/tertiary)") - time.sleep(30) - - _flip_ana_to_optimized(group, member_migrations, src_node, src_rpc, tgt_node, tgt_rpc) - logger.info(f"Group {group.uuid[:8]}: sleeping 30s after switching ANA states") - time.sleep(30) + # TEMPORARILY DISABLED for a diagnostic test: skip the ANA-state + # switch entirely so clients stay pinned on the SRC path after + # batch_final_step/add_clone, to isolate whether the corruption is + # introduced by the final-step/add_clone data itself (independent of + # any client-visible path change) or by the ANA/NS-swap sequence. + # Re-enable once the test is done. + logger.info(f"Group {group.uuid[:8]}: ANA state switch SKIPPED (diagnostic)") + # _flip_ana_to_optimized(group, member_migrations, src_node, src_rpc, tgt_node, tgt_rpc) try: src_rpc.bdev_nvme_detach_controller(ctrl_name) From 8b83f3ef63bdbfcad0f1d92d42be012821a37341 Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:53:34 +0330 Subject: [PATCH 033/122] disabling namespace swap --- .../services/tasks_runner_batch_migration.py | 148 +++++++++--------- 1 file changed, 75 insertions(+), 73 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index 6d8903fa14..30381f4333 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -450,75 +450,84 @@ def _flip_all_required(rpc, ips, port, trtype, state, label, attempts=3): "inaccessible", f"SRC-{src['node_id'][:8]}") # Step 4: namespace swap on overlap TGT paths. - # Each member has its own namespace in the shared NQN. We look up each - # member's nsid by matching ns['uuid'] == lvol.uuid so we never remove - # the wrong namespace (positional removal would corrupt I/O for other members). - # Query subsystem once per overlap node, then match by UUID for each member. - for tgt in tgt_paths: - if tgt['node_id'] not in overlap_ids: - continue - try: - s = tgt['rpc'].subsystem_get(nqn) - ns_by_uuid = { - ns.get('uuid'): ns['nsid'] - for ns in (s.get('namespaces', []) if s else []) - } - except Exception as e: - logger.warning( - f"Group {group.uuid[:8]}: subsystem_get on {tgt['node_id'][:8]} " - f"(non-fatal): {e}") - ns_by_uuid = {} - - # Two-pass swap: remove ALL old namespaces first, then add ALL new ones. - # A per-member remove+add loop would cause N sequential "namespace gone" - # events visible to initiators — each one triggering a reconnect. Batching - # the removes into one pass and the adds into a second pass collapses this - # into a single collective disruption, which initiators handle cleanly. - ns_adds = [] # (tgt_ns_bdev, uuid, guid) — collected during remove pass - for m in member_migrations: + # TEMPORARILY DISABLED for a diagnostic test: skip the remove/add-ns + # swap entirely -- just flip TGT optimized / SRC inaccessible (steps + # 2/3/5) and leave each overlap node's existing namespaces (still + # pointing at the pre-migration SRC bdev) untouched, to isolate + # whether the corruption is introduced by this swap or is already + # present in the final-step/add_clone data regardless of it. + # Re-enable once the test is done. + logger.info(f"Group {group.uuid[:8]}: overlap namespace swap SKIPPED (diagnostic)") + if False: + # Each member has its own namespace in the shared NQN. We look up each + # member's nsid by matching ns['uuid'] == lvol.uuid so we never remove + # the wrong namespace (positional removal would corrupt I/O for other members). + # Query subsystem once per overlap node, then match by UUID for each member. + for tgt in tgt_paths: + if tgt['node_id'] not in overlap_ids: + continue try: - lvol = db.get_lvol_by_id(m.lvol_id) - tgt_bdev_short = _lvol_tgt_bdev_name(lvol.lvol_bdev) - tgt_ns_bdev = ( - f"crypto_{tgt_bdev_short}" if lvol.crypto_bdev - else f"{tgt_node.lvstore}/{tgt_bdev_short}" - ) - nsid = ns_by_uuid.get(lvol.uuid) - if nsid: - try: - tgt['rpc'].nvmf_subsystem_remove_ns(nqn, nsid) - logger.info( - f"Group {group.uuid[:8]}: swap NS {tgt['node_id'][:8]}: " - f"removed nsid={nsid} for lvol {lvol.uuid[:8]}") - except Exception as e: - logger.warning( - f"Group {group.uuid[:8]}: remove ns {tgt['node_id'][:8]} " - f"nsid={nsid} (non-fatal): {e}") - else: - logger.warning( - f"Group {group.uuid[:8]}: no namespace for uuid={lvol.uuid[:8]} " - f"on {tgt['node_id'][:8]}; skipping remove") - ns_adds.append((tgt_ns_bdev, lvol.uuid, lvol.guid)) + s = tgt['rpc'].subsystem_get(nqn) + ns_by_uuid = { + ns.get('uuid'): ns['nsid'] + for ns in (s.get('namespaces', []) if s else []) + } except Exception as e: logger.warning( - f"Group {group.uuid[:8]}: namespace swap member {m.uuid[:8]} " - f"on {tgt['node_id'][:8]} (non-fatal): {e}") + f"Group {group.uuid[:8]}: subsystem_get on {tgt['node_id'][:8]} " + f"(non-fatal): {e}") + ns_by_uuid = {} + + # Two-pass swap: remove ALL old namespaces first, then add ALL new ones. + # A per-member remove+add loop would cause N sequential "namespace gone" + # events visible to initiators — each one triggering a reconnect. Batching + # the removes into one pass and the adds into a second pass collapses this + # into a single collective disruption, which initiators handle cleanly. + ns_adds = [] # (tgt_ns_bdev, uuid, guid) — collected during remove pass + for m in member_migrations: + try: + lvol = db.get_lvol_by_id(m.lvol_id) + tgt_bdev_short = _lvol_tgt_bdev_name(lvol.lvol_bdev) + tgt_ns_bdev = ( + f"crypto_{tgt_bdev_short}" if lvol.crypto_bdev + else f"{tgt_node.lvstore}/{tgt_bdev_short}" + ) + nsid = ns_by_uuid.get(lvol.uuid) + if nsid: + try: + tgt['rpc'].nvmf_subsystem_remove_ns(nqn, nsid) + logger.info( + f"Group {group.uuid[:8]}: swap NS {tgt['node_id'][:8]}: " + f"removed nsid={nsid} for lvol {lvol.uuid[:8]}") + except Exception as e: + logger.warning( + f"Group {group.uuid[:8]}: remove ns {tgt['node_id'][:8]} " + f"nsid={nsid} (non-fatal): {e}") + else: + logger.warning( + f"Group {group.uuid[:8]}: no namespace for uuid={lvol.uuid[:8]} " + f"on {tgt['node_id'][:8]}; skipping remove") + ns_adds.append((tgt_ns_bdev, lvol.uuid, lvol.guid)) + except Exception as e: + logger.warning( + f"Group {group.uuid[:8]}: namespace swap member {m.uuid[:8]} " + f"on {tgt['node_id'][:8]} (non-fatal): {e}") - for tgt_ns_bdev, uuid, guid in ns_adds: - try: - ret = tgt['rpc'].nvmf_subsystem_add_ns(nqn, tgt_ns_bdev, uuid, guid) - if not ret: - logger.error( - f"Group {group.uuid[:8]}: add ns {tgt_ns_bdev} failed " - f"on {tgt['node_id'][:8]}") - else: - logger.info( - f"Group {group.uuid[:8]}: swap NS {tgt['node_id'][:8]}: " - f"added {tgt_ns_bdev}") - except Exception as e: - logger.warning( - f"Group {group.uuid[:8]}: add ns {tgt_ns_bdev} " - f"on {tgt['node_id'][:8]} (non-fatal): {e}") + for tgt_ns_bdev, uuid, guid in ns_adds: + try: + ret = tgt['rpc'].nvmf_subsystem_add_ns(nqn, tgt_ns_bdev, uuid, guid) + if not ret: + logger.error( + f"Group {group.uuid[:8]}: add ns {tgt_ns_bdev} failed " + f"on {tgt['node_id'][:8]}") + else: + logger.info( + f"Group {group.uuid[:8]}: swap NS {tgt['node_id'][:8]}: " + f"added {tgt_ns_bdev}") + except Exception as e: + logger.warning( + f"Group {group.uuid[:8]}: add ns {tgt_ns_bdev} " + f"on {tgt['node_id'][:8]} (non-fatal): {e}") # Step 5: all TGT paths → correct ANA state at TGT port primary_tgt = tgt_paths[0] @@ -685,14 +694,7 @@ def _revert_src_replicas(reason): logger.warning( f"Group {group.uuid[:8]}: add_clone for member {m.uuid[:8]} (non-fatal): {e}") - # TEMPORARILY DISABLED for a diagnostic test: skip the ANA-state - # switch entirely so clients stay pinned on the SRC path after - # batch_final_step/add_clone, to isolate whether the corruption is - # introduced by the final-step/add_clone data itself (independent of - # any client-visible path change) or by the ANA/NS-swap sequence. - # Re-enable once the test is done. - logger.info(f"Group {group.uuid[:8]}: ANA state switch SKIPPED (diagnostic)") - # _flip_ana_to_optimized(group, member_migrations, src_node, src_rpc, tgt_node, tgt_rpc) + _flip_ana_to_optimized(group, member_migrations, src_node, src_rpc, tgt_node, tgt_rpc) try: src_rpc.bdev_nvme_detach_controller(ctrl_name) From 6095b7dddeca81f830ec57d4ec1ff380bc30ff86 Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:00:45 +0330 Subject: [PATCH 034/122] disbaling the delete of intermediate snapshots --- .../services/tasks_runner_lvol_migration.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_lvol_migration.py b/simplyblock_core/services/tasks_runner_lvol_migration.py index 8e691c24b5..5f8d4b7f7d 100644 --- a/simplyblock_core/services/tasks_runner_lvol_migration.py +++ b/simplyblock_core/services/tasks_runner_lvol_migration.py @@ -2614,10 +2614,16 @@ def _handle_cleanup_source(migration, src_node, src_rpc, tgt_node, tgt_rpc): tgt_ter_rpc = _make_rpc(tgt_ter) if tgt_ter else None try: if migration.intermediate_snaps: - _delete_intermediate_snaps_on_target( - migration, tgt_rpc, tgt_sec_rpc, tgt_ter_rpc, - tgt_all_nodes=[n for n in [tgt_node, tgt_sec, tgt_ter] if n], - tgt_lvs_name=tgt_node.lvstore) + # TEMPORARILY DISABLED for a diagnostic test: skip deleting the + # target-side intermediate ("_mig_*") snapshots so we can tell + # whether post-migration corruption still occurs with them left + # intact. Re-enable once the test is done. + logger.info(f"Intermediate snap delete on target SKIPPED (diagnostic): " + f"{migration.intermediate_snaps}") + # _delete_intermediate_snaps_on_target( + # migration, tgt_rpc, tgt_sec_rpc, tgt_ter_rpc, + # tgt_all_nodes=[n for n in [tgt_node, tgt_sec, tgt_ter] if n], + # tgt_lvs_name=tgt_node.lvstore) _rename_migrated_bdevs(migration, tgt_node, tgt_rpc, tgt_sec_rpc, tgt_ter_rpc, warnings=_warnings) except Exception as e: From 0ec0b5cdd6110760c9670b992e0f23583114ef04 Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:20:29 +0330 Subject: [PATCH 035/122] adding logs to trace the tree not reconstructing issue --- .../services/tasks_runner_batch_migration.py | 68 ++++++++++++++++++- .../services/tasks_runner_lvol_migration.py | 5 +- 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index 30381f4333..aa04b2c262 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -103,12 +103,20 @@ def _reconstruct_snap_tree(group, member_migrations, tgt_node, tgt_rpc) -> Optio committed: set = set() all_preexisting: set = set() for m in member_migrations: + logger.info( + f"Group {group.uuid[:8]}: DIAG reconstruct seed member {m.uuid[:8]} " + f"(lvol={m.lvol_id[:8] if m.lvol_id else None}): " + f"snaps_preexisting_on_target={list(m.snaps_preexisting_on_target)} " + f"snaps_migrated={list(m.snaps_migrated)} " + f"snaps_transferred_group={list(m.snaps_transferred_group)}") committed.update(m.snaps_preexisting_on_target) all_preexisting.update(m.snaps_preexisting_on_target) # Snaps already committed in a prior (crashed) run are in snaps_migrated. # Seeding committed from them prevents re-convert on re-entry (SPDK rejects # converting an already-immutable bdev, which would stall the group forever). committed.update(m.snaps_migrated) + logger.info(f"Group {group.uuid[:8]}: DIAG reconstruct initial committed set " + f"({len(committed)}): {list(committed)}") _lvstore_prefix = tgt_node.lvstore + "/" @@ -124,9 +132,16 @@ def _reconstruct_snap_tree(group, member_migrations, tgt_node, tgt_rpc) -> Optio for m in sorted(member_migrations, key=lambda x: getattr(x, '_sort_ns_id', 999)): chain = migration_controller.get_snapshot_chain(m.lvol_id, m.source_node_id) + logger.info( + f"Group {group.uuid[:8]}: DIAG reconstruct member {m.uuid[:8]} " + f"(ns_id={getattr(m, '_sort_ns_id', '?')}, lvol={m.lvol_id[:8] if m.lvol_id else None}): " + f"chain={list(chain)} snaps_transferred_group={list(m.snaps_transferred_group)}") for snap_uuid in m.snaps_transferred_group: if snap_uuid in committed: + logger.info( + f"Group {group.uuid[:8]}: DIAG reconstruct snap {snap_uuid[:8]} " + f"(member {m.uuid[:8]}) already in committed -- SKIPPING add_clone/convert") continue try: @@ -145,6 +160,10 @@ def _reconstruct_snap_tree(group, member_migrations, tgt_node, tgt_rpc) -> Optio break if sid in committed: pred_uuid = sid + logger.info( + f"Group {group.uuid[:8]}: DIAG reconstruct snap {snap_uuid[:8]} " + f"(member {m.uuid[:8]}, bdev={tgt_composite}): pred_uuid=" + f"{pred_uuid[:8] if pred_uuid else None}") if pred_uuid: try: @@ -171,6 +190,10 @@ def _reconstruct_snap_tree(group, member_migrations, tgt_node, tgt_rpc) -> Optio else: pred_short = _snap_tgt_short_name(pred_snap) pred_composite = f"{tgt_node.lvstore}/{pred_short}" + logger.info( + f"Group {group.uuid[:8]}: DIAG reconstruct add_clone " + f"{tgt_composite} -> parent {pred_composite} " + f"(preexisting={pred_uuid in all_preexisting})") if not tgt_rpc.bdev_lvol_add_clone(tgt_composite, pred_composite): return f"bdev_lvol_add_clone failed: {snap_uuid} → {pred_uuid}" if sec_rpc: @@ -179,12 +202,24 @@ def _reconstruct_snap_tree(group, member_migrations, tgt_node, tgt_rpc) -> Optio if ter_rpc: if not ter_rpc.bdev_lvol_add_clone(tgt_composite, pred_composite): return f"bdev_lvol_add_clone on tertiary failed: {snap_uuid} → {pred_uuid}" + logger.info( + f"Group {group.uuid[:8]}: DIAG reconstruct add_clone OK " + f"{tgt_composite} -> {pred_composite}") except KeyError: logger.warning(f"Predecessor {pred_uuid} not found; skipping add_clone") + else: + logger.info( + f"Group {group.uuid[:8]}: DIAG reconstruct snap {snap_uuid[:8]} " + f"has no committed predecessor -- converting as a root snapshot " + f"(no add_clone)") # Leadership gate: convert on a non-leader silently persists nothing. from simplyblock_core.controllers import lvol_controller as _lc - if not _lc.is_node_leader(tgt_node, tgt_composite.split("/")[0]): + _is_leader = _lc.is_node_leader(tgt_node, tgt_composite.split("/")[0]) + logger.info( + f"Group {group.uuid[:8]}: DIAG reconstruct convert {tgt_composite} " + f"is_leader={_is_leader}") + if not _is_leader: return f"target node not LVS leader for convert of {snap_uuid}, retrying" if not tgt_rpc.bdev_lvol_convert(tgt_composite): return f"bdev_lvol_convert failed for {snap_uuid}" @@ -194,6 +229,9 @@ def _reconstruct_snap_tree(group, member_migrations, tgt_node, tgt_rpc) -> Optio if ter_rpc: if not ter_rpc.bdev_lvol_convert(tgt_composite): return f"bdev_lvol_convert on tertiary failed for {snap_uuid}" + logger.info( + f"Group {group.uuid[:8]}: DIAG reconstruct convert OK {tgt_composite} " + f"(snap {snap_uuid[:8]} now committed)") # Early partial DB update: route health-check/delete to the target # node right away rather than waiting for apply_migration_to_db() @@ -222,6 +260,9 @@ def _reconstruct_snap_tree(group, member_migrations, tgt_node, tgt_rpc) -> Optio m.write_to_db(db.kv_store) + logger.info( + f"Group {group.uuid[:8]}: DIAG reconstruct done, final committed set " + f"({len(committed)}): {list(committed)}") return None # success @@ -258,6 +299,8 @@ def _build_batch_final_args(group, member_migrations, src_node, tgt_node, tgt_rp """ mid_to_migration = {m.uuid: m for m in member_migrations} ordered_ids = group.ordered_migration_ids() + logger.info(f"Group {group.uuid[:8]}: DIAG build_final_args ordered_ids " + f"({len(ordered_ids)}): {[mid[:8] for mid in ordered_ids]}") lvol_names = [] lvol_ids = [] @@ -288,6 +331,13 @@ def _build_batch_final_args(group, member_migrations, src_node, tgt_node, tgt_rp if map_id is None: raise ValueError(f"map_id missing for {tgt_bdev_short}") lvol_ids.append(map_id) + logger.info( + f"Group {group.uuid[:8]}: DIAG build_final_args migration {migration_id[:8]} " + f"lvol={m.lvol_id[:8] if m.lvol_id else None} src_bdev={src_composite} " + f"tgt_bdev_short={tgt_bdev_short} map_id={map_id} " + f"snaps_migrated={list(m.snaps_migrated)} " + f"snaps_preexisting_on_target={list(m.snaps_preexisting_on_target)} " + f"snaps_transferred_group={list(m.snaps_transferred_group)}") # Last transferred snap = last entry in snaps_migrated (the intermediate). tgt_snap_composite = "" @@ -325,8 +375,14 @@ def _build_batch_final_args(group, member_migrations, src_node, tgt_node, tgt_rp last_uuid, migration_id, ) + logger.info( + f"Group {group.uuid[:8]}: DIAG build_final_args migration {migration_id[:8]} " + f"resolved tgt_snap_composite={tgt_snap_composite!r}") snapshot_names.append(tgt_snap_composite) + logger.info( + f"Group {group.uuid[:8]}: DIAG build_final_args final pairing: " + f"{[(mid[:8], sn) for mid, sn in zip(ordered_ids, snapshot_names)]}") return lvol_names, lvol_ids, snapshot_names @@ -667,9 +723,17 @@ def _revert_src_replicas(reason): if sec_node or ter_node: sec_rpc_extra = _make_rpc(sec_node) if sec_node else None ter_rpc_extra = _make_rpc(ter_node) if ter_node else None - snap_by_migration_id = dict(zip(group.ordered_migration_ids(), snapshot_names)) + _reordered_ids = group.ordered_migration_ids() + snap_by_migration_id = dict(zip(_reordered_ids, snapshot_names)) + logger.info( + f"Group {group.uuid[:8]}: DIAG extra-add_clone snap_by_migration_id: " + f"{[(mid[:8], sn) for mid, sn in snap_by_migration_id.items()]}") for m in member_migrations: snap_composite = snap_by_migration_id.get(m.uuid, "") + logger.info( + f"Group {group.uuid[:8]}: DIAG extra-add_clone member {m.uuid[:8]} " + f"(lvol={m.lvol_id[:8] if m.lvol_id else None}) -> snap_composite=" + f"{snap_composite!r}") if not snap_composite: continue try: diff --git a/simplyblock_core/services/tasks_runner_lvol_migration.py b/simplyblock_core/services/tasks_runner_lvol_migration.py index 5f8d4b7f7d..0c734d400e 100644 --- a/simplyblock_core/services/tasks_runner_lvol_migration.py +++ b/simplyblock_core/services/tasks_runner_lvol_migration.py @@ -3150,7 +3150,10 @@ def _post_process_snap_group(snap, migration): if snap_uuid not in migration.snaps_transferred_group: migration.snaps_transferred_group.append(snap_uuid) migration_events.migration_snap_copied(migration, snap_uuid) - logger.info(f"Group worker: snap {snap_uuid} raw-transferred (pending tree reconstruction)") + logger.info( + f"Group worker {migration.uuid[:8]}: DIAG snap {snap_uuid[:8]} raw-transferred " + f"(pending tree reconstruction), lvol={migration.lvol_id[:8] if migration.lvol_id else None}, " + f"snaps_transferred_group now={list(migration.snaps_transferred_group)}") return True, None From 0b0c18d4026a21c949be4cafc98b933a181557d0 Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Wed, 19 Aug 2026 01:48:07 +0330 Subject: [PATCH 036/122] fix: batch migration skip building the tree on the target --- .../services/tasks_runner_batch_migration.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index aa04b2c262..ab2c4ea8ba 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -98,19 +98,36 @@ def _reconstruct_snap_tree(group, member_migrations, tgt_node, tgt_rpc) -> Optio tgt_ter, _ = _get_target_tertiary_node(tgt_node, "") ter_rpc = _make_rpc(tgt_ter) if tgt_ter else None + # A member's snaps_preexisting_on_target (set at create_batch_migration_continue + # time) conflates two very different things: snaps truly already on the + # target from OUTSIDE this group (a prior, unrelated migration), and + # "non_owned_preexisting" snaps -- ancestor snaps in this member's own + # chain that a DIFFERENT member of THIS SAME group owns and hasn't + # transferred/committed yet. Only the former may seed `committed` up + # front; seeding from the latter marks every ancestor snap "already + # committed" before its true owner ever gets a turn in the loop below, + # so add_clone/convert never runs for ANY snapshot in the tree. + owned_or_pending_uuids: set = set() + for m in member_migrations: + owned_or_pending_uuids.update(m.snap_migration_plan or []) + owned_or_pending_uuids.update(m.snaps_transferred_group or []) + # Global set of snaps that have been committed as immutable on the target, # either pre-existing or reconstructed in this call. committed: set = set() all_preexisting: set = set() for m in member_migrations: + truly_external_preexisting = [ + s for s in m.snaps_preexisting_on_target if s not in owned_or_pending_uuids] logger.info( f"Group {group.uuid[:8]}: DIAG reconstruct seed member {m.uuid[:8]} " f"(lvol={m.lvol_id[:8] if m.lvol_id else None}): " f"snaps_preexisting_on_target={list(m.snaps_preexisting_on_target)} " + f"truly_external_preexisting={truly_external_preexisting} " f"snaps_migrated={list(m.snaps_migrated)} " f"snaps_transferred_group={list(m.snaps_transferred_group)}") - committed.update(m.snaps_preexisting_on_target) - all_preexisting.update(m.snaps_preexisting_on_target) + committed.update(truly_external_preexisting) + all_preexisting.update(truly_external_preexisting) # Snaps already committed in a prior (crashed) run are in snaps_migrated. # Seeding committed from them prevents re-convert on re-entry (SPDK rejects # converting an already-immutable bdev, which would stall the group forever). From 2987130ce64272aebee40831dfeae4b64795938d Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Wed, 19 Aug 2026 05:06:08 +0330 Subject: [PATCH 037/122] fix: redundent add listener rpc call, explicit ns id passing to the target --- .../controllers/migration_controller.py | 40 ++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/simplyblock_core/controllers/migration_controller.py b/simplyblock_core/controllers/migration_controller.py index b723e25759..bfeb781711 100644 --- a/simplyblock_core/controllers/migration_controller.py +++ b/simplyblock_core/controllers/migration_controller.py @@ -1174,7 +1174,8 @@ def create_migration(lvol_id, target_node_id, f"create_migration: listener on overlap {_node_id[:8]} " f"(non-fatal): {_e}") else: - if not _rpc.subsystem_get(nqn): + _existing_subsys = _rpc.subsystem_get(nqn) + if not _existing_subsys: if _min_cntlid in subsys_min_cntlid_used: _min_cntlid = _min_cntlid + 10000 _rpc.subsystem_create( @@ -1190,22 +1191,51 @@ def create_migration(lvol_id, target_node_id, f"create_migration: allowed_hosts reapply on " f"{_node_id[:8]} (non-fatal): {_e}") + # For a shared-namespace batch group, create_migration() runs once + # per member against the SAME nqn/listener -- guard against + # re-adding a listener that a prior member's precreate already + # established (listeners_create had no existence check and its + # result was never inspected, so a real failure on a later + # member's redundant add would previously have been silent). + _existing_listeners = { + (_l.get('trtype', '').lower(), _l.get('traddr'), str(_l.get('trsvcid'))) + for _l in ((_existing_subsys or {}).get('listen_addresses') or []) + } for nic in _node.data_nics: if not nic.ip4_address or nic.trtype.lower() != lvol.fabric: continue + _listener_key = (nic.trtype.lower(), nic.ip4_address, str(_port)) + if _listener_key in _existing_listeners: + continue try: - _rpc.listeners_create(nqn, nic.trtype.lower(), nic.ip4_address, - _port, ana_state="inaccessible") + _ret_listener = _rpc.listeners_create( + nqn, nic.trtype.lower(), nic.ip4_address, + _port, ana_state="inaccessible") + if not _ret_listener: + logger.warning( + f"create_migration: listener add for {_node_id[:8]} " + f"{nic.ip4_address}:{_port} returned falsy") except Exception as _e: logger.warning( f"create_migration: listener on {_node_id[:8]} " f"(non-fatal): {_e}") - _ns = _rpc.nvmf_subsystem_add_ns(nqn, _ns_bdev, lvol.uuid, lvol.guid) + # Pin the target namespace to the SAME nsid the source already + # uses, rather than letting SPDK auto-assign on the target + # subsystem. Auto-assignment just happens to reproduce the + # source's numbering when adds land in the same order on an + # empty subsystem -- it isn't enforced, and any stale/leftover + # namespace occupying a low nsid on the target (or add_ns calls + # racing/reordering across nodes) would silently diverge the + # source and target nsid maps for this lvol. + _ns = _rpc.nvmf_subsystem_add_ns( + nqn, _ns_bdev, lvol.uuid, lvol.guid, + nsid=lvol.ns_id if lvol.ns_id else None) if _ns: logger.info( f"create_migration: namespace {_ns_bdev} added on " - f"{_tgt_label} {_node_id[:8]} nsid={_ns}") + f"{_tgt_label} {_node_id[:8]} nsid={_ns} " + f"(source nsid={lvol.ns_id})") else: logger.warning( f"create_migration: nvmf_subsystem_add_ns failed on " From 8a5286b10f528901a32b54f14d39cd782f5cccf3 Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Wed, 19 Aug 2026 06:40:39 +0330 Subject: [PATCH 038/122] testing with source paths going off for 5 second before opening target paths --- .../services/tasks_runner_batch_migration.py | 69 ++++++++----------- 1 file changed, 30 insertions(+), 39 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index ab2c4ea8ba..093324e51e 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -498,37 +498,39 @@ def _flip_all_required(rpc, ips, port, trtype, state, label, attempts=3): _flip_all(src['rpc'], src['ips'], src['port'], src['trtype'], "inaccessible", f"SRC-{src['node_id'][:8]}") else: - # Step 1: first non-overlap TGT → optimized before making any SRC inaccessible - non_overlap_tgt = next( - (t for t in tgt_paths if t['node_id'] not in overlap_ids), None) - if non_overlap_tgt: - if not _flip_all_required(non_overlap_tgt['rpc'], non_overlap_tgt['ips'], - non_overlap_tgt['port'], non_overlap_tgt['trtype'], - "optimized", - f"TGT-{non_overlap_tgt['node_id'][:8]}(pre)"): - logger.error( - f"Group {group.uuid[:8]}: ANA flip non-overlap TGT→optimized failed; " - f"proceeding anyway") - - # Step 2: overlap SRC paths → inaccessible at SRC port + # TEMPORARILY SIMPLIFIED for a diagnostic test: skip the overlap-aware + # TGT-first ordering and the namespace swap entirely. Just: + # 1. All SRC paths -> inaccessible (secondary/tertiary were already + # made inaccessible pre-final-step; this covers the primary and + # harmlessly re-covers secondary/tertiary). + # 2. Sleep 5s. + # 3. All TGT paths -> optimized (primary) / non_optimized (replicas). + # Re-enable the original overlap-aware sequence once this test is done. for src in src_paths: - if src['node_id'] in overlap_ids: - _flip_all(src['rpc'], src['ips'], src['port'], src['trtype'], - "inaccessible", f"SRC-{src['node_id'][:8]}(overlap)") + _flip_all(src['rpc'], src['ips'], src['port'], src['trtype'], + "inaccessible", f"SRC-{src['node_id'][:8]}") - # Step 3: non-overlap SRC paths → inaccessible - for src in src_paths: - if src['node_id'] not in overlap_ids: - _flip_all(src['rpc'], src['ips'], src['port'], src['trtype'], - "inaccessible", f"SRC-{src['node_id'][:8]}") + logger.info(f"Group {group.uuid[:8]}: sleeping 5s after SRC inaccessible " + f"before TGT flip (diagnostic)") + time.sleep(5) - # Step 4: namespace swap on overlap TGT paths. + primary_tgt = tgt_paths[0] + if not _flip_all_required(primary_tgt['rpc'], primary_tgt['ips'], primary_tgt['port'], + primary_tgt['trtype'], "optimized", + f"TGT-{primary_tgt['node_id'][:8]}"): + logger.error( + f"Group {group.uuid[:8]}: ANA flip TGT primary→optimized failed; " + f"clients may be on degraded path") + for tgt in tgt_paths[1:]: + _flip_all(tgt['rpc'], tgt['ips'], tgt['port'], tgt['trtype'], + "non_optimized", f"TGT-{tgt['node_id'][:8]}") + + # Namespace swap on overlap TGT paths. # TEMPORARILY DISABLED for a diagnostic test: skip the remove/add-ns - # swap entirely -- just flip TGT optimized / SRC inaccessible (steps - # 2/3/5) and leave each overlap node's existing namespaces (still - # pointing at the pre-migration SRC bdev) untouched, to isolate - # whether the corruption is introduced by this swap or is already - # present in the final-step/add_clone data regardless of it. + # swap entirely -- leave each overlap node's existing namespaces + # (still pointing at the pre-migration SRC bdev) untouched, to + # isolate whether the corruption is introduced by this swap or is + # already present in the final-step/add_clone data regardless of it. # Re-enable once the test is done. logger.info(f"Group {group.uuid[:8]}: overlap namespace swap SKIPPED (diagnostic)") if False: @@ -602,18 +604,7 @@ def _flip_all_required(rpc, ips, port, trtype, state, label, attempts=3): f"Group {group.uuid[:8]}: add ns {tgt_ns_bdev} " f"on {tgt['node_id'][:8]} (non-fatal): {e}") - # Step 5: all TGT paths → correct ANA state at TGT port - primary_tgt = tgt_paths[0] - if not _flip_all_required(primary_tgt['rpc'], primary_tgt['ips'], primary_tgt['port'], - primary_tgt['trtype'], "optimized", - f"TGT-{primary_tgt['node_id'][:8]}"): - logger.error( - f"Group {group.uuid[:8]}: ANA flip TGT primary→optimized (step 5) failed") - for tgt in tgt_paths[1:]: - _flip_all(tgt['rpc'], tgt['ips'], tgt['port'], tgt['trtype'], - "non_optimized", f"TGT-{tgt['node_id'][:8]}") - - # Step 6: remove old SRC-port listener from overlap TGT nodes if port changed + # Remove old SRC-port listener from overlap TGT nodes if port changed for tgt in tgt_paths: if tgt['node_id'] in overlap_ids: old_port = src_port_by_id.get(tgt['node_id']) From 01d31cbe5e5fdd8c074f2091e318798e1fa7e186 Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Wed, 19 Aug 2026 08:00:27 +0330 Subject: [PATCH 039/122] disabling rpc calls past final step in batch migration --- .../services/tasks_runner_batch_migration.py | 58 +++++++--------- .../services/tasks_runner_lvol_migration.py | 66 +++++++------------ 2 files changed, 46 insertions(+), 78 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index 093324e51e..d4ee179452 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -426,7 +426,6 @@ def _flip_ana_to_optimized(group, member_migrations, src_node, src_rpc, tgt_node """ nqn = group.target_nqn src_paths, tgt_paths, overlap_ids = _build_paths(src_node, tgt_node, src_rpc, tgt_rpc) - src_port_by_id = {p['node_id']: p['port'] for p in src_paths} # Detect and repair a target-side node restart that wiped the migration's # NVMe-oF subsystem/listener/namespace, right before this ANA-flip @@ -498,19 +497,18 @@ def _flip_all_required(rpc, ips, port, trtype, state, label, attempts=3): _flip_all(src['rpc'], src['ips'], src['port'], src['trtype'], "inaccessible", f"SRC-{src['node_id'][:8]}") else: - # TEMPORARILY SIMPLIFIED for a diagnostic test: skip the overlap-aware - # TGT-first ordering and the namespace swap entirely. Just: - # 1. All SRC paths -> inaccessible (secondary/tertiary were already - # made inaccessible pre-final-step; this covers the primary and - # harmlessly re-covers secondary/tertiary). - # 2. Sleep 5s. - # 3. All TGT paths -> optimized (primary) / non_optimized (replicas). + # TEMPORARILY SIMPLIFIED for a diagnostic test: the ONLY RPC calls + # this branch makes after batch_final_step are (1) SRC primary -> + # inaccessible, (2) a 5s sleep, (3) TGT primary -> optimized. No + # secondary/tertiary re-flip, no namespace swap, no old-listener + # cleanup -- everything else in the original overlap-aware sequence + # is skipped so those steps cannot be the source of the corruption. # Re-enable the original overlap-aware sequence once this test is done. - for src in src_paths: - _flip_all(src['rpc'], src['ips'], src['port'], src['trtype'], - "inaccessible", f"SRC-{src['node_id'][:8]}") + primary_src = src_paths[0] + _flip_all(primary_src['rpc'], primary_src['ips'], primary_src['port'], + primary_src['trtype'], "inaccessible", f"SRC-{primary_src['node_id'][:8]}") - logger.info(f"Group {group.uuid[:8]}: sleeping 5s after SRC inaccessible " + logger.info(f"Group {group.uuid[:8]}: sleeping 5s after SRC primary inaccessible " f"before TGT flip (diagnostic)") time.sleep(5) @@ -521,9 +519,6 @@ def _flip_all_required(rpc, ips, port, trtype, state, label, attempts=3): logger.error( f"Group {group.uuid[:8]}: ANA flip TGT primary→optimized failed; " f"clients may be on degraded path") - for tgt in tgt_paths[1:]: - _flip_all(tgt['rpc'], tgt['ips'], tgt['port'], tgt['trtype'], - "non_optimized", f"TGT-{tgt['node_id'][:8]}") # Namespace swap on overlap TGT paths. # TEMPORARILY DISABLED for a diagnostic test: skip the remove/add-ns @@ -604,21 +599,10 @@ def _flip_all_required(rpc, ips, port, trtype, state, label, attempts=3): f"Group {group.uuid[:8]}: add ns {tgt_ns_bdev} " f"on {tgt['node_id'][:8]} (non-fatal): {e}") - # Remove old SRC-port listener from overlap TGT nodes if port changed - for tgt in tgt_paths: - if tgt['node_id'] in overlap_ids: - old_port = src_port_by_id.get(tgt['node_id']) - if old_port and old_port != tgt['port']: - for _ip in tgt['ips']: - try: - tgt['rpc'].listeners_del(nqn, tgt['trtype'], _ip, old_port) - logger.info( - f"Group {group.uuid[:8]}: removed old SRC listener " - f"{_ip}:{old_port} from overlap {tgt['node_id'][:8]}") - except Exception as e: - logger.warning( - f"Group {group.uuid[:8]}: remove old SRC listener " - f"{tgt['node_id'][:8]} (non-fatal): {e}") + # TEMPORARILY DISABLED for a diagnostic test: skip removing the old + # SRC-port listener from overlap TGT nodes -- no RPC calls after the + # two ANA flips above. Re-enable once the test is done. + logger.info(f"Group {group.uuid[:8]}: old SRC-port listener cleanup SKIPPED (diagnostic)") def _handle_intermediate_barrier(group, member_migrations, src_node, tgt_node, src_rpc, tgt_rpc): @@ -768,12 +752,14 @@ def _revert_src_replicas(reason): _flip_ana_to_optimized(group, member_migrations, src_node, src_rpc, tgt_node, tgt_rpc) - try: - src_rpc.bdev_nvme_detach_controller(ctrl_name) - except Exception as e: - logger.warning(f"Group {group.uuid[:8]}: hub detach (non-fatal): {e}") - logger.info(f"Group {group.uuid[:8]}: sleeping 30s after hub detach") - time.sleep(30) + # TEMPORARILY DISABLED for a diagnostic test: no RPC calls at all after + # the two ANA flips above -- not even the hub controller detach. + # Re-enable once the test is done. + logger.info(f"Group {group.uuid[:8]}: hub detach SKIPPED (diagnostic)") + # try: + # src_rpc.bdev_nvme_detach_controller(ctrl_name) + # except Exception as e: + # logger.warning(f"Group {group.uuid[:8]}: hub detach (non-fatal): {e}") return batch_ok, batch_err diff --git a/simplyblock_core/services/tasks_runner_lvol_migration.py b/simplyblock_core/services/tasks_runner_lvol_migration.py index 0c734d400e..840b83e942 100644 --- a/simplyblock_core/services/tasks_runner_lvol_migration.py +++ b/simplyblock_core/services/tasks_runner_lvol_migration.py @@ -2483,37 +2483,12 @@ def _handle_cleanup_source(migration, src_node, src_rpc, tgt_node, tgt_rpc): "source bdev deletion will be skipped") _warnings.append("source_lvol_bdev not in ctx; source bdev not deleted") - # Build the safe-to-delete list, cross-checking each snap exists on the - # target before we touch the source. If the target is unreachable, skip - # all source snap deletions to avoid accidental data loss. + # TEMPORARILY DISABLED for a diagnostic test: skip the safe-to-delete + # verification (it queried tgt_rpc.bdev_lvol_get_lvols) since source + # snapshot deletion itself is disabled below -- no RPC calls here at + # all. Re-enable once the test is done. + logger.info("Source snapshot safe-to-delete verification SKIPPED (diagnostic)") _snaps_to_delete_src: list = [] - try: - to_delete_all = migration_controller.get_snaps_safe_to_delete_on_source(migration) - tgt_lvols = tgt_rpc.bdev_lvol_get_lvols(tgt_node.lvstore) or [] - tgt_names = {e.get('name', '').split('/')[-1] for e in tgt_lvols} - for snap_uuid in to_delete_all: - try: - snap = db.get_snapshot_by_id(snap_uuid) - _snap_bdev = snap.snap_bdev or '' - _primary = _snap_bdev.split('/', 1)[1] if '/' in _snap_bdev else _snap_bdev - _m_name = _snap_tgt_short_name(snap) - _canonical = _snap_short_name(snap) - _am_name = _canonical + _MIGRATION_BDEV_SUFFIX_DONE - if any(n in tgt_names for n in (_primary, _m_name, _canonical, _am_name)): - _snaps_to_delete_src.append(snap_uuid) - else: - logger.warning( - f"Target missing snapshot {_m_name} ({snap_uuid}); " - "skipping source delete to protect data") - _warnings.append(f"target missing snap {_m_name}; source copy kept") - except KeyError: - pass # already gone from DB; safe to skip - except Exception as _ve: - logger.warning( - f"Could not verify snapshots on target ({_ve}); " - "skipping all source snap deletions") - _warnings.append(f"snap target-verification failed: {_ve}") - _snaps_to_delete_src = [] ctx = { 'stage': 'cleanup_src', @@ -2562,16 +2537,19 @@ def _handle_cleanup_source(migration, src_node, src_rpc, tgt_node, tgt_rpc): lvol = None try: lvol = db.get_lvol_by_id(migration.lvol_id) - logger.info(f"Step 8: removing source NVMe-oF subsystem {lvol.nqn}") - _src_paths_cu, _, _overlap_ids_cu = _build_paths( - src_node, tgt_node, src_rpc, tgt_rpc) - for _sp in _src_paths_cu: - if _sp['node_id'] in _overlap_ids_cu: - logger.info( - f"Step 8: skip subsystem delete on overlap node " - f"{_sp['node_id'][:8]} (now serving TGT)") - else: - migration_controller.cleanup_subsystem_or_ns(lvol.nqn, lvol.uuid, True, _sp['rpc']) + # TEMPORARILY DISABLED for a diagnostic test: skip removing the + # source NVMe-oF subsystem/namespace entirely -- no RPC calls here + # at all. Re-enable once the test is done. + logger.info(f"Step 8: source NVMe-oF subsystem removal SKIPPED (diagnostic): {lvol.nqn}") + # _src_paths_cu, _, _overlap_ids_cu = _build_paths( + # src_node, tgt_node, src_rpc, tgt_rpc) + # for _sp in _src_paths_cu: + # if _sp['node_id'] in _overlap_ids_cu: + # logger.info( + # f"Step 8: skip subsystem delete on overlap node " + # f"{_sp['node_id'][:8]} (now serving TGT)") + # else: + # migration_controller.cleanup_subsystem_or_ns(lvol.nqn, lvol.uuid, True, _sp['rpc']) except Exception as e: logger.warning(f"Source subsystem cleanup failed: {e}") @@ -2624,8 +2602,12 @@ def _handle_cleanup_source(migration, src_node, src_rpc, tgt_node, tgt_rpc): # migration, tgt_rpc, tgt_sec_rpc, tgt_ter_rpc, # tgt_all_nodes=[n for n in [tgt_node, tgt_sec, tgt_ter] if n], # tgt_lvs_name=tgt_node.lvstore) - _rename_migrated_bdevs(migration, tgt_node, tgt_rpc, tgt_sec_rpc, tgt_ter_rpc, - warnings=_warnings) + # TEMPORARILY DISABLED for a diagnostic test: skip renaming migrated + # bdevs on the target back to their canonical names, so no RPC calls + # happen here at all. Re-enable once the test is done. + logger.info("Target bdev rename SKIPPED (diagnostic)") + # _rename_migrated_bdevs(migration, tgt_node, tgt_rpc, tgt_sec_rpc, tgt_ter_rpc, + # warnings=_warnings) except Exception as e: logger.warning(f"Target artifact cleanup (rename/intermediate snaps) failed: {e}") _warnings.append(f"target rename/intermediate-snap cleanup failed: {e}") From 7bcd4dfa3bed7901bb640d32261afb46cf41e3b0 Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Wed, 19 Aug 2026 13:37:15 +0330 Subject: [PATCH 040/122] making all paths inaccessible 10 second before fianl step --- .../services/tasks_runner_batch_migration.py | 69 ++++++++++++++----- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index d4ee179452..859c745fd3 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -639,17 +639,12 @@ def _handle_intermediate_barrier(group, member_migrations, src_node, tgt_node, s f"failure (non-fatal): {detach_exc}") return None, str(e) - # Pre-freeze: take SRC secondary/tertiary out of the read path before the - # synchronous final-step transfer below. bdev_lvol_batch_transfer_final_step - # freezes the SRC primary internally for the duration of the transfer, but - # a client sitting on a SRC replica path is not covered by that freeze — - # without this, a write accepted by a SRC replica during the transfer (or - # in the gap before cutover flips SRC paths inaccessible) never reaches - # the copy that already ran, and is silently lost. Mirrors the single-lvol - # path's identical pre-freeze (tasks_runner_lvol_migration.py). + # Pre-freeze: take SRC/TGT paths out of the read/write path before the + # synchronous final-step transfer below (see the diagnostic block further + # down for the current, temporarily-widened version of this). nqn = group.target_nqn - src_paths, _, _ = _build_paths(src_node, tgt_node, src_rpc, tgt_rpc) - src_replica_paths = src_paths[1:] # secondary/tertiary only; primary is frozen internally by the RPC below + src_paths, tgt_paths, _ = _build_paths(src_node, tgt_node, src_rpc, tgt_rpc) + src_replica_paths = src_paths[1:] # secondary/tertiary only; used for the failure-path revert below def _flip(rpc, ip, port, trtype, state, label): try: @@ -664,6 +659,18 @@ def _flip_all(rpc, ips, port, trtype, state, label): for _ip in ips: _flip(rpc, _ip, port, trtype, state, label) + def _flip_all_required(rpc, ips, port, trtype, state, label, attempts=3): + ok = True + for _ip in ips: + for i in range(attempts): + if _flip(rpc, _ip, port, trtype, state, label): + break + if i < attempts - 1: + time.sleep(1.0) + else: + ok = False + return ok + def _revert_src_replicas(reason): # Final step didn't complete — put SRC secondary/tertiary back into # the read path (their pre-freeze state) so clients keep multipath @@ -675,11 +682,23 @@ def _revert_src_replicas(reason): _flip_all(p['rpc'], p['ips'], p['port'], p['trtype'], "non_optimized", f"SRC-{p['node_id'][:8]}(revert)") - if src_replica_paths: - logger.info(f"Group {group.uuid[:8]}: setting SRC secondary/tertiary inaccessible pre-final-step") - for p in src_replica_paths: - _flip_all(p['rpc'], p['ips'], p['port'], p['trtype'], - "inaccessible", f"SRC-{p['node_id'][:8]}(pre-freeze)") + # TEMPORARILY CHANGED for a diagnostic test: instead of only freezing SRC + # secondary/tertiary pre-final-step (primary relied on the RPC's own + # internal freeze), make EVERY path -- all SRC (primary included) and all + # TGT -- inaccessible up front, wait 10s so any in-flight client I/O has + # time to fully settle/drain before the data actually moves, THEN call + # final_step. Re-enable the narrower pre-freeze once this test is done. + logger.info(f"Group {group.uuid[:8]}: setting ALL SRC and TGT paths inaccessible " + f"pre-final-step (diagnostic)") + for p in src_paths: + _flip_all(p['rpc'], p['ips'], p['port'], p['trtype'], + "inaccessible", f"SRC-{p['node_id'][:8]}(pre-freeze)") + for p in tgt_paths: + _flip_all(p['rpc'], p['ips'], p['port'], p['trtype'], + "inaccessible", f"TGT-{p['node_id'][:8]}(pre-freeze)") + logger.info(f"Group {group.uuid[:8]}: sleeping 10s after all-paths-inaccessible " + f"before batch_final_step (diagnostic)") + time.sleep(10) logger.info( f"Group {group.uuid[:8]}: batch_final_step " @@ -703,8 +722,9 @@ def _revert_src_replicas(reason): if not batch_ok: _revert_src_replicas("batch_final_step failed") - # else: left as-is — the Done handler's ANA sequence (_flip_ana_to_optimized) - # already drives every SRC path (including primary) to inaccessible on success. + # else: left as-is — all SRC/TGT paths were already driven inaccessible + # before final_step (diagnostic, see above); only TGT primary needs to + # come back optimized on success, handled below. if batch_ok: # bdev_lvol_batch_final_step handles add_clone on the primary internally. @@ -750,7 +770,20 @@ def _revert_src_replicas(reason): logger.warning( f"Group {group.uuid[:8]}: add_clone for member {m.uuid[:8]} (non-fatal): {e}") - _flip_ana_to_optimized(group, member_migrations, src_node, src_rpc, tgt_node, tgt_rpc) + # TEMPORARILY CHANGED for a diagnostic test: SRC/TGT were already + # driven inaccessible before final_step (see above) -- no need to + # re-flip SRC here. Just wait 2s, then bring TGT primary optimized. + # Re-enable _flip_ana_to_optimized() once this test is done. + logger.info(f"Group {group.uuid[:8]}: sleeping 2s after batch_final_step " + f"before TGT primary optimized (diagnostic)") + time.sleep(2) + primary_tgt = tgt_paths[0] + if not _flip_all_required(primary_tgt['rpc'], primary_tgt['ips'], primary_tgt['port'], + primary_tgt['trtype'], "optimized", + f"TGT-{primary_tgt['node_id'][:8]}"): + logger.error( + f"Group {group.uuid[:8]}: ANA flip TGT primary→optimized failed; " + f"clients may be on degraded path") # TEMPORARILY DISABLED for a diagnostic test: no RPC calls at all after # the two ANA flips above -- not even the hub controller detach. From 1456f01c461b891a581392f14e7d3fbe4405767c Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:09:57 +0330 Subject: [PATCH 041/122] reduced delay --- simplyblock_core/services/tasks_runner_batch_migration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index 859c745fd3..1ad51531d1 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -696,9 +696,9 @@ def _revert_src_replicas(reason): for p in tgt_paths: _flip_all(p['rpc'], p['ips'], p['port'], p['trtype'], "inaccessible", f"TGT-{p['node_id'][:8]}(pre-freeze)") - logger.info(f"Group {group.uuid[:8]}: sleeping 10s after all-paths-inaccessible " + logger.info(f"Group {group.uuid[:8]}: sleeping 2s after all-paths-inaccessible " f"before batch_final_step (diagnostic)") - time.sleep(10) + time.sleep(2) logger.info( f"Group {group.uuid[:8]}: batch_final_step " From 7f2b1e99303cbccebe9fddf14e7bc1e7e20dc8b9 Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:37:45 +0330 Subject: [PATCH 042/122] reenable the disabled features for the testing --- .../services/tasks_runner_batch_migration.py | 276 +++++++++--------- .../services/tasks_runner_lvol_migration.py | 137 ++++----- 2 files changed, 200 insertions(+), 213 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index 1ad51531d1..525a420a5d 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -426,6 +426,7 @@ def _flip_ana_to_optimized(group, member_migrations, src_node, src_rpc, tgt_node """ nqn = group.target_nqn src_paths, tgt_paths, overlap_ids = _build_paths(src_node, tgt_node, src_rpc, tgt_rpc) + src_port_by_id = {p['node_id']: p['port'] for p in src_paths} # Detect and repair a target-side node restart that wiped the migration's # NVMe-oF subsystem/listener/namespace, right before this ANA-flip @@ -497,112 +498,127 @@ def _flip_all_required(rpc, ips, port, trtype, state, label, attempts=3): _flip_all(src['rpc'], src['ips'], src['port'], src['trtype'], "inaccessible", f"SRC-{src['node_id'][:8]}") else: - # TEMPORARILY SIMPLIFIED for a diagnostic test: the ONLY RPC calls - # this branch makes after batch_final_step are (1) SRC primary -> - # inaccessible, (2) a 5s sleep, (3) TGT primary -> optimized. No - # secondary/tertiary re-flip, no namespace swap, no old-listener - # cleanup -- everything else in the original overlap-aware sequence - # is skipped so those steps cannot be the source of the corruption. - # Re-enable the original overlap-aware sequence once this test is done. - primary_src = src_paths[0] - _flip_all(primary_src['rpc'], primary_src['ips'], primary_src['port'], - primary_src['trtype'], "inaccessible", f"SRC-{primary_src['node_id'][:8]}") + # Step 1: first non-overlap TGT → optimized before making any SRC inaccessible + non_overlap_tgt = next( + (t for t in tgt_paths if t['node_id'] not in overlap_ids), None) + if non_overlap_tgt: + if not _flip_all_required(non_overlap_tgt['rpc'], non_overlap_tgt['ips'], + non_overlap_tgt['port'], non_overlap_tgt['trtype'], + "optimized", + f"TGT-{non_overlap_tgt['node_id'][:8]}(pre)"): + logger.error( + f"Group {group.uuid[:8]}: ANA flip non-overlap TGT→optimized failed; " + f"proceeding anyway") - logger.info(f"Group {group.uuid[:8]}: sleeping 5s after SRC primary inaccessible " - f"before TGT flip (diagnostic)") - time.sleep(5) + # Step 2: overlap SRC paths → inaccessible at SRC port + for src in src_paths: + if src['node_id'] in overlap_ids: + _flip_all(src['rpc'], src['ips'], src['port'], src['trtype'], + "inaccessible", f"SRC-{src['node_id'][:8]}(overlap)") - primary_tgt = tgt_paths[0] - if not _flip_all_required(primary_tgt['rpc'], primary_tgt['ips'], primary_tgt['port'], - primary_tgt['trtype'], "optimized", - f"TGT-{primary_tgt['node_id'][:8]}"): - logger.error( - f"Group {group.uuid[:8]}: ANA flip TGT primary→optimized failed; " - f"clients may be on degraded path") + # Step 3: non-overlap SRC paths → inaccessible + for src in src_paths: + if src['node_id'] not in overlap_ids: + _flip_all(src['rpc'], src['ips'], src['port'], src['trtype'], + "inaccessible", f"SRC-{src['node_id'][:8]}") + + # Step 4: namespace swap on overlap TGT paths. + # Each member has its own namespace in the shared NQN. We look up each + # member's nsid by matching ns['uuid'] == lvol.uuid so we never remove + # the wrong namespace (positional removal would corrupt I/O for other members). + # Query subsystem once per overlap node, then match by UUID for each member. + for tgt in tgt_paths: + if tgt['node_id'] not in overlap_ids: + continue + try: + s = tgt['rpc'].subsystem_get(nqn) + ns_by_uuid = { + ns.get('uuid'): ns['nsid'] + for ns in (s.get('namespaces', []) if s else []) + } + except Exception as e: + logger.warning( + f"Group {group.uuid[:8]}: subsystem_get on {tgt['node_id'][:8]} " + f"(non-fatal): {e}") + ns_by_uuid = {} + + # Two-pass swap: remove ALL old namespaces first, then add ALL new ones. + # A per-member remove+add loop would cause N sequential "namespace gone" + # events visible to initiators — each one triggering a reconnect. Batching + # the removes into one pass and the adds into a second pass collapses this + # into a single collective disruption, which initiators handle cleanly. + ns_adds = [] # (tgt_ns_bdev, uuid, guid) — collected during remove pass + for m in member_migrations: + try: + lvol = db.get_lvol_by_id(m.lvol_id) + tgt_bdev_short = _lvol_tgt_bdev_name(lvol.lvol_bdev) + tgt_ns_bdev = ( + f"crypto_{tgt_bdev_short}" if lvol.crypto_bdev + else f"{tgt_node.lvstore}/{tgt_bdev_short}" + ) + nsid = ns_by_uuid.get(lvol.uuid) + if nsid: + try: + tgt['rpc'].nvmf_subsystem_remove_ns(nqn, nsid) + logger.info( + f"Group {group.uuid[:8]}: swap NS {tgt['node_id'][:8]}: " + f"removed nsid={nsid} for lvol {lvol.uuid[:8]}") + except Exception as e: + logger.warning( + f"Group {group.uuid[:8]}: remove ns {tgt['node_id'][:8]} " + f"nsid={nsid} (non-fatal): {e}") + else: + logger.warning( + f"Group {group.uuid[:8]}: no namespace for uuid={lvol.uuid[:8]} " + f"on {tgt['node_id'][:8]}; skipping remove") + ns_adds.append((tgt_ns_bdev, lvol.uuid, lvol.guid)) + except Exception as e: + logger.warning( + f"Group {group.uuid[:8]}: namespace swap member {m.uuid[:8]} " + f"on {tgt['node_id'][:8]} (non-fatal): {e}") - # Namespace swap on overlap TGT paths. - # TEMPORARILY DISABLED for a diagnostic test: skip the remove/add-ns - # swap entirely -- leave each overlap node's existing namespaces - # (still pointing at the pre-migration SRC bdev) untouched, to - # isolate whether the corruption is introduced by this swap or is - # already present in the final-step/add_clone data regardless of it. - # Re-enable once the test is done. - logger.info(f"Group {group.uuid[:8]}: overlap namespace swap SKIPPED (diagnostic)") - if False: - # Each member has its own namespace in the shared NQN. We look up each - # member's nsid by matching ns['uuid'] == lvol.uuid so we never remove - # the wrong namespace (positional removal would corrupt I/O for other members). - # Query subsystem once per overlap node, then match by UUID for each member. - for tgt in tgt_paths: - if tgt['node_id'] not in overlap_ids: - continue + for tgt_ns_bdev, uuid, guid in ns_adds: try: - s = tgt['rpc'].subsystem_get(nqn) - ns_by_uuid = { - ns.get('uuid'): ns['nsid'] - for ns in (s.get('namespaces', []) if s else []) - } + ret = tgt['rpc'].nvmf_subsystem_add_ns(nqn, tgt_ns_bdev, uuid, guid) + if not ret: + logger.error( + f"Group {group.uuid[:8]}: add ns {tgt_ns_bdev} failed " + f"on {tgt['node_id'][:8]}") + else: + logger.info( + f"Group {group.uuid[:8]}: swap NS {tgt['node_id'][:8]}: " + f"added {tgt_ns_bdev}") except Exception as e: logger.warning( - f"Group {group.uuid[:8]}: subsystem_get on {tgt['node_id'][:8]} " - f"(non-fatal): {e}") - ns_by_uuid = {} - - # Two-pass swap: remove ALL old namespaces first, then add ALL new ones. - # A per-member remove+add loop would cause N sequential "namespace gone" - # events visible to initiators — each one triggering a reconnect. Batching - # the removes into one pass and the adds into a second pass collapses this - # into a single collective disruption, which initiators handle cleanly. - ns_adds = [] # (tgt_ns_bdev, uuid, guid) — collected during remove pass - for m in member_migrations: - try: - lvol = db.get_lvol_by_id(m.lvol_id) - tgt_bdev_short = _lvol_tgt_bdev_name(lvol.lvol_bdev) - tgt_ns_bdev = ( - f"crypto_{tgt_bdev_short}" if lvol.crypto_bdev - else f"{tgt_node.lvstore}/{tgt_bdev_short}" - ) - nsid = ns_by_uuid.get(lvol.uuid) - if nsid: - try: - tgt['rpc'].nvmf_subsystem_remove_ns(nqn, nsid) - logger.info( - f"Group {group.uuid[:8]}: swap NS {tgt['node_id'][:8]}: " - f"removed nsid={nsid} for lvol {lvol.uuid[:8]}") - except Exception as e: - logger.warning( - f"Group {group.uuid[:8]}: remove ns {tgt['node_id'][:8]} " - f"nsid={nsid} (non-fatal): {e}") - else: - logger.warning( - f"Group {group.uuid[:8]}: no namespace for uuid={lvol.uuid[:8]} " - f"on {tgt['node_id'][:8]}; skipping remove") - ns_adds.append((tgt_ns_bdev, lvol.uuid, lvol.guid)) - except Exception as e: - logger.warning( - f"Group {group.uuid[:8]}: namespace swap member {m.uuid[:8]} " - f"on {tgt['node_id'][:8]} (non-fatal): {e}") + f"Group {group.uuid[:8]}: add ns {tgt_ns_bdev} " + f"on {tgt['node_id'][:8]} (non-fatal): {e}") - for tgt_ns_bdev, uuid, guid in ns_adds: - try: - ret = tgt['rpc'].nvmf_subsystem_add_ns(nqn, tgt_ns_bdev, uuid, guid) - if not ret: - logger.error( - f"Group {group.uuid[:8]}: add ns {tgt_ns_bdev} failed " - f"on {tgt['node_id'][:8]}") - else: + # Step 5: all TGT paths → correct ANA state at TGT port + primary_tgt = tgt_paths[0] + if not _flip_all_required(primary_tgt['rpc'], primary_tgt['ips'], primary_tgt['port'], + primary_tgt['trtype'], "optimized", + f"TGT-{primary_tgt['node_id'][:8]}"): + logger.error( + f"Group {group.uuid[:8]}: ANA flip TGT primary→optimized (step 5) failed") + for tgt in tgt_paths[1:]: + _flip_all(tgt['rpc'], tgt['ips'], tgt['port'], tgt['trtype'], + "non_optimized", f"TGT-{tgt['node_id'][:8]}") + + # Step 6: remove old SRC-port listener from overlap TGT nodes if port changed + for tgt in tgt_paths: + if tgt['node_id'] in overlap_ids: + old_port = src_port_by_id.get(tgt['node_id']) + if old_port and old_port != tgt['port']: + for _ip in tgt['ips']: + try: + tgt['rpc'].listeners_del(nqn, tgt['trtype'], _ip, old_port) logger.info( - f"Group {group.uuid[:8]}: swap NS {tgt['node_id'][:8]}: " - f"added {tgt_ns_bdev}") - except Exception as e: - logger.warning( - f"Group {group.uuid[:8]}: add ns {tgt_ns_bdev} " - f"on {tgt['node_id'][:8]} (non-fatal): {e}") - - # TEMPORARILY DISABLED for a diagnostic test: skip removing the old - # SRC-port listener from overlap TGT nodes -- no RPC calls after the - # two ANA flips above. Re-enable once the test is done. - logger.info(f"Group {group.uuid[:8]}: old SRC-port listener cleanup SKIPPED (diagnostic)") + f"Group {group.uuid[:8]}: removed old SRC listener " + f"{_ip}:{old_port} from overlap {tgt['node_id'][:8]}") + except Exception as e: + logger.warning( + f"Group {group.uuid[:8]}: remove old SRC listener " + f"{tgt['node_id'][:8]} (non-fatal): {e}") def _handle_intermediate_barrier(group, member_migrations, src_node, tgt_node, src_rpc, tgt_rpc): @@ -659,25 +675,17 @@ def _flip_all(rpc, ips, port, trtype, state, label): for _ip in ips: _flip(rpc, _ip, port, trtype, state, label) - def _flip_all_required(rpc, ips, port, trtype, state, label, attempts=3): - ok = True - for _ip in ips: - for i in range(attempts): - if _flip(rpc, _ip, port, trtype, state, label): - break - if i < attempts - 1: - time.sleep(1.0) - else: - ok = False - return ok - def _revert_src_replicas(reason): - # Final step didn't complete — put SRC secondary/tertiary back into - # the read path (their pre-freeze state) so clients keep multipath - # access to the still-live source instead of being stuck on primary only. - if not src_replica_paths: - return - logger.warning(f"Group {group.uuid[:8]}: {reason}; reverting SRC secondary/tertiary to non_optimized") + # Final step didn't complete — put every SRC path back into the + # read/write path (their pre-freeze state) so clients keep access to + # the still-live source instead of being stuck with nothing reachable. + # Primary -> optimized (it was driven inaccessible pre-final-step by + # the diagnostic widened freeze above); secondary/tertiary -> non_optimized. + logger.warning(f"Group {group.uuid[:8]}: {reason}; reverting SRC paths " + f"(primary optimized, replicas non_optimized)") + primary_src = src_paths[0] + _flip_all(primary_src['rpc'], primary_src['ips'], primary_src['port'], + primary_src['trtype'], "optimized", f"SRC-{primary_src['node_id'][:8]}(revert)") for p in src_replica_paths: _flip_all(p['rpc'], p['ips'], p['port'], p['trtype'], "non_optimized", f"SRC-{p['node_id'][:8]}(revert)") @@ -685,7 +693,7 @@ def _revert_src_replicas(reason): # TEMPORARILY CHANGED for a diagnostic test: instead of only freezing SRC # secondary/tertiary pre-final-step (primary relied on the RPC's own # internal freeze), make EVERY path -- all SRC (primary included) and all - # TGT -- inaccessible up front, wait 10s so any in-flight client I/O has + # TGT -- inaccessible up front, wait 2s so any in-flight client I/O has # time to fully settle/drain before the data actually moves, THEN call # final_step. Re-enable the narrower pre-freeze once this test is done. logger.info(f"Group {group.uuid[:8]}: setting ALL SRC and TGT paths inaccessible " @@ -770,29 +778,12 @@ def _revert_src_replicas(reason): logger.warning( f"Group {group.uuid[:8]}: add_clone for member {m.uuid[:8]} (non-fatal): {e}") - # TEMPORARILY CHANGED for a diagnostic test: SRC/TGT were already - # driven inaccessible before final_step (see above) -- no need to - # re-flip SRC here. Just wait 2s, then bring TGT primary optimized. - # Re-enable _flip_ana_to_optimized() once this test is done. - logger.info(f"Group {group.uuid[:8]}: sleeping 2s after batch_final_step " - f"before TGT primary optimized (diagnostic)") - time.sleep(2) - primary_tgt = tgt_paths[0] - if not _flip_all_required(primary_tgt['rpc'], primary_tgt['ips'], primary_tgt['port'], - primary_tgt['trtype'], "optimized", - f"TGT-{primary_tgt['node_id'][:8]}"): - logger.error( - f"Group {group.uuid[:8]}: ANA flip TGT primary→optimized failed; " - f"clients may be on degraded path") + _flip_ana_to_optimized(group, member_migrations, src_node, src_rpc, tgt_node, tgt_rpc) - # TEMPORARILY DISABLED for a diagnostic test: no RPC calls at all after - # the two ANA flips above -- not even the hub controller detach. - # Re-enable once the test is done. - logger.info(f"Group {group.uuid[:8]}: hub detach SKIPPED (diagnostic)") - # try: - # src_rpc.bdev_nvme_detach_controller(ctrl_name) - # except Exception as e: - # logger.warning(f"Group {group.uuid[:8]}: hub detach (non-fatal): {e}") + try: + src_rpc.bdev_nvme_detach_controller(ctrl_name) + except Exception as e: + logger.warning(f"Group {group.uuid[:8]}: hub detach (non-fatal): {e}") return batch_ok, batch_err @@ -1140,12 +1131,7 @@ def task_runner(task): task.write_to_db(db.kv_store) return False - # TEMPORARILY DISABLED for a diagnostic test: skip deleting the source - # subsystem so we can tell whether post-migration corruption still - # occurs with the source side left completely intact. Re-enable once - # the test is done. - # _delete_source_subsystem(group, src_node, src_rpc, tgt_node, tgt_rpc) - logger.info(f"Group {group_id[:8]}: source subsystem cleanup SKIPPED (diagnostic)") + _delete_source_subsystem(group, src_node, src_rpc, tgt_node, tgt_rpc) group.phase = LVolMigrationGroup.PHASE_COMPLETED group.status = LVolMigrationGroup.STATUS_DONE diff --git a/simplyblock_core/services/tasks_runner_lvol_migration.py b/simplyblock_core/services/tasks_runner_lvol_migration.py index 840b83e942..81c7710f52 100644 --- a/simplyblock_core/services/tasks_runner_lvol_migration.py +++ b/simplyblock_core/services/tasks_runner_lvol_migration.py @@ -2483,12 +2483,37 @@ def _handle_cleanup_source(migration, src_node, src_rpc, tgt_node, tgt_rpc): "source bdev deletion will be skipped") _warnings.append("source_lvol_bdev not in ctx; source bdev not deleted") - # TEMPORARILY DISABLED for a diagnostic test: skip the safe-to-delete - # verification (it queried tgt_rpc.bdev_lvol_get_lvols) since source - # snapshot deletion itself is disabled below -- no RPC calls here at - # all. Re-enable once the test is done. - logger.info("Source snapshot safe-to-delete verification SKIPPED (diagnostic)") + # Build the safe-to-delete list, cross-checking each snap exists on the + # target before we touch the source. If the target is unreachable, skip + # all source snap deletions to avoid accidental data loss. _snaps_to_delete_src: list = [] + try: + to_delete_all = migration_controller.get_snaps_safe_to_delete_on_source(migration) + tgt_lvols = tgt_rpc.bdev_lvol_get_lvols(tgt_node.lvstore) or [] + tgt_names = {e.get('name', '').split('/')[-1] for e in tgt_lvols} + for snap_uuid in to_delete_all: + try: + snap = db.get_snapshot_by_id(snap_uuid) + _snap_bdev = snap.snap_bdev or '' + _primary = _snap_bdev.split('/', 1)[1] if '/' in _snap_bdev else _snap_bdev + _m_name = _snap_tgt_short_name(snap) + _canonical = _snap_short_name(snap) + _am_name = _canonical + _MIGRATION_BDEV_SUFFIX_DONE + if any(n in tgt_names for n in (_primary, _m_name, _canonical, _am_name)): + _snaps_to_delete_src.append(snap_uuid) + else: + logger.warning( + f"Target missing snapshot {_m_name} ({snap_uuid}); " + "skipping source delete to protect data") + _warnings.append(f"target missing snap {_m_name}; source copy kept") + except KeyError: + pass # already gone from DB; safe to skip + except Exception as _ve: + logger.warning( + f"Could not verify snapshots on target ({_ve}); " + "skipping all source snap deletions") + _warnings.append(f"snap target-verification failed: {_ve}") + _snaps_to_delete_src = [] ctx = { 'stage': 'cleanup_src', @@ -2512,44 +2537,36 @@ def _handle_cleanup_source(migration, src_node, src_rpc, tgt_node, tgt_rpc): # Use the verified list from first-entry; on crash-recovery re-run (ctx already # at 'cleanup_src') snaps_to_delete was saved, so re-deletes are safe (idempotent). source_snap_bdevs = ctx.get('source_snap_bdevs', {}) - # TEMPORARILY DISABLED for a diagnostic test: skip deleting source - # snapshots so they're left intact for inspection. Re-enable once the - # test is done. for snap_uuid in ctx.get('snaps_to_delete', []): - logger.info(f"Source snapshot delete SKIPPED (diagnostic): {snap_uuid}") - # for snap_uuid in ctx.get('snaps_to_delete', []): - # try: - # snap = db.get_snapshot_by_id(snap_uuid) - # bdev_name = (source_snap_bdevs.get(snap_uuid) - # or f"{src_node.lvstore}/{_snap_short_name(snap)}") - # try: - # _delete_bdev_blocking(bdev_name, src_rpc, - # secondary_rpc=src_sec_rpc, tertiary_rpc=src_ter_rpc, - # all_nodes=[n for n in [src_node, src_sec, src_ter] if n], - # lvs_name=src_node.lvstore) - # logger.info(f"Deleted source bdev {bdev_name}") - # except Exception as e: - # logger.warning(f"delete source bdev {bdev_name}: {e}") - # except KeyError: - # logger.warning(f"Source snapshot {snap_uuid} not found in DB; skipping") + try: + snap = db.get_snapshot_by_id(snap_uuid) + bdev_name = (source_snap_bdevs.get(snap_uuid) + or f"{src_node.lvstore}/{_snap_short_name(snap)}") + try: + _delete_bdev_blocking(bdev_name, src_rpc, + secondary_rpc=src_sec_rpc, tertiary_rpc=src_ter_rpc, + all_nodes=[n for n in [src_node, src_sec, src_ter] if n], + lvs_name=src_node.lvstore) + logger.info(f"Deleted source bdev {bdev_name}") + except Exception as e: + logger.warning(f"delete source bdev {bdev_name}: {e}") + except KeyError: + logger.warning(f"Source snapshot {snap_uuid} not found in DB; skipping") # --- Source NVMe-oF subsystem teardown (best-effort) --- lvol = None try: lvol = db.get_lvol_by_id(migration.lvol_id) - # TEMPORARILY DISABLED for a diagnostic test: skip removing the - # source NVMe-oF subsystem/namespace entirely -- no RPC calls here - # at all. Re-enable once the test is done. - logger.info(f"Step 8: source NVMe-oF subsystem removal SKIPPED (diagnostic): {lvol.nqn}") - # _src_paths_cu, _, _overlap_ids_cu = _build_paths( - # src_node, tgt_node, src_rpc, tgt_rpc) - # for _sp in _src_paths_cu: - # if _sp['node_id'] in _overlap_ids_cu: - # logger.info( - # f"Step 8: skip subsystem delete on overlap node " - # f"{_sp['node_id'][:8]} (now serving TGT)") - # else: - # migration_controller.cleanup_subsystem_or_ns(lvol.nqn, lvol.uuid, True, _sp['rpc']) + logger.info(f"Step 8: removing source NVMe-oF subsystem {lvol.nqn}") + _src_paths_cu, _, _overlap_ids_cu = _build_paths( + src_node, tgt_node, src_rpc, tgt_rpc) + for _sp in _src_paths_cu: + if _sp['node_id'] in _overlap_ids_cu: + logger.info( + f"Step 8: skip subsystem delete on overlap node " + f"{_sp['node_id'][:8]} (now serving TGT)") + else: + migration_controller.cleanup_subsystem_or_ns(lvol.nqn, lvol.uuid, True, _sp['rpc']) except Exception as e: logger.warning(f"Source subsystem cleanup failed: {e}") @@ -2557,23 +2574,17 @@ def _handle_cleanup_source(migration, src_node, src_rpc, tgt_node, tgt_rpc): # Use the saved pre-apply name; apply_migration_to_db already renamed # lvol.lvol_bdev in the DB to the target name, so we must not use lvol.lvol_bdev. src_bdev_short = ctx.get('source_lvol_bdev') - # TEMPORARILY DISABLED for a diagnostic test: skip deleting the source - # lvol bdev so it's left intact for inspection. Re-enable once the test - # is done. if lvol is not None and src_bdev_short: src_lvol_composite = f"{src_node.lvstore}/{src_bdev_short}" - logger.info(f"Source lvol bdev delete SKIPPED (diagnostic): {src_lvol_composite}") - # if lvol is not None and src_bdev_short: - # src_lvol_composite = f"{src_node.lvstore}/{src_bdev_short}" - # try: - # _delete_bdev_blocking( - # src_lvol_composite, src_rpc, - # secondary_rpc=src_sec_rpc, tertiary_rpc=src_ter_rpc, - # all_nodes=[n for n in [src_node, src_sec, src_ter] if n], - # lvs_name=src_node.lvstore) - # logger.info(f"Deleted source lvol bdev {src_lvol_composite}") - # except Exception as e: - # logger.warning(f"Source lvol delete failed: {e}") + try: + _delete_bdev_blocking( + src_lvol_composite, src_rpc, + secondary_rpc=src_sec_rpc, tertiary_rpc=src_ter_rpc, + all_nodes=[n for n in [src_node, src_sec, src_ter] if n], + lvs_name=src_node.lvstore) + logger.info(f"Deleted source lvol bdev {src_lvol_composite}") + except Exception as e: + logger.warning(f"Source lvol delete failed: {e}") # --- DB update --- tgt_lvol_uuid = ctx.get('tgt_lvol_uuid') @@ -2592,22 +2603,12 @@ def _handle_cleanup_source(migration, src_node, src_rpc, tgt_node, tgt_rpc): tgt_ter_rpc = _make_rpc(tgt_ter) if tgt_ter else None try: if migration.intermediate_snaps: - # TEMPORARILY DISABLED for a diagnostic test: skip deleting the - # target-side intermediate ("_mig_*") snapshots so we can tell - # whether post-migration corruption still occurs with them left - # intact. Re-enable once the test is done. - logger.info(f"Intermediate snap delete on target SKIPPED (diagnostic): " - f"{migration.intermediate_snaps}") - # _delete_intermediate_snaps_on_target( - # migration, tgt_rpc, tgt_sec_rpc, tgt_ter_rpc, - # tgt_all_nodes=[n for n in [tgt_node, tgt_sec, tgt_ter] if n], - # tgt_lvs_name=tgt_node.lvstore) - # TEMPORARILY DISABLED for a diagnostic test: skip renaming migrated - # bdevs on the target back to their canonical names, so no RPC calls - # happen here at all. Re-enable once the test is done. - logger.info("Target bdev rename SKIPPED (diagnostic)") - # _rename_migrated_bdevs(migration, tgt_node, tgt_rpc, tgt_sec_rpc, tgt_ter_rpc, - # warnings=_warnings) + _delete_intermediate_snaps_on_target( + migration, tgt_rpc, tgt_sec_rpc, tgt_ter_rpc, + tgt_all_nodes=[n for n in [tgt_node, tgt_sec, tgt_ter] if n], + tgt_lvs_name=tgt_node.lvstore) + _rename_migrated_bdevs(migration, tgt_node, tgt_rpc, tgt_sec_rpc, tgt_ter_rpc, + warnings=_warnings) except Exception as e: logger.warning(f"Target artifact cleanup (rename/intermediate snaps) failed: {e}") _warnings.append(f"target rename/intermediate-snap cleanup failed: {e}") From c360a249138553fe0b44ca36a299065fe2d7ac9f Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:12:59 +0330 Subject: [PATCH 043/122] removing debug logs, and redudent calls --- .../services/tasks_runner_batch_migration.py | 110 +++--------------- .../services/tasks_runner_lvol_migration.py | 30 +++-- 2 files changed, 36 insertions(+), 104 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index 525a420a5d..f144ee8cca 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -119,21 +119,12 @@ def _reconstruct_snap_tree(group, member_migrations, tgt_node, tgt_rpc) -> Optio for m in member_migrations: truly_external_preexisting = [ s for s in m.snaps_preexisting_on_target if s not in owned_or_pending_uuids] - logger.info( - f"Group {group.uuid[:8]}: DIAG reconstruct seed member {m.uuid[:8]} " - f"(lvol={m.lvol_id[:8] if m.lvol_id else None}): " - f"snaps_preexisting_on_target={list(m.snaps_preexisting_on_target)} " - f"truly_external_preexisting={truly_external_preexisting} " - f"snaps_migrated={list(m.snaps_migrated)} " - f"snaps_transferred_group={list(m.snaps_transferred_group)}") committed.update(truly_external_preexisting) all_preexisting.update(truly_external_preexisting) # Snaps already committed in a prior (crashed) run are in snaps_migrated. # Seeding committed from them prevents re-convert on re-entry (SPDK rejects # converting an already-immutable bdev, which would stall the group forever). committed.update(m.snaps_migrated) - logger.info(f"Group {group.uuid[:8]}: DIAG reconstruct initial committed set " - f"({len(committed)}): {list(committed)}") _lvstore_prefix = tgt_node.lvstore + "/" @@ -149,16 +140,9 @@ def _reconstruct_snap_tree(group, member_migrations, tgt_node, tgt_rpc) -> Optio for m in sorted(member_migrations, key=lambda x: getattr(x, '_sort_ns_id', 999)): chain = migration_controller.get_snapshot_chain(m.lvol_id, m.source_node_id) - logger.info( - f"Group {group.uuid[:8]}: DIAG reconstruct member {m.uuid[:8]} " - f"(ns_id={getattr(m, '_sort_ns_id', '?')}, lvol={m.lvol_id[:8] if m.lvol_id else None}): " - f"chain={list(chain)} snaps_transferred_group={list(m.snaps_transferred_group)}") for snap_uuid in m.snaps_transferred_group: if snap_uuid in committed: - logger.info( - f"Group {group.uuid[:8]}: DIAG reconstruct snap {snap_uuid[:8]} " - f"(member {m.uuid[:8]}) already in committed -- SKIPPING add_clone/convert") continue try: @@ -177,10 +161,6 @@ def _reconstruct_snap_tree(group, member_migrations, tgt_node, tgt_rpc) -> Optio break if sid in committed: pred_uuid = sid - logger.info( - f"Group {group.uuid[:8]}: DIAG reconstruct snap {snap_uuid[:8]} " - f"(member {m.uuid[:8]}, bdev={tgt_composite}): pred_uuid=" - f"{pred_uuid[:8] if pred_uuid else None}") if pred_uuid: try: @@ -207,10 +187,6 @@ def _reconstruct_snap_tree(group, member_migrations, tgt_node, tgt_rpc) -> Optio else: pred_short = _snap_tgt_short_name(pred_snap) pred_composite = f"{tgt_node.lvstore}/{pred_short}" - logger.info( - f"Group {group.uuid[:8]}: DIAG reconstruct add_clone " - f"{tgt_composite} -> parent {pred_composite} " - f"(preexisting={pred_uuid in all_preexisting})") if not tgt_rpc.bdev_lvol_add_clone(tgt_composite, pred_composite): return f"bdev_lvol_add_clone failed: {snap_uuid} → {pred_uuid}" if sec_rpc: @@ -219,24 +195,12 @@ def _reconstruct_snap_tree(group, member_migrations, tgt_node, tgt_rpc) -> Optio if ter_rpc: if not ter_rpc.bdev_lvol_add_clone(tgt_composite, pred_composite): return f"bdev_lvol_add_clone on tertiary failed: {snap_uuid} → {pred_uuid}" - logger.info( - f"Group {group.uuid[:8]}: DIAG reconstruct add_clone OK " - f"{tgt_composite} -> {pred_composite}") except KeyError: logger.warning(f"Predecessor {pred_uuid} not found; skipping add_clone") - else: - logger.info( - f"Group {group.uuid[:8]}: DIAG reconstruct snap {snap_uuid[:8]} " - f"has no committed predecessor -- converting as a root snapshot " - f"(no add_clone)") # Leadership gate: convert on a non-leader silently persists nothing. from simplyblock_core.controllers import lvol_controller as _lc - _is_leader = _lc.is_node_leader(tgt_node, tgt_composite.split("/")[0]) - logger.info( - f"Group {group.uuid[:8]}: DIAG reconstruct convert {tgt_composite} " - f"is_leader={_is_leader}") - if not _is_leader: + if not _lc.is_node_leader(tgt_node, tgt_composite.split("/")[0]): return f"target node not LVS leader for convert of {snap_uuid}, retrying" if not tgt_rpc.bdev_lvol_convert(tgt_composite): return f"bdev_lvol_convert failed for {snap_uuid}" @@ -246,9 +210,6 @@ def _reconstruct_snap_tree(group, member_migrations, tgt_node, tgt_rpc) -> Optio if ter_rpc: if not ter_rpc.bdev_lvol_convert(tgt_composite): return f"bdev_lvol_convert on tertiary failed for {snap_uuid}" - logger.info( - f"Group {group.uuid[:8]}: DIAG reconstruct convert OK {tgt_composite} " - f"(snap {snap_uuid[:8]} now committed)") # Early partial DB update: route health-check/delete to the target # node right away rather than waiting for apply_migration_to_db() @@ -277,9 +238,6 @@ def _reconstruct_snap_tree(group, member_migrations, tgt_node, tgt_rpc) -> Optio m.write_to_db(db.kv_store) - logger.info( - f"Group {group.uuid[:8]}: DIAG reconstruct done, final committed set " - f"({len(committed)}): {list(committed)}") return None # success @@ -316,8 +274,6 @@ def _build_batch_final_args(group, member_migrations, src_node, tgt_node, tgt_rp """ mid_to_migration = {m.uuid: m for m in member_migrations} ordered_ids = group.ordered_migration_ids() - logger.info(f"Group {group.uuid[:8]}: DIAG build_final_args ordered_ids " - f"({len(ordered_ids)}): {[mid[:8] for mid in ordered_ids]}") lvol_names = [] lvol_ids = [] @@ -348,13 +304,6 @@ def _build_batch_final_args(group, member_migrations, src_node, tgt_node, tgt_rp if map_id is None: raise ValueError(f"map_id missing for {tgt_bdev_short}") lvol_ids.append(map_id) - logger.info( - f"Group {group.uuid[:8]}: DIAG build_final_args migration {migration_id[:8]} " - f"lvol={m.lvol_id[:8] if m.lvol_id else None} src_bdev={src_composite} " - f"tgt_bdev_short={tgt_bdev_short} map_id={map_id} " - f"snaps_migrated={list(m.snaps_migrated)} " - f"snaps_preexisting_on_target={list(m.snaps_preexisting_on_target)} " - f"snaps_transferred_group={list(m.snaps_transferred_group)}") # Last transferred snap = last entry in snaps_migrated (the intermediate). tgt_snap_composite = "" @@ -392,14 +341,8 @@ def _build_batch_final_args(group, member_migrations, src_node, tgt_node, tgt_rp last_uuid, migration_id, ) - logger.info( - f"Group {group.uuid[:8]}: DIAG build_final_args migration {migration_id[:8]} " - f"resolved tgt_snap_composite={tgt_snap_composite!r}") snapshot_names.append(tgt_snap_composite) - logger.info( - f"Group {group.uuid[:8]}: DIAG build_final_args final pairing: " - f"{[(mid[:8], sn) for mid, sn in zip(ordered_ids, snapshot_names)]}") return lvol_names, lvol_ids, snapshot_names @@ -407,22 +350,23 @@ def _flip_ana_to_optimized(group, member_migrations, src_node, src_rpc, tgt_node """ After a successful bdev_lvol_batch_final_step, drive clients to the new target. - Mirrors the single-lvol Done-handler ANA sequence exactly: + Mirrors the single-lvol Done-handler ANA sequence, minus the SRC-inaccessible + step: _handle_intermediate_barrier's pre-final-step freeze already drives + every SRC path (primary + secondary/tertiary) inaccessible before + batch_final_step even runs, so re-flipping them here would just repeat + the same RPC calls with no effect. No-overlap: 1. TGT primary → optimized (required; logs error on failure but continues since bdev_lvol_batch_final_step cannot be undone) 2. TGT secondary/tertiary → non_optimized - 3. All SRC paths → inaccessible Overlap: 1. First non-overlap TGT → optimized (live path before touching overlap) - 2. Overlap SRC paths → inaccessible (at SRC port) - 3. Non-overlap SRC paths → inaccessible - 4. Namespace swap on overlap TGT paths: SRC bdev → migrated TGT bdev + 2. Namespace swap on overlap TGT paths: SRC bdev → migrated TGT bdev (uses _swap_namespace which dynamically re-queries nsid; respects crypto_bdev) - 5. All TGT paths → correct ANA state at TGT port - 6. Remove old SRC-port listener from overlap TGT nodes if port changed + 3. All TGT paths → correct ANA state at TGT port + 4. Remove old SRC-port listener from overlap TGT nodes if port changed """ nqn = group.target_nqn src_paths, tgt_paths, overlap_ids = _build_paths(src_node, tgt_node, src_rpc, tgt_rpc) @@ -492,13 +436,10 @@ def _flip_all_required(rpc, ips, port, trtype, state, label, attempts=3): # Step 2: TGT secondary/tertiary → non_optimized for i, tp in enumerate(tgt_paths[1:], 1): _flip_all(tp['rpc'], tp['ips'], tp['port'], tp['trtype'], "non_optimized", f"TGT-rep{i}") - - # Step 3: all SRC paths → inaccessible - for src in src_paths: - _flip_all(src['rpc'], src['ips'], src['port'], src['trtype'], - "inaccessible", f"SRC-{src['node_id'][:8]}") else: - # Step 1: first non-overlap TGT → optimized before making any SRC inaccessible + # Step 1: first non-overlap TGT → optimized. SRC paths (overlap and + # non-overlap alike) are already inaccessible from the pre-final-step + # freeze -- no need to re-flip them here. non_overlap_tgt = next( (t for t in tgt_paths if t['node_id'] not in overlap_ids), None) if non_overlap_tgt: @@ -510,19 +451,7 @@ def _flip_all_required(rpc, ips, port, trtype, state, label, attempts=3): f"Group {group.uuid[:8]}: ANA flip non-overlap TGT→optimized failed; " f"proceeding anyway") - # Step 2: overlap SRC paths → inaccessible at SRC port - for src in src_paths: - if src['node_id'] in overlap_ids: - _flip_all(src['rpc'], src['ips'], src['port'], src['trtype'], - "inaccessible", f"SRC-{src['node_id'][:8]}(overlap)") - - # Step 3: non-overlap SRC paths → inaccessible - for src in src_paths: - if src['node_id'] not in overlap_ids: - _flip_all(src['rpc'], src['ips'], src['port'], src['trtype'], - "inaccessible", f"SRC-{src['node_id'][:8]}") - - # Step 4: namespace swap on overlap TGT paths. + # Step 2: namespace swap on overlap TGT paths. # Each member has its own namespace in the shared NQN. We look up each # member's nsid by matching ns['uuid'] == lvol.uuid so we never remove # the wrong namespace (positional removal would corrupt I/O for other members). @@ -593,18 +522,18 @@ def _flip_all_required(rpc, ips, port, trtype, state, label, attempts=3): f"Group {group.uuid[:8]}: add ns {tgt_ns_bdev} " f"on {tgt['node_id'][:8]} (non-fatal): {e}") - # Step 5: all TGT paths → correct ANA state at TGT port + # Step 3: all TGT paths → correct ANA state at TGT port primary_tgt = tgt_paths[0] if not _flip_all_required(primary_tgt['rpc'], primary_tgt['ips'], primary_tgt['port'], primary_tgt['trtype'], "optimized", f"TGT-{primary_tgt['node_id'][:8]}"): logger.error( - f"Group {group.uuid[:8]}: ANA flip TGT primary→optimized (step 5) failed") + f"Group {group.uuid[:8]}: ANA flip TGT primary→optimized (step 3) failed") for tgt in tgt_paths[1:]: _flip_all(tgt['rpc'], tgt['ips'], tgt['port'], tgt['trtype'], "non_optimized", f"TGT-{tgt['node_id'][:8]}") - # Step 6: remove old SRC-port listener from overlap TGT nodes if port changed + # Step 4: remove old SRC-port listener from overlap TGT nodes if port changed for tgt in tgt_paths: if tgt['node_id'] in overlap_ids: old_port = src_port_by_id.get(tgt['node_id']) @@ -745,15 +674,8 @@ def _revert_src_replicas(reason): ter_rpc_extra = _make_rpc(ter_node) if ter_node else None _reordered_ids = group.ordered_migration_ids() snap_by_migration_id = dict(zip(_reordered_ids, snapshot_names)) - logger.info( - f"Group {group.uuid[:8]}: DIAG extra-add_clone snap_by_migration_id: " - f"{[(mid[:8], sn) for mid, sn in snap_by_migration_id.items()]}") for m in member_migrations: snap_composite = snap_by_migration_id.get(m.uuid, "") - logger.info( - f"Group {group.uuid[:8]}: DIAG extra-add_clone member {m.uuid[:8]} " - f"(lvol={m.lvol_id[:8] if m.lvol_id else None}) -> snap_composite=" - f"{snap_composite!r}") if not snap_composite: continue try: diff --git a/simplyblock_core/services/tasks_runner_lvol_migration.py b/simplyblock_core/services/tasks_runner_lvol_migration.py index 81c7710f52..b4f94ad25a 100644 --- a/simplyblock_core/services/tasks_runner_lvol_migration.py +++ b/simplyblock_core/services/tasks_runner_lvol_migration.py @@ -2554,19 +2554,29 @@ def _handle_cleanup_source(migration, src_node, src_rpc, tgt_node, tgt_rpc): logger.warning(f"Source snapshot {snap_uuid} not found in DB; skipping") # --- Source NVMe-oF subsystem teardown (best-effort) --- + # Batch group workers share ONE subsystem across every member -- deleting + # it here, per worker, would mean up to N-1 redundant attempts before the + # group orchestrator's own _delete_source_subsystem() runs once, after + # the barrier, on the master thread. Skip it here for group workers; + # single-lvol migrations (no group) still own their own subsystem and + # must still delete it themselves. lvol = None try: lvol = db.get_lvol_by_id(migration.lvol_id) - logger.info(f"Step 8: removing source NVMe-oF subsystem {lvol.nqn}") - _src_paths_cu, _, _overlap_ids_cu = _build_paths( - src_node, tgt_node, src_rpc, tgt_rpc) - for _sp in _src_paths_cu: - if _sp['node_id'] in _overlap_ids_cu: - logger.info( - f"Step 8: skip subsystem delete on overlap node " - f"{_sp['node_id'][:8]} (now serving TGT)") - else: - migration_controller.cleanup_subsystem_or_ns(lvol.nqn, lvol.uuid, True, _sp['rpc']) + if migration.migration_group_id: + logger.info(f"Step 8: source subsystem delete deferred to group " + f"orchestrator (worker of group {migration.migration_group_id[:8]})") + else: + logger.info(f"Step 8: removing source NVMe-oF subsystem {lvol.nqn}") + _src_paths_cu, _, _overlap_ids_cu = _build_paths( + src_node, tgt_node, src_rpc, tgt_rpc) + for _sp in _src_paths_cu: + if _sp['node_id'] in _overlap_ids_cu: + logger.info( + f"Step 8: skip subsystem delete on overlap node " + f"{_sp['node_id'][:8]} (now serving TGT)") + else: + migration_controller.cleanup_subsystem_or_ns(lvol.nqn, lvol.uuid, True, _sp['rpc']) except Exception as e: logger.warning(f"Source subsystem cleanup failed: {e}") From d940afc924248e061a743647ae6f9174248350ac Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Thu, 20 Aug 2026 08:49:42 +0330 Subject: [PATCH 044/122] fix: batch migration cleanup target fail to detect overlap nodes --- .../services/tasks_runner_lvol_migration.py | 75 +++++++++++++++---- 1 file changed, 62 insertions(+), 13 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_lvol_migration.py b/simplyblock_core/services/tasks_runner_lvol_migration.py index b4f94ad25a..99305de1cb 100644 --- a/simplyblock_core/services/tasks_runner_lvol_migration.py +++ b/simplyblock_core/services/tasks_runner_lvol_migration.py @@ -2631,7 +2631,7 @@ def _handle_cleanup_source(migration, src_node, src_rpc, tgt_node, tgt_rpc): return True, False, None -def _handle_cleanup_target(migration, tgt_node, tgt_rpc, src_rpc=None): +def _handle_cleanup_target(migration, tgt_node, tgt_rpc, src_rpc=None, src_node=None): """ Roll back a failed or cancelled migration: remove any partially-created target lvol/subsystem, then delete all snapshots copied to the target. @@ -2640,6 +2640,13 @@ def _handle_cleanup_target(migration, tgt_node, tgt_rpc, src_rpc=None): on primary and secondary). Idempotent: "not found" (status 2) is treated as already done, so a crash-recovery re-run is safe. + Overlap safety: a target node that is also one of this lvol's SOURCE + replica paths (shared/overlap topology) still has its namespace pointing + at the SRC bdev pre-cutover -- it is the live path a client is currently + using, not a spare "target" namespace. Rollback must never touch the + subsystem/namespace on such a node; only non-overlap target-only nodes + are safe to tear down. + Returns (done: bool, suspend: bool, error: str|None). """ @@ -2653,6 +2660,15 @@ def _handle_cleanup_target(migration, tgt_node, tgt_rpc, src_rpc=None): tgt_ter, _ = _get_target_tertiary_node(tgt_node, migration.source_node_id) tgt_ter_rpc = _make_rpc(tgt_ter) if tgt_ter else None + overlap_ids = set() + if src_node is not None: + try: + _, _, overlap_ids = _build_paths(src_node, tgt_node, src_rpc, tgt_rpc) + except Exception as e: + logger.warning( + f"cleanup_target: could not compute overlap nodes, treating " + f"none as overlap (safer would be all -- proceeding with caution): {e}") + # --- Step 0: delete dangling target lvol/subsystems from a failed LVOL_MIGRATE --- # Also handles the pre-create case where bdev/subsystems were set up by # create_migration() but migration was cancelled before LVOL_MIGRATE completed. @@ -2682,17 +2698,27 @@ def _handle_cleanup_target(migration, tgt_node, tgt_rpc, src_rpc=None): # Clean up NVMe-oF subsystem — from ctx (LVOL_MIGRATE failure) or from pre-create. _nqn_to_clean = nqn or _pre_nqn if _nqn_to_clean: - try: - migration_controller.cleanup_subsystem_or_ns( - _nqn_to_clean, migration.lvol_id, - tgt_node.get_id() in owned_node_ids, tgt_rpc) - except Exception as e: - logger.warning(f"cleanup target subsystem {_nqn_to_clean}: {e}") + if tgt_node.get_id() in overlap_ids: + logger.info( + f"cleanup_target: skip subsystem/ns teardown on overlap " + f"node {tgt_node.get_id()[:8]} (still serving live SRC path)") + else: + try: + migration_controller.cleanup_subsystem_or_ns( + _nqn_to_clean, migration.lvol_id, + tgt_node.get_id() in owned_node_ids, tgt_rpc) + except Exception as e: + logger.warning(f"cleanup target subsystem {_nqn_to_clean}: {e}") for _label, _extra_node, _extra_rpc in [ ("secondary", tgt_sec, tgt_sec_rpc), ("tertiary", tgt_ter, tgt_ter_rpc), ]: if _extra_rpc and _extra_node: + if _extra_node.get_id() in overlap_ids: + logger.info( + f"cleanup_target: skip {_label} subsystem/ns teardown on " + f"overlap node {_extra_node.get_id()[:8]} (still serving live SRC path)") + continue try: migration_controller.cleanup_subsystem_or_ns( _nqn_to_clean, migration.lvol_id, @@ -2999,7 +3025,7 @@ def task_runner(task): next_phase = LVolMigration.PHASE_COMPLETED elif phase == LVolMigration.PHASE_CLEANUP_TARGET: - done, suspend, error = _handle_cleanup_target(migration, tgt_node, tgt_rpc, src_rpc=src_rpc) + done, suspend, error = _handle_cleanup_target(migration, tgt_node, tgt_rpc, src_rpc=src_rpc, src_node=src_node) next_phase = "" # terminal — done-handler always sets STATUS_FAILED/CANCELLED else: @@ -3496,8 +3522,14 @@ def _group_worker_phase_dispatch(task, migration, phase, src_node, tgt_node, src if phase == LVolMigration.PHASE_SNAP_COPY: if migration_id not in group.snap_copy_done: # Still transferring owned snaps. - done, suspend, error = _handle_group_snap_copy( - migration, src_node, tgt_node, src_rpc, tgt_rpc) + try: + done, suspend, error = _handle_group_snap_copy( + migration, src_node, tgt_node, src_rpc, tgt_rpc) + except RPCException as exc: + # Charge this worker's own retry budget and report failure to + # the group -- never decide/roll back unilaterally (see + # _group_worker_budget_suspend's docstring). + return _group_worker_budget_suspend(task, migration, group_id, str(exc)) if error: return _group_worker_budget_suspend(task, migration, group_id, error) if suspend: @@ -3539,8 +3571,25 @@ def _group_worker_phase_dispatch(task, migration, phase, src_node, tgt_node, src # --- LVOL_MIGRATE (group worker: take 1 intermediate + wait for batch_result) --- if phase == LVolMigration.PHASE_LVOL_MIGRATE: if migration_id not in group.intermediates_done: - done, suspend, error = _handle_group_intermediate( - migration, src_node, tgt_node, src_rpc, tgt_rpc) + # A sibling may have already failed and told the group to roll + # back while we were mid-retry ourselves -- notice it immediately + # instead of continuing to loop until our own budget runs out. + group = db.get_migration_group_by_id(group_id) + if group.phase == LVolMigrationGroup.PHASE_CLEANUP_TARGET: + migration.phase = LVolMigration.PHASE_CLEANUP_TARGET + migration.write_to_db(db.kv_store) + return _group_worker_phase_dispatch( + task, migration, LVolMigration.PHASE_CLEANUP_TARGET, + src_node, tgt_node, src_rpc, tgt_rpc) + + try: + done, suspend, error = _handle_group_intermediate( + migration, src_node, tgt_node, src_rpc, tgt_rpc) + except RPCException as exc: + # Charge this worker's own retry budget and report failure to + # the group -- never decide/roll back unilaterally (see + # _group_worker_budget_suspend's docstring). + return _group_worker_budget_suspend(task, migration, group_id, str(exc)) if error: return _group_worker_budget_suspend(task, migration, group_id, error) if suspend: @@ -3619,7 +3668,7 @@ def _group_worker_phase_dispatch(task, migration, phase, src_node, tgt_node, src if phase == LVolMigration.PHASE_CLEANUP_TARGET: try: done, suspend, error = _handle_cleanup_target( - migration, tgt_node, tgt_rpc, src_rpc=src_rpc) + migration, tgt_node, tgt_rpc, src_rpc=src_rpc, src_node=src_node) except RPCException as exc: return _suspend_task(task, migration, str(exc)) From 5b1ab52dda9727239ba93c858f5a745814f7d9e5 Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Thu, 20 Aug 2026 10:15:08 +0330 Subject: [PATCH 045/122] removing any hub lvol detach call from the retry path of batch lvol migration --- .../services/hub_controller_manager.py | 2 +- .../services/tasks_runner_batch_migration.py | 17 +++------ .../services/tasks_runner_lvol_migration.py | 37 ++++++++++--------- 3 files changed, 26 insertions(+), 30 deletions(-) diff --git a/simplyblock_core/services/hub_controller_manager.py b/simplyblock_core/services/hub_controller_manager.py index 9e5b9a0a34..092b89558e 100644 --- a/simplyblock_core/services/hub_controller_manager.py +++ b/simplyblock_core/services/hub_controller_manager.py @@ -93,7 +93,7 @@ class HubControllerManager: # Seconds since the last acquire() before the GC triggers a detach. # Refreshed on every acquire() so concurrent migrations naturally keep # the controller alive without any reference counting. - IDLE_TIMEOUT = 300 # 5 minutes + IDLE_TIMEOUT = 1200 # 20 minutes # GC sweep period. GC_INTERVAL = 30 diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index f144ee8cca..b68f9aa670 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -576,12 +576,8 @@ def _handle_intermediate_barrier(group, member_migrations, src_node, tgt_node, s lvol_names, lvol_ids, snapshot_names = _build_batch_final_args( group, member_migrations, src_node, tgt_node, tgt_rpc) except (ValueError, KeyError) as e: - try: - src_rpc.bdev_nvme_detach_controller(ctrl_name) - except Exception as detach_exc: - logger.warning( - f"Group {group.uuid[:8]}: hub controller detach after build-args " - f"failure (non-fatal): {detach_exc}") + # Hub controller left attached — hub_manager owns its lifecycle + # entirely via its own idle timeout. return None, str(e) # Pre-freeze: take SRC/TGT paths out of the read/write path before the @@ -702,11 +698,10 @@ def _revert_src_replicas(reason): _flip_ana_to_optimized(group, member_migrations, src_node, src_rpc, tgt_node, tgt_rpc) - try: - src_rpc.bdev_nvme_detach_controller(ctrl_name) - except Exception as e: - logger.warning(f"Group {group.uuid[:8]}: hub detach (non-fatal): {e}") - + # Hub controller left attached on both success and failure — hub_manager + # owns its lifecycle entirely via its own idle timeout. Detaching it here + # unconditionally, on every group's final step, defeated the whole point + # of keeping it warm for the next group to reuse. return batch_ok, batch_err diff --git a/simplyblock_core/services/tasks_runner_lvol_migration.py b/simplyblock_core/services/tasks_runner_lvol_migration.py index 99305de1cb..a832acc3d6 100644 --- a/simplyblock_core/services/tasks_runner_lvol_migration.py +++ b/simplyblock_core/services/tasks_runner_lvol_migration.py @@ -865,12 +865,12 @@ def _cleanup_final_migration(src_rpc, ctx, tgt_rpc=None, rollback_target=False, tgt_all_nodes=None, tgt_lvs_name=None): """Clean up after a final lvol migration attempt. - On the success path (rollback_target=False) the hub controller is kept - attached on source — detaching it would drop the migration path before - clients have switched to the new target path. + The hub controller is never touched here on either path — it is owned + and lifecycle-managed entirely by hub_manager's own activity-based idle + timeout, not by this function. - On the rollback path (rollback_target=True) the hub controller IS detached - and the target lvol/subsystem are torn down so a retry starts clean. + On the rollback path (rollback_target=True) the target lvol/subsystem + are torn down so a retry starts clean. ``nqn``/``lvol_uuid``/``subsystem_created_on_target`` must come from the caller (the lvol record and migration.target_subsystem_node_ids) — @@ -878,13 +878,12 @@ def _cleanup_final_migration(src_rpc, ctx, tgt_rpc=None, rollback_target=False, stage, so reading them from ``ctx`` here silently no-ops the subsystem cleanup entirely. """ - ctrl_name = ctx.get('ctrl_name') - if ctrl_name and rollback_target: - try: - src_rpc.bdev_nvme_detach_controller(ctrl_name) - except Exception as e: - logger.warning(f"detach hub ctrl {ctrl_name}: {e}") - + # The hub controller is intentionally left attached here, even on + # rollback: it's managed entirely by hub_manager's own activity-based + # idle timeout (IDLE_TIMEOUT with no acquire()s). A retry of this same + # migration will just reuse it via acquire() instead of paying the + # reattach + DETACH_COOLDOWN cost, and a sibling migration to the same + # target isn't disrupted. if rollback_target and tgt_rpc: tgt_composite = ctx.get('tgt_lvol_composite') _nqn = ctx.get('nqn') or nqn @@ -1953,7 +1952,8 @@ def _revert_src_replicas(reason): try: last_snap = db.get_snapshot_by_id(last_snap_uuid) except KeyError: - src_rpc.bdev_nvme_detach_controller(ctrl_name) + # Hub controller left attached — hub_manager owns its + # lifecycle entirely via its own idle timeout. try: _delete_bdev_blocking(tgt_lvol_composite, tgt_rpc, secondary_rpc=tgt_sec_rpc, tertiary_rpc=tgt_ter_rpc, @@ -1979,7 +1979,7 @@ def _revert_src_replicas(reason): tgt_snap_composite = snap_bdev break if not tgt_snap_composite: - src_rpc.bdev_nvme_detach_controller(ctrl_name) + # Hub controller left attached — see comment above. try: _delete_bdev_blocking(tgt_lvol_composite, tgt_rpc, secondary_rpc=tgt_sec_rpc, tertiary_rpc=tgt_ter_rpc, @@ -2041,7 +2041,8 @@ def _revert_src_replicas(reason): if not ret: if state not in ('Done', 'No process'): _revert_src_replicas("final migration failed") - src_rpc.bdev_nvme_detach_controller(ctrl_name) + # Hub controller left attached — see comment above; this is a + # retryable suspend, not an abandoned migration. # Do NOT delete the target bdev on transfer failure — the bdev is # still valid and retaining it keeps the map_id stable across retries. # Deleting it would force a recreate at a higher map_id (due to @@ -2650,9 +2651,9 @@ def _handle_cleanup_target(migration, tgt_node, tgt_rpc, src_rpc=None, src_node= Returns (done: bool, suspend: bool, error: str|None). """ - # Immediately detach the hub controller on failure/cancel — don't leave it - # connected to a target whose snapshots we're about to roll back. - hub_manager.detach_now(migration.source_node_id, tgt_node.get_id(), src_rpc=src_rpc) + # Hub controller left attached here too — hub_manager owns its lifecycle + # entirely via its own idle timeout now; nothing in the migration runners + # calls detach_now() any more. ctx = migration.transfer_context or {} tgt_sec, _ = _get_target_secondary_node(tgt_node, migration.source_node_id) From 6cb2b232dcfb26f04372d4848762fc98542d2bda Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Thu, 20 Aug 2026 19:00:29 +0330 Subject: [PATCH 046/122] batch migration now may take extra intermediate snapshots if the delta on the lvol is still big in size --- simplyblock_core/constants.py | 2 +- .../models/lvol_migration_group.py | 26 ++++++-- .../services/tasks_runner_batch_migration.py | 23 ++++++- .../services/tasks_runner_lvol_migration.py | 60 +++++++++++++++---- 4 files changed, 92 insertions(+), 19 deletions(-) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 911b63c3e3..c655a16369 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -592,7 +592,7 @@ def get_config_var(name, default=None): LVOL_MIG_MAX_RETRIES = 5 # max retries before entering cleanup_target LVOL_MIG_DEADLINE_SEC = 3600 # 1-hour deadline (0 = no deadline) LVOL_MIG_MAX_INTERMEDIATE_SNAPS = 3 # max recursive "shrink" snapshot rounds -LVOL_MIG_INTERMEDIATE_SNAP_THRESHOLD_BYTES = 500 * 1024 * 1024 # 500 MiB — skip if delta is smaller +LVOL_MIG_INTERMEDIATE_SNAP_THRESHOLD_BYTES = 2 * 1024 * 1024 * 1024 # 2 GiB — skip if delta is smaller LVOL_MIG_BDEV_SUFFIX = 'm' # appended to every migration bdev on the target to avoid collision with real bdevs # NVMe-oF TLS / DH-HMAC-CHAP security diff --git a/simplyblock_core/models/lvol_migration_group.py b/simplyblock_core/models/lvol_migration_group.py index 2641cf4527..09d29dab2b 100644 --- a/simplyblock_core/models/lvol_migration_group.py +++ b/simplyblock_core/models/lvol_migration_group.py @@ -14,8 +14,14 @@ N worker tasks copy their owned snapshot chains in parallel. snap_copy_done tracks which workers have finished. INTERMEDIATE - All workers take exactly one intermediate ('shrink') snapshot each. - intermediates_done tracks which workers have finished. + All workers take one intermediate ('shrink') snapshot each, in lockstep + rounds. intermediates_done tracks which workers have finished the + current round (intermediate_round). If any worker's dirty delta is + still above the threshold after a round, it flags itself in + intermediate_more_needed; once every worker has finished the round, the + orchestrator starts another synchronized round (all members retake a + snapshot together, even ones whose own delta was already low) if + intermediate_more_needed is non-empty and the round cap hasn't been hit. BATCH_MIGRATE Main calls bdev_lvol_batch_final_step with all lvols ordered by ns_id. batch_result is set to True on success, False on failure. @@ -80,10 +86,22 @@ class LVolMigrationGroup(BaseModel): # waiting for the INTERMEDIATE phase signal from the main orchestrator. snap_copy_done: List[str] = [] - # migration_ids that have taken and transferred their single intermediate - # snapshot and are waiting for batch_result. + # migration_ids that have taken and transferred their intermediate + # snapshot for the CURRENT intermediate_round and are waiting for either + # another round or batch_result. Cleared when a new round starts. intermediates_done: List[str] = [] + # Which intermediate round is currently in flight (0-indexed; round 0 is + # always taken unconditionally). Incremented when the orchestrator starts + # another synchronized round. + intermediate_round: int = 0 + + # migration_ids that reported their dirty delta still exceeded the + # threshold after finishing intermediate_round. Cleared when a new round + # starts. Non-empty at the end of a round (and under the round cap) + # triggers another synchronized round for every member. + intermediate_more_needed: List[str] = [] + # migration_ids that have completed CLEANUP_SOURCE. cleanup_source_done: List[str] = [] diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index b68f9aa670..f5920f067e 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -17,8 +17,11 @@ Advance group to PHASE_INTERMEDIATE. PHASE_INTERMEDIATE (orchestrator: wait + batch-final) - Wait for all workers to signal intermediates_done. - Build the batch-final-step argument lists (one entry per member, ordered + Wait for all workers to signal intermediates_done for the current round. + If any worker's dirty delta is still above the threshold, start another + synchronized round (every member retakes a snapshot together) up to + LVOL_MIG_MAX_INTERMEDIATE_SNAPS rounds. Once no more rounds are needed, + build the batch-final-step argument lists (one entry per member, ordered by ns_id), acquire a shared hub connection via hub_manager, and call bdev_lvol_batch_final_step on the source node. Set group.batch_result = True/False. @@ -562,6 +565,22 @@ def _handle_intermediate_barrier(group, member_migrations, src_node, tgt_node, s logger.debug(f"intermediates barrier: waiting for {len(waiting)} workers") return None, None # None = still waiting + # Every member finished this round. If any of them still has too much + # dirty delta to freeze quickly at cutover, start another synchronized + # round -- every member retakes a snapshot together, even ones whose own + # delta was already low -- up to the round cap. + if (group.intermediate_more_needed + and group.intermediate_round + 1 < constants.LVOL_MIG_MAX_INTERMEDIATE_SNAPS): + group.intermediate_round += 1 + group.intermediates_done = [] + group.intermediate_more_needed = [] + group.write_to_db(db.kv_store) + logger.info( + f"Group {group.uuid[:8]}: dirty delta still high after round " + f"{group.intermediate_round}/{constants.LVOL_MIG_MAX_INTERMEDIATE_SNAPS}; " + f"starting another synchronized intermediate round") + return None, None # None = still waiting -- workers will redo this round + logger.info( f"Group {group.uuid[:8]}: all workers reached intermediates_done; " f"calling bdev_lvol_batch_final_step") diff --git a/simplyblock_core/services/tasks_runner_lvol_migration.py b/simplyblock_core/services/tasks_runner_lvol_migration.py index a832acc3d6..171f2330c1 100644 --- a/simplyblock_core/services/tasks_runner_lvol_migration.py +++ b/simplyblock_core/services/tasks_runner_lvol_migration.py @@ -3323,22 +3323,34 @@ def _handle_group_snap_copy(migration, src_node, tgt_node, src_rpc, tgt_rpc): return True, False, None -def _handle_group_intermediate(migration, src_node, tgt_node, src_rpc, tgt_rpc): +def _handle_group_intermediate(migration, src_node, tgt_node, src_rpc, tgt_rpc, target_round=0): """ INTERMEDIATE phase for a group worker. - Takes exactly one intermediate ("shrink") snapshot and transfers it to the - target, skipping add_clone/convert (same as snap_copy). After this the - worker signals intermediates_done to the group and waits for batch_result. + Takes one intermediate ("shrink") snapshot per round and transfers it to + the target, skipping add_clone/convert (same as snap_copy). After this + the worker signals intermediates_done to the group and waits for either + another round or batch_result. + + ``target_round`` is the group's current intermediate_round. If this + worker has already completed that round (migration.intermediate_snap_rounds + > target_round), it's done for now. Otherwise -- including when it was + previously done for an earlier round but the group has since asked for + another synchronized round -- it resets and takes a fresh snapshot. Returns (done: bool, suspend: bool, error: str|None). """ trtype, _ = _get_migration_nic(tgt_node) ctx = migration.transfer_context or {} - # If we already took and transferred the intermediate snap, we're done. + # If we already took and transferred the intermediate snap for the round + # the group is currently on, we're done. Otherwise the group has asked + # for another round since we last finished -- fall through and redo. if ctx.get('stage') == 'intermediate_done': - return True, False, None + if migration.intermediate_snap_rounds > target_round: + return True, False, None + ctx = {} + migration.transfer_context = {} # Take the intermediate snapshot if not already in flight. if ctx.get('stage') != 'intermediate_transfer': @@ -3503,9 +3515,10 @@ def _group_worker_phase_dispatch(task, migration, phase, src_node, tgt_node, src Manages the group worker state machine: SNAP_COPY → transfer owned snaps (no add_clone/convert) → signal snap_copy_done to group → wait for INTERMEDIATE - LVOL_MIGRATE (repurposed as the single-intermediate phase for workers) - → take + transfer 1 intermediate snap - → signal intermediates_done → wait for batch_result + LVOL_MIGRATE (repurposed as the intermediate phase for workers) + → take + transfer 1 intermediate snap for the current round + → signal intermediates_done → wait for either another + synchronized round or batch_result CLEANUP_SOURCE → normal source cleanup + signal cleanup_source_done CLEANUP_TARGET → normal target rollback @@ -3569,7 +3582,7 @@ def _group_worker_phase_dispatch(task, migration, phase, src_node, tgt_node, src task.write_to_db(db.kv_store) return False - # --- LVOL_MIGRATE (group worker: take 1 intermediate + wait for batch_result) --- + # --- LVOL_MIGRATE (group worker: take intermediate round(s) + wait for batch_result) --- if phase == LVolMigration.PHASE_LVOL_MIGRATE: if migration_id not in group.intermediates_done: # A sibling may have already failed and told the group to roll @@ -3585,7 +3598,8 @@ def _group_worker_phase_dispatch(task, migration, phase, src_node, tgt_node, src try: done, suspend, error = _handle_group_intermediate( - migration, src_node, tgt_node, src_rpc, tgt_rpc) + migration, src_node, tgt_node, src_rpc, tgt_rpc, + target_round=group.intermediate_round) except RPCException as exc: # Charge this worker's own retry budget and report failure to # the group -- never decide/roll back unilaterally (see @@ -3598,11 +3612,33 @@ def _group_worker_phase_dispatch(task, migration, phase, src_node, tgt_node, src if done: group = db.get_migration_group_by_id(group_id) if migration_id not in group.intermediates_done: + # Below the round cap, check whether this worker's dirty + # delta is still too large to freeze quickly at cutover -- + # if so, flag it so the orchestrator starts another + # synchronized round for every member (see + # LVolMigrationGroup's INTERMEDIATE docstring). + needs_more = False + if group.intermediate_round + 1 < constants.LVOL_MIG_MAX_INTERMEDIATE_SNAPS: + try: + lvol = db.get_lvol_by_id(migration.lvol_id) + src_composite = f"{src_node.lvstore}/{lvol.lvol_bdev}" + delta = _get_lvol_delta_bytes(src_rpc, src_composite) + needs_more = ( + delta is None + or delta > constants.LVOL_MIG_INTERMEDIATE_SNAP_THRESHOLD_BYTES) + except Exception as e: + logger.warning( + f"Group worker {migration_id[:8]}: delta check failed " + f"(assuming another round is needed): {e}") + needs_more = True + if needs_more and migration_id not in group.intermediate_more_needed: + group.intermediate_more_needed.append(migration_id) group.intermediates_done.append(migration_id) group.write_to_db(db.kv_store) logger.info( f"Group worker {migration_id[:8]}: signalled intermediates_done " - f"({len(group.intermediates_done)}/{group.member_count()})") + f"({len(group.intermediates_done)}/{group.member_count()})" + + (" [delta still high, requesting another round]" if needs_more else "")) migration.write_to_db(db.kv_store) task.write_to_db(db.kv_store) return False From 8e586cf52b0f1c481c39cf1efc48b90280afa6f4 Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:56:16 +0330 Subject: [PATCH 047/122] on retry batch migration will take extra intermediate snapshots if needed, the timeout for batch final step increased to 15 seconds --- simplyblock_core/constants.py | 2 +- .../services/tasks_runner_batch_migration.py | 58 ++++++++++++++++++- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index c655a16369..911b63c3e3 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -592,7 +592,7 @@ def get_config_var(name, default=None): LVOL_MIG_MAX_RETRIES = 5 # max retries before entering cleanup_target LVOL_MIG_DEADLINE_SEC = 3600 # 1-hour deadline (0 = no deadline) LVOL_MIG_MAX_INTERMEDIATE_SNAPS = 3 # max recursive "shrink" snapshot rounds -LVOL_MIG_INTERMEDIATE_SNAP_THRESHOLD_BYTES = 2 * 1024 * 1024 * 1024 # 2 GiB — skip if delta is smaller +LVOL_MIG_INTERMEDIATE_SNAP_THRESHOLD_BYTES = 500 * 1024 * 1024 # 500 MiB — skip if delta is smaller LVOL_MIG_BDEV_SUFFIX = 'm' # appended to every migration bdev on the target to avoid collision with real bdevs # NVMe-oF TLS / DH-HMAC-CHAP security diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index f5920f067e..f237878bc6 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -658,8 +658,12 @@ def _revert_src_replicas(reason): batch_ok = False batch_err = None try: - ret = src_rpc.bdev_lvol_batch_transfer_final_step( - lvol_names, lvol_ids, snapshot_names, 2, hub_bdev, "migrate") + # This call moves real data and can legitimately run longer than the + # 5s blanket timeout _make_rpc()/src_rpc uses for every other RPC in + # this file -- use a dedicated, longer-timeout client just for it. + final_step_rpc = src_node.rpc_client(timeout=15, retry=2) + ret = final_step_rpc.bdev_lvol_batch_transfer_final_step( + lvol_names, lvol_ids, snapshot_names, 16, hub_bdev, "migrate") logger.info(f"Group {group.uuid[:8]}: bdev_lvol_batch_transfer_final_step returned {ret!r}") batch_ok = True except RPCRemoteError as e: @@ -674,6 +678,56 @@ def _revert_src_replicas(reason): if not batch_ok: _revert_src_replicas("batch_final_step failed") + # The revert above reopened SRC to live client I/O. Retrying with the + # snapshots taken before this reopen would silently miss whatever the + # client writes in the meantime -- force every member through one + # more synchronized intermediate round first, same mechanism as the + # dirty-delta trigger above, so the retry's snapshots actually cover + # the reopen window. Falls through to the normal suspend/retry-budget + # path once the round cap is hit, so a persistently failing group + # still eventually resolves instead of looping forever. + if group.intermediate_round + 1 < constants.LVOL_MIG_MAX_INTERMEDIATE_SNAPS: + group.intermediate_round += 1 + group.intermediates_done = [] + group.intermediate_more_needed = [] + group.write_to_db(db.kv_store) + + # bdev_lvol_set_migration_flag drives the distrib-level special_io + # machinery for the target bdev (see snapshot_replication.py's + # comment on the same flag); it's only ever set once, at initial + # target-bdev creation (migration_controller.create_migration). + # A failed/aborted final_step attempt may clear it on the target, + # so re-assert it on every member's target bdev before retrying — + # otherwise the retry's cutover could run without the target + # being treated as migration-aware. + tgt_sec_node, _ = _get_target_secondary_node(tgt_node, src_node.get_id()) + tgt_ter_node, _ = _get_target_tertiary_node(tgt_node, src_node.get_id()) + tgt_sec_rpc_reflag = _make_rpc(tgt_sec_node) if tgt_sec_node else None + tgt_ter_rpc_reflag = _make_rpc(tgt_ter_node) if tgt_ter_node else None + for m in member_migrations: + try: + m_lvol = db.get_lvol_by_id(m.lvol_id) + m_tgt_composite = f"{tgt_node.lvstore}/{_lvol_tgt_bdev_name(m_lvol.lvol_bdev)}" + except KeyError: + continue + if not tgt_rpc.bdev_lvol_set_migration_flag(m_tgt_composite): + logger.warning( + f"Group {group.uuid[:8]}: re-assert migration flag on primary " + f"failed for {m_tgt_composite} (may already be flagged)") + for _extra_rpc in (tgt_sec_rpc_reflag, tgt_ter_rpc_reflag): + if _extra_rpc: + try: + _extra_rpc.bdev_lvol_set_migration_flag(m_tgt_composite) + except Exception as e: + logger.warning( + f"Group {group.uuid[:8]}: re-assert migration flag on " + f"replica failed for {m_tgt_composite} (non-fatal): {e}") + + logger.warning( + f"Group {group.uuid[:8]}: batch_final_step failed; forcing another " + f"synchronized intermediate round {group.intermediate_round}/" + f"{constants.LVOL_MIG_MAX_INTERMEDIATE_SNAPS} before retrying") + return None, None # None = still waiting -- workers will redo this round # else: left as-is — all SRC/TGT paths were already driven inaccessible # before final_step (diagnostic, see above); only TGT primary needs to # come back optimized on success, handled below. From dbed3fa874c2c35c0031e2bbb04f8a1512cbb568 Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:01:58 +0330 Subject: [PATCH 048/122] temp: manually blockiong the port during lvol migration for testing --- .../services/tasks_runner_batch_migration.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index f237878bc6..ecf9f05d10 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -39,6 +39,7 @@ NVMe-oF subsystem and marks group FAILED/CANCELLED. """ +import threading import time from typing import Optional @@ -50,6 +51,7 @@ from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.rpc_client import RPCErrorCode, RPCRemoteError, RPCException from simplyblock_core.services.hub_controller_manager import HubControllerManager +from simplyblock_core.utils import port_block from simplyblock_core.services.tasks_runner_lvol_migration import ( _make_rpc, _snap_tgt_short_name, @@ -652,6 +654,36 @@ def _revert_src_replicas(reason): f"before batch_final_step (diagnostic)") time.sleep(2) + # TEMPORARILY ADDED for a diagnostic test: block the target's transfer-hub + # NVMe-oF listener port right as batch_final_step is about to be called, + # then unblock it 5s later from a background thread while the call is + # in flight -- reproduces on demand the "final_step legitimately takes + # ~5-6s and races its own client-side timeout" race this session has + # been chasing, instead of waiting for it to happen naturally. Remove + # once this test is done. + _hub_port = getattr(tgt_node.transfer_hublvol, 'nvmf_port', None) if tgt_node.transfer_hublvol else None + if _hub_port: + logger.info(f"Group {group.uuid[:8]}: blocking hub port {_hub_port} on " + f"target {tgt_node.get_id()[:8]} for 5s (diagnostic)") + try: + port_block.set_port(tgt_node, _hub_port, block=True) + except Exception as e: + logger.warning(f"Group {group.uuid[:8]}: hub port block failed (non-fatal): {e}") + + def _unblock_hub_port_later(): + time.sleep(5) + try: + port_block.set_port(tgt_node, _hub_port, block=False) + logger.info(f"Group {group.uuid[:8]}: unblocked hub port {_hub_port} " + f"on target {tgt_node.get_id()[:8]} (diagnostic)") + except Exception as e: + logger.warning(f"Group {group.uuid[:8]}: hub port unblock failed: {e}") + + threading.Thread(target=_unblock_hub_port_later, daemon=True).start() + else: + logger.warning(f"Group {group.uuid[:8]}: no transfer_hublvol port found on " + f"target -- skipping diagnostic port block") + logger.info( f"Group {group.uuid[:8]}: batch_final_step " f"lvols={len(lvol_names)} hub={hub_bdev}") From 16eea54b96fb60cc0425a452c918d4f5653402cd Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Fri, 21 Aug 2026 21:55:06 +0330 Subject: [PATCH 049/122] removing temp port block --- .../services/tasks_runner_batch_migration.py | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index ecf9f05d10..f237878bc6 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -39,7 +39,6 @@ NVMe-oF subsystem and marks group FAILED/CANCELLED. """ -import threading import time from typing import Optional @@ -51,7 +50,6 @@ from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.rpc_client import RPCErrorCode, RPCRemoteError, RPCException from simplyblock_core.services.hub_controller_manager import HubControllerManager -from simplyblock_core.utils import port_block from simplyblock_core.services.tasks_runner_lvol_migration import ( _make_rpc, _snap_tgt_short_name, @@ -654,36 +652,6 @@ def _revert_src_replicas(reason): f"before batch_final_step (diagnostic)") time.sleep(2) - # TEMPORARILY ADDED for a diagnostic test: block the target's transfer-hub - # NVMe-oF listener port right as batch_final_step is about to be called, - # then unblock it 5s later from a background thread while the call is - # in flight -- reproduces on demand the "final_step legitimately takes - # ~5-6s and races its own client-side timeout" race this session has - # been chasing, instead of waiting for it to happen naturally. Remove - # once this test is done. - _hub_port = getattr(tgt_node.transfer_hublvol, 'nvmf_port', None) if tgt_node.transfer_hublvol else None - if _hub_port: - logger.info(f"Group {group.uuid[:8]}: blocking hub port {_hub_port} on " - f"target {tgt_node.get_id()[:8]} for 5s (diagnostic)") - try: - port_block.set_port(tgt_node, _hub_port, block=True) - except Exception as e: - logger.warning(f"Group {group.uuid[:8]}: hub port block failed (non-fatal): {e}") - - def _unblock_hub_port_later(): - time.sleep(5) - try: - port_block.set_port(tgt_node, _hub_port, block=False) - logger.info(f"Group {group.uuid[:8]}: unblocked hub port {_hub_port} " - f"on target {tgt_node.get_id()[:8]} (diagnostic)") - except Exception as e: - logger.warning(f"Group {group.uuid[:8]}: hub port unblock failed: {e}") - - threading.Thread(target=_unblock_hub_port_later, daemon=True).start() - else: - logger.warning(f"Group {group.uuid[:8]}: no transfer_hublvol port found on " - f"target -- skipping diagnostic port block") - logger.info( f"Group {group.uuid[:8]}: batch_final_step " f"lvols={len(lvol_names)} hub={hub_bdev}") From 57895596fdca520dc62ad2d6bbcea93199f91545 Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Mon, 24 Aug 2026 04:37:28 +0330 Subject: [PATCH 050/122] fix: ns id not passed explicitly, chain between intermediates, add clone/convert for intermediates, redudent rpc calls, bad delete --- .../controllers/migration_controller.py | 6 +- .../services/tasks_runner_batch_migration.py | 150 +++++++++++++++- .../services/tasks_runner_lvol_migration.py | 161 ++++++++++++++---- 3 files changed, 278 insertions(+), 39 deletions(-) diff --git a/simplyblock_core/controllers/migration_controller.py b/simplyblock_core/controllers/migration_controller.py index bfeb781711..bc6964292f 100644 --- a/simplyblock_core/controllers/migration_controller.py +++ b/simplyblock_core/controllers/migration_controller.py @@ -896,9 +896,9 @@ def create_migration(lvol_id, target_node_id, 1. Create migration bdev m in the target lvstore. Idempotent: skipped if the bdev already exists. 2. Create NVMe-oF subsystem with the same NQN as the source lvol. - 3. Add inaccessible listeners on every data NIC. - No namespace is added — the task runner wires it up when migration - actually begins (PHASE_LVOL_MIGRATE). + 3. Add inaccessible listeners on every data NIC, then add the namespace + itself, pinned to the source's nsid (lvol.ns_id) rather than letting + SPDK auto-assign — see the nvmf_subsystem_add_ns call below for why. 4. Create an LVolMigration record in PHASE_PRE_CREATED so that cancel_migration can tear everything down on request. diff --git a/simplyblock_core/services/tasks_runner_batch_migration.py b/simplyblock_core/services/tasks_runner_batch_migration.py index f237878bc6..6890a41805 100644 --- a/simplyblock_core/services/tasks_runner_batch_migration.py +++ b/simplyblock_core/services/tasks_runner_batch_migration.py @@ -349,6 +349,121 @@ def _build_batch_final_args(group, member_migrations, src_node, tgt_node, tgt_rp return lvol_names, lvol_ids, snapshot_names +def _commit_intermediate_snapshot_chain(group, member_migrations, tgt_node, tgt_rpc): + """ + Link each member's intermediate ("shrink") snapshots into the target + ancestry chain and freeze them immutable, before bdev_lvol_batch_final_step + runs. + + _handle_group_intermediate transfers each round's data to the target (and + registers it on secondary/tertiary via _setup_snap_transfer) but + deliberately skips add_clone/convert -- same as the snap_copy phase -- + deferring tree-building to the orchestrator. _reconstruct_snap_tree is the + orchestrator step that normally does that linking, but it only covers the + snap_copy chain (via snaps_transferred_group) and only runs once, at the + SNAP_COPY -> INTERMEDIATE transition, before any intermediate round + exists -- it can never reach forward to link them. _build_batch_final_args + then picks only the LAST intermediate snapshot (snaps_migrated[-1]) as the + one boundary snapshot, and the post-final_step code below links only that + single snapshot to the live bdev. Nothing anywhere ever linked the + intermediate snapshots to each other, or round 0 to the snap_copy chain's + last snapshot. + + With exactly one intermediate round this was invisible: the only link + needed (live bdev -> round 0) was the one link that existed. With two or + more rounds, every earlier round's target blob was left parentless -- + reads falling outside what that specific round itself captured returned + zeros instead of falling through to the real predecessor data (observed + as fio's "bad magic header 0" checksum failures). + + Called once per group, before batch_final_step, so the whole chain is + committed immutable ahead of the live cutover -- mirrors + _reconstruct_snap_tree's own add_clone-on-all-replicas-then-convert + ordering. Returns None on success, or an error string. + """ + tgt_sec_node, _ = _get_target_secondary_node(tgt_node, "") + sec_rpc = _make_rpc(tgt_sec_node) if tgt_sec_node else None + tgt_ter_node, _ = _get_target_tertiary_node(tgt_node, "") + ter_rpc = _make_rpc(tgt_ter_node) if tgt_ter_node else None + from simplyblock_core.controllers import lvol_controller as _lc + + for m in member_migrations: + intermediate_snaps = m.intermediate_snaps or [] + if not intermediate_snaps: + continue + + # Predecessor for round 0: the last snap_copy-chain snapshot already + # committed on target -- mirrors _reconstruct_snap_tree's own + # preexisting-vs-freshly-transferred predecessor lookup. + pred_composite = None + pred_uuid = (m.snaps_transferred_group or m.snaps_preexisting_on_target or [None])[-1] + if pred_uuid: + try: + pred_snap = db.get_snapshot_by_id(pred_uuid) + if pred_uuid in (m.snaps_preexisting_on_target or []): + _lvstore_prefix = tgt_node.lvstore + '/' + pred_short = None + if pred_snap.snap_bdev and pred_snap.snap_bdev.startswith(_lvstore_prefix): + pred_short = pred_snap.snap_bdev.split('/', 1)[1] + else: + for _inst in pred_snap.instances or []: + _inst_bdev = _inst.get('snap_bdev', '') + if _inst_bdev.startswith(_lvstore_prefix): + pred_short = _inst_bdev.split('/', 1)[1] + break + pred_short = pred_short or _snap_tgt_short_name(pred_snap) + else: + pred_short = _snap_tgt_short_name(pred_snap) + pred_composite = f"{tgt_node.lvstore}/{pred_short}" + except KeyError: + logger.warning( + f"Group {group.uuid[:8]}: intermediate-chain predecessor " + f"{pred_uuid} not found for member {m.uuid[:8]}; linking round 0 " + f"without a parent") + + for snap_uuid in intermediate_snaps: + try: + snap = db.get_snapshot_by_id(snap_uuid) + except KeyError: + return f"Intermediate snapshot {snap_uuid} not found while committing chain" + tgt_composite = f"{tgt_node.lvstore}/{_snap_tgt_short_name(snap)}" + + # Same known SPDK behavior _reconstruct_snap_tree already guards + # against: converting an already-immutable bdev is rejected, so a + # retry that reaches an earlier round already committed here would + # otherwise fail every time. bdev_lvol_get_bdevs reports + # is_snapshot on the primary; treat that as "already done" and + # just advance the predecessor pointer. + _existing = tgt_rpc.get_bdevs(tgt_composite) + if _existing and _existing[0].get('driver_specific', {}).get('lvol', {}).get('is_snapshot'): + pred_composite = tgt_composite + continue + + if pred_composite: + if not tgt_rpc.bdev_lvol_add_clone(tgt_composite, pred_composite): + return f"bdev_lvol_add_clone failed for intermediate snap {snap_uuid}" + if sec_rpc and not sec_rpc.bdev_lvol_add_clone(tgt_composite, pred_composite): + return f"bdev_lvol_add_clone on secondary failed for intermediate snap {snap_uuid}" + if ter_rpc and not ter_rpc.bdev_lvol_add_clone(tgt_composite, pred_composite): + return f"bdev_lvol_add_clone on tertiary failed for intermediate snap {snap_uuid}" + + if not _lc.is_node_leader(tgt_node, tgt_composite.split("/")[0]): + return f"target node not LVS leader for convert of intermediate snap {snap_uuid}, retrying" + if not tgt_rpc.bdev_lvol_convert(tgt_composite): + return f"bdev_lvol_convert failed for intermediate snap {snap_uuid}" + if sec_rpc and not sec_rpc.bdev_lvol_convert(tgt_composite): + return f"bdev_lvol_convert on secondary failed for intermediate snap {snap_uuid}" + if ter_rpc and not ter_rpc.bdev_lvol_convert(tgt_composite): + return f"bdev_lvol_convert on tertiary failed for intermediate snap {snap_uuid}" + + logger.info( + f"Group {group.uuid[:8]}: committed intermediate snap {snap_uuid[:8]} " + f"({tgt_composite}) parent={pred_composite}") + pred_composite = tgt_composite + + return None + + def _flip_ana_to_optimized(group, member_migrations, src_node, src_rpc, tgt_node, tgt_rpc): """ After a successful bdev_lvol_batch_final_step, drive clients to the new target. @@ -479,7 +594,7 @@ def _flip_all_required(rpc, ips, port, trtype, state, label, attempts=3): # events visible to initiators — each one triggering a reconnect. Batching # the removes into one pass and the adds into a second pass collapses this # into a single collective disruption, which initiators handle cleanly. - ns_adds = [] # (tgt_ns_bdev, uuid, guid) — collected during remove pass + ns_adds = [] # (tgt_ns_bdev, uuid, guid, nsid) — collected during remove pass for m in member_migrations: try: lvol = db.get_lvol_by_id(m.lvol_id) @@ -503,15 +618,21 @@ def _flip_all_required(rpc, ips, port, trtype, state, label, attempts=3): logger.warning( f"Group {group.uuid[:8]}: no namespace for uuid={lvol.uuid[:8]} " f"on {tgt['node_id'][:8]}; skipping remove") - ns_adds.append((tgt_ns_bdev, lvol.uuid, lvol.guid)) + # Re-add under the SAME nsid it just had here, instead of letting + # SPDK auto-assign the next free one. Client-side identity across + # the swap is carried by uuid/nguid regardless (Linux NVMe + # multipath groups paths by NGUID, not nsid), so this isn't a + # correctness fix -- it just keeps the namespace's nsid stable on + # this node across the swap instead of drifting to a new value. + ns_adds.append((tgt_ns_bdev, lvol.uuid, lvol.guid, nsid)) except Exception as e: logger.warning( f"Group {group.uuid[:8]}: namespace swap member {m.uuid[:8]} " f"on {tgt['node_id'][:8]} (non-fatal): {e}") - for tgt_ns_bdev, uuid, guid in ns_adds: + for tgt_ns_bdev, uuid, guid, nsid in ns_adds: try: - ret = tgt['rpc'].nvmf_subsystem_add_ns(nqn, tgt_ns_bdev, uuid, guid) + ret = tgt['rpc'].nvmf_subsystem_add_ns(nqn, tgt_ns_bdev, uuid, guid, nsid=nsid) if not ret: logger.error( f"Group {group.uuid[:8]}: add ns {tgt_ns_bdev} failed " @@ -599,6 +720,11 @@ def _handle_intermediate_barrier(group, member_migrations, src_node, tgt_node, s # entirely via its own idle timeout. return None, str(e) + chain_err = _commit_intermediate_snapshot_chain(group, member_migrations, tgt_node, tgt_rpc) + if chain_err: + # Hub controller left attached — see comment above. + return None, chain_err + # Pre-freeze: take SRC/TGT paths out of the read/write path before the # synchronous final-step transfer below (see the diagnostic block further # down for the current, temporarily-widened version of this). @@ -665,7 +791,21 @@ def _revert_src_replicas(reason): ret = final_step_rpc.bdev_lvol_batch_transfer_final_step( lvol_names, lvol_ids, snapshot_names, 16, hub_bdev, "migrate") logger.info(f"Group {group.uuid[:8]}: bdev_lvol_batch_transfer_final_step returned {ret!r}") - batch_ok = True + # The RPC can return normally (no exception) while still reporting the + # transfer itself failed -- transfer_state is one of "No process" | + # "In progress" | "Failed" | "Done" (see bdev_lvol_transfer_stat). + # Treating any non-exception response as success let a "Failed" + # transfer proceed straight to the ANA flip and source cleanup as if + # the data had actually moved (observed run 2026-08-22, group + # 911aa7af/mig-108: 'Failed' logged then treated as success, target + # listeners never came up, checksum corruption followed). + transfer_state = ret.get("transfer_state") if isinstance(ret, dict) else None + if transfer_state == "Done": + batch_ok = True + else: + batch_err = f"transfer_state={transfer_state!r} (expected 'Done'): {ret!r}" + logger.error(f"Group {group.uuid[:8]}: bdev_lvol_batch_transfer_final_step " + f"did not report success: {batch_err}") except RPCRemoteError as e: logger.error(f"Group {group.uuid[:8]}: bdev_lvol_batch_transfer_final_step RPC error code={e.code}: {e}") batch_err = str(e) diff --git a/simplyblock_core/services/tasks_runner_lvol_migration.py b/simplyblock_core/services/tasks_runner_lvol_migration.py index 171f2330c1..25b6991e86 100644 --- a/simplyblock_core/services/tasks_runner_lvol_migration.py +++ b/simplyblock_core/services/tasks_runner_lvol_migration.py @@ -698,7 +698,7 @@ def _swap_namespace(rpc, nqn, new_bdev, uuid, guid, label): logger.info(f"Swap NS {label}: removed nsid={nsid}") except Exception as e: logger.warning(f"Swap NS remove (non-fatal) on {label}: {e}") - ret = rpc.nvmf_subsystem_add_ns(nqn, new_bdev, uuid, guid) + ret = rpc.nvmf_subsystem_add_ns(nqn, new_bdev, uuid, guid, nsid=nsid) if not ret: logger.error(f"Swap NS add failed on {label}") @@ -748,7 +748,7 @@ def _ensure_nvmf_state_on_node(migration, lvol, nqn, path, label, owns_subsystem for _ip in path['ips']: rpc.listeners_create(nqn, path['trtype'], _ip, path['port'], ana_state="inaccessible") - ns = rpc.nvmf_subsystem_add_ns(nqn, ns_composite, lvol.uuid, lvol.guid) + ns = rpc.nvmf_subsystem_add_ns(nqn, ns_composite, lvol.uuid, lvol.guid, nsid=lvol.ns_id) if not ns: logger.warning( f"_ensure_nvmf_state_on_node: namespace add failed on " @@ -785,7 +785,7 @@ def _ensure_nvmf_state_on_node(migration, lvol, nqn, path, label, owns_subsystem logger.warning( f"_ensure_nvmf_state_on_node: namespace for {lvol.uuid} " f"missing on {label} target node {node_id[:8]} — re-adding") - ns = rpc.nvmf_subsystem_add_ns(nqn, ns_composite, lvol.uuid, lvol.guid) + ns = rpc.nvmf_subsystem_add_ns(nqn, ns_composite, lvol.uuid, lvol.guid, nsid=lvol.ns_id) if not ns: logger.warning( f"_ensure_nvmf_state_on_node: namespace re-add failed " @@ -906,10 +906,17 @@ def _cleanup_final_migration(src_rpc, ctx, tgt_rpc=None, rollback_target=False, # --------------------------------------------------------------------------- +# Sentinel distinguishing "caller has no answer, query fresh" from a caller- +# supplied get_bdevs() result (including an explicit [], i.e. "confirmed +# absent") for _setup_snap_transfer's existing_bdev_info param below. +_BDEV_INFO_UNSET = object() + + def _setup_snap_transfer(snap, snap_index, src_node, tgt_node, src_rpc, tgt_rpc, trtype, tgt_sec=None, sec_rpc=None, tgt_ter=None, ter_rpc=None, - lvol_size_mib=None, migration=None): + lvol_size_mib=None, migration=None, + existing_bdev_info=_BDEV_INFO_UNSET): """ Prepare a single snapshot for async transfer: 1. Create writable lvol on target primary @@ -921,6 +928,14 @@ def _setup_snap_transfer(snap, snap_index, src_node, tgt_node, Returns a transfer-dict on success or (None, error_string) on failure. Callers are responsible for rolling back any previously launched transfers. + + ``existing_bdev_info``: every caller already runs its own get_bdevs(tgt_composite) + pre-check (to decide whether to reuse an owned bdev or clean up a stale one) + immediately before calling this function, which then repeated the identical + query for its own reuse-vs-create decision -- two RPC round-trips for the + same fact. Callers that already have a trustworthy answer (the bdev was + confirmed absent, or confirmed present and owned) can pass that result + straight through here instead of paying for a second lookup. """ snap_uuid = snap.uuid snap_short = _snap_tgt_short_name(snap) @@ -950,7 +965,10 @@ def _setup_snap_transfer(snap, snap_index, src_node, tgt_node, # Step 1: create target lvol on primary, or reuse if already owned by this migration. # Pre-cleanup skips deletion of owned bdevs so we can reuse them here on retry # rather than paying the create cost again. - _bdev_info = tgt_rpc.get_bdevs(tgt_composite) + if existing_bdev_info is _BDEV_INFO_UNSET: + _bdev_info = tgt_rpc.get_bdevs(tgt_composite) + else: + _bdev_info = existing_bdev_info if _bdev_info: logger.info( f"[REUSE] snap={snap_uuid[:8]} reusing owned writable bdev {tgt_composite}") @@ -987,8 +1005,8 @@ def _setup_snap_transfer(snap, snap_index, src_node, tgt_node, except Exception as e: logger.warning(f"cleanup target lvol {tgt_composite} (non-fatal): {e}") return None, f"Could not get bdev info for {tgt_composite} after creation" - snap_blobid = _bdev_info[0]['driver_specific']['lvol']['blobid'] - snap_uuid_on_tgt = _bdev_info[0]['uuid'] + snap_blobid = _bdev_info[0]['driver_specific']['lvol']['blobid'] # type: ignore[index] + snap_uuid_on_tgt = _bdev_info[0]['uuid'] # type: ignore[index] if sec_rpc.get_bdevs(tgt_composite): sec_registered = True logger.info(f"Secondary already has {tgt_composite}; skipping registration") @@ -1370,7 +1388,8 @@ def _handle_snap_copy(migration, src_node, tgt_node, src_rpc, tgt_rpc): # Pre-existing (immutable) bdevs were caught by the pre-scan above and # excluded from unprocessed. Anything still found here is a writable # leftover from a previous failed attempt — delete and retry. - if tgt_rpc.get_bdevs(tgt_composite): + _existing_bdev = tgt_rpc.get_bdevs(tgt_composite) + if _existing_bdev: if tgt_composite in (migration.target_snap_bdevs or []): logger.info( f"Owned writable bdev {tgt_composite} found — reusing for retry") @@ -1383,10 +1402,16 @@ def _handle_snap_copy(migration, src_node, tgt_node, src_rpc, tgt_rpc): lvs_name=tgt_node.lvstore) for _ in range(10): if not tgt_rpc.get_bdevs(tgt_composite): + _existing_bdev = [] break time.sleep(0.2) + else: + # Deletion never confirmed within the polling window — + # state is uncertain, let _setup_snap_transfer re-query. + _existing_bdev = _BDEV_INFO_UNSET except Exception as e: logger.warning(f"Pre-cleanup of {tgt_composite} failed (continuing): {e}") + _existing_bdev = _BDEV_INFO_UNSET t, err = _setup_snap_transfer( snap, snap_index, src_node, tgt_node, @@ -1394,7 +1419,8 @@ def _handle_snap_copy(migration, src_node, tgt_node, src_rpc, tgt_rpc): tgt_sec=tgt_sec, sec_rpc=sec_rpc, tgt_ter=tgt_ter, ter_rpc=ter_rpc, lvol_size_mib=_snap_lvol_size_mib, - migration=migration) + migration=migration, + existing_bdev_info=_existing_bdev) if t is None: return False, True, err @@ -1422,9 +1448,18 @@ def _handle_snap_copy(migration, src_node, tgt_node, src_rpc, tgt_rpc): # ── B. Poll all in-flight transfers; post-process completed ones ────────── if ctx.get('stage') == 'parallel_transfer': transfers = ctx['transfers'] - # Resolve secondary once for the whole poll pass + # Resolve secondary and tertiary once for the whole poll pass. This + # branch runs on a fresh function invocation (tgt_sec/tgt_ter default + # to None at the top of this function) whenever a transfer launched + # on a prior tick is still being polled -- the common case, since + # transfers essentially never finish within the same tick they start. + # Tertiary was previously left unresolved here, silently skipping its + # add_clone/convert in _post_process_snap below (the `if tgt_ter and + # ter_rpc:` guard just evaluated false, with no error logged). tgt_sec, _sec_err = _get_target_secondary_node(tgt_node, src_node.get_id()) sec_rpc = _make_rpc(tgt_sec) if tgt_sec and not _sec_err else None + tgt_ter, _ter_err = _get_target_tertiary_node(tgt_node, src_node.get_id()) + ter_rpc = _make_rpc(tgt_ter) if tgt_ter and not _ter_err else None # Process in snap_index order: add_clone requires predecessor to be # converted first. prev_post_done tracks whether the predecessor has # been post-processed; if not, we must not post-process the current snap @@ -1578,7 +1613,8 @@ def _handle_snap_copy(migration, src_node, tgt_node, src_rpc, tgt_rpc): # Pre-cleanup: if a bdev exists on the target it is a writable leftover # from a previous crashed run — intermediate snaps are always freshly # created by this migration so they can never be pre-existing. - if tgt_rpc.get_bdevs(tgt_composite): + _existing_bdev = tgt_rpc.get_bdevs(tgt_composite) + if _existing_bdev: if tgt_composite in (migration.target_snap_bdevs or []): logger.info( f"Owned writable intermediate bdev {tgt_composite} found — reusing for retry") @@ -1591,10 +1627,14 @@ def _handle_snap_copy(migration, src_node, tgt_node, src_rpc, tgt_rpc): lvs_name=tgt_node.lvstore) for _ in range(10): if not tgt_rpc.get_bdevs(tgt_composite): + _existing_bdev = [] break time.sleep(0.2) + else: + _existing_bdev = _BDEV_INFO_UNSET except Exception as e: logger.warning(f"Pre-cleanup of {tgt_composite} failed (continuing): {e}") + _existing_bdev = _BDEV_INFO_UNSET t, err = _setup_snap_transfer( snap, snap_index, src_node, tgt_node, @@ -1602,7 +1642,8 @@ def _handle_snap_copy(migration, src_node, tgt_node, src_rpc, tgt_rpc): tgt_sec=tgt_sec, sec_rpc=sec_rpc, tgt_ter=tgt_ter, ter_rpc=ter_rpc, lvol_size_mib=_snap_lvol_size_mib, - migration=migration) + migration=migration, + existing_bdev_info=_existing_bdev) if t is None: return False, True, err @@ -2230,13 +2271,27 @@ def _revert_src_replicas(reason): def _delete_intermediate_snaps_on_target(migration, tgt_rpc, tgt_sec_rpc=None, tgt_ter_rpc=None, - tgt_all_nodes=None, tgt_lvs_name=None): + tgt_all_nodes=None, tgt_lvs_name=None, + src_rpc=None, src_sec_rpc=None, src_ter_rpc=None, + src_all_nodes=None, src_lvs_name=None): """ - Delete migration-created intermediate ('shrink') snapshots from the target - after a successful migration. - - Must be called AFTER apply_migration_to_db() — at that point snap.snap_bdev - already holds the target composite path (e.g. LVS_TGT/SNAP_xxxm). + Delete migration-created intermediate ('shrink') snapshots from wherever + each one actually lives after a successful migration. + + Most rounds' snap.snap_bdev holds the target composite path (e.g. + LVS_TGT/SNAP_xxxm), updated by apply_migration_to_db(). But a round whose + OWN transfer never completed (superseded by a later round, e.g. a + multi-round intermediate sequence) never got that update -- snap.snap_bdev + is still the ORIGINAL source composite. Routing that delete through the + current target's lvstore name/node-list makes the leader-routed async + delete's lvstore mismatch what it's operating on, which SPDK rejects -- + _delete_bdev_blocking then raises before ever reaching its poll/sync + phases, silently caught below, leaking the blob's metadata on every + replica (observed run 2026-08-22, group B LVOL_30's round-0 snap + SNAP_611: async delete fired and was rejected, zero follow-up poll or + sync calls ever appeared in the SPDK logs). Resolve the lvstore actually + present in snap.snap_bdev and route to the matching node-list/rpc set + (target's or source's) instead of assuming it's always the target's. Delegates to _delete_bdev_blocking(coalescing=True): the intermediate snapshot's clusters must be merged into its child bdev before being freed @@ -2250,21 +2305,44 @@ def _delete_intermediate_snaps_on_target(migration, tgt_rpc, tgt_sec_rpc=None, t logger.info(f"Intermediate snap {snap_uuid} already removed from DB; skipping") continue - tgt_composite = snap.snap_bdev # updated to target path by apply_migration_to_db + composite = snap.snap_bdev # target path if its round's transfer completed, else still source + actual_lvs = composite.split('/', 1)[0] if composite and '/' in composite else None + + if actual_lvs == tgt_lvs_name: + _rpc, _sec_rpc, _ter_rpc, _all_nodes, _lvs_name = ( + tgt_rpc, tgt_sec_rpc, tgt_ter_rpc, tgt_all_nodes, tgt_lvs_name) + elif src_lvs_name and actual_lvs == src_lvs_name: + logger.info( + f"Intermediate snap {composite}: this round's transfer never completed " + f"(still on source lvstore {actual_lvs}); routing delete to source") + _rpc, _sec_rpc, _ter_rpc, _all_nodes, _lvs_name = ( + src_rpc, src_sec_rpc, src_ter_rpc, src_all_nodes, src_lvs_name) + else: + logger.warning( + f"Intermediate snap {composite}: lvstore {actual_lvs!r} matches neither " + f"target ({tgt_lvs_name!r}) nor source ({src_lvs_name!r}); skipping delete " + f"to avoid routing it against the wrong lvstore") + continue + + if _rpc is None: + logger.warning( + f"Intermediate snap {composite}: no RPC client available for lvstore " + f"{actual_lvs!r} (caller did not supply source routing info); skipping delete") + continue - if not tgt_rpc.get_bdevs(tgt_composite): + if not _rpc.get_bdevs(composite): logger.info( - f"Intermediate snap bdev {tgt_composite} absent from target; skipping SPDK delete") + f"Intermediate snap bdev {composite} absent; skipping SPDK delete") else: try: - _delete_bdev_blocking(tgt_composite, tgt_rpc, - secondary_rpc=tgt_sec_rpc, tertiary_rpc=tgt_ter_rpc, + _delete_bdev_blocking(composite, _rpc, + secondary_rpc=_sec_rpc, tertiary_rpc=_ter_rpc, coalescing=True, - all_nodes=tgt_all_nodes, lvs_name=tgt_lvs_name) - logger.info(f"Deleted intermediate snap bdev {tgt_composite} from target") + all_nodes=_all_nodes, lvs_name=_lvs_name) + logger.info(f"Deleted intermediate snap bdev {composite}") except Exception as e: logger.warning( - f"Could not delete intermediate snap {tgt_composite} from target: {e}") + f"Could not delete intermediate snap {composite}: {e}") try: snap.remove(db.kv_store) @@ -2617,7 +2695,15 @@ def _handle_cleanup_source(migration, src_node, src_rpc, tgt_node, tgt_rpc): _delete_intermediate_snaps_on_target( migration, tgt_rpc, tgt_sec_rpc, tgt_ter_rpc, tgt_all_nodes=[n for n in [tgt_node, tgt_sec, tgt_ter] if n], - tgt_lvs_name=tgt_node.lvstore) + tgt_lvs_name=tgt_node.lvstore, + # A round whose own transfer never completed (superseded by a + # later round) leaves snap.snap_bdev on the SOURCE composite -- + # pass source routing too so that case gets deleted from the + # right place instead of being mis-routed against the target's + # lvstore and silently rejected. See the function's docstring. + src_rpc=src_rpc, src_sec_rpc=src_sec_rpc, src_ter_rpc=src_ter_rpc, + src_all_nodes=[n for n in [src_node, src_sec, src_ter] if n], + src_lvs_name=src_node.lvstore) _rename_migrated_bdevs(migration, tgt_node, tgt_rpc, tgt_sec_rpc, tgt_ter_rpc, warnings=_warnings) except Exception as e: @@ -3242,7 +3328,8 @@ def _handle_group_snap_copy(migration, src_node, tgt_node, src_rpc, tgt_rpc): _g_sec_rpc = _make_rpc(_g_tgt_sec) if _g_tgt_sec else None _g_ter_rpc = _make_rpc(_g_tgt_ter) if _g_tgt_ter else None - if tgt_rpc.get_bdevs(tgt_composite): + _existing_bdev = tgt_rpc.get_bdevs(tgt_composite) + if _existing_bdev: if tgt_composite in (migration.target_snap_bdevs or []): logger.info( f"Owned writable bdev {tgt_composite} found — reusing for retry") @@ -3251,8 +3338,13 @@ def _handle_group_snap_copy(migration, src_node, tgt_node, src_rpc, tgt_rpc): _delete_bdev_blocking(tgt_composite, tgt_rpc, _g_sec_rpc, _g_ter_rpc, all_nodes=[n for n in [tgt_node, _g_tgt_sec, _g_tgt_ter] if n], lvs_name=tgt_node.lvstore) + # No post-delete confirmation poll here (unlike the solo-path + # callers) — state after this point is uncertain, so let + # _setup_snap_transfer re-query rather than assuming deleted. + _existing_bdev = _BDEV_INFO_UNSET except Exception as e: logger.warning(f"Group worker: pre-cleanup of {tgt_composite} failed: {e}") + _existing_bdev = _BDEV_INFO_UNSET t, err = _setup_snap_transfer( snap, plan.index(snap_uuid), src_node, tgt_node, @@ -3260,7 +3352,8 @@ def _handle_group_snap_copy(migration, src_node, tgt_node, src_rpc, tgt_rpc): tgt_sec=_g_tgt_sec, sec_rpc=_g_sec_rpc, tgt_ter=_g_tgt_ter, ter_rpc=_g_ter_rpc, lvol_size_mib=_snap_lvol_size_mib, - migration=migration) + migration=migration, + existing_bdev_info=_existing_bdev) if t is None: return False, True, err @@ -3388,7 +3481,8 @@ def _handle_group_intermediate(migration, src_node, tgt_node, src_rpc, tgt_rpc, _g_sec_rpc = _make_rpc(_g_tgt_sec) if _g_tgt_sec else None _g_ter_rpc = _make_rpc(_g_tgt_ter) if _g_tgt_ter else None - if tgt_rpc.get_bdevs(tgt_composite): + _existing_bdev = tgt_rpc.get_bdevs(tgt_composite) + if _existing_bdev: if tgt_composite in (migration.target_snap_bdevs or []): logger.info( f"Owned writable intermediate bdev {tgt_composite} found — reusing for retry") @@ -3397,8 +3491,12 @@ def _handle_group_intermediate(migration, src_node, tgt_node, src_rpc, tgt_rpc, _delete_bdev_blocking(tgt_composite, tgt_rpc, _g_sec_rpc, _g_ter_rpc, all_nodes=[n for n in [tgt_node, _g_tgt_sec, _g_tgt_ter] if n], lvs_name=tgt_node.lvstore) + # No post-delete confirmation poll here — state after this + # point is uncertain, so let _setup_snap_transfer re-query. + _existing_bdev = _BDEV_INFO_UNSET except Exception as e: logger.warning(f"Group intermediate: pre-cleanup of {tgt_composite} failed: {e}") + _existing_bdev = _BDEV_INFO_UNSET t, err = _setup_snap_transfer( snap, snap_index, src_node, tgt_node, @@ -3406,7 +3504,8 @@ def _handle_group_intermediate(migration, src_node, tgt_node, src_rpc, tgt_rpc, tgt_sec=_g_tgt_sec, sec_rpc=_g_sec_rpc, tgt_ter=_g_tgt_ter, ter_rpc=_g_ter_rpc, lvol_size_mib=_snap_lvol_size_mib, - migration=migration) + migration=migration, + existing_bdev_info=_existing_bdev) if t is None: return False, True, err From b24c3c7d9746a04431413c4a2541015f749bad67 Mon Sep 17 00:00:00 2001 From: ebrahim savari <106956085+EbiRider@users.noreply.github.com> Date: Mon, 24 Aug 2026 18:34:37 +0330 Subject: [PATCH 051/122] fixed bad delete, duplicate bdev get calls --- .../controllers/migration_bdev_ops.py | 93 ++++++++++++ .../controllers/migration_controller.py | 140 ++++++++++++------ .../services/tasks_runner_lvol_migration.py | 91 +++--------- 3 files changed, 212 insertions(+), 112 deletions(-) create mode 100644 simplyblock_core/controllers/migration_bdev_ops.py diff --git a/simplyblock_core/controllers/migration_bdev_ops.py b/simplyblock_core/controllers/migration_bdev_ops.py new file mode 100644 index 0000000000..4284056bd6 --- /dev/null +++ b/simplyblock_core/controllers/migration_bdev_ops.py @@ -0,0 +1,93 @@ +# coding=utf-8 +""" +migration_bdev_ops.py -- shared bdev-delete primitive for migration code. + +``delete_bdev_blocking`` used to live in tasks_runner_lvol_migration.py, with +migration_controller.py pulling it in via a local (function-scoped) import at +every call site. That local-import dance is required there because +tasks_runner_lvol_migration.py imports migration_controller at module level +(``from simplyblock_core.controllers import (migration_controller, ...)``), +so the reverse import at module level would be a real cycle: +tasks_runner_lvol_migration -> migration_controller -> tasks_runner_lvol_migration. + +Living in its own module sidesteps that: nothing importing this module needs +to worry about the direction of the migration_controller <-> task-runner +relationship, since this module doesn't import either of them. All three +migration files (migration_controller.py, tasks_runner_lvol_migration.py, +tasks_runner_batch_migration.py) import it directly at module level. +""" + +import logging +import time + +from tenacity import RetryError, Retrying, before_sleep_log, stop_after_attempt, wait_fixed + +from simplyblock_core import utils + +logger = utils.get_logger(__name__) + + +def delete_bdev_blocking(bdev_name, primary_rpc, secondary_rpc=None, tertiary_rpc=None, + timeout_s=120, coalescing=False, all_nodes=None, lvs_name=None): + """ + Two-phase blocking bdev delete. + + Phase 1 — leader node, sync=False: initiates the async delete. When + all_nodes + lvs_name are supplied, the actual LVS leader is resolved via + execute_on_leader_with_failover so the delete goes to the secondary or + tertiary if the primary is down. Without them the call falls back to + primary_rpc directly (original behaviour). By default (coalescing=False) + special_delete=True tells SPDK to free the bdev's own clusters without + merging them into any child — correct for source cleanup, rollback, and + any path where no child needs to inherit data. Pass coalescing=True when + the bdev's child must inherit its clusters (e.g. deleting a migration + intermediate snapshot). + Wait — poll bdev_lvol_get_lvol_delete_status on the leader until done. + Phase 2 — all nodes (primary + secondary + tertiary), sync=True + (sync=True, special_delete=False): finalises the deletion on every replica. + """ + if all_nodes and lvs_name: + # Local import: storage_node_ops transitively imports migration_controller + # (storage_node_ops -> snapshot_controller -> migration_controller), so + # importing it at this module's top level would recreate the exact cycle + # this module exists to avoid, just one hop further out. + from simplyblock_core.storage_node_ops import execute_on_leader_with_failover + + def _async_delete(leader): + ret, _ = leader.rpc_client().delete_lvol( + bdev_name, sync=False, special_delete=not coalescing) + return ret or False + ok, leader_node, _ = execute_on_leader_with_failover(all_nodes, lvs_name, _async_delete) + if not ok or leader_node is None: + raise RuntimeError(f"delete bdev {bdev_name}: initiation failed") + leader_rpc = leader_node.rpc_client() + else: + ret, _ = primary_rpc.delete_lvol(bdev_name, sync=False, special_delete=not coalescing) + if not ret: + raise RuntimeError(f"delete bdev {bdev_name}: initiation failed") + leader_rpc = primary_rpc + + deadline = time.monotonic() + timeout_s + while leader_rpc.bdev_lvol_get_lvol_delete_status(bdev_name) == 1: + if time.monotonic() > deadline: + if not leader_rpc.get_bdevs(bdev_name): + logger.warning( + f"delete bdev {bdev_name}: poll timed out after {timeout_s}s " + f"but bdev is gone — treating as success") + break + raise RuntimeError( + f"delete bdev {bdev_name}: timed out after {timeout_s}s, bdev still present") + time.sleep(0.2) + + for rpc in filter(None, [primary_rpc, secondary_rpc, tertiary_rpc]): + try: + Retrying( + stop=stop_after_attempt(3), + wait=wait_fixed(1), + before_sleep=before_sleep_log(logger, logging.WARNING), + )(rpc.delete_lvol, bdev_name, sync=True, special_delete=False) + except RetryError: + logger.exception( + f"delete bdev {bdev_name} sync finalize STILL failing after 3 attempts " + f"(non-fatal, blob metadata may not be cleared on this replica)" + ) diff --git a/simplyblock_core/controllers/migration_controller.py b/simplyblock_core/controllers/migration_controller.py index bc6964292f..49a173f874 100644 --- a/simplyblock_core/controllers/migration_controller.py +++ b/simplyblock_core/controllers/migration_controller.py @@ -45,6 +45,7 @@ from simplyblock_core import constants from simplyblock_core.controllers import migration_events, tasks_controller +from simplyblock_core.controllers.migration_bdev_ops import delete_bdev_blocking as _delete_bdev_blocking from simplyblock_core.exceptions import MigrationConflictError, PreconditionError from simplyblock_core.controllers.host_auth import _reapply_allowed_hosts from simplyblock_core.kms import create_kms_connection, lvol_dek_path, pool_kek_name @@ -260,11 +261,27 @@ def _cleanup_created(migration): tgt_rpc = tgt_node.rpc_client() tgt_port = tgt_node.get_lvol_subsys_port(tgt_node.lvstore) - # Secondary cleanup + # Resolved once, up front, so the final migration-bdev delete below can + # reuse them regardless of whether the listener/subsystem cleanup above + # hit an exception partway through. + sec_node = sec_rpc = None if tgt_node.secondary_node_id: try: sec_node = db.get_storage_node_by_id(tgt_node.secondary_node_id) - sec_rpc = sec_node.rpc_client() + sec_rpc = sec_node.rpc_client() + except Exception as e: + logger.warning(f"_cleanup_created: could not reach TGT-sec: {e}") + ter_node = ter_rpc = None + if tgt_node.tertiary_node_id: + try: + ter_node = db.get_storage_node_by_id(tgt_node.tertiary_node_id) + ter_rpc = ter_node.rpc_client() + except Exception as e: + logger.warning(f"_cleanup_created: could not reach TGT-ter: {e}") + + # Secondary cleanup + if sec_node is not None and sec_rpc is not None: + try: sec_port = sec_node.get_lvol_subsys_port(tgt_node.lvstore) if sec_node.get_id() in overlap_ids: for nic in sec_node.data_nics: @@ -306,9 +323,25 @@ def _cleanup_created(migration): except Exception as e: logger.warning(f"_cleanup_created: could not clean TGT-prim subsystem: {e}") - # Migration bdev (always delete — we always created it) + # Migration bdev (always delete — we always created it). + # + # This used to be a single tgt_rpc.delete_lvol(composite) call -- sync=False, + # special_delete=False by that method's own defaults, i.e. exactly the + # async-only half of the two-phase delete protocol, with no completion + # poll, no sync finalize, and no secondary/tertiary cleanup at all (despite + # this function's own docstring claiming "deleted ... on the target + # primary and secondary"). That leaked the bdev's blob metadata on every + # replica whenever a PRE_CREATED migration got cancelled -- observed run + # 2026-08-24: LVS_5/LVOL_37m async-deleted with zero follow-up anywhere in + # the SPDK logs, then reused unconditioned 11 minutes later by the next + # migration attempt targeting the same lvstore. _delete_bdev_blocking is + # the established two-phase (async + poll + sync-on-every-replica) + # primitive used everywhere else in the migration code for exactly this. try: - tgt_rpc.delete_lvol(composite) + _delete_bdev_blocking( + composite, tgt_rpc, secondary_rpc=sec_rpc, tertiary_rpc=ter_rpc, + all_nodes=[n for n in [tgt_node, sec_node, ter_node] if n], + lvs_name=tgt_node.lvstore) logger.info(f"_cleanup_created: deleted migration bdev {composite} on {tgt_node.get_id()}") except Exception as e: logger.warning(f"_cleanup_created: could not clean migration bdev: {e}") @@ -680,21 +713,17 @@ def cleanup_migration_target(migration_id): skipped = [] errors = [] - def _try_delete_bdev(rpc, bdev_path, tag): - try: - if rpc.get_bdevs(bdev_path): - rpc.bdev_lvol_delete(bdev_path) - deleted.append({**tag, "bdev": bdev_path}) - else: - not_found.append({**tag, "bdev": bdev_path}) - except Exception: - logger.exception("cleanup_migration_target: failed to delete bdev %s", bdev_path) - errors.append({**tag, "bdev": bdev_path, "error": "Internal error during cleanup operation"}) - - # Build RPC clients for primary + HA peers. - rpc_clients = [] + # Delete via the same leader-aware, replica-syncing primitive the + # migration runner itself uses (async delete + poll + sync=True on every + # replica). The previous version of this function did an independent + # sync=True delete_lvol per replica, called directly against whichever + # node happened to hold that replica -- that skips leader routing + # entirely, so a sync delete issued straight at a non-leader replica + # while the lvstore leader has failed over elsewhere is exactly the kind + # of case _delete_bdev_blocking exists to handle correctly. + primary_rpc = None try: - rpc_clients.append((tgt_node.get_id(), tgt_node.rpc_client(), "primary")) + primary_rpc = tgt_node.rpc_client() except Exception: logger.exception( "cleanup_migration_target: failed to create RPC client for node %s", @@ -702,22 +731,44 @@ def _try_delete_bdev(rpc, bdev_path, tag): errors.append({"type": "rpc_connect", "node": migration.target_node_id[:8], "error": "Internal error during cleanup operation"}) - for attr, label in [("secondary_node_id", "secondary"), - ("tertiary_node_id", "tertiary")]: - peer_id = getattr(tgt_node, attr, None) - if not peer_id: - continue + sec_node = sec_rpc = None + if tgt_node.secondary_node_id: + try: + sec_node = db.get_storage_node_by_id(tgt_node.secondary_node_id) + sec_rpc = sec_node.rpc_client() + except Exception: + pass # peer unreachable — skip gracefully + + ter_node = ter_rpc = None + if tgt_node.tertiary_node_id: try: - peer = db.get_storage_node_by_id(peer_id) - rpc_clients.append((peer.get_id(), peer.rpc_client(), label)) + ter_node = db.get_storage_node_by_id(tgt_node.tertiary_node_id) + ter_rpc = ter_node.rpc_client() except Exception: pass # peer unreachable — skip gracefully + all_nodes = [n for n in [tgt_node, sec_node, ter_node] if n] + + def _try_delete_bdev(bdev_path, tag): + if primary_rpc is None: + errors.append({**tag, "bdev": bdev_path, "error": "Internal error during cleanup operation"}) + return + try: + if not primary_rpc.get_bdevs(bdev_path): + not_found.append({**tag, "bdev": bdev_path}) + return + lvs_name = bdev_path.split('/', 1)[0] + _delete_bdev_blocking(bdev_path, primary_rpc, + secondary_rpc=sec_rpc, tertiary_rpc=ter_rpc, + all_nodes=all_nodes, lvs_name=lvs_name) + deleted.append({**tag, "bdev": bdev_path}) + except Exception: + logger.exception("cleanup_migration_target: failed to delete bdev %s", bdev_path) + errors.append({**tag, "bdev": bdev_path, "error": "Internal error during cleanup operation"}) + # ── 1. Migration lvol bdev ──────────────────────────────────────────────── if migration.target_lvol_bdev: - for _, rpc, label in rpc_clients: - _try_delete_bdev(rpc, migration.target_lvol_bdev, - {"type": "lvol_bdev", "node": label}) + _try_delete_bdev(migration.target_lvol_bdev, {"type": "lvol_bdev"}) # ── 2. Snapshot bdevs (reverse order: children before parents) ──────────── # target_snap_bdevs stores the exact path at creation time ("LVS_TGT/SNAP_xxx_m"). @@ -754,20 +805,20 @@ def _try_delete_bdev(rpc, bdev_path, tag): "reason": "referenced by another lvol on target"}) continue - for _, rpc, label in rpc_clients: - bdev_name = next( - (f"{lvstore}/{n}" - for n in (short_m, short_base, short_base + _DONE_SUFFIX) - if rpc.get_bdevs(f"{lvstore}/{n}")), - None, - ) - if bdev_name: - _try_delete_bdev(rpc, bdev_name, - {"type": "snap_bdev", "stored_path": stored_path, - "node": label}) - else: - not_found.append({"type": "snap_bdev", "stored_path": stored_path, - "node": label}) + if primary_rpc is None: + errors.append({"type": "snap_bdev", "stored_path": stored_path, + "error": "Internal error during cleanup operation"}) + continue + bdev_name = next( + (f"{lvstore}/{n}" + for n in (short_m, short_base, short_base + _DONE_SUFFIX) + if primary_rpc.get_bdevs(f"{lvstore}/{n}")), + None, + ) + if bdev_name: + _try_delete_bdev(bdev_name, {"type": "snap_bdev", "stored_path": stored_path}) + else: + not_found.append({"type": "snap_bdev", "stored_path": stored_path}) # ── 3. Subsystems — only on nodes where we called subsystem_create ───────── # Delegates the delete-vs-detach-namespace decision to cleanup_subsystem_or_ns @@ -970,7 +1021,8 @@ def create_migration(lvol_id, target_node_id, tgt_port = tgt_node.get_lvol_subsys_port(tgt_node.lvstore) # ── 1. Bdev ────────────────────────────────────────────────────────────── - if not tgt_rpc.get_bdevs(composite): + _bdev_info = tgt_rpc.get_bdevs(composite) + if not _bdev_info: ok, err = _ensure_lvstore_primary_leader(tgt_rpc, tgt_node.lvstore, target_node_id) if not ok: raise PreconditionError(f"Cannot create target lvol {composite}: {err}") @@ -981,11 +1033,11 @@ def create_migration(lvol_id, target_node_id, if not ret: raise ValueError(f"bdev_lvol_create failed for {composite} on {target_node_id}") logger.info(f"create_migration: created bdev {composite}") + _bdev_info = tgt_rpc.get_bdevs(composite) else: logger.info(f"create_migration: bdev {composite} already exists — skipping create") # ── 1b. Get bdev info for secondary registration ────────────────────────── - _bdev_info = tgt_rpc.get_bdevs(composite) _tgt_blobid = None _tgt_uuid = None if _bdev_info and isinstance(_bdev_info[0], dict): diff --git a/simplyblock_core/services/tasks_runner_lvol_migration.py b/simplyblock_core/services/tasks_runner_lvol_migration.py index 25b6991e86..5b8cf53024 100644 --- a/simplyblock_core/services/tasks_runner_lvol_migration.py +++ b/simplyblock_core/services/tasks_runner_lvol_migration.py @@ -83,13 +83,10 @@ """ import datetime -import logging import random import time from typing import Optional -from tenacity import RetryError, Retrying, before_sleep_log, stop_after_attempt, wait_fixed - from simplyblock_core import db_controller as db_mod, utils, constants from simplyblock_core.utils import convert_size from simplyblock_core.controllers import ( @@ -104,7 +101,7 @@ from simplyblock_core.models.snapshot import SnapShot from simplyblock_core.rpc_client import RPCErrorCode, RPCRemoteError, RPCException, RPCClient from simplyblock_core.services.hub_controller_manager import HubControllerManager -from simplyblock_core.storage_node_ops import execute_on_leader_with_failover +from simplyblock_core.controllers.migration_bdev_ops import delete_bdev_blocking as _delete_bdev_blocking logger = utils.get_logger(__name__) db = db_mod.DBController() @@ -423,20 +420,32 @@ def _bytes_to_mib(nbytes): return max(1, utils.convert_size(nbytes, 'MiB', round_up=False)) -def _log_spdk_bdev_size(rpc, composite_name, label): +# Sentinel distinguishing "caller has no answer, query fresh" from a +# caller-supplied get_bdevs() result -- including an explicit [] (confirmed +# absent). Used by _log_spdk_bdev_size and _setup_snap_transfer to avoid +# repeating an RPC round-trip the caller already paid for. +_BDEV_INFO_UNSET = object() + + +def _log_spdk_bdev_size(rpc, composite_name, label, bdev_info=_BDEV_INFO_UNSET): """Query SPDK for *composite_name* and emit a [BDEV SIZE] log line. Reports num_blocks × block_size → actual_mib and sectors@512 (the sector count the client sees via the NVMe namespace). Never raises. + + ``bdev_info``: pass an already-fetched get_bdevs() result to log against + it instead of paying for a second identical RPC round-trip -- callers + that are about to query (or just queried) the same composite for their + own purposes should pass that result through here. """ _MIB = 1048576 try: - info = rpc.get_bdevs(composite_name) + info = rpc.get_bdevs(composite_name) if bdev_info is _BDEV_INFO_UNSET else bdev_info if not info: logger.warning( f"[BDEV SIZE] {label}: {composite_name} — bdev not found in SPDK") return None - b = info[0] + b = info[0] # type: ignore[index] num_blocks = b.get('num_blocks', 0) block_size = b.get('block_size', 512) actual_bytes = num_blocks * block_size @@ -455,64 +464,8 @@ def _log_spdk_bdev_size(rpc, composite_name, label): return None -def _delete_bdev_blocking(bdev_name, primary_rpc, secondary_rpc=None, tertiary_rpc=None, - timeout_s=120, coalescing=False, all_nodes=None, lvs_name=None): - """ - Two-phase blocking bdev delete. - - Phase 1 — leader node, sync=False: initiates the async delete. When - all_nodes + lvs_name are supplied, the actual LVS leader is resolved via - execute_on_leader_with_failover so the delete goes to the secondary or - tertiary if the primary is down. Without them the call falls back to - primary_rpc directly (original behaviour). By default (coalescing=False) - special_delete=True tells SPDK to free the bdev's own clusters without - merging them into any child — correct for source cleanup, rollback, and - any path where no child needs to inherit data. Pass coalescing=True when - the bdev's child must inherit its clusters (e.g. deleting a migration - intermediate snapshot). - Wait — poll bdev_lvol_get_lvol_delete_status on the leader until done. - Phase 2 — all nodes (primary + secondary + tertiary), sync=True - (sync=True, special_delete=False): finalises the deletion on every replica. - """ - if all_nodes and lvs_name: - def _async_delete(leader): - ret, _ = leader.rpc_client().delete_lvol( - bdev_name, sync=False, special_delete=not coalescing) - return ret or False - ok, leader_node, _ = execute_on_leader_with_failover(all_nodes, lvs_name, _async_delete) - if not ok or leader_node is None: - raise RuntimeError(f"delete bdev {bdev_name}: initiation failed") - leader_rpc = leader_node.rpc_client() - else: - ret, _ = primary_rpc.delete_lvol(bdev_name, sync=False, special_delete=not coalescing) - if not ret: - raise RuntimeError(f"delete bdev {bdev_name}: initiation failed") - leader_rpc = primary_rpc - - deadline = time.monotonic() + timeout_s - while leader_rpc.bdev_lvol_get_lvol_delete_status(bdev_name) == 1: - if time.monotonic() > deadline: - if not leader_rpc.get_bdevs(bdev_name): - logger.warning( - f"delete bdev {bdev_name}: poll timed out after {timeout_s}s " - f"but bdev is gone — treating as success") - break - raise RuntimeError( - f"delete bdev {bdev_name}: timed out after {timeout_s}s, bdev still present") - time.sleep(0.2) - - for rpc in filter(None, [primary_rpc, secondary_rpc, tertiary_rpc]): - try: - Retrying( - stop=stop_after_attempt(3), - wait=wait_fixed(1), - before_sleep=before_sleep_log(logger, logging.WARNING), - )(rpc.delete_lvol, bdev_name, sync=True, special_delete=False) - except RetryError: - logger.exception( - f"delete bdev {bdev_name} sync finalize STILL failing after 3 attempts " - f"(non-fatal, blob metadata may not be cleared on this replica)" - ) +# _delete_bdev_blocking lives in controllers/migration_bdev_ops.py (imported +# above as delete_bdev_blocking) -- see that module's docstring for why. # --------------------------------------------------------------------------- @@ -972,7 +925,8 @@ def _setup_snap_transfer(snap, snap_index, src_node, tgt_node, if _bdev_info: logger.info( f"[REUSE] snap={snap_uuid[:8]} reusing owned writable bdev {tgt_composite}") - _log_spdk_bdev_size(tgt_rpc, tgt_composite, f"TGT snap[{snap_uuid[:8]}] reuse") + _log_spdk_bdev_size(tgt_rpc, tgt_composite, f"TGT snap[{snap_uuid[:8]}] reuse", + bdev_info=_bdev_info) if migration is not None and tgt_composite not in migration.target_snap_bdevs: migration.target_snap_bdevs.append(tgt_composite) migration.write_to_db(db.kv_store) @@ -984,11 +938,12 @@ def _setup_snap_transfer(snap, snap_index, src_node, tgt_node, ret = tgt_rpc.create_lvol(snap_short, size_in_mib, tgt_node.lvstore, ndcs=_ndcs, npcs=_npcs) if not ret: return None, f"Failed to create target lvol for snap {snap_uuid}" - _log_spdk_bdev_size(tgt_rpc, tgt_composite, f"TGT snap[{snap_uuid[:8]}] post-create") + _bdev_info = tgt_rpc.get_bdevs(tgt_composite) + _log_spdk_bdev_size(tgt_rpc, tgt_composite, f"TGT snap[{snap_uuid[:8]}] post-create", + bdev_info=_bdev_info) if migration is not None and tgt_composite not in migration.target_snap_bdevs: migration.target_snap_bdevs.append(tgt_composite) migration.write_to_db(db.kv_store) - _bdev_info = tgt_rpc.get_bdevs(tgt_composite) # Step 2: register on secondary/tertiary if not already there. # On a normal first pass the secondary has no knowledge of this bdev yet; From b26ebfd105a8b424681fa9a8de17a5138fc94ad4 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 10:38:35 +0200 Subject: [PATCH 052/122] test: pin the ECR digest for main-d91ff03a-amd64, not docker.io's The CI push log prints docker.io's digest; public.ecr.aws stores a different one for the same tag, and pulling the docker.io digest from ECR fails manifest resolution (401 on the blob HEAD; deploy 2026-08-26 08:37). Resolve the pin against the registry the deployer actually pulls from. Co-Authored-By: Claude Fable 5 --- scripts/setup_repl_test_2clusters.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/setup_repl_test_2clusters.py b/scripts/setup_repl_test_2clusters.py index fc7751b1a3..1f693418ab 100644 --- a/scripts/setup_repl_test_2clusters.py +++ b/scripts/setup_repl_test_2clusters.py @@ -109,12 +109,14 @@ # race that leaves its amd64 entry pointing at the PREVIOUS build (observed # 2026-08-17, -21 and -22). This digest = main-d91ff03a-amd64, 2026-08-25: # the first ultra build FROM spdk-core:master-latest (spdk master = R26.3 -# merged + the ANA-transition change reverted). Previous pin -- +# merged + the ANA-transition change reverted). NOTE the digest printed in +# the CI push log is docker.io's; ECR's differs -- resolve it against +# public.ecr.aws (docker manifest inspect -v ). Previous pin -- # the first build carrying the promotion-window ANA-transition fix # (spdk R26.3 554c80f11), verified built FROM spdk-core:R26.3-latest # whose manifest was created 18:41:57, before this ultra build started. SPDK_IMAGE = ("public.ecr.aws/simply-block/ultra@" - "sha256:a3854cd445a6c356db26cb51f30b21e33880ce085f6f04a27715fc21165aeed1") + "sha256:0d631068e3add220d9198f212cf78d1e732ad9f0f92061dbebc413a9a6550e3b") SN_COUNT = sum(c["nodes"] for c in CLUSTERS) SBCTL = "sudo /usr/local/bin/sbctl" From fc12620b034d60472fbe801638d98d44bd668d38 Mon Sep 17 00:00:00 2001 From: Waleed Mousa <32266980+wmousa@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:57:10 +0200 Subject: [PATCH 053/122] Feat/distrib priority over alceml (#1268) * feat(cluster): distrib/poller take priority over alceml in core allocation calculate_core_allocations reserved alceml_cpu_cores -- scaled by the node's actual device count -- before distrib/poller ever saw the budget. distrib's share was therefore accidental: a byproduct of however many devices this specific node happened to have, not a capacity decision. Below 22 vCPUs this never mattered -- alceml's share there was already a fixed constant (1 core under 12 vCPUs, 2 from 12-21), never scaled by the device count -- so that range is untouched. At 22+ vCPUs, where alceml really did scale with the device count and ate into distrib/poller's budget first: distrib now claims its share first, as a pure function of vCPU count (the same tiered dp = floor(remaining/2), capped at 12 then 24, that already existed -- just no longer reduced by alceml_count). alceml then takes its real device-scaled count from whatever's left, clipped if there genuinely isn't room. poller -- already the 'whatever's left' role -- absorbs the true remainder; it no longer needs the separate 'add the one odd leftover core' patch for this tier since 'take everything left' already covers that case. Also adds explicit, hand-specified layouts for 2-5 vCPU hosts: too few cores for the general formula's role co-location choices to mean anything -- every role has to double up somewhere -- so these are literal per the product's own spec rather than derived. tests/unit/test_distrib_priority_over_alceml.py covers: the four tiny layouts (including a real-core-id case, not just 0..N-1), the <22 vCPU tiers being unchanged, distrib's count at 22+ being alceml_count- independent across the tier boundaries (9/10/11/12-plateau/24-jump), the clipping behavior when alceml's real count exceeds what's left, and that no core is ever double-booked outside the deliberate lvol_poller/ jc_singleton co-location. * fix(cluster): restart must not re-derive core-role allocation, only l_cores identity alceml_cpu_cores/distrib_cpu_cores/poller_cpu_cores and every derived mask are l-core INDICES, not physical core ids -- SPDK addresses reactors by their position in -l. That role-to-index split is decided once, at add time, by whichever allocation policy is in effect then. Restart called recalculate_cores_distribution on every restart where the OS-reported core count still matched (the common case), which re-ran calculate_core_ allocations from scratch -- so upgrading the node agent to a build with a changed policy (this branch's distrib/alceml reorder, concretely) would have silently re-pinned an already-provisioned, >=22-vCPU node's roles the next time it merely restarted. Not a deliberate re-provisioning action -- a routine restart, post-upgrade. What legitimately can go stale across a restart is which *physical* core sits at each index: the OS/k8s CPU manager can hand back a different specific set (same count) than before. That's all restart refreshes now -- the index@physical_core pairing in l_cores -- via the same generate_l_cores helper add_node/sn configure already build it with. A genuine core-count mismatch now warns instead of silently leaving l_cores stale. generate_l_cores also replaces five near-identical inline copies of the same '{i}@{core}' pairing already in utils/__init__.py -- the exact drift-prone shape that let number_of_distribs disagree across generate_configs/regenerate_config/calculate_hp_only earlier; one shared definition now, covered by a regression test that fails if the inline idiom reappears anywhere else. * fix(cluster): restart re-places physical cores sibling-aware, not by plain sort Restart's fresh cpuset from read_allowed_list() could land distrib/ poller/alceml's frozen index sets on physical cores that are not hyperthread siblings of each other, with no way to notice -- a plain sorted(cores) trusts the OS/k8s CPU manager to keep whole physical cores together and never checks. Add reassign_l_cores_for_restart(), which keeps every role's index set (size, and any sharing with another role) exactly as decided at add time, but chooses which fresh physical core fills each index by letting distrib, then poller, then alceml -- the existing allocation priority -- claim intact sibling pairs first, using the real sysfs topology (parse_thread_siblings) rather than calculate_core_allocations' machine-wide pair_hyperthreads()/os.cpu_count() guess. Falls back to unpaired singles rather than failing the restart when pairs run out. Also short-circuit when the fresh cpuset is identical to what's already recorded, so a no-op restart doesn't gratuitously reshuffle which physical core each role lands on. * fix(cluster): rederive number_of_distribs when resizing a node's core layout apply_cluster_vcpu_count resizes a host's CPU layout (isolated cores, distribution, l-cores) to the cluster's vcpu_count at add time, but left number_of_distribs untouched. That field is only ever computed once, at `sn configure` time, against the host's full core count -- before the node belongs to any cluster and before vcpu_count is known. A node configured with e.g. 18 cores gets number_of_distribs=6; resizing it down to an 8-vcpu cluster budget shrinks distrib_cpu_cores to 2, but the stale 6 persisted through both add_node (snode.number_of_distribs) and apply_cluster_hugepages' pool-count sizing, which reads the same field. Rederive number_of_distribs from the resized distrib_cpu_cores count in apply_cluster_vcpu_count, the same way regenerate_config already does for the sn configure-upgrade path, and persist it via a new number_of_distribs field on persist_node_config (snode_client + the internal API's PersistNodeConfigParams/handler). --- simplyblock_core/snode_client.py | 5 +- simplyblock_core/storage_node_ops.py | 71 +++-- simplyblock_core/utils/__init__.py | 250 ++++++++++++++---- .../api/internal/storage_node/docker.py | 3 + tests/unit/test_cluster_spdk_sizing.py | 29 ++ .../unit/test_distrib_priority_over_alceml.py | 157 +++++++++++ .../unit/test_reassign_l_cores_for_restart.py | 109 ++++++++ ...st_restart_does_not_rederive_core_roles.py | 84 ++++++ 8 files changed, 630 insertions(+), 78 deletions(-) create mode 100644 tests/unit/test_distrib_priority_over_alceml.py create mode 100644 tests/unit/test_reassign_l_cores_for_restart.py create mode 100644 tests/unit/test_restart_does_not_rederive_core_roles.py diff --git a/simplyblock_core/snode_client.py b/simplyblock_core/snode_client.py index 5f5f8b5711..8dd976be4a 100644 --- a/simplyblock_core/snode_client.py +++ b/simplyblock_core/snode_client.py @@ -230,7 +230,8 @@ def set_hugepages(self): def persist_node_config(self, max_lvol, huge_page_memory, numa_node, ssd_list, cpu_mask=None, isolated=None, l_cores=None, distribution=None, core_to_index=None, - small_pool_count=None, large_pool_count=None): + small_pool_count=None, large_pool_count=None, + number_of_distribs=None): payload = { "max_lvol": max_lvol, "huge_page_memory": huge_page_memory, @@ -251,6 +252,8 @@ def persist_node_config(self, max_lvol, huge_page_memory, numa_node, ssd_list, payload["small_pool_count"] = small_pool_count if large_pool_count is not None: payload["large_pool_count"] = large_pool_count + if number_of_distribs is not None: + payload["number_of_distribs"] = number_of_distribs return self._request("POST", "persist_node_config", payload) def ifc_is_roce(self, nic): diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index f9a4b04218..fb4ac02c2b 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -2713,12 +2713,28 @@ def apply_cluster_vcpu_count(snode_api, node_info, nodes, vcpu_count): entry["distribution"] = _resolve_core_distribution( replacement["distribution"], replacement["core_to_index"]) entry["core_to_index"] = replacement["core_to_index"] + + # number_of_distribs is sized off distrib_cpu_cores at configure + # time (generate_configs), before the host belongs to any cluster + # -- so it reflects the host's full core count, not the budget + # vcpu_count just clamped distrib_cpu_cores down to above. Rederive + # it the same way regenerate_config does, or add_node persists a + # distrib count sized for the pre-resize layout. + number_of_distribs = 2 + number_of_distribs_cores = len(entry["distribution"]["distrib_cpu_cores"]) + if 12 >= number_of_distribs_cores > 2: + number_of_distribs = number_of_distribs_cores + elif number_of_distribs_cores > 12: + number_of_distribs = 12 + entry["number_of_distribs"] = number_of_distribs + ok, err = snode_api.persist_node_config( max_lvol=None, huge_page_memory=None, numa_node=numa_socket, ssd_list=entry.get("ssd_pcis"), cpu_mask=entry["cpu_mask"], isolated=entry["isolated"], l_cores=entry["l-cores"], distribution=entry["distribution"], - core_to_index={str(k): v for k, v in entry["core_to_index"].items()}) + core_to_index={str(k): v for k, v in entry["core_to_index"].items()}, + number_of_distribs=number_of_distribs) if not ok: logger.error( "Failed to persist the resized CPU layout for socket %s: %s", @@ -4988,26 +5004,41 @@ def _restart_storage_node_impl( cores, _ = snode_api.read_allowed_list() logger.info(f"read_allowed list is {cores}") + # alceml_cpu_cores/distrib_cpu_cores/poller_cpu_cores and every mask below + # are l-core INDICES (0..req_cpu_count-1), not physical core ids -- SPDK + # addresses reactors by their position in -l, and that role-to-index split + # was decided once, at add time (see apply_cluster_vcpu_count/add_node), + # using whatever allocation policy existed then. Restart must not + # re-derive it: calling recalculate_cores_distribution here re-ran + # calculate_core_allocations fresh on every restart, so upgrading the + # node agent to a build with a changed allocation policy silently + # re-pinned an already-provisioned node's roles the next time it merely + # restarted -- not a deliberate re-provisioning action. + # + # What legitimately can go stale across a restart is which *physical* + # core sits at each index -- the OS/k8s CPU manager can hand back a + # different specific set (same count) than before. That's all this + # refreshes: the index@physical_core pairing in l_cores, nothing else -- + # reassign_l_cores_for_restart keeps every role's index set (hence its + # size and any sharing with other roles) exactly as decided at add time, + # only choosing which fresh physical core fills each index, preferring + # to keep distrib/poller/alceml's own cores mutual hyperthread siblings. if len(cores) == req_cpu_count: - new_distribution, _ = snode_api.recalculate_cores_distribution(cores, snode.number_of_alceml_devices) - poller_cpu_cores = new_distribution.get("poller_cpu_cores") - snode.alceml_cpu_cores = new_distribution.get("alceml_cpu_cores") - snode.distrib_cpu_cores = new_distribution.get("distrib_cpu_cores") - snode.alceml_worker_cpu_cores = new_distribution.get("alceml_worker_cpu_cores") - jc_singleton_core = new_distribution.get("jc_singleton_core") - app_thread_core = new_distribution.get("app_thread_core") - jm_cpu_core = new_distribution.get("jm_cpu_core") - snode.pollers_mask = utils.generate_mask(poller_cpu_cores) - snode.app_thread_mask = utils.generate_mask(app_thread_core) - lvol_poller_core = new_distribution.get("lvol_poller_core") - snode.lvol_poller_mask = utils.generate_mask(lvol_poller_core) - - if jc_singleton_core: - snode.jc_singleton_mask = utils.decimal_to_hex_power_of_2(jc_singleton_core[0]) - compression_core = new_distribution.get("compression_core") - if compression_core: - snode.compression_cpu_mask = utils.generate_mask(compression_core) - snode.jm_cpu_mask = utils.generate_mask(jm_cpu_core) + prior_physical_cores = {int(pair.split("@")[1]) for pair in snode.l_cores.split(",") if pair} + if set(cores) == prior_physical_cores: + # Identical cpuset -- nothing to reassign; leave l_cores exactly + # as it was rather than have reassign_l_cores_for_restart pick an + # arbitrary (if equally valid) sibling pairing that only churns + # which physical core each role lands on for no operational gain. + logger.info(f"Node {node_id}: cpuset unchanged, keeping existing l_cores") + else: + placement = utils.reassign_l_cores_for_restart( + cores, snode.distrib_cpu_cores, snode.poller_cpu_cores, snode.alceml_cpu_cores) + snode.l_cores = utils.generate_l_cores(placement) + else: + logger.warning( + "Node %s: read_allowed_list returned %d core(s), expected %d -- " + "leaving l_cores as-is", node_id, len(cores), req_cpu_count) if not results: logger.error(f"Failed to start spdk: {err}") diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index 474e9dedb6..63d2afdfd2 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -445,6 +445,8 @@ def calculate_core_allocations(vcpu_list, alceml_count=2): is_hyperthreaded = is_hyperthreading_enabled_via_siblings() pairs = pair_hyperthreads() if is_hyperthreaded else {} remaining = set(vcpu_list) + cores = sorted(vcpu_list) + v = len(vcpu_list) def reserve(vcpu, get_sibling=False): if vcpu in remaining: @@ -460,85 +462,139 @@ def reserve(vcpu, get_sibling=False): def reserve_n(count): vcpus: list = [] if count > 0: - for v in sorted(remaining): + for c in sorted(remaining): if (count - len(vcpus)) >= 2: - vcpus += reserve(v, True) + vcpus += reserve(c, True) else: - vcpus += reserve(v) + vcpus += reserve(c) if len(vcpus) >= count: break return vcpus[:count] + def finalize(assigned): + # Return the individual threads as separate values + return ( + assigned.get("app_thread_core", []), + assigned.get("jm_cpu_core", []), + assigned.get("poller_cpu_cores", []), + assigned.get("alceml_cpu_cores", []), + assigned.get("alceml_worker_cpu_cores", []), + assigned.get("distrib_cpu_cores", []), + assigned.get("jc_singleton_core", []), + assigned.get("lvol_poller_core", []), + # Reserved: always empty now that the compression-thread feature + # this slot backed is gone. Kept for shape/backport compatibility + # with positional consumers (e.g. distribution[8] callers) elsewhere. + assigned.get("compression_core", []), + ) + + def split_distrib_and_poller(): + """distrib/poller split the cores left after the base roles (and, on + this path, after alceml -- see the v<22 tiers below where alceml is + reserved first). Only used where alceml still comes first; the v>=22 + tier below reorders that and does its own split.""" + dp = int(len(remaining) / 2) + if 17 > dp >= 12: + poller_n = len(remaining) - 12 + distrib = reserve_n(12) + poller = reserve_n(poller_n) + elif dp >= 17: + poller_n = len(remaining) - 24 + distrib = reserve_n(24) + poller = reserve_n(poller_n) + else: + distrib = reserve_n(dp) + poller = reserve_n(dp) + if len(remaining) > 0: + if len(poller) == 0: + distrib = poller = reserve_n(1) + else: + poller = poller + reserve_n(1) + return distrib, poller + + # Too few cores for the general formula's role co-location to make sense + # -- every role has to double up somewhere, so these tiny counts are + # hand-specified rather than derived. + if v <= 1: + only = cores[:1] + return finalize({ + "app_thread_core": only, "jm_cpu_core": only, "jc_singleton_core": only, + "lvol_poller_core": only, "alceml_cpu_cores": only, + "distrib_cpu_cores": only, "poller_cpu_cores": only, + }) + if v == 2: + return finalize({ + "app_thread_core": cores[0:1], "jc_singleton_core": cores[0:1], + "jm_cpu_core": cores[0:1], "lvol_poller_core": cores[0:1], + "alceml_cpu_cores": cores[0:2], "distrib_cpu_cores": cores[1:2], + }) + if v == 3: + return finalize({ + "app_thread_core": cores[0:1], "jc_singleton_core": cores[0:1], + "jm_cpu_core": cores[0:1], "alceml_cpu_cores": cores[0:1], + "lvol_poller_core": cores[1:2], "poller_cpu_cores": cores[1:2], + "distrib_cpu_cores": cores[2:3], + }) + if v == 4: + return finalize({ + "app_thread_core": cores[0:1], "jc_singleton_core": cores[0:1], + "jm_cpu_core": cores[1:2], "alceml_cpu_cores": cores[1:2], + "lvol_poller_core": cores[2:3], "poller_cpu_cores": cores[2:3], + "distrib_cpu_cores": cores[3:4], + }) + if v == 5: + return finalize({ + "app_thread_core": cores[0:1], "jc_singleton_core": cores[0:1], + "jm_cpu_core": cores[1:2], "lvol_poller_core": cores[1:2], + "poller_cpu_cores": cores[2:3], "distrib_cpu_cores": cores[3:4], + "alceml_cpu_cores": cores[4:5], + }) + assigned = {} # lvol_poller co-locates with jc_singleton's core below 32 vCPU to save a # core; at/above 32 vCPU it gets its own dedicated core. - colocate_lvs = len(vcpu_list) < 32 - if (len(vcpu_list) < 12): + colocate_lvs = v < 32 + if v < 12: vcpu = reserve_n(4 if colocate_lvs else 5) assigned["app_thread_core"] = vcpu[0:1] assigned["jm_cpu_core"] = vcpu[1:2] assigned["jc_singleton_core"] = vcpu[2:3] assigned["alceml_cpu_cores"] = vcpu[3:4] assigned["lvol_poller_core"] = vcpu[2:3] if colocate_lvs else vcpu[4:5] - elif (len(vcpu_list) < 22): + assigned["distrib_cpu_cores"], assigned["poller_cpu_cores"] = split_distrib_and_poller() + elif v < 22: vcpu = reserve_n(5 if colocate_lvs else 6) assigned["app_thread_core"] = vcpu[0:1] assigned["jm_cpu_core"] = vcpu[1:2] assigned["jc_singleton_core"] = vcpu[2:3] assigned["alceml_cpu_cores"] = vcpu[3:5] assigned["lvol_poller_core"] = vcpu[2:3] if colocate_lvs else vcpu[5:6] + assigned["distrib_cpu_cores"], assigned["poller_cpu_cores"] = split_distrib_and_poller() else: - # base threads: app, jm, jc (+ own lvol_poller unless co-located) + # Reordered: distrib claims its share first, as a pure function of + # what's left after the base roles -- no longer reduced by however + # many devices this node happens to have. alceml then takes its real + # device-scaled count from what's left (clipped if there genuinely + # isn't room), and poller -- already the "whatever's left" role -- + # absorbs the true remainder. base = 3 if colocate_lvs else 4 - vcpus = reserve_n(base + alceml_count) + vcpus = reserve_n(base) assigned["app_thread_core"] = vcpus[0:1] assigned["jm_cpu_core"] = vcpus[1:2] assigned["jc_singleton_core"] = vcpus[2:3] - idx = 3 - if colocate_lvs: - assigned["lvol_poller_core"] = vcpus[2:3] - else: - assigned["lvol_poller_core"] = vcpus[idx:idx + 1] - idx += 1 - assigned["alceml_cpu_cores"] = vcpus[idx:idx + alceml_count] - dp = int(len(remaining) / 2) - if 17 > dp >= 12: - poller_n = len(remaining) - 12 - vcpus = reserve_n(12) - assigned["distrib_cpu_cores"] = vcpus - vcpus = reserve_n(poller_n) - assigned["poller_cpu_cores"] = vcpus - elif dp >= 17: - poller_n = len(remaining) - 24 - vcpus = reserve_n(24) - assigned["distrib_cpu_cores"] = vcpus - vcpus = reserve_n(poller_n) - assigned["poller_cpu_cores"] = vcpus - else: - vcpus = reserve_n(dp) - assigned["distrib_cpu_cores"] = vcpus - vcpus = reserve_n(dp) - assigned["poller_cpu_cores"] = vcpus - if len(remaining) > 0: - if len(assigned["poller_cpu_cores"]) == 0: - assigned["distrib_cpu_cores"] = assigned["poller_cpu_cores"] = reserve_n(1) + assigned["lvol_poller_core"] = vcpus[2:3] if colocate_lvs else vcpus[3:4] + + dp = int(len(remaining) / 2) + if 17 > dp >= 12: + distrib_n = 12 + elif dp >= 17: + distrib_n = 24 else: - assigned["poller_cpu_cores"] = assigned["poller_cpu_cores"] + reserve_n(1) - # Return the individual threads as separate values - return ( - assigned.get("app_thread_core", []), - assigned.get("jm_cpu_core", []), - assigned.get("poller_cpu_cores", []), - assigned.get("alceml_cpu_cores", []), - assigned.get("alceml_worker_cpu_cores", []), - assigned.get("distrib_cpu_cores", []), - assigned.get("jc_singleton_core", []), - assigned.get("lvol_poller_core", []), - # Reserved: always empty now that the compression-thread feature this - # slot backed is gone. Kept for shape/backport compatibility with - # positional consumers (e.g. distribution[8] callers) elsewhere. - assigned.get("compression_core", []), - ) + distrib_n = dp + assigned["distrib_cpu_cores"] = reserve_n(distrib_n) + assigned["alceml_cpu_cores"] = reserve_n(min(alceml_count, len(remaining))) + assigned["poller_cpu_cores"] = reserve_n(len(remaining)) + return finalize(assigned) def isolate_cores(spdk_cpu_mask): @@ -562,6 +618,15 @@ def generate_mask(cores): return f'0x{mask:X}' +def generate_l_cores(cores): + """The SPDK -l argument: "@" pairs, one + per reactor. Every role (app_thread/jm/jc_singleton/alceml/distrib/ + poller/...) is addressed by its l-core INDEX afterwards, never by the + physical id directly -- this is the one place that pairing is made. + """ + return ",".join(f"{i}@{core}" for i, core in enumerate(cores)) + + def calculate_pool_count(alceml_count, number_of_distribs, cpu_count, poller_count, max_lvol=0): ''' Small pool count Large pool count @@ -1790,7 +1855,7 @@ def generate_core_allocation(cores_by_numa, sockets_to_use, nodes_per_socket, vc if nodes_per_socket == 1: # If there's only one node, assign all available cores to it node_cores = available_cores - l_cores = ",".join([f"{i}@{core}" for i, core in enumerate(node_cores)]) + l_cores = generate_l_cores(node_cores) core_to_index = {core: idx for idx, core in enumerate(node_cores)} node_distribution[numa_node].append({ "cpu_mask": hex(sum([1 << c for c in node_cores])), @@ -1811,7 +1876,7 @@ def generate_core_allocation(cores_by_numa, sockets_to_use, nodes_per_socket, vc min_isolated_cores = min(len(node_0_cores), len(node_1_cores)) # Generate l-cores for node 0 - l_cores_0 = ",".join([f"{i}@{core}" for i, core in enumerate(node_0_cores[:min_isolated_cores])]) + l_cores_0 = generate_l_cores(node_0_cores[:min_isolated_cores]) core_to_index = {core: idx for idx, core in enumerate(node_0_cores)} isolated_cores = node_0_cores[:min_isolated_cores] node_distribution[numa_node].append({ @@ -1823,7 +1888,7 @@ def generate_core_allocation(cores_by_numa, sockets_to_use, nodes_per_socket, vc }) # Generate l-cores for node 1 - l_cores_1 = ",".join([f"{i}@{core}" for i, core in enumerate(node_1_cores[:min_isolated_cores])]) + l_cores_1 = generate_l_cores(node_1_cores[:min_isolated_cores]) core_to_index = {core: idx for idx, core in enumerate(node_1_cores)} isolated_cores = node_1_cores[:min_isolated_cores] node_distribution[numa_node].append({ @@ -1857,7 +1922,7 @@ def regenerate_config(new_config, old_config, force=False): return False old_config["nodes"][i]["number_of_alcemls"] = number_of_alcemls old_config["nodes"][i]["cpu_mask"] = new_config["nodes"][i]["cpu_mask"] - old_config["nodes"][i]["l-cores"] = ",".join([f"{i}@{core}" for i, core in enumerate(isolated_cores)]) + old_config["nodes"][i]["l-cores"] = generate_l_cores(isolated_cores) old_config["nodes"][i]["isolated"] = isolated_cores distribution = calculate_core_allocations(isolated_cores, number_of_alcemls + 1) core_to_index = {core: idx for idx, core in enumerate(isolated_cores)} @@ -2022,7 +2087,7 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a "socket": nid, "cpu_mask": core_group["cpu_mask"], "isolated": core_group["isolated"], - "l-cores": ",".join([f"{i}@{core}" for i, core in enumerate(core_group["isolated"])]), + "l-cores": generate_l_cores(core_group["isolated"]), "number_of_alcemls": 0, "distribution": { "app_thread_core": get_core_indexes(core_group["core_to_index"], core_group["distribution"][0]), @@ -3743,6 +3808,77 @@ def recalculate_cores_distribution(cores, number_of_alcemls): "compression_core": get_core_indexes(core_to_index, distribution[8])} +def _take_sibling_aware(cores_remaining, count, siblings): + """Pull up to `count` cores out of the `cores_remaining` set, preferring + to grab a real hyperthread sibling alongside whenever at least 2 more + are still needed -- mirrors calculate_core_allocations' own reserve_n, + but driven by real sysfs sibling data instead of pair_hyperthreads()' + os.cpu_count()-wide guess, since here the pool is already scoped to one + node's own fresh core list.""" + chosen: List[int] = [] + for core in sorted(cores_remaining): + if len(chosen) >= count: + break + if core not in cores_remaining: + continue # already claimed as someone else's sibling this pass + if count - len(chosen) >= 2: + sibling = next((s for s in siblings.get(core, [core]) if s != core), None) + if sibling is not None and sibling in cores_remaining: + cores_remaining.discard(core) + cores_remaining.discard(sibling) + chosen += [core, sibling] + continue + cores_remaining.discard(core) + chosen.append(core) + return chosen[:count] + + +def reassign_l_cores_for_restart(cores, distrib_indices, poller_indices, alceml_indices): + """Rebuild the l-core index -> physical-core mapping at restart, when the + OS/k8s CPU manager may have handed back a different (same-count) cpuset + than the node was added with. + + Every role's INDEX SET -- hence its core count, and any sharing between + roles that already point at the same index -- was decided once, at add + time, and must not change here (see _restart_storage_node_impl); this + only chooses which of the fresh physical cores fills which index. + Unlike a plain numeric sort, it tries to keep each sibling-sensitive + role's own cores mutual hyperthread pairs -- using the real sysfs + topology (parse_thread_siblings), not calculate_core_allocations' + machine-wide pair_hyperthreads() guess -- so distrib/poller/alceml, in + that priority order, get first claim on intact sibling pairs from + whatever the fresh cpuset actually contains. Falls back to unpaired + singles when pairs run out rather than failing: a restart must not + block on an imperfect cpuset. + + Returns the physical-core list to pass to generate_l_cores(), ordered by + index (result[i] is the physical core for l-core index i). + """ + n = len(cores) + remaining = set(cores) + siblings = parse_thread_siblings() + placement: List[Optional[int]] = [None] * n + + for role, indices in (("distrib", distrib_indices), ("poller", poller_indices), + ("alceml", alceml_indices)): + if not indices: + continue + picked = _take_sibling_aware(remaining, len(indices), siblings) + if len(picked) < len(indices): + logger.warning( + "restart core placement: only found %d/%d core(s) for %s " + "in the fresh cpuset; the rest will be unpaired", + len(picked), len(indices), role) + for idx, core in zip(sorted(indices), picked): + placement[idx] = core + + leftover_indices = [i for i in range(n) if placement[i] is None] + for idx, core in zip(leftover_indices, sorted(remaining)): + placement[idx] = core + + return placement + + def resolve_address(host_port: str) -> str: """Resolves an host:port string to its IP address diff --git a/simplyblock_web/api/internal/storage_node/docker.py b/simplyblock_web/api/internal/storage_node/docker.py index e2e98a6cd0..b58ba49ec2 100644 --- a/simplyblock_web/api/internal/storage_node/docker.py +++ b/simplyblock_web/api/internal/storage_node/docker.py @@ -718,6 +718,7 @@ class PersistNodeConfigParams(BaseModel): l_cores: Optional[str] = None distribution: Optional[dict] = None core_to_index: Optional[dict] = None + number_of_distribs: Optional[int] = Field(None, ge=0) @api.post('/persist_node_config', responses={ @@ -755,6 +756,8 @@ def persist_node_config(body: PersistNodeConfigParams): node_config["distribution"] = body.distribution if body.core_to_index is not None: node_config["core_to_index"] = body.core_to_index + if body.number_of_distribs is not None: + node_config["number_of_distribs"] = body.number_of_distribs matched = True break diff --git a/tests/unit/test_cluster_spdk_sizing.py b/tests/unit/test_cluster_spdk_sizing.py index 728e634e6d..dee148ad96 100644 --- a/tests/unit/test_cluster_spdk_sizing.py +++ b/tests/unit/test_cluster_spdk_sizing.py @@ -191,6 +191,35 @@ def test_resizes_to_the_cluster_budget_and_persists_once(self): persisted_kwargs = snode_api.persist_node_config.call_args.kwargs self.assertIsInstance(persisted_kwargs["distribution"], dict) + def test_number_of_distribs_is_rederived_from_the_resized_layout(self): + """number_of_distribs is sized off distrib_cpu_cores at configure time, + against the host's full core count -- before the host belongs to any + cluster. Resizing the layout down to the cluster's vcpu_count must + rederive it too, or add_node persists a distrib count sized for the + pre-resize (much larger) layout instead of the one actually running. + + Configured against isolated_len=18 (no cluster yet) gives + distrib_cpu_cores=6 (table: V=17-18 -> 6 distribs). Resized to the + cluster's vcpu_count=8, distrib_cpu_cores drops to 2 (table: V=8-9 -> + 2 distribs); number_of_distribs must follow it down to 2, not stay 6. + """ + snode_api = MagicMock() + snode_api.persist_node_config.return_value = (True, None) + node_info = self._node_info({0: list(range(32))}) + node = self._node_config(0, isolated_len=18) + node["number_of_distribs"] = 6 # stale, from configure-time sizing + nodes = [node] + + ok = storage_node_ops.apply_cluster_vcpu_count(snode_api, node_info, nodes, 8) + + self.assertTrue(ok) + self.assertEqual(len(nodes[0]["distribution"]["distrib_cpu_cores"]), 2) + self.assertEqual(nodes[0]["number_of_distribs"], 2, + "must be rederived from the resized layout, not left at the stale 6") + persisted_kwargs = snode_api.persist_node_config.call_args.kwargs + self.assertEqual(persisted_kwargs["number_of_distribs"], 2, + "the rederived count must also be persisted to the node's config file") + def test_already_correct_is_a_no_op(self): """A retried add_node re-fetches the file its own earlier attempt already resized; it must not refetch topology or rewrite it again.""" diff --git a/tests/unit/test_distrib_priority_over_alceml.py b/tests/unit/test_distrib_priority_over_alceml.py new file mode 100644 index 0000000000..5fd66cd266 --- /dev/null +++ b/tests/unit/test_distrib_priority_over_alceml.py @@ -0,0 +1,157 @@ +# coding=utf-8 +"""calculate_core_allocations() gives distrib/poller priority over alceml. + +Previously alceml claimed its cores (scaled by the node's actual device +count) before distrib/poller ever saw the budget, so a node with many +devices could starve distrib down to almost nothing even on a host with +plenty of vCPUs -- distrib's share was accidental, not a capacity decision. +Below 22 vCPUs this never actually mattered: alceml's share there was +already a fixed constant (1 core under 12 vCPUs, 2 from 12-21), never +scaled by the device count, so nothing changes in that range. At 22+ +vCPUs, where alceml really did scale with the device count, distrib now +claims its share first, as a pure function of vCPU count; alceml takes +its real device-scaled count from whatever's left (clipped if there +genuinely isn't room); poller -- already the "whatever's left" role -- +absorbs the true remainder. + +Below 6 vCPUs there's no room left for the general formula's role +co-location choices to make sense at all, so 2-5 vCPU hosts get literal, +hand-specified layouts instead of a derived split. +""" + +from unittest.mock import patch + +from simplyblock_core import utils + +_FIELDS = ("app_thread_core", "jm_cpu_core", "poller_cpu_cores", "alceml_cpu_cores", + "alceml_worker_cpu_cores", "distrib_cpu_cores", "jc_singleton_core", + "lvol_poller_core", "compression_core") + + +def _calc(vcpu_list, alceml_count=2): + with patch("simplyblock_core.utils.is_hyperthreading_enabled_via_siblings", return_value=False): + result = utils.calculate_core_allocations(vcpu_list, alceml_count=alceml_count) + return dict(zip(_FIELDS, result)) + + +class TestTinyNodeLayouts: + """2-5 vCPU hosts: every role has to double up somewhere, so these are + the literal layouts the product wants, not a derived split.""" + + def test_two_vcpus(self): + assigned = _calc([0, 1]) + assert assigned["app_thread_core"] == [0] + assert assigned["jc_singleton_core"] == [0] + assert assigned["jm_cpu_core"] == [0] + assert assigned["lvol_poller_core"] == [0] + assert assigned["alceml_cpu_cores"] == [0, 1] + assert assigned["distrib_cpu_cores"] == [1] + assert assigned["poller_cpu_cores"] == [] + + def test_three_vcpus(self): + assigned = _calc([0, 1, 2]) + assert assigned["app_thread_core"] == [0] + assert assigned["jc_singleton_core"] == [0] + assert assigned["jm_cpu_core"] == [0] + assert assigned["alceml_cpu_cores"] == [0] + assert assigned["lvol_poller_core"] == [1] + assert assigned["poller_cpu_cores"] == [1] + assert assigned["distrib_cpu_cores"] == [2] + + def test_four_vcpus(self): + assigned = _calc([0, 1, 2, 3]) + assert assigned["app_thread_core"] == [0] + assert assigned["jc_singleton_core"] == [0] + assert assigned["jm_cpu_core"] == [1] + assert assigned["alceml_cpu_cores"] == [1] + assert assigned["lvol_poller_core"] == [2] + assert assigned["poller_cpu_cores"] == [2] + assert assigned["distrib_cpu_cores"] == [3] + + def test_five_vcpus(self): + assigned = _calc([0, 1, 2, 3, 4]) + assert assigned["app_thread_core"] == [0] + assert assigned["jc_singleton_core"] == [0] + assert assigned["jm_cpu_core"] == [1] + assert assigned["lvol_poller_core"] == [1] + assert assigned["poller_cpu_cores"] == [2] + assert assigned["distrib_cpu_cores"] == [3] + assert assigned["alceml_cpu_cores"] == [4] + + def test_layouts_use_real_core_ids_not_positional_indices(self): + """The layout must key off the actual core ids handed in, not + assume 0..N-1 -- add_node's isolated-core lists are never that.""" + assigned = _calc([5, 9, 14]) + assert assigned["app_thread_core"] == [5] + assert assigned["jc_singleton_core"] == [5] + assert assigned["jm_cpu_core"] == [5] + assert assigned["alceml_cpu_cores"] == [5] + assert assigned["lvol_poller_core"] == [9] + assert assigned["poller_cpu_cores"] == [9] + assert assigned["distrib_cpu_cores"] == [14] + + +class TestUnder22VcpusUnchanged: + """alceml's share below 22 vCPUs was never scaled by the device count, + so there's nothing to reorder here -- confirms the boundary tiers still + match what shipped before this change.""" + + def test_below_12_vcpus_alceml_gets_one_fixed_core(self): + for alceml_count in (1, 3, 8): + assigned = _calc(list(range(10)), alceml_count) + assert len(assigned["alceml_cpu_cores"]) == 1, alceml_count + + def test_12_to_21_vcpus_alceml_gets_two_fixed_cores(self): + for alceml_count in (1, 3, 8): + assigned = _calc(list(range(16)), alceml_count) + assert len(assigned["alceml_cpu_cores"]) == 2, alceml_count + + +class TestDistribPriorityAt22PlusVcpus: + """22+ vCPUs is the tier where alceml used to scale with the device + count and eat into distrib/poller's budget before they saw it.""" + + # V, expected distrib-core count -- independent of alceml_count. + DISTRIB_CORES_BY_VCPU = { + 22: 9, 23: 10, 24: 10, 25: 11, 26: 11, + 27: 12, 30: 12, 37: 12, # capped at 12 through this range + 38: 24, 40: 24, # jumps straight to 24, no ramp + } + + def test_distrib_count_is_independent_of_alceml_count(self): + for vcpu_count, expected in self.DISTRIB_CORES_BY_VCPU.items(): + for alceml_count in (1, 2, 5, 10): + assigned = _calc(list(range(vcpu_count)), alceml_count) + got = len(assigned["distrib_cpu_cores"]) + assert got == expected, ( + f"V={vcpu_count} A={alceml_count}: expected {expected} " + f"distrib cores, got {got}") + + def test_alceml_gets_its_real_share_when_there_is_room(self): + assigned = _calc(list(range(22)), alceml_count=2) + assert len(assigned["distrib_cpu_cores"]) == 9 + assert len(assigned["alceml_cpu_cores"]) == 2 + assert len(assigned["poller_cpu_cores"]) == 8, "poller absorbs the true remainder" + + def test_alceml_is_clipped_when_the_request_exceeds_what_is_left(self): + """V=22: base=3, remaining=19, distrib takes 9, leaving 10 for + alceml+poller. Ask for 15 -- more than exists -- and it must clip, + not raise or overrun into cores distrib/poller already hold.""" + assigned = _calc(list(range(22)), alceml_count=15) + assert len(assigned["distrib_cpu_cores"]) == 9 + assert len(assigned["alceml_cpu_cores"]) == 10 + assert len(assigned["poller_cpu_cores"]) == 0 + + def test_no_core_is_assigned_to_more_than_one_role(self): + for vcpu_count in range(22, 45): + assigned = _calc(list(range(vcpu_count)), alceml_count=3) + exclusive = (assigned["app_thread_core"] + assigned["jm_cpu_core"] + + assigned["jc_singleton_core"] + assigned["alceml_cpu_cores"] + + assigned["distrib_cpu_cores"] + assigned["poller_cpu_cores"]) + # lvol_poller co-locates with jc_singleton by design below 32 vCPU + # -- only count it separately once it has its own core. + if assigned["lvol_poller_core"] != assigned["jc_singleton_core"]: + exclusive += assigned["lvol_poller_core"] + dupes = {c for c in exclusive if exclusive.count(c) > 1} + assert not dupes, f"V={vcpu_count}: {dupes} assigned to more than one role" + assert len(exclusive) <= vcpu_count diff --git a/tests/unit/test_reassign_l_cores_for_restart.py b/tests/unit/test_reassign_l_cores_for_restart.py new file mode 100644 index 0000000000..9f5f63668e --- /dev/null +++ b/tests/unit/test_reassign_l_cores_for_restart.py @@ -0,0 +1,109 @@ +# coding=utf-8 +"""reassign_l_cores_for_restart(): restart-time index->physical placement. + +A restart must never change a role's INDEX SET (its core count, and any +sharing with another role at the same index) -- that was decided once, at +add time (see test_restart_does_not_rederive_core_roles.py). What legitimately +changes across a restart is which physical core the OS/k8s CPU manager hands +back for each index. This function chooses that mapping so that +distrib/poller/alceml -- in that priority order, matching the product's +existing allocation priority -- get first claim on any intact hyperthread +sibling pairs present in the fresh cpuset, using the real sysfs topology +(parse_thread_siblings) rather than calculate_core_allocations' machine-wide +pair_hyperthreads() guess. +""" +from unittest.mock import patch + +from simplyblock_core import utils + +# A 40-logical-CPU host, siblings at i / i+20 (the common low-half/high-half +# hyperthread numbering convention). +SIBLINGS_40 = {i: sorted([i, i + 20]) for i in range(20)} +SIBLINGS_40.update({i + 20: sorted([i, i + 20]) for i in range(20)}) + + +def _reassign(cores, distrib, poller, alceml, siblings=SIBLINGS_40): + with patch("simplyblock_core.utils.parse_thread_siblings", return_value=siblings): + return utils.reassign_l_cores_for_restart(cores, distrib, poller, alceml) + + +class TestPreservesShapeAndCompleteness: + + def test_every_index_gets_exactly_one_core_no_duplicates(self): + cores = list(range(20)) + placement = _reassign(cores, distrib=[0, 1, 2, 3, 4, 5, 6], poller=[7, 8, 9, 10, 11, 12, 13, 14], + alceml=[15, 16]) + assert len(placement) == 20 + assert sorted(placement) == sorted(cores), "must be a bijection onto the fresh cpuset" + + def test_role_counts_are_unchanged_by_construction(self): + """The function only ever fills the index positions it's handed -- + it can't grow or shrink a role's slot count.""" + cores = list(range(10)) + placement = _reassign(cores, distrib=[0, 1], poller=[2, 3, 4], alceml=[5, 6]) + assert len({placement[i] for i in (0, 1)}) == 2 + assert len({placement[i] for i in (2, 3, 4)}) == 3 + assert len({placement[i] for i in (5, 6)}) == 2 + + +class TestSiblingPreferenceAndPriority: + + def test_cpuset_with_intact_pairs_gives_distrib_a_real_sibling_pair(self): + """The function doesn't promise to reuse the SAME physical cores a + role had before (that's a stability nice-to-have the caller can add + on top, e.g. by short-circuiting when the cpuset hasn't changed at + all) -- only that whatever pair it picks is a REAL sibling pair.""" + isolated = sorted(c for c in range(40) if c % 2 == 1) # sibling-closed subset + placement = _reassign(isolated, distrib=[3, 13], poller=[0], alceml=[1]) + a, b = placement[3], placement[13] + assert SIBLINGS_40[a] == sorted([a, b]), "distrib's pair must be real siblings" + + def test_different_but_whole_core_cpuset_still_pairs_distrib(self): + """k8s hands back a DIFFERENT 10 physical cores than before, but + still both hyperthreads of each -- distrib's own two indices must + still land on a real sibling pair, even though the specific cores + changed entirely.""" + new_cores = sorted(list(range(0, 5)) + list(range(20, 25))) # 5 whole cores + placement = _reassign(new_cores, distrib=[0, 1], poller=[2, 3], alceml=[4]) + a, b = placement[0], placement[1] + assert {a, b} in ({0, 20}, {1, 21}, {2, 22}, {3, 23}, {4, 24}) + + def test_distrib_takes_priority_over_poller_and_alceml(self): + """Only ONE real sibling pair exists in the fresh cpuset; distrib + must get it even though poller is asked for first... no, distrib is + asked for first by priority and must win it.""" + # cores: one true pair (0,20), plus three unrelated singles + cores = [0, 20, 5, 9, 13] + siblings = {0: [0, 20], 20: [0, 20], 5: [5], 9: [9], 13: [13]} + placement = _reassign(cores, distrib=[0, 1], poller=[2, 3], alceml=[4], siblings=siblings) + assert {placement[0], placement[1]} == {0, 20}, "distrib must claim the only real sibling pair" + + def test_poller_takes_priority_over_alceml(self): + """Same scarcity setup one priority level down: with only one real + pair for two multi-core roles that each need one, poller (checked + first) must be the one that ends up paired, not alceml.""" + cores = [0, 20, 5, 9] + siblings = {0: [0, 20], 20: [0, 20], 5: [5], 9: [9]} + placement = _reassign(cores, distrib=[], poller=[0, 1], alceml=[2, 3], siblings=siblings) + assert {placement[0], placement[1]} == {0, 20} + assert {placement[2], placement[3]} == {5, 9} + + def test_broken_nonsibling_closed_cpuset_still_completes_without_raising(self): + """No real sibling pairs at all in the fresh cpuset -- every role + just gets unpaired singles instead of failing the restart.""" + cores = [1, 3, 5, 7] + siblings = {c: [c] for c in cores} # nobody has a sibling present + placement = _reassign(cores, distrib=[0, 1], poller=[2], alceml=[3], siblings=siblings) + assert sorted(placement) == sorted(cores) + + +class TestLeftoverSingleCoreRoles: + + def test_indices_outside_the_three_named_roles_are_still_filled(self): + """app_thread/jm/jc_singleton/lvol_poller/compression aren't passed + in by name (sibling-pairing is moot for a 1-core role) -- indices + they occupy must still end up with a real physical core.""" + cores = list(range(6)) + placement = _reassign(cores, distrib=[0, 1], poller=[2], alceml=[3]) + assert placement[4] is not None and placement[5] is not None + assert sorted(placement) == sorted(cores) diff --git a/tests/unit/test_restart_does_not_rederive_core_roles.py b/tests/unit/test_restart_does_not_rederive_core_roles.py new file mode 100644 index 0000000000..827a0fc765 --- /dev/null +++ b/tests/unit/test_restart_does_not_rederive_core_roles.py @@ -0,0 +1,84 @@ +# coding=utf-8 +"""Restart must not re-derive the node's core-role allocation. + +alceml_cpu_cores/distrib_cpu_cores/poller_cpu_cores and every derived mask +are l-core INDICES (0..req_cpu_count-1), decided once at add time by +whichever allocation policy was in effect then (calculate_core_allocations, +via add_node/recalculate_cores_distribution's first-ever call). Restart +used to call recalculate_cores_distribution again on every restart where +the OS-reported core count still matched -- which re-ran that policy from +scratch, so upgrading the node agent to a build with a changed allocation +policy (e.g. distrib/poller now taking priority over alceml) silently +re-pinned an already-provisioned node's roles the next time it merely +restarted, not as a deliberate re-provisioning action. + +What legitimately can go stale across a restart is which *physical* core +sits at each index -- the OS/k8s CPU manager can hand back a different +specific set (same count) than before. That's all restart may still +refresh: the index@physical_core pairing in l_cores, via +reassign_l_cores_for_restart() -- which keeps every role's index set (its +size, and any sharing with another role at the same index) exactly as +decided at add time, only choosing which fresh physical core fills each +index, preferring to keep distrib/poller/alceml's own cores mutual +hyperthread siblings -- and generate_l_cores(), nothing else. +""" +import inspect + +from simplyblock_core import storage_node_ops, utils + + +class TestRestartDoesNotRederiveCoreRoles: + + def test_restart_never_calls_recalculate_cores_distribution(self): + """add_node is the one legitimate place this runs -- restart must + not call it again. Checks for an actual call, not just the name -- + the explanatory comment above the fix mentions it by name too.""" + src = inspect.getsource(storage_node_ops._restart_storage_node_impl) + assert "recalculate_cores_distribution(" not in src + + def test_restart_still_refreshes_l_cores_from_the_fresh_core_list(self): + src = inspect.getsource(storage_node_ops._restart_storage_node_impl) + assert "read_allowed_list" in src + assert "snode.l_cores = utils.generate_l_cores(" in src + + def test_restart_places_physical_cores_via_the_sibling_aware_helper(self): + """Not a plain sort -- distrib/poller/alceml's saved index sets are + handed to reassign_l_cores_for_restart so it can prefer keeping + each role's own cores real hyperthread siblings.""" + src = inspect.getsource(storage_node_ops._restart_storage_node_impl) + assert "utils.reassign_l_cores_for_restart(" in src + + def test_restart_skips_reassignment_on_an_unchanged_cpuset(self): + """A no-op restart (identical cpuset) must not churn which physical + core each role lands on for no operational reason.""" + src = inspect.getsource(storage_node_ops._restart_storage_node_impl) + assert "prior_physical_cores" in src + + def test_restart_warns_rather_than_silently_stales_on_a_core_count_mismatch(self): + """A genuine mismatch (host lost cores) must be visible, not + swallowed -- a stale l_cores left in place is exactly the kind of + thing that should show up in the logs, not just happen quietly.""" + src = inspect.getsource(storage_node_ops._restart_storage_node_impl) + assert "leaving l_cores as-is" in src + + +class TestGenerateLCores: + """The shared helper restart/add_node/sn configure all key off, so a + fix to one path can't drift from the others the way number_of_distribs + already had across generate_configs/regenerate_config/calculate_hp_only.""" + + def test_pairs_index_with_physical_core_in_order(self): + assert utils.generate_l_cores([5, 9, 14]) == "0@5,1@9,2@14" + + def test_empty_list_is_empty_string(self): + assert utils.generate_l_cores([]) == "" + + def test_used_by_every_l_cores_call_site(self): + """Regression guard for the drift class of bug this refactor keeps + running into: the inline '{i}@{core}' idiom must appear exactly + once in the whole module -- inside generate_l_cores itself -- not + reintroduced ad hoc at some other call site.""" + src = inspect.getsource(utils) + assert src.count('{i}@{core}"') == 1, ( + "the l-cores pairing idiom should live in exactly one place " + "(generate_l_cores); found it duplicated again") From 28af35be93c1752988562f6f394393855f2f36c3 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 15:23:43 +0200 Subject: [PATCH 054/122] test: run the outage soak on a non-multipath 2+2 cluster, deployed in one command The 2026-08-25 multipath soak produced three data corruptions (two bad-magic verify failures and an off-by-one-block read, plus an EIO storm). Whether that is multipath-specific or a general regression is answerable only by running the same outage pattern on a single-path cluster, so the multipath soak now supports one. Reusing that soak rather than the mixed-churn one is deliberate: the outage timing is the part worth keeping. It runs one thread per node with its own mgmt connection -- established before the offset sleep so an SSH handshake cannot smear the planned offset -- and each node holds its own outage and issues its own restart at its own boundary, so a pair genuinely overlaps. Mixed churn applies both outages and then restarts serially, retrying against the CP's concurrent-restart guard, which is far slower and a different scenario. Two changes make it single-path capable; everything else already derives from --data-nics (path counts, listener counts) or is learned at baseline (client path counts), and the multipath metadata check is only a warning: --no-nic-phase phase 1 takes "one" data NIC down on every node at once. With one NIC per node that isolates the whole cluster instead of exercising path redundancy. --nic-phase-every 0 does NOT disable it -- it means "once, on iteration 1". the active_active / rr_min_io assertions are skipped when --data-nics names fewer than two NICs, where a bdev legitimately reports active_passive and asserting otherwise would make every bdev a "problem" so the heal gate could never converge. setup_perf_test1.py becomes one command instead of a deployment plus a hand-assembled second step, which is where runs get lost: * Phase 7 stages the soak, its launcher, collect_logs.py, the metadata this script writes and the private key onto mgmt -- the soak runs there and SSHes to the nodes and client, so it needs all of it. Missing files fail loudly rather than staging a half-set. * Phase 8 starts the soak, with placement dumps on. --no-soak stages only and prints the start command; --iterations / --runtime / --restart-timeout / --start-iteration pass through. * SPDK_IMAGE pin, because sn add-node otherwise takes the control-plane default, which drifts as main is rebuilt -- useless when the point of the run is to hold the build constant and compare against another run. * stdout/stderr reconfigured to UTF-8 at import. Streamed remote stderr is arbitrary UTF-8 (systemctl alone prints "Created symlink ... -> ..." with U+2192) and on Windows a redirected stdout defaults to cp1252, so that one byte killed a 6-node deployment at phase 2a with UnicodeEncodeError. Log formatting must never abort a deployment. start_soak_base.sh is written with LF endings on purpose: a CRLF shell script dies on the node with a bare-CR "set" option error, which cost a run earlier in this campaign. Untested end to end: the deployment this was written against died before phase 7, so phases 7-8 have not yet run against a live mgmt. Co-Authored-By: Claude Opus 5 (1M context) --- .../aws_dual_node_outage_soak_multipath.py | 23 +++- scripts/setup_perf_test1.py | 104 ++++++++++++++++++ scripts/start_soak_base.sh | 37 +++++++ 3 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 scripts/start_soak_base.sh diff --git a/scripts/aws_dual_node_outage_soak_multipath.py b/scripts/aws_dual_node_outage_soak_multipath.py index 7939d708cc..6b816bc7f7 100755 --- a/scripts/aws_dual_node_outage_soak_multipath.py +++ b/scripts/aws_dual_node_outage_soak_multipath.py @@ -218,6 +218,14 @@ def parse_args(): help="Seconds the NIC stays down on all nodes (default 30).") nic.add_argument("--nic-phase-settle", type=int, default=30, help="Seconds to wait after NIC restore before verifying (default 30).") + nic.add_argument("--no-nic-phase", action="store_true", + help="Disable phase 1 (the all-nodes single-NIC outage) " + "entirely. REQUIRED on a single-data-NIC, non-multipath " + "cluster: with one data NIC per node, phase 1 takes that " + "NIC down on every node at once and isolates the whole " + "cluster instead of exercising path redundancy. Note that " + "--nic-phase-every 0 does NOT disable phase 1 -- it means " + "'once, on iteration 1'.") nic.add_argument("--nic-phase-every", type=int, default=1, help="Run the NIC phase every N iterations. 0 = once, before the " "first pair only (default 1).") @@ -1621,6 +1629,12 @@ def _rpc_json(subcmd, what): # node passive. active_passive means one NIC carries all hub IO. to_check = [(b, "hublvol") for b in hublvol_bdevs] to_check += [(b, "remote") for b in remote_bdevs[:max(0, self.args.policy_sample)]] + # Only meaningful with more than one path: on a single-data-NIC + # (non-multipath) cluster a bdev legitimately reports active_passive, + # and asserting active_active there would make every bdev a "problem", + # so the heal gate could never converge. + if len(self.args.data_nics) < 2: + to_check = [] for bdev_name, kind in to_check: try: bdevs = _rpc_json(f"bdev_get_bdevs -b {shlex.quote(bdev_name)}", @@ -2171,8 +2185,13 @@ def run(self): + ", ".join(f"{n['uuid'][:12]}:{n['status']}" for n in current)) uuids = [n["uuid"] for n in current] - nic_due = (iteration == 1 if args.nic_phase_every == 0 - else (iteration - 1) % args.nic_phase_every == 0) + # --nic-phase-every 0 means "once, on iteration 1"; only + # --no-nic-phase disables phase 1 outright, which is required on a + # single-data-NIC cluster where taking "one" NIC down on every node + # isolates the whole cluster instead of testing path redundancy. + nic_due = ((not args.no_nic_phase) + and (iteration == 1 if args.nic_phase_every == 0 + else (iteration - 1) % args.nic_phase_every == 0)) if nic_due: nic = args.data_nics[(iteration - 1) % len(args.data_nics)] if self.run_nic_phase(iteration, nic, uuids): diff --git a/scripts/setup_perf_test1.py b/scripts/setup_perf_test1.py index 4f9b8e3492..0bf87a2935 100644 --- a/scripts/setup_perf_test1.py +++ b/scripts/setup_perf_test1.py @@ -5,6 +5,18 @@ import select import time from concurrent.futures import ThreadPoolExecutor +import sys + +# Streamed remote stderr is arbitrary UTF-8 -- systemctl alone prints +# "Created symlink ... -> ..." with U+2192. On Windows a redirected stdout +# defaults to cp1252, so the first such byte killed a 6-node deployment at +# phase 2a with UnicodeEncodeError (2026-08-26). Never let log formatting +# abort a deployment. +for _stream in (sys.stdout, sys.stderr): + try: + _stream.reconfigure(encoding="utf-8", errors="replace") + except Exception: + pass import boto3 import paramiko @@ -26,6 +38,12 @@ SUBNET_ID = "subnet-0593459d6b931ee4c" STORAGE_SG_ID = "sg-02e89a1372e9f39e9" SN_TYPE = "i3en.2xlarge" +#: Pin the SPDK/ultra image so a run is reproducible and comparable against +#: another run. Without it, sn add-node takes the control-plane default, which +#: drifts as main is rebuilt -- useless when the whole point of a run is to +#: hold the build constant. Empty string = control-plane default. +SPDK_IMAGE = "public.ecr.aws/simply-block/ultra:main-d91ff03a-amd64" + SN_COUNT = 6 MGMT_TYPE = "m6i.2xlarge" # --- Selectable Client Specification --- @@ -372,6 +390,63 @@ def close(self): +#: Soak wiring. The deployment is only useful if the soak actually starts, and +#: getting the mgmt node ready by hand is where runs get lost: the soak runs ON +#: mgmt (--run-on-mgmt) and SSHes to the nodes and client, so it needs the key, +#: the metadata this script writes, and its own source there. +SOAK_SCRIPT = "aws_dual_node_outage_soak_multipath.py" +SOAK_LAUNCHER = "start_soak_base.sh" +SOAK_EXTRA_FILES = ("collect_logs.py",) +METADATA_FILE = "cluster_metadata_base.json" + + +def stage_soak(mgmt_ip): + """Put the soak, its launcher, the ssh key and the metadata on mgmt.""" + here = os.path.dirname(os.path.abspath(__file__)) + payload = [os.path.join(here, f) for f in + (SOAK_SCRIPT, SOAK_LAUNCHER) + SOAK_EXTRA_FILES] + payload.append(os.path.abspath(METADATA_FILE)) + missing = [f for f in payload if not os.path.isfile(f)] + if missing: + raise RuntimeError(f"cannot stage soak, missing files: {missing}") + + print(f"\n--- Phase 7: Stage soak on mgmt {mgmt_ip} ---") + ssh_exec(mgmt_ip, ["mkdir -p ~/.ssh"], check=True) + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(mgmt_ip, username="ec2-user", key_filename=KEY_PATH, + allow_agent=False, look_for_keys=False) + try: + sftp = ssh.open_sftp() + try: + for src in payload: + dst = "/home/ec2-user/" + os.path.basename(src) + sftp.put(src, dst) + print(f" staged {os.path.basename(src)}") + # The soak reaches the storage nodes and the client from mgmt, so + # the private key has to travel with it. + sftp.put(KEY_PATH, "/home/ec2-user/.ssh/mtes01.pem") + print(" staged ssh key") + finally: + sftp.close() + finally: + ssh.close() + # The launcher is written with LF endings on purpose; a CRLF shell script + # fails on the node with a bare-CR "set" option error. + ssh_exec(mgmt_ip, [ + "chmod 600 ~/.ssh/mtes01.pem", + f"chmod +x ~/{SOAK_LAUNCHER}", + f"sed -i 's/\r$//' ~/{SOAK_LAUNCHER}", + ], check=True) + + +def start_soak(mgmt_ip, env_prefix=""): + print(f"\n--- Phase 8: Start soak on mgmt {mgmt_ip} ---") + out = ssh_exec(mgmt_ip, [f"{env_prefix}bash ~/{SOAK_LAUNCHER}"], + get_output=True, check=True)[0] + print(out) + return out + def main(): @@ -487,6 +562,7 @@ def add_one_node(priv_ip): try: ssh_exec(mgmt_ip, [ f"sudo /usr/local/bin/sbctl -d sn add-node {cluster_uuid} {priv_ip}:5000 {IFACE} --ha-jm-count 4" + + (f" --spdk-image {SPDK_IMAGE}" if SPDK_IMAGE else "") ], check=True) return except RuntimeError: @@ -592,6 +668,34 @@ def add_one_node(priv_ip): print("\n--- Setup Complete ---") print(f"Cluster {cluster_uuid} is active. Metadata saved.") + # A deployment that stops here needs a second, hand-assembled step before + # it produces any data, and that is where runs get lost. Stage the soak and + # start it, unless explicitly told not to. + if "--no-soak" in sys.argv: + stage_soak(mgmt_ip) + print("--no-soak: files staged, soak NOT started. Start it with:") + print(f" ssh -i {KEY_PATH} ec2-user@{mgmt_ip} " + f"\"PLACEMENT_DUMPS=1 bash ~/{SOAK_LAUNCHER}\"") + return + + stage_soak(mgmt_ip) + env = "PLACEMENT_DUMPS=1 " + for flag, var in (("--iterations", "ITERATIONS"), + ("--runtime", "RUNTIME"), + ("--restart-timeout", "RESTART_TIMEOUT"), + ("--start-iteration", "START_ITERATION")): + if flag in sys.argv: + env += f"{var}={sys.argv[sys.argv.index(flag) + 1]} " + start_soak(mgmt_ip, env_prefix=env) + print("") + print("Soak running on mgmt. Follow it with:") + print(f" ssh -i {KEY_PATH} ec2-user@{mgmt_ip} " + "\"tail -f ~/soak_base_$(cat ~/soak_ts).out\"") + print("Check verdicts / faults with:") + print(f" ssh -i {KEY_PATH} ec2-user@{mgmt_ip} " + "\"grep -E '####|PASS|FAIL|ERROR|verify:' " + "~/soak_base_$(cat ~/soak_ts).out | tail -20\"") + diff --git a/scripts/start_soak_base.sh b/scripts/start_soak_base.sh new file mode 100644 index 0000000000..c7bc354059 --- /dev/null +++ b/scripts/start_soak_base.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Launch the outage soak on a NON-multipath (single data NIC) 2+2 cluster. +# +# Same script as the multipath soak -- we deliberately reuse it rather than the +# mixed-churn soak, because its outage timing is the part we want: one thread +# per node with its own mgmt connection, each node holding its own outage and +# issuing its own restart at its own boundary, so the pair genuinely overlaps. +# The mixed-churn soak applies both outages first and then restarts serially, +# retrying against the CP's concurrent-restart guard, which is far slower. +# +# Non-multipath adaptations (both needed, see the soak's own --help): +# --data-nics eth0 the base deploy adds nodes on eth0, so there is ONE path; +# path-count, listener-count and client-path expectations +# all derive from this, and the active_active policy +# assertions are skipped when it names fewer than 2 NICs. +# --no-nic-phase phase 1 takes "one" data NIC down on every node at once. +# With a single NIC per node that isolates the whole +# cluster instead of testing path redundancy. Note +# --nic-phase-every 0 does NOT disable it (it means +# "once, on iteration 1"). +# +# Env overrides: ITERATIONS, RUNTIME, START_ITERATION, RESTART_TIMEOUT, +# PLACEMENT_DUMPS=1, DATA_NIC. +set -u +cd "$HOME" +TS=$(date +%Y%m%d_%H%M%S) +echo "$TS" > "$HOME/soak_ts" +LOG="$HOME/soak_base_${TS}.log" +OUT="$HOME/soak_base_${TS}.out" +setsid nohup python3 "$HOME/aws_dual_node_outage_soak_multipath.py" --run-on-mgmt --metadata "$HOME/cluster_metadata_base.json" --ssh-key "$HOME/.ssh/mtes01.pem" --data-nics "${DATA_NIC:-eth0}" --no-nic-phase --iterations "${ITERATIONS:-75}" --start-iteration "${START_ITERATION:-1}" ${PLACEMENT_DUMPS:+--placement-dumps} ${RESTART_TIMEOUT:+--restart-timeout $RESTART_TIMEOUT} --runtime "${RUNTIME:-52000}" --log-file "$LOG" > "$OUT" 2>&1 < /dev/null & +PID=$! +echo "$PID" > "$HOME/soak_pid" +sleep 3 +echo "launched pid=$PID ts=$TS" +echo "log=$LOG" +echo "out=$OUT" +ps -p "$PID" -o pid=,etime=,cmd= | cut -c1-110 From 064f61a7fc4ffa94ec70aed6d0e9b3d7a4677db1 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 15:31:06 +0200 Subject: [PATCH 055/122] fix(scripts): cap --max-subsys at the product ceiling so cluster create succeeds Four deploy scripts passed --max-subsys 100 against a product ceiling of 75 (constants.MAX_SUBSYSTEMS_PER_NODE, set in c404ff871). That was accepted until 8bcedce79 added validate_spdk_sizing() to the cluster-create path, which now rejects it: ValueError: max_subsys must be between 1 and 75 (0 = product default) The scripts have been wrong since the ceiling existed; before the validation they were silently clamped at placement time, which is exactly the failure mode the ceiling was introduced to stop -- a node reserving huge pages for subsystems it could never serve while operators believed a limit that did not hold. So the validation is right and the callers were wrong; capping them at 75 keeps the original "as many as allowed" intent. Fixes a 2026-08-26 2+2 deployment that died at phase 2a on cluster create. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/setup_lab_perf_test1.py | 2 +- scripts/setup_perf_test1.py | 4 ++-- scripts/setup_perf_test_3node.py | 4 ++-- scripts/setup_perf_test_failure_domain.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/setup_lab_perf_test1.py b/scripts/setup_lab_perf_test1.py index 305d836fd9..1fd47806af 100644 --- a/scripts/setup_lab_perf_test1.py +++ b/scripts/setup_lab_perf_test1.py @@ -42,7 +42,7 @@ IFACE = "eth0" DATA_IFACE = "eth1" BRANCH = "inline-checksum-validation" -MAX_LVOL = "100" +MAX_LVOL = "75" # capped by constants.MAX_SUBSYSTEMS_PER_NODE (75): above it, values were silently clamped at placement time, so a node reserved huge pages for subsystems it could never serve. cluster create rejects it since 8bcedce79. # Same volume plan layout as the AWS variant; consumed by downstream perf tooling. VOLUME_PLAN = [ diff --git a/scripts/setup_perf_test1.py b/scripts/setup_perf_test1.py index 0bf87a2935..ec527b526c 100644 --- a/scripts/setup_perf_test1.py +++ b/scripts/setup_perf_test1.py @@ -32,7 +32,7 @@ AZ = "us-east-1a" SG_NAME = "default" BRANCH = "main" -MAX_LVOL = "100" +MAX_LVOL = "75" # capped by constants.MAX_SUBSYSTEMS_PER_NODE (75): above it, values were silently clamped at placement time, so a node reserved huge pages for subsystems it could never serve. cluster create rejects it since 8bcedce79. # --- Manual Network Config --- # Replace this with your actual Subnet ID (e.g., "subnet-0593459d6b931ee4c") SUBNET_ID = "subnet-0593459d6b931ee4c" @@ -55,7 +55,7 @@ USER = "ec2-user" AZ = "us-east-1a" IFACE = "eth0" -MAX_LVOL = "100" +MAX_LVOL = "75" VOLUME_PLAN = [ {"idx": 0, "node_idx": 0, "qty": 5, "size": "100G", "client": "client1", "io_queues": 12}, diff --git a/scripts/setup_perf_test_3node.py b/scripts/setup_perf_test_3node.py index 20eddb4621..b80f1550ba 100644 --- a/scripts/setup_perf_test_3node.py +++ b/scripts/setup_perf_test_3node.py @@ -22,7 +22,7 @@ AZ = "us-east-1a" SG_NAME = "default" BRANCH = "main" -MAX_LVOL = "100" +MAX_LVOL = "75" # capped by constants.MAX_SUBSYSTEMS_PER_NODE (75): above it, values were silently clamped at placement time, so a node reserved huge pages for subsystems it could never serve. cluster create rejects it since 8bcedce79. # --- Manual Network Config --- SUBNET_ID = "subnet-0593459d6b931ee4c" STORAGE_SG_ID = "sg-02e89a1372e9f39e9" @@ -45,7 +45,7 @@ USER = "ec2-user" AZ = "us-east-1a" IFACE = "eth0" -MAX_LVOL = "100" +MAX_LVOL = "75" # --- Helper: Management Node with 30GB Root --- diff --git a/scripts/setup_perf_test_failure_domain.py b/scripts/setup_perf_test_failure_domain.py index 5827e270dd..136b73aa0e 100644 --- a/scripts/setup_perf_test_failure_domain.py +++ b/scripts/setup_perf_test_failure_domain.py @@ -22,7 +22,7 @@ # --failure-domain) lives on this branch. Installing from it also pulls the # matching SPDK ultra image pinned in simplyblock_core/env_var. BRANCH = "main" -MAX_LVOL = "100" +MAX_LVOL = "75" # capped by constants.MAX_SUBSYSTEMS_PER_NODE (75): above it, values were silently clamped at placement time, so a node reserved huge pages for subsystems it could never serve. cluster create rejects it since 8bcedce79. # --- Manual Network Config --- # Replace this with your actual Subnet ID (e.g., "subnet-0593459d6b931ee4c") SUBNET_ID = "subnet-0593459d6b931ee4c" @@ -82,7 +82,7 @@ USER = "ec2-user" AZ = "us-east-1a" IFACE = "eth0" -MAX_LVOL = "100" +MAX_LVOL = "75" VOLUME_PLAN = [ {"idx": 0, "node_idx": 0, "qty": 5, "size": "100G", "client": "client1", "io_queues": 12}, From 48c0e2f6ee57fab09928157c0e250789c919823c Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 26 Aug 2026 14:36:23 +0200 Subject: [PATCH 056/122] fix: resolve API v1 metric leak The global gauges persisted data for deleted entities, maintaining them perpetually. Gauges are the meant for long-term indicators, this use is unidiomatic. Reconstructing them on each call is cheap and fixes the immediate problem though. --- simplyblock_web/api/v1/metrics.py | 79 +++++++++++++++---------------- 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/simplyblock_web/api/v1/metrics.py b/simplyblock_web/api/v1/metrics.py index 8a9f525b78..9017ed9c32 100644 --- a/simplyblock_web/api/v1/metrics.py +++ b/simplyblock_web/api/v1/metrics.py @@ -50,65 +50,60 @@ "write_latency_ticks", ] -ng: dict[str, Gauge] = {} -cg: dict[str, Gauge] = {} -dg: dict[str, Gauge] = {} -lg: dict[str, Gauge] = {} -pg: dict[str, Gauge] = {} - def get_device_metrics(): - global dg - if not dg: - labels = ['cluster', "cluster_name", "snode", "device"] - for k in io_stats_keys + ["status_code", "health_check"]: - dg["device_" + k] = Gauge("device_" + k, "device_" + k, labelnames=labels, registry=registry) - return dg + labels = ['cluster', "cluster_name", "snode", "device"] + return { + "device_" + k: Gauge("device_" + k, "device_" + k, labelnames=labels, registry=registry) + for k in + io_stats_keys + ["status_code", "health_check"] + } + def get_snode_metrics(): - global ng - if not ng: - labels = ['cluster', "cluster_name", "snode", "hostname"] - for k in io_stats_keys + ["status_code", "health_check"]: - ng["snode_" + k] = Gauge("snode_" + k, "snode_" + k, labelnames=labels, registry=registry) - - # Additional SPDK-specific metrics - ng["snode_cpu_busy_percentage"] = Gauge( + labels = ['cluster', "cluster_name", "snode", "hostname"] + return { + **{ + "snode_" + k: Gauge("snode_" + k, "snode_" + k, labelnames=labels, registry=registry) + for k in + io_stats_keys + ["status_code", "health_check"] + }, + "snode_cpu_busy_percentage": Gauge( "snode_cpu_busy_percentage", "Per-thread CPU Busy %", labelnames=['cluster', "cluster_name", 'snode', 'hostname', 'thread_name'], registry=registry - ) - ng["snode_cpu_core_utilization"] = Gauge( + ), + "snode_cpu_core_utilization": Gauge( "snode_cpu_core_utilization", "Per-core CPU Utilization %", labelnames=['cluster', "cluster_name", 'snode', 'hostname', 'core_id', 'thread_names'], registry=registry - ) - return ng + ), + } + def get_cluster_metrics(): - global cg - if not cg: - labels = ['cluster', "cluster_name"] - for k in io_stats_keys + ["status_code", "prov_cap_crit", "cap_crit"]: - cg["cluster_" + k] = Gauge("cluster_" + k, "cluster_" + k, labelnames=labels, registry=registry) - return cg + labels = ['cluster', "cluster_name"] + return { + "cluster_" + k: Gauge("cluster_" + k, "cluster_" + k, labelnames=labels, registry=registry) + for k in io_stats_keys + ["status_code", "prov_cap_crit", "cap_crit"] + } + def get_lvol_metrics(): - global lg - if not lg: - labels = ['cluster', "cluster_name", "pool", "lvol", "lvol_name", "pvc_name"] - for k in io_stats_keys + ["status_code", "health_check"]: - lg["lvol_" + k] = Gauge("lvol_" + k, "lvol_" + k, labelnames=labels, registry=registry) - return lg + labels = ['cluster', "cluster_name", "pool", "lvol", "lvol_name", "pvc_name"] + return { + "lvol_" + k: Gauge("lvol_" + k, "lvol_" + k, labelnames=labels, registry=registry) + for k in io_stats_keys + ["status_code", "health_check"] + } + def get_pool_metrics(): - global pg - if not pg: - labels = ['cluster', "cluster_name", "pool", "name"] - for k in io_stats_keys + ["status_code"]: - pg["pool_" + k] = Gauge("pool_" + k, "pool_" + k, labelnames=labels, registry=registry) - return pg + labels = ['cluster', "cluster_name", "pool", "name"] + return { + "pool_" + k: Gauge("pool_" + k, "pool_" + k, labelnames=labels, registry=registry) + for k in io_stats_keys + ["status_code"] + } @bp.route('/cluster/metrics', methods=['GET']) From c6682023ded30afe2448868decb2e38802b9d4db Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 15:43:13 +0200 Subject: [PATCH 057/122] test: repl_soak.py -- one-liner to deploy a lab from a branch and run cases python scripts/repl_soak.py --cases case6,case7 --branch main python scripts/repl_soak.py --cases case9 --env CHAOS_EVENTS=100 --teardown on-pass python scripts/repl_soak.py --cases case3 --reuse-lab --branch selects the sbcli code actually under test: it is pip-installed on every lab node, mounted over the running control-plane services by the hotfix, and staged to the management node as the driver -- so a case runs that branch's code regardless of what the pre-built CP image lags behind. --spdk-image / --cp-image override the images independently, --env passes driver knobs (CHAOS_EVENTS, CASE11_RUNTIME_MIN, ...), --reuse-lab skips deployment, --teardown on-pass keeps a failed lab for diagnosis. Exit code is 0 only when every case passed. Each branch gets its own clone under soak-labs/ so a run never touches a developer's working tree, and the deployer now reads SBCLI_BRANCH / SPDK_IMAGE / SIMPLYBLOCK_DOCKER_IMAGE from the environment. Co-Authored-By: Claude Fable 5 --- scripts/repl_soak.py | 195 +++++++++++++++++++++++++++ scripts/setup_repl_test_2clusters.py | 10 +- 2 files changed, 202 insertions(+), 3 deletions(-) create mode 100644 scripts/repl_soak.py diff --git a/scripts/repl_soak.py b/scripts/repl_soak.py new file mode 100644 index 0000000000..36554e1517 --- /dev/null +++ b/scripts/repl_soak.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""One-liner for the async-replication soak: deploy a lab from a branch, run cases, report. + + python scripts/repl_soak.py --cases case6,case7 --branch main + python scripts/repl_soak.py --cases case9 --env CHAOS_EVENTS=100 --teardown on-pass + python scripts/repl_soak.py --cases case10,case11 --branch main --spdk-image public.ecr.aws/simply-block/ultra:main-latest-amd64 + python scripts/repl_soak.py --cases case3 --reuse-lab # existing lab, no redeploy + +What --branch means: the sbcli checkout that is (1) pip-installed on every +lab node by the deployer, (2) the source of the control-plane hotfix mounted +over the running services, and (3) the copy of the test driver that is staged +to the management node. So a case runs against exactly that branch's code, +whatever the pre-built CP image lags behind. The SPDK image is independent +(--spdk-image, default: the digest pinned in setup_repl_test_2clusters.py); +--cp-image overrides the control-plane image (SIMPLYBLOCK_DOCKER_IMAGE). + +Steps: clone/refresh the branch -> deploy (unless --reuse-lab) -> hotfix +(unless --no-hotfix) -> stage + run the cases -> wait for the driver -> +print the SUMMARY -> optional teardown. Exit code 0 only when every case +passed. +""" +import argparse +import json +import os +import re +import shlex +import subprocess +import sys +import time +from pathlib import Path + +KEY = os.environ.get("REPL_KEY", "C:/Users/Michael/.ssh/mtes01.pem") +REPO_URL = "github.com/simplyblock/sbcli.git" +WORK_ROOT = Path(os.environ.get("REPL_SOAK_ROOT", + Path(__file__).resolve().parent.parent.parent / "soak-labs")) +SSH_OPTS = ["-o", "StrictHostKeyChecking=no", "-o", "LogLevel=ERROR", + "-o", "ConnectTimeout=30", "-i", KEY] + + +def log(msg): + print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) + + +def sh(cmd, cwd=None, env=None, check=True, capture=False): + log("$ " + (cmd if isinstance(cmd, str) else " ".join(shlex.quote(c) for c in cmd))) + r = subprocess.run(cmd, cwd=cwd, env=env, shell=isinstance(cmd, str), + text=True, capture_output=capture) + if check and r.returncode != 0: + if capture: + print(r.stdout[-2000:], r.stderr[-2000:]) + raise SystemExit(f"step failed (rc={r.returncode}): {cmd}") + return r + + +def gh_token(): + r = subprocess.run(["C:/Users/Michael/.local/bin/gh.exe", "auth", "token"], + text=True, capture_output=True, check=True) + return r.stdout.strip() + + +def ssh(mgmt, remote_cmd, check=True): + r = subprocess.run(["ssh", *SSH_OPTS, f"ec2-user@{mgmt}", remote_cmd], + text=True, capture_output=True) + if check and r.returncode != 0: + raise SystemExit(f"ssh failed on {mgmt}: {r.stderr.strip()[-500:]}") + return r.stdout + + +def checkout(branch): + """A dedicated clone per branch under WORK_ROOT (never the developer's tree).""" + WORK_ROOT.mkdir(parents=True, exist_ok=True) + dest = WORK_ROOT / f"sbcli-{re.sub(r'[^A-Za-z0-9._-]', '_', branch)}" + url = f"https://x-access-token:{gh_token()}@{REPO_URL}" + if not (dest / ".git").exists(): + sh(["git", "clone", "-q", "--branch", branch, url, str(dest)]) + else: + sh(["git", "-C", str(dest), "fetch", "-q", url, branch]) + sh(["git", "-C", str(dest), "checkout", "-q", "-B", branch, "FETCH_HEAD"]) + head = sh(["git", "-C", str(dest), "log", "--oneline", "-1"], capture=True).stdout.strip() + log(f"branch {branch} @ {head}") + return dest + + +def load_meta(scripts_dir): + p = scripts_dir / "cluster_metadata_repl.json" + if not p.exists(): + raise SystemExit(f"no lab metadata at {p} (deploy first, or drop --reuse-lab)") + return json.loads(p.read_text()) + + +def deploy(scripts_dir, branch, spdk_image, cp_image): + env = dict(os.environ, SBCLI_BRANCH=branch) + if spdk_image: + env["SPDK_IMAGE"] = spdk_image + if cp_image: + env["SIMPLYBLOCK_DOCKER_IMAGE"] = cp_image + sh([sys.executable, "-u", "setup_repl_test_2clusters.py", "-d"], cwd=scripts_dir, env=env) + + +def hotfix(scripts_dir): + sh([sys.executable, "hotfix_repl_lab.py"], cwd=scripts_dir) + + +def run_cases(scripts_dir, cases, env_kv): + sh([sys.executable, "stage_and_run_repl_cases.py", cases, *env_kv], cwd=scripts_dir) + + +def wait_for_driver(mgmt, poll=120): + """Follow the remote driver to === DONE ===, then return (summary, passed).""" + logfile = "" + for _ in range(20): + logfile = ssh(mgmt, "cat ~/repl_log 2>/dev/null", check=False).strip() + if logfile: + break + time.sleep(15) + if not logfile: + raise SystemExit("driver never registered a log file on the management node") + log(f"following {logfile}") + while True: + out = ssh(mgmt, + f"if grep -q '=== DONE ===' {logfile}; then echo FINISHED; " + f"grep -A12 '=== SUMMARY ===' {logfile}; " + f"elif ! pgrep -f '[t]est_async_replication.py' >/dev/null; then echo DIED; " + f"tail -15 {logfile}; else echo RUNNING; " + f"grep -E '^==========' {logfile} | tail -1; fi", check=False) + if out.startswith("FINISHED"): + summary = out.split("\n", 1)[1] + passed = ("FAIL" not in summary) and ("PASS" in summary) + return summary, passed + if out.startswith("DIED"): + return out, False + log("running: " + (out.split("\n", 1)[1].strip() if "\n" in out else "...")) + time.sleep(poll) + + +def teardown(meta): + ips = [meta["mgmt"]["private_ip"]] + [c["private_ip"] for c in meta.get("clients", [])] + for cl in meta["clusters"].values(): + ips += cl["storage_private_ips"] + r = sh(["aws", "ec2", "describe-instances", "--filters", + f"Name=private-ip-address,Values={','.join(ips)}", + "Name=instance-state-name,Values=running,stopped,pending", + "--query", "Reservations[].Instances[].InstanceId", "--output", "text"], + capture=True) + ids = r.stdout.split() + if ids: + sh(["aws", "ec2", "terminate-instances", "--instance-ids", *ids], capture=True) + log(f"terminated {len(ids)} lab instances") + else: + log("no lab instances found to terminate") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--cases", required=True, + help="comma list or group name for test_async_replication.py (e.g. case6,case7 | all9 | features)") + ap.add_argument("--branch", default="main", help="sbcli branch to deploy/hotfix/stage (default main)") + ap.add_argument("--spdk-image", default="", help="ultra image ref; default = digest pinned in the deployer") + ap.add_argument("--cp-image", default="", help="control-plane docker image (SIMPLYBLOCK_DOCKER_IMAGE)") + ap.add_argument("--env", action="append", default=[], metavar="KEY=VAL", + help="driver knobs, e.g. CHAOS_EVENTS=100, CASE11_RUNTIME_MIN=30 (repeatable)") + ap.add_argument("--reuse-lab", action="store_true", help="skip deploy; use the branch clone's existing metadata") + ap.add_argument("--no-hotfix", action="store_true", help="skip mounting the branch's python over the CP services") + ap.add_argument("--teardown", choices=["never", "on-pass", "always"], default="never") + args = ap.parse_args() + for kv in args.env: + if "=" not in kv: + ap.error(f"--env expects KEY=VAL, got {kv!r}") + + clone = checkout(args.branch) + scripts_dir = clone / "scripts" + + if not args.reuse_lab: + deploy(scripts_dir, args.branch, args.spdk_image, args.cp_image) + meta = load_meta(scripts_dir) + mgmt = meta["mgmt"]["public_ip"] + log(f"lab mgmt {mgmt}: {', '.join(f'{k}={v['cluster_uuid'][:8]}({v['nodes']}n)' for k, v in meta['clusters'].items())}") + + if not args.no_hotfix: + hotfix(scripts_dir) + + run_cases(scripts_dir, args.cases, args.env) + summary, passed = wait_for_driver(mgmt) + print("\n" + summary.strip() + "\n") + log("RESULT: " + ("ALL PASSED" if passed else "FAILED")) + + if args.teardown == "always" or (args.teardown == "on-pass" and passed): + teardown(meta) + elif not passed: + log(f"lab kept for diagnosis: ssh -i {KEY} ec2-user@{mgmt} (log: $(cat ~/repl_log))") + raise SystemExit(0 if passed else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/setup_repl_test_2clusters.py b/scripts/setup_repl_test_2clusters.py index 1f693418ab..edb6eee94e 100644 --- a/scripts/setup_repl_test_2clusters.py +++ b/scripts/setup_repl_test_2clusters.py @@ -40,7 +40,10 @@ # tasks_runner_replication_final, replication_final_step, cluster # add-replication), so main is what we test; the default # SIMPLY_BLOCK_DOCKER_IMAGE (simplyblock/simplyblock:main) matches it. -BRANCH = "main" +# Overridable for the repl_soak.py one-liner: SBCLI_BRANCH selects the sbcli +# checkout installed on every node (and the hotfix source), SPDK_IMAGE the +# pinned ultra image, SIMPLYBLOCK_DOCKER_IMAGE the control-plane image. +BRANCH = os.environ.get("SBCLI_BRANCH", "main") SN_TYPE = "i3en.2xlarge" MGMT_TYPE = "m6i.2xlarge" @@ -115,8 +118,9 @@ # the first build carrying the promotion-window ANA-transition fix # (spdk R26.3 554c80f11), verified built FROM spdk-core:R26.3-latest # whose manifest was created 18:41:57, before this ultra build started. -SPDK_IMAGE = ("public.ecr.aws/simply-block/ultra@" - "sha256:0d631068e3add220d9198f212cf78d1e732ad9f0f92061dbebc413a9a6550e3b") +SPDK_IMAGE = os.environ.get( + "SPDK_IMAGE", + "public.ecr.aws/simply-block/ultra@sha256:0d631068e3add220d9198f212cf78d1e732ad9f0f92061dbebc413a9a6550e3b") SN_COUNT = sum(c["nodes"] for c in CLUSTERS) SBCTL = "sudo /usr/local/bin/sbctl" From a30308056ae096dddc94519f71066a9d7efeeb5f Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 15:48:04 +0200 Subject: [PATCH 058/122] fix(scripts): pass --dev on add-node, or --spdk-image is rejected The image pin added in 28af35be9 made every add-node fail with sbctl: error: unrecognized arguments: --spdk-image public.ecr.aws/... --spdk-image is declared for add-node with `private: true` in cli-reference.yaml, and private arguments are only registered in developer mode. `-d` is debug output; `--dev` is what sets developer_mode. The base script already passed --dev on cluster create but not on add-node, which is why this only surfaced once a pin was added. setup_perf_test_multipath.py has the same call with --dev and a comment saying exactly this; I appended the flag without carrying the mode over. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/setup_perf_test1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup_perf_test1.py b/scripts/setup_perf_test1.py index ec527b526c..7069041f6e 100644 --- a/scripts/setup_perf_test1.py +++ b/scripts/setup_perf_test1.py @@ -561,7 +561,7 @@ def add_one_node(priv_ip): for attempt in range(5): try: ssh_exec(mgmt_ip, [ - f"sudo /usr/local/bin/sbctl -d sn add-node {cluster_uuid} {priv_ip}:5000 {IFACE} --ha-jm-count 4" + f"sudo /usr/local/bin/sbctl -d --dev sn add-node {cluster_uuid} {priv_ip}:5000 {IFACE} --ha-jm-count 4" + (f" --spdk-image {SPDK_IMAGE}" if SPDK_IMAGE else "") ], check=True) return From 08a2e4a7e76438c7d3a5c9246b8bbcf4cc103c95 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 15:59:51 +0200 Subject: [PATCH 059/122] test: repl_soak must probe the lab before using stale metadata scripts/cluster_metadata_repl.json is TRACKED, so every fresh branch clone arrives carrying whichever lab was alive when it was last committed. --reuse-lab therefore looked healthy and drove straight into the hotfix against a long-dead management node, failing deep inside with an ssh timeout. Probe the management node first and say exactly what is wrong, naming the stale-metadata case. Also drop a nested-quote f-string (3.12+ only) from the lab-shape line. Co-Authored-By: Claude Fable 5 --- scripts/repl_soak.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/scripts/repl_soak.py b/scripts/repl_soak.py index 36554e1517..7e2217430e 100644 --- a/scripts/repl_soak.py +++ b/scripts/repl_soak.py @@ -88,6 +88,27 @@ def load_meta(scripts_dir): return json.loads(p.read_text()) +def lab_is_reachable(mgmt, timeout=20): + r = subprocess.run(["ssh", "-o", "StrictHostKeyChecking=no", "-o", "LogLevel=ERROR", + "-o", f"ConnectTimeout={timeout}", "-i", KEY, + f"ec2-user@{mgmt}", "true"], capture_output=True, text=True) + return r.returncode == 0 + + +def require_reachable(mgmt, reused): + """A stale metadata file is the normal case, not the exception: the repo + TRACKS scripts/cluster_metadata_repl.json, so every fresh clone arrives + carrying whichever lab was alive when it was last committed. Probe before + doing anything that would otherwise fail deep inside the hotfix.""" + if lab_is_reachable(mgmt): + return + hint = ("that metadata is the copy committed in the repo, pointing at a lab " + "that no longer exists -- drop --reuse-lab to deploy a fresh one" + if reused else + "the deploy reported success but the management node is unreachable") + raise SystemExit(f"management node {mgmt} is not reachable: {hint}") + + def deploy(scripts_dir, branch, spdk_image, cp_image): env = dict(os.environ, SBCLI_BRANCH=branch) if spdk_image: @@ -174,7 +195,10 @@ def main(): deploy(scripts_dir, args.branch, args.spdk_image, args.cp_image) meta = load_meta(scripts_dir) mgmt = meta["mgmt"]["public_ip"] - log(f"lab mgmt {mgmt}: {', '.join(f'{k}={v['cluster_uuid'][:8]}({v['nodes']}n)' for k, v in meta['clusters'].items())}") + shape = ", ".join("{}={}({}n)".format(k, v["cluster_uuid"][:8], v["nodes"]) + for k, v in meta["clusters"].items()) + log(f"lab mgmt {mgmt}: {shape}") + require_reachable(mgmt, args.reuse_lab) if not args.no_hotfix: hotfix(scripts_dir) From 38bc4567c231fc89cd7588bfb78134f8e572329f Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 16:31:07 +0200 Subject: [PATCH 060/122] test: default the soak to this branch and its own ultra image On replication-features the defaults should BE replication-features: SBCLI_BRANCH and repl_soak's --branch now default to it, and SPDK_IMAGE pins the ultra build made from this branch's spdk (replication-features-latest-amd64, ECR digest sha256:d929b4d7..., resolved via the ECR API -- the CI push log prints docker.io's digest and the two differ). So a bare 'python scripts/repl_soak.py --cases ...' on this branch deploys and tests THIS feature set end to end. Co-Authored-By: Claude Fable 5 --- scripts/repl_soak.py | 5 +++-- scripts/setup_repl_test_2clusters.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/repl_soak.py b/scripts/repl_soak.py index 7e2217430e..e17bd9f07b 100644 --- a/scripts/repl_soak.py +++ b/scripts/repl_soak.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """One-liner for the async-replication soak: deploy a lab from a branch, run cases, report. - python scripts/repl_soak.py --cases case6,case7 --branch main + python scripts/repl_soak.py --cases case7,case9 --env CHAOS_EVENTS=100 python scripts/repl_soak.py --cases case9 --env CHAOS_EVENTS=100 --teardown on-pass python scripts/repl_soak.py --cases case10,case11 --branch main --spdk-image public.ecr.aws/simply-block/ultra:main-latest-amd64 python scripts/repl_soak.py --cases case3 --reuse-lab # existing lab, no redeploy @@ -175,7 +175,8 @@ def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--cases", required=True, help="comma list or group name for test_async_replication.py (e.g. case6,case7 | all9 | features)") - ap.add_argument("--branch", default="main", help="sbcli branch to deploy/hotfix/stage (default main)") + ap.add_argument("--branch", default="replication-features", + help="sbcli branch to deploy/hotfix/stage (default replication-features)") ap.add_argument("--spdk-image", default="", help="ultra image ref; default = digest pinned in the deployer") ap.add_argument("--cp-image", default="", help="control-plane docker image (SIMPLYBLOCK_DOCKER_IMAGE)") ap.add_argument("--env", action="append", default=[], metavar="KEY=VAL", diff --git a/scripts/setup_repl_test_2clusters.py b/scripts/setup_repl_test_2clusters.py index edb6eee94e..5c8436cd96 100644 --- a/scripts/setup_repl_test_2clusters.py +++ b/scripts/setup_repl_test_2clusters.py @@ -43,7 +43,7 @@ # Overridable for the repl_soak.py one-liner: SBCLI_BRANCH selects the sbcli # checkout installed on every node (and the hotfix source), SPDK_IMAGE the # pinned ultra image, SIMPLYBLOCK_DOCKER_IMAGE the control-plane image. -BRANCH = os.environ.get("SBCLI_BRANCH", "main") +BRANCH = os.environ.get("SBCLI_BRANCH", "replication-features") SN_TYPE = "i3en.2xlarge" MGMT_TYPE = "m6i.2xlarge" @@ -120,7 +120,7 @@ # whose manifest was created 18:41:57, before this ultra build started. SPDK_IMAGE = os.environ.get( "SPDK_IMAGE", - "public.ecr.aws/simply-block/ultra@sha256:0d631068e3add220d9198f212cf78d1e732ad9f0f92061dbebc413a9a6550e3b") + "public.ecr.aws/simply-block/ultra@sha256:d929b4d7ececee0fa0e1ad5973f87fa4e4cf79f7079cdf454bfbb27e2df51cb6") SN_COUNT = sum(c["nodes"] for c in CLUSTERS) SBCTL = "sudo /usr/local/bin/sbctl" From 6e29c679b41218312cf0df282cec12abc5073cc2 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 26 Aug 2026 15:40:05 +0100 Subject: [PATCH 061/122] fix: preserve original source lvolID through failback cutover by swapping new clone UUID in FN_REPLICATION_FINAL --- .../tasks_runner_replication_final.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 761002a90d..caf2124bef 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -41,6 +41,7 @@ def _finalize(task, ok, err): if ok: replication_id = task.function_params.get("replication_id") final_state = task.function_params.get("final_state", LVolReplication.STATE_CUTOVER_DONE) + rep = None if replication_id: try: rep = db.get_lvol_replication_by_id(replication_id) @@ -48,6 +49,10 @@ def _finalize(task, ok, err): rep.write_to_db(db.kv_store) except Exception as e: logger.error(f"Failed to update replication state: {e}") + + failback_source_id = task.function_params.get("failback_source_lvol_id") + if failback_source_id and rep is not None: + _swap_failback_lvol_uuid(rep, failback_source_id) task.function_result = "cutover done" task.status = JobSchedule.STATUS_DONE task.function_params["end_time"] = int(time.time()) @@ -106,6 +111,56 @@ def _finalize(task, ok, err): return False +def _swap_failback_lvol_uuid(rep, failback_source_id): + """After a successful failback cutover, reassign the new clone's UUID to + the original source lvol's UUID. + + The operator stores the original source lvolID in the ReplicationSlot's + Spec.VolumeID. Preserving that UUID through the failback means the slot + stays valid with no update required. + + The old source lvol DB record is removed first (its NVMf subsystem was + already evicted by _evict_stale_namespace during _create_target_lvol_clone). + The stale clone record at the old UUID is cleared last so there is never a + window with two records under the same UUID. + """ + try: + new_lvol = db.get_lvol_by_id(rep.target_lvol.get_id()) + except KeyError: + logger.error( + "failback UUID swap: new clone %s not found in DB; skipping", + rep.target_lvol.get_id()) + return + + stale_uuid = new_lvol.get_id() + + # Remove the old source lvol record so its DB key is free. + try: + old_lvol = db.get_lvol_by_id(failback_source_id) + old_lvol.remove(db.kv_store) + except KeyError: + logger.warning( + "failback UUID swap: original source lvol %s already absent from DB", + failback_source_id) + + # Write the clone under the original source UUID. + new_lvol.uuid = failback_source_id + new_lvol.write_to_db(db.kv_store) + + # Clear the stale record at the old clone UUID. + stale = LVol() + stale.uuid = stale_uuid + stale.remove(db.kv_store) + + # Keep the relationship's target reference consistent. + rep.target_lvol.uuid = failback_source_id + rep.write_to_db(db.kv_store) + + logger.info( + "failback UUID swap complete: clone %s reassigned to original source UUID %s", + stale_uuid, failback_source_id) + + def task_runner(task: JobSchedule): params = task.function_params lvol_id = params.get("lvol_id") @@ -292,6 +347,21 @@ def _prepare_cutover(task, lvol, src_node, tgt_node): rep.target_ns_id = new_lvol.ns_id rep.write_to_db(db.kv_store) + # Detect failback: if the current replication source (lvol) was previously + # the TARGET in a completed relationship whose source cluster is now tgt_node, + # this is a failback cutover. Store the original source UUID so _finalize can + # reassign it to the new clone, keeping the operator's VolumeID valid. + for prior in db.get_lvol_replication_objects(): + if (prior.target_lvol and prior.target_lvol.get_id() == lvol.get_id() + and prior.source_cluster_id == tgt_node.cluster_id + and prior.source_lvol): + task.function_params["failback_source_lvol_id"] = prior.source_lvol.get_id() + logger.info( + "failback cutover detected: original source UUID %s will be " + "preserved on new clone %s after cutover", + prior.source_lvol.get_id(), new_lvol.get_id()) + break + task.function_params.update({ "tgt_lvol_composite": new_lvol.top_bdev, "tgt_map_id": tgt_map_id, From 8d2305dff72e7419ab9c20a35bc0216ca1b91ed7 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 17:11:44 +0200 Subject: [PATCH 062/122] test: cases 13-15 -- a single node dies DURING fail-over / fail-back / cutover Three phase-targeted chaos cases, each repeating CHAOS_PHASE_ROUNDS (20) times with the kill landing at a RANDOM instant inside the phase under test, so the kills sweep across the whole phase rather than one point: case 13 a target-side node dies while the fail-over runs case 14 a source-side node dies while the fail-back runs case 15 either side dies during the online cutover -- the final migration step, so these volumes carry a migration-mode policy and replication-commit on the SOURCE is the operation under attack The kill delay is randomized over the phase's own measured duration (round 1 times it), and the victim is picked from the side the phase actually builds on. Chaos is ALLOWED to make the operation fail; what is not allowed is losing data, wedging the cluster, or leaving a state a retry cannot escape. Each round therefore has to end in one of two verdicts -- completed despite the kill, or failed and then succeeded on retry once the cluster was healthy again -- and every round verifies the baseline md5 on the volumes that resulted, then restores the home side so the next round attacks from the same shape. Cluster recovery is the health-aware gate (nodes online AND healthy, clusters active), not just node status. Co-Authored-By: Claude Fable 5 --- scripts/test_async_replication.py | 268 ++++++++++++++++++++++++++++++ 1 file changed, 268 insertions(+) diff --git a/scripts/test_async_replication.py b/scripts/test_async_replication.py index 8bc1b20761..61709aff66 100644 --- a/scripts/test_async_replication.py +++ b/scripts/test_async_replication.py @@ -2612,6 +2612,270 @@ def test_case_12(meta): print("CASE 12 PASSED: CG snapshots complete per generation, retention " "correct, both fail-overs crash-consistent and generation-exact.") + + +# --------------------------------------------------------------------------- # +# Cases 13-15: a single node dies DURING fail-over / fail-back / cutover +# --------------------------------------------------------------------------- # +# One node is killed at a RANDOM instant inside the phase under test, repeated +# so the kills land at many different points of it. Chaos is allowed to make +# the operation fail -- what is NOT allowed is losing data, wedging the +# cluster, or leaving a state a retry cannot get out of. Each round therefore +# ends in one of two accepted verdicts (completed despite the kill / failed +# then succeeded on retry after recovery) and any third outcome fails the case. +CHAOS_PHASE_ROUNDS = int(os.environ.get("CHAOS_PHASE_ROUNDS", "20")) +CHAOS_PHASE_VOLS = int(os.environ.get("CHAOS_PHASE_VOLS", "3")) +#: cap for the randomized kill delay when a phase turns out to be slow +CHAOS_PHASE_MAX_DELAY = float(os.environ.get("CHAOS_PHASE_MAX_DELAY", "45")) + + +def _kill_after(delay_s, ip, key_path, sink): + """Kill SPDK on *ip* after *delay_s*, off the main thread.""" + import threading + + def _run(): + time.sleep(delay_s) + try: + kill_spdk(ip, key_path) + sink.append(("killed", ip, time.time())) + except Exception as exc: # noqa: BLE001 - recorded + sink.append(("kill-failed", ip, str(exc))) + + t = threading.Thread(target=_run, daemon=True) + t.start() + return t + + +def _await_cluster_healthy(mgmt_ip, key_path, timeout=NODE_STATE_TIMEOUT): + """Every node online AND healthy, every cluster active/degraded.""" + deadline = time.time() + timeout + last = {} + while time.time() < deadline: + last = _all_nodes_online(mgmt_ip, key_path) + if last.get("all_online"): + return True, last + time.sleep(20) + return False, last + + +def _phase_victims(meta, phase): + """Nodes worth killing for the phase under test, as (label, public_ip). + + Fail-over and fail-back BUILD the copy on the destination, so the + destination cluster is where a kill bites; the cutover freezes and flips + the source, so both sides matter there. + """ + _src_uuid, src, _tgt_uuid, tgt = _src_target(meta) + if phase == "failover": + pool = [("tgt", ip) for ip in tgt["storage_public_ips"]] + elif phase == "failback": + pool = [("src", ip) for ip in src["storage_public_ips"]] + else: + pool = ([("src", ip) for ip in src["storage_public_ips"]] + + [("tgt", ip) for ip in tgt["storage_public_ips"]]) + return pool + + +def _failover_all(mgmt_ip, key_path, lvols): + """Returns (ok, target_lvols, error). Never raises on a chaos failure.""" + out = [] + for lv in lvols: + fo = do_failover(mgmt_ip, key_path, lv) + if not isinstance(fo, dict) or not fo.get("connection_strings"): + return False, out, (f"{lv[:8]}: " + f"{(fo or {}).get('error', '')} " + f"{(fo or {}).get('log', '')}".strip()[:300]) + out.append(fo["lvol_id"]) + return True, out, "" + + +def _commit_all(mgmt_ip, key_path, lvols, timeout=None): + """Drive replication-commit for every volume and wait for the cutovers.""" + timeout = timeout or CUTOVER_WAIT_TIMEOUT + for lv in lvols: + run(mgmt_ip, key_path, "%s -d volume replication-commit %s" % (SBCTL, lv), + check=False) + deadline = time.time() + timeout + while time.time() < deadline: + states = replication_states(mgmt_ip, key_path, lvols) + done = sum(1 for x in states.values() if x in ("cutover_done", "failed_over")) + if done == len(lvols): + return True, "" + time.sleep(15) + return False, "only %d/%d cutovers completed in %ds" % (done, len(lvols), timeout) + + +def _failback_all(mgmt_ip, key_path, meta, tgt_lvols): + """Point replication home, fail back, commit, and return the new lvol ids.""" + src_uuid, src, tgt_uuid, _tgt = _src_target(meta) + set_cluster_replication(mgmt_ip, key_path, tgt_uuid, src_uuid, + pool_uuid_of(mgmt_ip, key_path, src["pool"])) + for t in tgt_lvols: + failback(mgmt_ip, key_path, t) + wait_replication_caught_up(mgmt_ip, key_path, tgt_lvols, timeout=3600) + ok, why = _commit_all(mgmt_ip, key_path, tgt_lvols) + if not ok: + return False, [], why + back = failed_over_targets(mgmt_ip, key_path, tgt_lvols) + new = [back[t] for t in tgt_lvols if t in back] + if len(new) != len(tgt_lvols): + return False, new, "fail-back returned %d/%d volumes" % (len(new), len(tgt_lvols)) + return True, new, "" + + +def _chaos_phase_case(meta, phase, title): + """Kill ONE node at a random instant inside *phase*, CHAOS_PHASE_ROUNDS times.""" + import random + print("\n========== %s ==========" % title) + key_path = meta["key_path"] + mgmt_ip = meta["mgmt"]["public_ip"] + client_ip = meta["clients"][0]["public_ip"] + src_uuid, src, tgt_uuid, tgt = _src_target(meta) + seed = int(os.environ.get("CHAOS_PHASE_SEED") or time.time()) + rng = random.Random(seed) + print(" seed: %d, rounds: %d, phase: %s" % (seed, CHAOS_PHASE_ROUNDS, phase)) + + prepare_mount_points(client_ip, key_path) + delete_test_volumes(mgmt_ip, key_path, _all_test_pools(meta)) + # The cutover under test in case 15 is the FINAL MIGRATION STEP -- that is + # a migration-mode policy committed on the SOURCE volumes while IO runs. + mode = "migration" if phase == "commit" else "failover" + lvols = create_volumes(mgmt_ip, key_path, src_uuid, src["pool"], tgt_uuid, + tgt["pool"], mode=mode, count=CHAOS_PHASE_VOLS) + mounts = connect_and_mount(client_ip, key_path, mgmt_ip, lvols, fmt=True) + baseline = write_baseline(client_ip, key_path, mounts) + baseline_ts = time.time() + wait_replication_caught_up(mgmt_ip, key_path, lvols) + wait_data_replicated(mgmt_ip, key_path, lvols, baseline_ts) + cleanup_client(client_ip, key_path, mounts) + + victims = _phase_victims(meta, phase) + phase_secs = 20.0 # replaced by the first round's measurement + verdicts, retries, kills = [], 0, [] + home = list(lvols) # volumes currently on the SOURCE side + #: md5 per volume, positionally aligned with `home` -- every hop below + #: preserves order, so the baseline follows a volume across fail-overs + #: without depending on dict iteration order. + base_list = [baseline[lv] for lv in lvols] + + for rnd in range(1, CHAOS_PHASE_ROUNDS + 1): + delay = round(rng.uniform(0.0, min(phase_secs * 1.2, CHAOS_PHASE_MAX_DELAY)), 1) + label, victim = rng.choice(victims) + print("--- round %d/%d: kill %s node %s at T+%.1fs of the %s phase ---" + % (rnd, CHAOS_PHASE_ROUNDS, label, victim, delay, phase)) + sink = [] + started = time.time() + + if phase == "failover": + _kill_after(delay, victim, key_path, sink) + ok, tgt_lvols, why = _failover_all(mgmt_ip, key_path, home) + elif phase == "failback": + ok, tgt_lvols, why = _failover_all(mgmt_ip, key_path, home) + if not ok: + raise RuntimeError("FAIL: setup fail-over failed before the " + "fail-back under test: %s" % why) + _kill_after(delay, victim, key_path, sink) + ok, back, why = _failback_all(mgmt_ip, key_path, meta, tgt_lvols) + else: # commit / online cutover + # No fail-over first: the volumes are live on the source under a + # migration policy, and replication-commit IS the online cutover. + _kill_after(delay, victim, key_path, sink) + ok, why = _commit_all(mgmt_ip, key_path, home) + tgt_lvols = home + measured = time.time() - started + if rnd == 1: + phase_secs = max(5.0, measured) + print(" measured %s phase: %.1fs (kill delays randomize over it)" + % (phase, phase_secs)) + kills.extend(sink) + + healthy, state = _await_cluster_healthy(mgmt_ip, key_path) + if not healthy: + raise RuntimeError( + "FAIL: round %d -- cluster did not recover %ds after killing %s: " + "unhealthy=%s clusters=%s" + % (rnd, NODE_STATE_TIMEOUT, victim, state.get("offline"), + state.get("clusters"))) + + if ok: + verdicts.append("completed") + else: + # Chaos may legitimately fail the operation -- a RETRY after the + # cluster is healthy again must then succeed. Anything else is a + # wedged state, which is the bug this case hunts. + print(" operation failed under chaos (%s); retrying after recovery" + % why[:160]) + retries += 1 + if phase == "failover": + ok, tgt_lvols, why = _failover_all(mgmt_ip, key_path, home) + elif phase == "failback": + ok, back, why = _failback_all(mgmt_ip, key_path, meta, tgt_lvols) + else: + ok, why = _commit_all(mgmt_ip, key_path, tgt_lvols) + if not ok: + raise RuntimeError("FAIL: round %d -- %s did not succeed even on " + "retry after recovery: %s" % (rnd, phase, why)) + verdicts.append("retry") + + # Verify the data that arrived, then put the volumes back HOME so the + # next round starts from the same shape. + if phase == "failover": + check_lvols = tgt_lvols + elif phase == "failback": + check_lvols = back + else: + after = failed_over_targets(mgmt_ip, key_path, home) + check_lvols = [after[h] for h in home if h in after] + if len(check_lvols) != len(home): + raise RuntimeError("FAIL: round %d -- cutover produced %d/%d " + "target volumes" % (rnd, len(check_lvols), len(home))) + check_base = dict(zip(check_lvols, base_list)) + vmounts = connect_and_mount(client_ip, key_path, mgmt_ip, check_lvols, + fmt=False, mount_base=MOUNT_BASE + "_ph") + good, _ = verify_baseline(client_ip, key_path, vmounts, check_base) + cleanup_client(client_ip, key_path, vmounts) + if not good: + raise RuntimeError("FAIL: round %d -- data not intact after %s with a " + "node killed at T+%.1fs" % (rnd, phase, delay)) + + if phase in ("failover", "commit"): + # The volumes now live on the TARGET side; bring them home so the + # next round attacks the same phase from the same shape. + okb, back, whyb = _failback_all(mgmt_ip, key_path, meta, check_lvols) + if not okb: + raise RuntimeError("FAIL: round %d -- could not restore the home " + "side after the %s under test: %s" + % (rnd, phase, whyb)) + home = back + else: + home = check_lvols + set_cluster_replication(mgmt_ip, key_path, src_uuid, tgt_uuid, + pool_uuid_of(mgmt_ip, key_path, tgt["pool"]), + mode=mode) + wait_replication_caught_up(mgmt_ip, key_path, home, timeout=3600) + + completed = verdicts.count("completed") + print(" %d rounds: %d completed under the kill, %d needed a retry after " + "recovery, %d kills delivered" % (CHAOS_PHASE_ROUNDS, completed, retries, + sum(1 for k in kills if k[0] == "killed"))) + print("%s PASSED: every round ended intact and unwedged (seed %d)." + % (title.split(":")[0], seed)) + + +def test_case_13(meta): + _chaos_phase_case(meta, "failover", + "CASE 13: single node dies DURING fail-over") + + +def test_case_14(meta): + _chaos_phase_case(meta, "failback", + "CASE 14: single node dies DURING fail-back") + + +def test_case_15(meta): + _chaos_phase_case(meta, "commit", + "CASE 15: single node dies DURING the online cutover") + CASES = { "case1": test_case_1, # online migration cutover, no IO interruption "case2": test_case_2, # DR fail-over on source-cluster loss @@ -2625,6 +2889,9 @@ def test_case_12(meta): "case10": test_case_10, # migration under heavy IO + cutover freeze timing "case11": test_case_11, # retention ladder + random-generation fail-overs "case12": test_case_12, # consistency groups (needs the CG build) + "case13": test_case_13, # node dies during fail-over, x20 random instants + "case14": test_case_14, # node dies during fail-back, x20 random instants + "case15": test_case_15, # node dies during the online cutover, x20 } GROUPS = { "both": ["case1", "case2"], @@ -2632,6 +2899,7 @@ def test_case_12(meta): "errors": ["case5", "case6"], "extended": ["case7", "case8", "case9"], "features": ["case10", "case11", "case12"], + "phase-chaos": ["case13", "case14", "case15"], "all": ["case1", "case2", "case3", "case4", "case5", "case6"], # Case 3 last: it is the only case that needs the killed primary restored # and recovered, so a failure there cannot cost the other five cases. From d461f3c7a5b67cbd02cc61c0beee13a57fd5c90e Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 26 Aug 2026 16:20:37 +0100 Subject: [PATCH 063/122] fix: evict stale failback namespace from HA peer nodes to prevent -32602 on nvmf_subsystem_add_ns --- .../controllers/lvol_controller.py | 54 +++++++++++-------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index ca491c9a66..3f97a0ef44 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -3594,30 +3594,40 @@ def _evict_stale_namespace(new_lvol, target_node): retry: 2026-08-24, 5/5 fail-back cutovers, 40x "Failed to add bdev to subsystem". Evict a namespace occupying the clone's nsid unless it is already the clone's own bdev (idempotent re-run). + + The eviction must cover all HA peer nodes (secondary, tertiary) as well as + the primary. add_lvol_on_node registers the new namespace on every peer, so + any peer that still holds the old namespace at nsid also fails with -32602 + and blocks the entire cutover. """ - try: - rpc = target_node.rpc_client() - subsystems = rpc.subsystem_get(new_lvol.nqn) - if not subsystems: - return - # subsystem_get returns a list of subsystem dicts; take the first entry. - subsystem = subsystems[0] if isinstance(subsystems, list) else subsystems - for ns in (subsystem.get("namespaces") or []): - if ns.get("nsid") != new_lvol.ns_id: + peer_ids = [target_node.secondary_node_id, target_node.tertiary_node_id] + nodes_to_evict = [target_node] + [ + db_controller.get_storage_node_by_id(pid) + for pid in peer_ids if pid + ] + for node in nodes_to_evict: + try: + rpc = node.rpc_client() + subsystems = rpc.subsystem_get(new_lvol.nqn) + if not subsystems: continue - if ns.get("bdev_name") == new_lvol.top_bdev: - return # already ours (re-run) - logger.info( - f"Fail-back cutover: evicting stale namespace nsid={ns.get('nsid')} " - f"(bdev {ns.get('bdev_name')}) from {new_lvol.nqn} on " - f"{target_node.get_id()} -- superseded by the failed-over data") - rpc.nvmf_subsystem_remove_ns(new_lvol.nqn, ns.get("nsid")) - return - except Exception as e: - # Best effort: if the subsystem is not there, add_lvol_on_node creates - # it; if the eviction genuinely failed, add_ns will say so loudly. - logger.warning(f"Stale-namespace check on {target_node.get_id()} for " - f"{new_lvol.nqn} raised: {e}") + subsystem = subsystems[0] if isinstance(subsystems, list) else subsystems + for ns in (subsystem.get("namespaces") or []): + if ns.get("nsid") != new_lvol.ns_id: + continue + if ns.get("bdev_name") == new_lvol.top_bdev: + break # already ours on this node (re-run) + logger.info( + f"Fail-back cutover: evicting stale namespace nsid={ns.get('nsid')} " + f"(bdev {ns.get('bdev_name')}) from {new_lvol.nqn} on " + f"{node.get_id()} -- superseded by the failed-over data") + rpc.nvmf_subsystem_remove_ns(new_lvol.nqn, ns.get("nsid")) + break + except Exception as e: + # Best effort: if the subsystem is not there, add_lvol_on_node creates + # it; if the eviction genuinely failed, add_ns will say so loudly. + logger.warning(f"Stale-namespace check on {node.get_id()} for " + f"{new_lvol.nqn} raised: {e}") def _clone_from_last_replicated(db_controller, lvol_id, lvol, target_node, pool_uuid, From 5d716b587688d42af00b12a79997a944ef70b817 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 26 Aug 2026 17:21:59 +0200 Subject: [PATCH 064/122] ci: apply checks to release PRs as well --- .github/workflows/python-checks.yml | 1 + .github/workflows/tests.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/python-checks.yml b/.github/workflows/python-checks.yml index 06aaab69de..a161a0a0a8 100644 --- a/.github/workflows/python-checks.yml +++ b/.github/workflows/python-checks.yml @@ -7,6 +7,7 @@ on: branches: - main - R26.2-PRE + - R26.3-PRE permissions: contents: read diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 71083bc0d6..d936397bae 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,6 +7,7 @@ on: branches: - main - R26.2-PRE + - R26.3-PRE permissions: contents: read From d474afaee3f1c136605fa939575e3c6be397feff Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 17:45:45 +0200 Subject: [PATCH 065/122] test: give each phase-chaos round its own policy and volumes Per review: 20 rounds do not need to share volumes. Cases 13-15 now set up CHAOS_PHASE_ROUNDS policies with CHAOS_PHASE_VOLS_PER_POLICY volumes each (default 20 x 2 = 40 x 10G), replicate them ONCE, then walk the policies one at a time and attack the phase on that policy's volumes with the kill at a random instant. That removes the whole restore leg the previous shape needed between rounds -- a fail-back, its commit and a re-sync after every single round, purely to hand the next round the same volumes back. Now a round ends when its data has been verified; the next round owns different volumes entirely. Same coverage, a fraction of the wall-clock, and rounds no longer inherit any state from their predecessor -- so a failure is attributable to the kill that round, not to residue from earlier ones. set_cluster_replication takes an explicit policy_name so the per-round policies are distinct. Co-Authored-By: Claude Fable 5 --- scripts/test_async_replication.py | 145 ++++++++++++++++-------------- 1 file changed, 76 insertions(+), 69 deletions(-) diff --git a/scripts/test_async_replication.py b/scripts/test_async_replication.py index 61709aff66..a6e0333e30 100644 --- a/scripts/test_async_replication.py +++ b/scripts/test_async_replication.py @@ -795,7 +795,7 @@ def ensure_replication_policy(mgmt_ip, key_path, from_cluster, target_name, mode def set_cluster_replication(mgmt_ip, key_path, from_cluster, to_cluster, to_pool_uuid, - mode="migration", extra_flags=""): + mode="migration", extra_flags="", policy_name=None): """Create the target + policy that let `from_cluster` replicate to `to_cluster`. Replication is NEVER started per volume any more: `volume replication-start` @@ -811,7 +811,7 @@ def set_cluster_replication(mgmt_ip, key_path, from_cluster, to_cluster, to_pool target = ensure_replication_target(mgmt_ip, key_path, from_cluster, to_cluster, to_pool_uuid) policy = ensure_replication_policy(mgmt_ip, key_path, from_cluster, target, mode, - extra_flags=extra_flags) + name=policy_name, extra_flags=extra_flags) # PRODUCT GAP (bridge, delete once the readers consult the policy): # replicate_lvol_on_target_cluster() and tasks_runner_replication_final still @@ -2624,7 +2624,8 @@ def test_case_12(meta): # ends in one of two accepted verdicts (completed despite the kill / failed # then succeeded on retry after recovery) and any third outcome fails the case. CHAOS_PHASE_ROUNDS = int(os.environ.get("CHAOS_PHASE_ROUNDS", "20")) -CHAOS_PHASE_VOLS = int(os.environ.get("CHAOS_PHASE_VOLS", "3")) +CHAOS_PHASE_VOLS_PER_POLICY = int(os.environ.get("CHAOS_PHASE_VOLS_PER_POLICY", "2")) +CHAOS_PHASE_VOL_SIZE = os.environ.get("CHAOS_PHASE_VOL_SIZE", "10G") #: cap for the randomized kill delay when a phase turns out to be slow CHAOS_PHASE_MAX_DELAY = float(os.environ.get("CHAOS_PHASE_MAX_DELAY", "45")) @@ -2724,7 +2725,15 @@ def _failback_all(mgmt_ip, key_path, meta, tgt_lvols): def _chaos_phase_case(meta, phase, title): - """Kill ONE node at a random instant inside *phase*, CHAOS_PHASE_ROUNDS times.""" + """Kill ONE node at a random instant inside *phase*, once per POLICY. + + Each round owns its own policy and its own volumes, so a round never has + to undo what the previous one did: no fail-back, no re-sync, no restore to + a "home shape". Set up CHAOS_PHASE_ROUNDS policies with + CHAOS_PHASE_VOLS_PER_POLICY volumes each, replicate them all once, then + walk the policies one at a time and attack the phase on that policy's + volumes with the kill landing at a random instant. + """ import random print("\n========== %s ==========" % title) key_path = meta["key_path"] @@ -2733,55 +2742,71 @@ def _chaos_phase_case(meta, phase, title): src_uuid, src, tgt_uuid, tgt = _src_target(meta) seed = int(os.environ.get("CHAOS_PHASE_SEED") or time.time()) rng = random.Random(seed) - print(" seed: %d, rounds: %d, phase: %s" % (seed, CHAOS_PHASE_ROUNDS, phase)) + # The cutover under test in case 15 is the FINAL MIGRATION STEP: a + # migration-mode policy committed on the SOURCE volumes. + mode = "migration" if phase == "commit" else "failover" + print(" seed=%d rounds=%d vols/policy=%d mode=%s phase=%s" + % (seed, CHAOS_PHASE_ROUNDS, CHAOS_PHASE_VOLS_PER_POLICY, mode, phase)) prepare_mount_points(client_ip, key_path) delete_test_volumes(mgmt_ip, key_path, _all_test_pools(meta)) - # The cutover under test in case 15 is the FINAL MIGRATION STEP -- that is - # a migration-mode policy committed on the SOURCE volumes while IO runs. - mode = "migration" if phase == "commit" else "failover" - lvols = create_volumes(mgmt_ip, key_path, src_uuid, src["pool"], tgt_uuid, - tgt["pool"], mode=mode, count=CHAOS_PHASE_VOLS) - mounts = connect_and_mount(client_ip, key_path, mgmt_ip, lvols, fmt=True) + + # --- one-time setup: N policies x M volumes, all replicating ------------ + groups, all_lvols = [], [] + for r in range(CHAOS_PHASE_ROUNDS): + policy = set_cluster_replication( + mgmt_ip, key_path, src_uuid, tgt_uuid, + pool_uuid_of(mgmt_ip, key_path, tgt["pool"]), mode=mode, + policy_name="pol_chaos_%s_%02d" % (phase, r)) + vols = [] + for v in range(CHAOS_PHASE_VOLS_PER_POLICY): + name = "replvol%02d_%d" % (r, v) + run(mgmt_ip, key_path, + "%s -d volume add %s %s %s --replication-policy %s" + % (SBCTL, name, CHAOS_PHASE_VOL_SIZE, src["pool"], policy)) + vols.append(resolve_lvol(mgmt_ip, key_path, name)["uuid"]) + groups.append({"policy": policy, "vols": vols}) + all_lvols.extend(vols) + print(" created %d policies x %d volumes = %d volumes" + % (CHAOS_PHASE_ROUNDS, CHAOS_PHASE_VOLS_PER_POLICY, len(all_lvols))) + + mounts = connect_and_mount(client_ip, key_path, mgmt_ip, all_lvols, fmt=True) baseline = write_baseline(client_ip, key_path, mounts) baseline_ts = time.time() - wait_replication_caught_up(mgmt_ip, key_path, lvols) - wait_data_replicated(mgmt_ip, key_path, lvols, baseline_ts) cleanup_client(client_ip, key_path, mounts) + wait_replication_caught_up(mgmt_ip, key_path, all_lvols, timeout=7200) + wait_data_replicated(mgmt_ip, key_path, all_lvols, baseline_ts, timeout=7200) + print(" all %d volumes replicated; starting the rounds" % len(all_lvols)) victims = _phase_victims(meta, phase) phase_secs = 20.0 # replaced by the first round's measurement verdicts, retries, kills = [], 0, [] - home = list(lvols) # volumes currently on the SOURCE side - #: md5 per volume, positionally aligned with `home` -- every hop below - #: preserves order, so the baseline follows a volume across fail-overs - #: without depending on dict iteration order. - base_list = [baseline[lv] for lv in lvols] - for rnd in range(1, CHAOS_PHASE_ROUNDS + 1): + for rnd, grp in enumerate(groups, start=1): + vols = grp["vols"] delay = round(rng.uniform(0.0, min(phase_secs * 1.2, CHAOS_PHASE_MAX_DELAY)), 1) label, victim = rng.choice(victims) - print("--- round %d/%d: kill %s node %s at T+%.1fs of the %s phase ---" - % (rnd, CHAOS_PHASE_ROUNDS, label, victim, delay, phase)) + print("--- round %d/%d (policy %s): kill %s node %s at T+%.1fs of %s ---" + % (rnd, CHAOS_PHASE_ROUNDS, grp["policy"], label, victim, delay, phase)) sink = [] started = time.time() if phase == "failover": _kill_after(delay, victim, key_path, sink) - ok, tgt_lvols, why = _failover_all(mgmt_ip, key_path, home) + ok, result_lvols, why = _failover_all(mgmt_ip, key_path, vols) elif phase == "failback": - ok, tgt_lvols, why = _failover_all(mgmt_ip, key_path, home) + # The fail-BACK is under test, so the fail-over that sets it up runs + # clean; only then does the node die. + ok, tgt_lvols, why = _failover_all(mgmt_ip, key_path, vols) if not ok: - raise RuntimeError("FAIL: setup fail-over failed before the " - "fail-back under test: %s" % why) + raise RuntimeError("FAIL: round %d -- the setup fail-over failed " + "before the fail-back under test: %s" % (rnd, why)) _kill_after(delay, victim, key_path, sink) - ok, back, why = _failback_all(mgmt_ip, key_path, meta, tgt_lvols) + ok, result_lvols, why = _failback_all(mgmt_ip, key_path, meta, tgt_lvols) else: # commit / online cutover - # No fail-over first: the volumes are live on the source under a - # migration policy, and replication-commit IS the online cutover. _kill_after(delay, victim, key_path, sink) - ok, why = _commit_all(mgmt_ip, key_path, home) - tgt_lvols = home + ok, why = _commit_all(mgmt_ip, key_path, vols) + result_lvols = [] measured = time.time() - started if rnd == 1: phase_secs = max(5.0, measured) @@ -2800,64 +2825,46 @@ def _chaos_phase_case(meta, phase, title): if ok: verdicts.append("completed") else: - # Chaos may legitimately fail the operation -- a RETRY after the - # cluster is healthy again must then succeed. Anything else is a + # Chaos may legitimately fail the operation. A RETRY once the + # cluster is healthy again must then succeed -- anything else is a # wedged state, which is the bug this case hunts. print(" operation failed under chaos (%s); retrying after recovery" % why[:160]) retries += 1 if phase == "failover": - ok, tgt_lvols, why = _failover_all(mgmt_ip, key_path, home) + ok, result_lvols, why = _failover_all(mgmt_ip, key_path, vols) elif phase == "failback": - ok, back, why = _failback_all(mgmt_ip, key_path, meta, tgt_lvols) + ok, result_lvols, why = _failback_all(mgmt_ip, key_path, meta, tgt_lvols) else: - ok, why = _commit_all(mgmt_ip, key_path, tgt_lvols) + ok, why = _commit_all(mgmt_ip, key_path, vols) if not ok: - raise RuntimeError("FAIL: round %d -- %s did not succeed even on " + raise RuntimeError("FAIL: round %d -- %s did not succeed even on a " "retry after recovery: %s" % (rnd, phase, why)) verdicts.append("retry") - # Verify the data that arrived, then put the volumes back HOME so the - # next round starts from the same shape. - if phase == "failover": - check_lvols = tgt_lvols - elif phase == "failback": - check_lvols = back - else: - after = failed_over_targets(mgmt_ip, key_path, home) - check_lvols = [after[h] for h in home if h in after] - if len(check_lvols) != len(home): - raise RuntimeError("FAIL: round %d -- cutover produced %d/%d " - "target volumes" % (rnd, len(check_lvols), len(home))) - check_base = dict(zip(check_lvols, base_list)) - vmounts = connect_and_mount(client_ip, key_path, mgmt_ip, check_lvols, + # Where the data ended up, and whether it is intact. + if phase == "commit": + after = failed_over_targets(mgmt_ip, key_path, vols) + result_lvols = [after[v] for v in vols if v in after] + if len(result_lvols) != len(vols): + raise RuntimeError("FAIL: round %d -- cutover produced %d/%d target " + "volumes" % (rnd, len(result_lvols), len(vols))) + check_base = dict(zip(result_lvols, [baseline[v] for v in vols])) + vmounts = connect_and_mount(client_ip, key_path, mgmt_ip, result_lvols, fmt=False, mount_base=MOUNT_BASE + "_ph") good, _ = verify_baseline(client_ip, key_path, vmounts, check_base) cleanup_client(client_ip, key_path, vmounts) if not good: raise RuntimeError("FAIL: round %d -- data not intact after %s with a " - "node killed at T+%.1fs" % (rnd, phase, delay)) - - if phase in ("failover", "commit"): - # The volumes now live on the TARGET side; bring them home so the - # next round attacks the same phase from the same shape. - okb, back, whyb = _failback_all(mgmt_ip, key_path, meta, check_lvols) - if not okb: - raise RuntimeError("FAIL: round %d -- could not restore the home " - "side after the %s under test: %s" - % (rnd, phase, whyb)) - home = back - else: - home = check_lvols - set_cluster_replication(mgmt_ip, key_path, src_uuid, tgt_uuid, - pool_uuid_of(mgmt_ip, key_path, tgt["pool"]), - mode=mode) - wait_replication_caught_up(mgmt_ip, key_path, home, timeout=3600) + "node killed at T+%.1fs (policy %s)" + % (rnd, phase, delay, grp["policy"])) + # No restore: the next round owns different volumes entirely. completed = verdicts.count("completed") + delivered = sum(1 for k in kills if k[0] == "killed") print(" %d rounds: %d completed under the kill, %d needed a retry after " - "recovery, %d kills delivered" % (CHAOS_PHASE_ROUNDS, completed, retries, - sum(1 for k in kills if k[0] == "killed"))) + "recovery, %d/%d kills delivered" + % (CHAOS_PHASE_ROUNDS, completed, retries, delivered, len(kills))) print("%s PASSED: every round ended intact and unwedged (seed %d)." % (title.split(":")[0], seed)) From 115ae1f86b1096370f695453e64892a8c212fda1 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 17:52:26 +0200 Subject: [PATCH 066/122] test: accept --cases in every shape a shell produces '--cases case7, case9' died with 'unrecognized arguments: case9': the space makes the shell pass two argv entries, and a single-value option only consumed the first. Now nargs='+' plus comma splitting, so 'case7,case9', 'case7, case9' and 'case7 case9' all mean the same list. Co-Authored-By: Claude Fable 5 --- scripts/repl_soak.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/repl_soak.py b/scripts/repl_soak.py index e17bd9f07b..5d7ae46b39 100644 --- a/scripts/repl_soak.py +++ b/scripts/repl_soak.py @@ -173,8 +173,9 @@ def teardown(meta): def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--cases", required=True, - help="comma list or group name for test_async_replication.py (e.g. case6,case7 | all9 | features)") + ap.add_argument("--cases", required=True, nargs="+", metavar="CASE", + help="cases or a group name: 'case6,case7', 'case6 case7', " + "'case6, case7' and 'all9' all work") ap.add_argument("--branch", default="replication-features", help="sbcli branch to deploy/hotfix/stage (default replication-features)") ap.add_argument("--spdk-image", default="", help="ultra image ref; default = digest pinned in the deployer") @@ -185,6 +186,11 @@ def main(): ap.add_argument("--no-hotfix", action="store_true", help="skip mounting the branch's python over the CP services") ap.add_argument("--teardown", choices=["never", "on-pass", "always"], default="never") args = ap.parse_args() + # Accept every shape a shell hands us: "a,b", "a, b" (the space makes the + # shell pass two argv entries) and "a b" all mean the same list. + args.cases = ",".join(c for tok in args.cases for c in tok.split(",") if c) + if not args.cases: + ap.error("--cases got no case names") for kv in args.env: if "=" not in kv: ap.error(f"--env expects KEY=VAL, got {kv!r}") From a284f73d10f712450194cc41ef744fcf626ad7c6 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 18:25:19 +0200 Subject: [PATCH 067/122] test: hotfix mounts every module that differs from the image, not a hand-kept list The lab run died with 'ImportError: cannot import name ConsistencyGroup from simplyblock_core.models.replication' crash-looping app_SnapshotReplication: the mounted branch code imported a model the deployed image (built from main) does not have, and models/replication.py was not in the mount list. Third time this class has bitten -- ops_gate, lvol_monitor, now ConsistencyGroup -- because the list is maintained by hand and must be extended for every module a fix happens to touch. discover_drift() now asks the image for the md5 of every simplyblock_core module, compares against the checkout, and mounts everything that differs or is missing (15 modules for replication-features vs a main-built image). Also refuse to stage two modules with the same basename, which the flat staging dir would silently collapse into one. Co-Authored-By: Claude Fable 5 --- scripts/hotfix_repl_lab.py | 63 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/scripts/hotfix_repl_lab.py b/scripts/hotfix_repl_lab.py index aa79eeb640..bc00e73589 100644 --- a/scripts/hotfix_repl_lab.py +++ b/scripts/hotfix_repl_lab.py @@ -21,6 +21,7 @@ Usage: python hotfix_repl_lab.py [--verify-only] """ import json +import shlex import subprocess import sys import time @@ -86,6 +87,56 @@ HOST_FILES.update({p: v for s in SERVICES.values() for p, v in s.items()}) HOST_CLI = "simplyblock_cli/clibase.py" +CRLF, LF = bytes([13, 10]), bytes([10]) + +#: filled in by discover_drift() at run time: repo path -> path under +#: simplyblock_core, for every module that differs from the deployed image +DRIFT = {} + + +def discover_drift(mgmt): + """Every simplyblock_core module that differs from the one in the IMAGE. + + A hand-maintained mount list is wrong by construction: it must be extended + for every new module a fix happens to touch. Forgetting one fails loudly + (ImportError: cannot import name 'ConsistencyGroup' -- the service + crash-looping, lab 2026-08-26) or, worse, quietly: the fix mounted while a + module it depends on stays at the image's version. Ask the image what it + has and mount everything that does not match. + """ + import hashlib + ref = ssh(mgmt, "sudo docker ps --format '{{.Names}}' | grep -m1 " + "TasksRunnerReplicationFinal", check=False).strip() + if not ref: + log("no reference container; falling back to the static mount list") + return {} + probe = ("cd /usr/local/lib/python3.12/site-packages && " + 'find simplyblock_core -name "*.py" | sort | xargs md5sum') + listing = ssh(mgmt, f"sudo docker exec {ref} sh -c {shlex.quote(probe)}", + check=False) + image = {} + for line in listing.splitlines(): + parts = line.split() + if len(parts) == 2 and parts[1].endswith(".py"): + image[parts[1]] = parts[0] + if not image: + log("could not read the image's module hashes; using the static list") + return {} + + drift, core = {}, REPO / "simplyblock_core" + for path in core.rglob("*.py"): + rel = path.relative_to(REPO).as_posix() + if "__pycache__" in rel or rel.startswith("simplyblock_core/test"): + continue + raw = path.read_bytes().replace(CRLF, LF) # image files are LF + digest = hashlib.md5(raw).hexdigest() + if image.get(rel) != digest: + drift[rel] = path.relative_to(core).as_posix() + log(f"image drift: {len(drift)} module(s) differ from the deployed image") + for rel in sorted(drift): + log(f" {rel}{'' if rel in image else ' (new)'}") + return drift + def log(msg): print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) @@ -105,6 +156,11 @@ def ssh(host, cmd, timeout=600, check=True): def stage(mgmt): log(f"staging {len(HOST_FILES) + 1} files in {HOTFIX_DIR}") + names = [Path(x).name for x in HOST_FILES] + dupes = sorted({n for n in names if names.count(n) > 1}) + if dupes: + raise RuntimeError(f"modules share a basename and would collide in the " + f"flat staging dir: {dupes}") ssh(mgmt, f"sudo mkdir -p {HOTFIX_DIR}/backup && sudo chown ec2-user {HOTFIX_DIR}") locals_ = [str(REPO / p) for p in HOST_FILES] + [str(REPO / HOST_CLI)] run(["scp", *SSH_OPTS, *locals_, f"ec2-user@{mgmt}:{HOTFIX_DIR}/"], timeout=900) @@ -175,10 +231,17 @@ def verify(mgmt): def main(): + global DRIFT, HOST_FILES meta = json.loads((HERE / "cluster_metadata_repl.json").read_text()) mgmt = meta["mgmt"]["public_ip"] log(f"mgmt={mgmt}") if "--verify-only" not in sys.argv: + # Mount everything that differs from the image, not a hand-kept list. + DRIFT = discover_drift(mgmt) + SHARED.update(DRIFT) + HOST_FILES = dict(SHARED) + HOST_FILES.update({p: v for sv in SERVICES.values() + for p, v in sv.items()}) stage(mgmt) mount_services(mgmt) patch_host(mgmt) From 90989c7363b3df7170d958c49d21352bb61938fc Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 18:58:58 +0200 Subject: [PATCH 068/122] fix: say WHY a replica namespace add was rejected A replica add_ns cannot re-claim a slot -- its nsid is dictated by the primary -- so a -32602 there ends the whole create or fail-over with the bare 'Failed to add bdev to subsystem'. That message has now cost several lab runs: the cause is always in the node's own namespace table (the nsid already taken by another bdev, or max_namespaces below the requested nsid) and nothing logged it. The replica failure path now dumps what that node actually holds for the subsystem -- requested nsid, max_namespaces, and the (nsid, bdev) pairs present -- into both the log and the returned error. Co-Authored-By: Claude Fable 5 --- .../controllers/lvol_controller.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 6c29a67a9b..4a9998208f 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -1371,8 +1371,31 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0): return _fail_after_bdev(lvol, rpc_client, str(e)) return add_lvol_on_node(lvol, snode, is_primary=is_primary, secondary_index=secondary_index) else: + # A REPLICA add cannot re-claim a slot (its nsid is dictated by the + # primary), so -32602 here ends the whole create/fail-over. Say WHY: + # dump what the node actually has for this subsystem, because the + # bare message costs a full lab run to diagnose and the cause is + # always in this table -- nsid already taken by another bdev, or a + # max_namespaces smaller than the requested nsid. + detail = "" + try: + subsys = rpc_client.subsystem_get(lvol.nqn) + if subsys: + existing = sorted( + (n.get("nsid"), n.get("bdev_name")) + for n in (subsys.get("namespaces") or [])) + detail = (f" [node {snode.get_id()[:8]} wanted nsid=" + f"{requested_nsid} max_namespaces=" + f"{subsys.get('max_namespaces')} holds={existing}]") + else: + detail = (f" [subsystem {lvol.nqn} does not exist on " + f"{snode.get_id()[:8]}]") + except Exception as diag_exc: # noqa: BLE001 + detail = f" [could not read the subsystem: {diag_exc}]" + logger.error("Namespace add rejected on %s for %s:%s", + snode.get_id()[:8], lvol.get_id(), detail) return _fail_after_bdev( - lvol, rpc_client, "Failed to add bdev to subsystem") + lvol, rpc_client, "Failed to add bdev to subsystem" + detail) if is_primary: # Persist the target-assigned nsid; replicas re-add with exactly From dab4bb05db2d6e793e2d32d3ceaddec368fda260 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 19:25:07 +0200 Subject: [PATCH 069/122] fix: never evict a sibling's namespace from a shared subsystem _evict_stale_namespace matched a namespace to remove by nsid OR uuid. On a SHARED (namespaced) subsystem nsid is not this volume's identity: siblings occupy the other slots, and new_lvol.ns_id at that moment is still the SOURCE cluster's number, because the destination primary auto-assigns its own and only then overwrites the record. An nsid match could therefore remove a sibling's live, already-failed-over namespace -- taking a healthy volume's block device away from its client. Cross-cluster nsid equality buys nothing anyway: the client looks its paths up through connect_lvol. Only the HA paths WITHIN one subsystem must agree on nsid, and that is enforced elsewhere (primary assigns, replicas reuse). Eviction now matches on the volume's own uuid, keeping the nsid match only for a single-namespace subsystem -- which cannot hold anyone else's namespace and is the fail-back-to-a-recovered-source case the eviction was written for. Also corrected two comments still claiming a fail-over copy 'preserves the NQN and nsid': only the NQN is preserved. The existing fake had no uuid at all, so the new matcher raised into the best-effort except and evicted nothing -- the same silent-skip failure this suite exists to catch. The fake now carries one, and the matcher uses getattr so a malformed record degrades to 'no match' instead of an exception. Co-Authored-By: Claude Fable 5 --- .../controllers/lvol_controller.py | 31 +++++++++++++---- .../test_replication_chain_completeness.py | 34 +++++++++++++++++++ 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 4a9998208f..3aa350575b 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -680,7 +680,8 @@ def add_lvol_ha(name, size, host_id_or_name, ha_type, pool_id_or_name, use_comp= else: replication_cluster_id = cl.snapshot_replication_target_cluster # Namespaced siblings MUST replicate to the same target node. - # A fail-over copy preserves the volume's NQN and nsid, so all + # A fail-over copy preserves the volume's NQN (its nsid is assigned + # afresh by the destination), so all # volumes sharing a subsystem land in the SAME subsystem on the # target. Picking the destination purely by capacity scattered # siblings across the target cluster's nodes, which splits one @@ -2608,7 +2609,8 @@ def _connect_path_volumes(db_controller, lvol): cutover_done -> ONLY the post-move volume; the pre-migration paths are not handed out any more - The clone preserves the source NQN and ns_id, so every path returned here + The clone preserves the source NQN -- its nsid is assigned by the + destination primary and may differ -- so every path returned here aggregates into one multipath device on the client. """ from simplyblock_core.models.lvol_model import LVolReplication @@ -3595,11 +3597,26 @@ def _evict_stale_namespace(new_lvol, target_node): subsystem = rpc.subsystem_get(new_lvol.nqn) if not subsystem: return - # Match by nsid OR by uuid: the stale namespace carries the volume's - # preserved identity on both axes, and either collides with add_ns. - stale = [ns for ns in (subsystem.get("namespaces") or []) - if (ns.get("nsid") == new_lvol.ns_id - or ns.get("uuid") == new_lvol.uuid) + # Match on the volume's UUID -- that is what identifies THIS volume's + # own stale namespace. An nsid match is only safe when the subsystem + # cannot hold anyone else's namespace. + # + # nsid is NOT identity on a SHARED (namespaced) subsystem: siblings + # occupy the other slots, and new_lvol.ns_id here is still the SOURCE + # cluster's number (the destination primary auto-assigns and only then + # overwrites the record). Matching on it could evict a sibling's live, + # already-failed-over namespace -- taking a healthy volume's device + # away from its client. Cross-cluster nsid equality is not required + # anyway: the client looks its paths up through connect_lvol. + namespaces = subsystem.get("namespaces") or [] + single_namespace_subsystem = len(namespaces) <= 1 + own_uuid = getattr(new_lvol, "uuid", None) + own_nsid = getattr(new_lvol, "ns_id", None) + stale = [ns for ns in namespaces + if ((own_uuid is not None and ns.get("uuid") == own_uuid) + or (single_namespace_subsystem + and own_nsid is not None + and ns.get("nsid") == own_nsid)) and ns.get("bdev_name") != new_lvol.top_bdev] if not stale: return diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index 7df8a03353..4e38073ff5 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -304,6 +304,11 @@ def rpc_client(self): class _CloneLvol: nqn = "nqn.test:lvol:ORIG" ns_id = 7 + # A real LVol always carries a uuid, and the eviction matches on it -- + # a fake without one made the matcher raise into the best-effort except + # and silently evict nothing, the exact failure mode this suite exists + # to catch. + uuid = "11111111-1111-1111-1111-111111111111" top_bdev = "LVS_1/LVOL_CLONE" @@ -536,3 +541,32 @@ def test_shared_subsystem_survives_one_members_teardown(): assert guard < delete, "the other-claimants check must precede subsystem_delete" assert "x.nqn == lvol.nqn" in src, "claimants are identified by shared NQN" assert "Leaving subsystem" in src + + +def test_eviction_never_removes_a_siblings_namespace(): + """On a SHARED subsystem the nsid is not this volume's identity: siblings + hold the other slots, and new_lvol.ns_id is still the SOURCE cluster's + number (the destination primary auto-assigns and only then overwrites the + record). Matching on nsid would evict a sibling's live, already-failed-over + namespace -- taking a healthy volume's device away from its client.""" + from simplyblock_core.controllers import lvol_controller as lc + rpc = _EvictRPC([ + {"nsid": 7, "bdev_name": "LVS_1/LVOL_SIBLING", + "uuid": "22222222-2222-2222-2222-222222222222"}, # same nsid, other volume + {"nsid": 3, "bdev_name": "LVS_1/LVOL_MINE_OLD", + "uuid": "11111111-1111-1111-1111-111111111111"}, # THIS volume, stale + ]) + lc._evict_stale_namespace(_CloneLvol(), _EvictNode(rpc)) + assert rpc.removed == [("nqn.test:lvol:ORIG", 3)], \ + "must evict only this volume's own namespace, never the sibling at nsid 7" + + +def test_eviction_still_uses_nsid_on_a_single_namespace_subsystem(): + """A dedicated subsystem cannot hold anyone else's namespace, so nsid stays + a safe match there -- which is the fail-back-to-a-recovered-source case the + eviction was written for, where the old record may carry a different uuid.""" + from simplyblock_core.controllers import lvol_controller as lc + rpc = _EvictRPC([{"nsid": 7, "bdev_name": "LVS_1/LVOL_ORIG", + "uuid": "99999999-9999-9999-9999-999999999999"}]) + lc._evict_stale_namespace(_CloneLvol(), _EvictNode(rpc)) + assert rpc.removed == [("nqn.test:lvol:ORIG", 7)] From d3b11a9152af16458b1c950e9f38e05865efb40b Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 26 Aug 2026 18:36:42 +0100 Subject: [PATCH 070/122] fix: use DBController() instance in _evict_stale_namespace instead of undefined db_controller --- simplyblock_core/controllers/lvol_controller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 3f97a0ef44..92de7d2d08 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -3601,8 +3601,9 @@ def _evict_stale_namespace(new_lvol, target_node): and blocks the entire cutover. """ peer_ids = [target_node.secondary_node_id, target_node.tertiary_node_id] + db = DBController() nodes_to_evict = [target_node] + [ - db_controller.get_storage_node_by_id(pid) + db.get_storage_node_by_id(pid) for pid in peer_ids if pid ] for node in nodes_to_evict: From 3aebabceb6e15c6aa441b3709137fe2bdb2c0249 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 26 Aug 2026 19:20:54 +0100 Subject: [PATCH 071/122] fix: preserve do_replicate and replication config fields from original source during failback UUID swap --- .../services/tasks_runner_replication_final.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index caf2124bef..bec1f6b3a6 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -134,9 +134,14 @@ def _swap_failback_lvol_uuid(rep, failback_source_id): stale_uuid = new_lvol.get_id() - # Remove the old source lvol record so its DB key is free. + # Copy replication config from the original source before removing its record. try: old_lvol = db.get_lvol_by_id(failback_source_id) + new_lvol.do_replicate = old_lvol.do_replicate + new_lvol.replication_node_id = old_lvol.replication_node_id + new_lvol.replication_mode = old_lvol.replication_mode + new_lvol.replication_interval_min = old_lvol.replication_interval_min + new_lvol.replication_policy_id = old_lvol.replication_policy_id old_lvol.remove(db.kv_store) except KeyError: logger.warning( From 2bc3d76ba7ab6db2805ccfe63fd82a06340046de Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 26 Aug 2026 19:30:51 +0100 Subject: [PATCH 072/122] test: add secondary_node_id and tertiary_node_id to _EvictNode stub to match HA peer eviction changes --- simplyblock_core/test/test_replication_chain_completeness.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index c31691090f..df67128c1e 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -282,6 +282,9 @@ def nvmf_subsystem_remove_ns(self, nqn, nsid): class _EvictNode: + secondary_node_id = None + tertiary_node_id = None + def __init__(self, rpc): self._rpc = rpc From 92fdaa7688b0d3b00d8d91165f9424bdffebf20c Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 21:36:40 +0200 Subject: [PATCH 073/122] test: case 7 runs its 20 namespaced volumes on a 5-minute cadence 20 volumes on a one-minute cadence means 20 transfers a minute competing for the same hub. The lag settles around 240s -- above the 180s gate -- so the case spent 35 minutes oscillating between 216s and 285s and would have timed out in setup without ever reaching the fail-over it exists to test (run 20260826_205051). That is the workload's arithmetic, not a regression: the earlier attempts only cleared the gate because they got there before the backlog built. Cadence is now per case (create_volumes/set_cluster_replication/ ensure_replication_policy take interval_min, and a non-default cadence gets its own policy name so a 1-minute policy is never reused for it), and the readiness gate is derived from the cadence rather than fixed: lag_gate_for(interval) = 3 cadence periods, so case 7 waits on <= 900s. Relaxing only the gate would have hidden real lag instead. Co-Authored-By: Claude Fable 5 --- scripts/test_async_replication.py | 34 ++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/scripts/test_async_replication.py b/scripts/test_async_replication.py index a6e0333e30..a110af1dd8 100644 --- a/scripts/test_async_replication.py +++ b/scripts/test_async_replication.py @@ -350,7 +350,8 @@ def replication_states(mgmt_ip, key_path, lvol_uuids): """, replayable=True) -def wait_replication_caught_up(mgmt_ip, key_path, lvol_uuids, timeout=REPL_WAIT_TIMEOUT): +def wait_replication_caught_up(mgmt_ip, key_path, lvol_uuids, timeout=REPL_WAIT_TIMEOUT, + max_lag=None): """Wait until every volume is replicating steadily with a bounded lag. NOT outstanding_count == 0. `outstanding_count` counts internal snapshots @@ -364,7 +365,7 @@ def wait_replication_caught_up(mgmt_ip, key_path, lvol_uuids, timeout=REPL_WAIT_ keeping up; the residual delta is `replication-commit`'s job, which is documented to "minimize delta then fail the client over". """ - max_lag = MAX_LAG_SECONDS + max_lag = max_lag or MAX_LAG_SECONDS print(f"Waiting for replication to reach a steady state (lag <= {max_lag}s) on all volumes...") start = time.time() stable = 0 @@ -783,6 +784,8 @@ def ensure_replication_policy(mgmt_ip, key_path, from_cluster, target_name, mode plain one another case left behind. """ suffix = "_x%08x" % (hash(extra_flags) & 0xffffffff) if extra_flags else "" + if interval_min != REPL_INTERVAL_MIN: + suffix += f"_i{interval_min}" name = name or f"pol_{mode}_{target_name}{suffix}" for row in _replication_list(mgmt_ip, key_path, "policy", from_cluster): if row.get("Name") == name: @@ -795,7 +798,8 @@ def ensure_replication_policy(mgmt_ip, key_path, from_cluster, target_name, mode def set_cluster_replication(mgmt_ip, key_path, from_cluster, to_cluster, to_pool_uuid, - mode="migration", extra_flags="", policy_name=None): + mode="migration", extra_flags="", policy_name=None, + interval_min=REPL_INTERVAL_MIN): """Create the target + policy that let `from_cluster` replicate to `to_cluster`. Replication is NEVER started per volume any more: `volume replication-start` @@ -811,6 +815,7 @@ def set_cluster_replication(mgmt_ip, key_path, from_cluster, to_cluster, to_pool target = ensure_replication_target(mgmt_ip, key_path, from_cluster, to_cluster, to_pool_uuid) policy = ensure_replication_policy(mgmt_ip, key_path, from_cluster, target, mode, + interval_min=interval_min, name=policy_name, extra_flags=extra_flags) # PRODUCT GAP (bridge, delete once the readers consult the policy): @@ -1008,9 +1013,14 @@ def delete_test_volumes(mgmt_ip, key_path, pools): f"after {drain_polls * 10}s") +def lag_gate_for(interval_min): + """The steady-state lag a cadence can actually hold: 3 cadence periods.""" + return max(1, int(interval_min)) * 60 * 3 + + def create_volumes(mgmt_ip, key_path, src_uuid, pool, tgt_uuid, tgt_pool, mode, count=NUM_VOLUMES, prefix="replvol", size=VOL_SIZE, - extra_flags=""): + extra_flags="", interval_min=REPL_INTERVAL_MIN): """Create the test volumes already following a replication policy. The policy IS the start: `volume add --replication-policy` attaches it, and @@ -1019,7 +1029,7 @@ def create_volumes(mgmt_ip, key_path, src_uuid, pool, tgt_uuid, tgt_pool, mode, """ policy = set_cluster_replication(mgmt_ip, key_path, src_uuid, tgt_uuid, pool_uuid_of(mgmt_ip, key_path, tgt_pool), - mode=mode) + mode=mode, interval_min=interval_min) lvols = [] for i in range(count): name = f"{prefix}{i}" @@ -1649,6 +1659,13 @@ def test_case_6(meta): NS_VOLUMES = int(os.environ.get("NS_VOLUMES", "20")) NS_PER_SUBSYS = int(os.environ.get("NS_PER_SUBSYS", "10")) NS_VOL_SIZE = os.environ.get("NS_VOL_SIZE", "20G") +#: 20 volumes on a one-minute cadence means 20 transfers a minute +#: competing for the same hub: the lag settles at ~240s, above the +#: 180s gate, and the case times out in setup without ever reaching +#: the fail-over it exists to test (run 20260826_205051, lag +#: oscillating 216-285s for 35 minutes). Five minutes gives the same +#: coverage with a backlog the cluster can actually hold. +NS_INTERVAL_MIN = int(os.environ.get("NS_INTERVAL_MIN", "5")) PRESSURE_VOLUMES = int(os.environ.get("PRESSURE_VOLUMES", "2")) PRESSURE_VOL_SIZE = os.environ.get("PRESSURE_VOL_SIZE", "120G") @@ -1766,6 +1783,7 @@ def test_case_7(meta): lvols = create_volumes( mgmt_ip, key_path, src_uuid, src["pool"], tgt_uuid, tgt["pool"], mode="failover", count=NS_VOLUMES, prefix="nsvol", size=NS_VOL_SIZE, + interval_min=NS_INTERVAL_MIN, extra_flags=f"--namespaced True --max-namespace-per-subsys {NS_PER_SUBSYS}") idents = lvol_identities(mgmt_ip, key_path, lvols) @@ -1798,7 +1816,8 @@ def test_case_7(meta): for ip in assign: start_fio(ip, key_path, write_fio_jobfile(ip, key_path, mounts_by_client[ip], size="1G")) - wait_replication_caught_up(mgmt_ip, key_path, lvols, timeout=3600) + ns_gate = lag_gate_for(NS_INTERVAL_MIN) + wait_replication_caught_up(mgmt_ip, key_path, lvols, timeout=3600, max_lag=ns_gate) wait_data_replicated(mgmt_ip, key_path, lvols, baseline_ts, timeout=3600) print("Killing the source cluster (both nodes)...") @@ -1855,7 +1874,8 @@ def test_case_7(meta): pool_uuid_of(mgmt_ip, key_path, src["pool"])) for lv in tgt_lvols: failback(mgmt_ip, key_path, lv) - wait_replication_caught_up(mgmt_ip, key_path, tgt_lvols, timeout=3600) + wait_replication_caught_up(mgmt_ip, key_path, tgt_lvols, timeout=3600, + max_lag=ns_gate) for lv in tgt_lvols: run(mgmt_ip, key_path, f"{SBCTL} -d volume replication-commit {lv}") From ffc69b1f830ca74c96417cabae4d98319bf46ae7 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 21:53:06 +0200 Subject: [PATCH 074/122] test: an ssh -f timeout is inconclusive, not a launch failure The 60s budget covers the ssh HANDSHAKE, not the driver, but a busy management node can exceed it while the driver has already started. The wrapper then exited non-zero on two healthy runs (20260826_205051 and _214011), both of which were progressing normally on the node -- a false failure that hides real ones. On timeout the launcher now asks the node what happened: it polls for up to 150s for ~/repl_log naming THIS run and a live driver process, and only fails when neither appears. Co-Authored-By: Claude Fable 5 --- scripts/stage_and_run_repl_cases.py | 31 +++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/scripts/stage_and_run_repl_cases.py b/scripts/stage_and_run_repl_cases.py index 1b34f5f229..b0c5980065 100644 --- a/scripts/stage_and_run_repl_cases.py +++ b/scripts/stage_and_run_repl_cases.py @@ -82,10 +82,33 @@ def main(): # fd and using setsid is still needed so the driver survives the client # going away; without -f, ssh sat on the channel until it timed out (twice: # 2026-08-19 with nohup, 2026-08-20 with setsid) while the driver ran fine. - run(["ssh", "-f", *SSH_OPTS, f"ec2-user@{mgmt}", - f"cd ~ && setsid env {env_prefix}python3 -u test_async_replication.py {cases} " - f"> {remote_log} 2>&1 < /dev/null & echo $! > ~/repl_pid; " - f"echo ~/repl_cases_{ts}.log > ~/repl_log"], timeout=60) + # The 60s budget is for the ssh HANDSHAKE, not for the driver -- but a busy + # management node can exceed it while the driver has already started, and + # treating that as a launch failure reported a healthy run as dead twice + # (runs 20260826_205051 and _214011, both progressing normally while the + # wrapper exited non-zero). A timeout here is inconclusive, so ask the node + # what actually happened instead of guessing. + try: + run(["ssh", "-f", *SSH_OPTS, f"ec2-user@{mgmt}", + f"cd ~ && setsid env {env_prefix}python3 -u test_async_replication.py {cases} " + f"> {remote_log} 2>&1 < /dev/null & echo $! > ~/repl_pid; " + f"echo ~/repl_cases_{ts}.log > ~/repl_log"], timeout=60) + except subprocess.TimeoutExpired: + log("ssh -f exceeded its 60s budget; checking whether the driver started") + started = False + for _ in range(10): + time.sleep(15) + probe = ssh(mgmt, + f"test -f ~/repl_log && grep -q {ts} ~/repl_log && " + f"pgrep -f '[t]est_async_replication.py' >/dev/null " + f"&& echo STARTED || echo NOT_YET", check=False) + if "STARTED" in probe: + started = True + break + if not started: + raise RuntimeError( + f"ssh -f timed out AND no driver for {ts} is running on {mgmt}") + log("driver is running despite the ssh timeout") time.sleep(45) status = ssh(mgmt, "P=$(cat ~/repl_pid); L=$(cat ~/repl_log); " "echo \"pid=$P etime=$(ps -p $P -o etime= | tr -d ' ')\"; " From 19501bd449c3b1bb460300cc8970263b76ffca4f Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 21:56:11 +0200 Subject: [PATCH 075/122] fix: create the shared subsystem on a node that does not have it yet The create-vs-attach decision was 'return not lvol.namespace' -- read from the DB alone. The record says the volume SHARES a subsystem; it says nothing about whether THIS node has that subsystem yet. On the primary a wrong guess self-heals (the -32602 fallback re-resolves and retries), but a REPLICA has no fallback -- its nsid is dictated by the primary -- so the whole create/fail-over dies. Case 7, run 20260826_214011: the very FIRST namespaced fail-over failed with 'subsystem ... does not exist on 5198fb03'. The peer had never had that subsystem created, and every attempt to add a namespace to it was rejected. (Named outright by the diagnostic added in 90989c736 -- the previous runs only said 'Failed to add bdev to subsystem'.) A shared subsystem now probes the node itself with the nqn-FILTERED nvmf_get_subsystems -- not the full dump the docstring warns about -- and creates the subsystem when it is genuinely absent. A dedicated subsystem still short-circuits without an RPC, and a probe that raises falls back to the record's implication rather than deciding. Co-Authored-By: Claude Fable 5 --- .../controllers/lvol_controller.py | 21 +++++- .../test_replication_chain_completeness.py | 66 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 3aa350575b..9c30e8ee0b 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -1123,8 +1123,27 @@ def _resolve_namespaced_subsystem(lvol, rpc_client, snode): whose response grows with total lvol count, paid on EVERY create to catch a race that occurs at most once per max_namespaces creates. Trust the CP's own record and let the error path pay the dump only when it actually fires. + + EXCEPT that "someone else already created it" is only true PER NODE. The + record says the volume shares a subsystem; it says nothing about whether + THIS node has that subsystem yet. On the primary a wrong guess self-heals + (the -32602 fallback re-resolves and retries), but a REPLICA has no such + fallback -- its nsid is dictated by the primary -- so it fails the whole + create/fail-over with "subsystem does not exist on " (case 7, + run 20260826_214011: the very first namespaced fail-over died this way, + on a peer whose subsystem had never been created). So for a shared + subsystem, ask the node itself. The probe is the nqn-FILTERED + nvmf_get_subsystems, not the full dump this docstring warns about. """ - return not lvol.namespace + if not lvol.namespace: + return True # dedicated subsystem: always create + try: + return not rpc_client.subsystem_get(lvol.nqn) + except Exception as e: # noqa: BLE001 - probe must not decide + logger.warning("Could not probe subsystem %s on %s (%s); assuming it " + "exists, as the record implies", lvol.nqn, + snode.get_id()[:8], e) + return False def _fail_after_bdev(lvol, rpc_client, msg): diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index 4e38073ff5..8997177fe9 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -570,3 +570,69 @@ def test_eviction_still_uses_nsid_on_a_single_namespace_subsystem(): "uuid": "99999999-9999-9999-9999-999999999999"}]) lc._evict_stale_namespace(_CloneLvol(), _EvictNode(rpc)) assert rpc.removed == [("nqn.test:lvol:ORIG", 7)] + + +class _SubsysProbeRPC: + def __init__(self, existing_nqns): + self._existing = set(existing_nqns) + self.probed = [] + + def subsystem_get(self, nqn): + self.probed.append(nqn) + return {"nqn": nqn, "namespaces": []} if nqn in self._existing else None + + +class _ProbeNode: + def get_id(self): + return "NODE_PEER" + + +class _NsLvol: + nqn = "nqn.test:lvol:SHARED" + namespace = "SHARED" # truthy => shared/namespaced subsystem + + +class _DedicatedLvol: + nqn = "nqn.test:lvol:OWN" + namespace = "" # dedicated subsystem + + +def test_shared_subsystem_is_created_on_a_node_that_lacks_it(): + """Case 7, run 20260826_214011: the very first namespaced fail-over died + with 'subsystem does not exist on '. The create-vs-attach decision + was read from the DB alone -- which says the volume SHARES a subsystem, + not whether THIS node has one. The primary self-heals through the -32602 + fallback; a replica cannot (its nsid is fixed by the primary), so it must + create the subsystem when the node genuinely lacks it.""" + from simplyblock_core.controllers import lvol_controller as lc + rpc = _SubsysProbeRPC(existing_nqns=[]) # peer has nothing + assert lc._resolve_namespaced_subsystem(_NsLvol(), rpc, _ProbeNode()) is True + assert rpc.probed == ["nqn.test:lvol:SHARED"] + + +def test_shared_subsystem_is_reused_where_it_already_exists(): + from simplyblock_core.controllers import lvol_controller as lc + rpc = _SubsysProbeRPC(existing_nqns=["nqn.test:lvol:SHARED"]) + assert lc._resolve_namespaced_subsystem(_NsLvol(), rpc, _ProbeNode()) is False + + +def test_dedicated_subsystem_needs_no_probe(): + """A non-namespaced volume always creates its own -- do not spend an RPC.""" + from simplyblock_core.controllers import lvol_controller as lc + rpc = _SubsysProbeRPC(existing_nqns=[]) + assert lc._resolve_namespaced_subsystem(_DedicatedLvol(), rpc, _ProbeNode()) is True + assert rpc.probed == [] + + +def test_probe_failure_falls_back_to_the_record(): + """A probe that raises must not decide: assume the record is right + (attach), which is the pre-existing behaviour.""" + from simplyblock_core.controllers import lvol_controller as lc + + class _Boom: + def subsystem_get(self, nqn): + raise RuntimeError("rpc down") + + from simplyblock_core.controllers import lvol_controller + assert lvol_controller._resolve_namespaced_subsystem( + _NsLvol(), _Boom(), _ProbeNode()) is False From 5eded38780d92809a4ffb43cfa3359c33fd3d4c2 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 22:32:29 +0200 Subject: [PATCH 076/122] fix: a failed replica leg must leave nothing behind in the subsystem The rollback issued the leader-gated ASYNC delete on a replica, which a non-leader refuses (Deleting async lvol on non-leader lvs), so a failed replica leg left BOTH its bdev and its namespace on that node. The leftovers poison every later attempt: the peer's shared subsystem keeps namespaces at nsids the primary -- starting clean -- then hands to OTHER volumes, and the replica add is rejected with wanted nsid=1 ... holds=[(1, ), ...]. Case 7, run 20260826_221806 showed exactly that: 4 stale namespaces on the peer, nsid 1 occupied by a different uuid, first fail-over rejected. The node-aware subsystem fix 19501bd44 got the subsystem created there; this is the residue that fix then exposed. _fail_after_bdev now removes the namespace belonging to THIS attempt before tearing the stack down, and rolls a replica back with the SYNC delete a non-leader accepts. Every rollback site in add_lvol_on_node passes is_primary so the right semantics are used. Co-Authored-By: Claude Fable 5 --- .../controllers/lvol_controller.py | 44 ++++++++++++---- .../test_replication_chain_completeness.py | 52 +++++++++++++++++++ 2 files changed, 86 insertions(+), 10 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 9c30e8ee0b..edb12ba6a1 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -1146,15 +1146,39 @@ def _resolve_namespaced_subsystem(lvol, rpc_client, snode): return False -def _fail_after_bdev(lvol, rpc_client, msg): +def _fail_after_bdev(lvol, rpc_client, msg, is_primary=True): """Rollback an in-progress add_lvol_on_node after _create_bdev_stack has already produced a bdev/blob. Without this, a post-bdev-stack failure (a missing namespaced subsystem, a listener add error, an add_ns error) leaves the SPDK clone-blob in place, which then blocks the parent snapshot delete with "vbdev_lvol_destroy: ... has N clones". Logs but does not raise on - rollback failure so the caller still sees the original error.""" + rollback failure so the caller still sees the original error. + + ``is_primary`` decides HOW the bdev goes away. A REPLICA's blob is a + registration, not the leader's copy, and the leader-gated async delete is + refused there -- "Deleting async lvol on non-leader lvs" -- so the + rollback silently left the bdev AND its namespace behind. Those leftovers + poison the next attempt: the peer's shared subsystem keeps namespaces at + nsids the primary (starting clean) later hands to OTHER volumes, and every + subsequent fail-over is rejected with "wanted nsid=1 ... holds=[(1, ), ...]" -- 4 stale namespaces on the peer in case 7 run + 20260826_221806. Replicas therefore roll back with a SYNC delete, and the + namespace is dropped first so nothing of this attempt survives in the + subsystem. + """ try: - _remove_bdev_stack(lvol.bdev_stack[::-1], rpc_client) + try: + subsystem = rpc_client.subsystem_get(lvol.nqn) + for ns in ((subsystem or {}).get("namespaces") or []): + if ns.get("bdev_name") == lvol.top_bdev: + logger.info("rollback: removing namespace nsid=%s (%s) left " + "by the failed attempt", + ns.get("nsid"), lvol.top_bdev) + rpc_client.nvmf_subsystem_remove_ns(lvol.nqn, ns.get("nsid")) + except Exception: # noqa: BLE001 - best effort + logger.exception("rollback: could not clear the namespace for %s", + lvol.get_id()) + _remove_bdev_stack(lvol.bdev_stack[::-1], rpc_client, sync=not is_primary) lvol.status = LVol.STATUS_IN_DELETION lvol.write_to_db(DBController().kv_store) except Exception: @@ -1238,7 +1262,7 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0): ret, msg = _create_bdev_stack(lvol, snode, is_primary=is_primary) if not ret: - return _fail_after_bdev(lvol, rpc_client, msg) + return _fail_after_bdev(lvol, rpc_client, msg, is_primary=is_primary) db_controller = DBController() pool = db_controller.get_pool_by_id(lvol.pool_uuid) @@ -1248,7 +1272,7 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0): try: resolve_subsys = _resolve_namespaced_subsystem(lvol, rpc_client, snode) except Exception as e: - return _fail_after_bdev(lvol, rpc_client, str(e)) + return _fail_after_bdev(lvol, rpc_client, str(e), is_primary=is_primary) if resolve_subsys: min_cntlid = lvol_min_cntlid(0 if is_primary else secondary_index + 1) @@ -1334,7 +1358,7 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0): else: return _fail_after_bdev( lvol, rpc_client, - f"Failed to create listener for {lvol.get_id()}") + f"Failed to create listener for {lvol.get_id()}", is_primary=is_primary) elif iface.ip4_address and lvol.fabric == "tcp" and snode.active_tcp: logger.info("adding listener for %s on IP %s, fabric TCP port %s" % (lvol.nqn, iface.ip4_address, listener_port)) ret, err = rpc_client.nvmf_subsystem_add_listener( @@ -1345,7 +1369,7 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0): else: return _fail_after_bdev( lvol, rpc_client, - f"Failed to create listener for {lvol.get_id()}") + f"Failed to create listener for {lvol.get_id()}", is_primary=is_primary) logger.info("Add BDev to subsystem") # Cluster-consistent namespace IDs: the PRIMARY add lets the target @@ -1368,7 +1392,7 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0): lvol, rpc_client, f"Replica namespace add for {lvol.get_id()} has no primary-" f"assigned ns_id; refusing auto-assignment (divergent nsid " - f"maps across the shared subsystem's paths)") + f"maps across the shared subsystem's paths)", is_primary=is_primary) requested_nsid = lvol.ns_id ret, err = rpc_client.nvmf_subsystem_add_ns2( lvol.nqn, lvol.top_bdev, lvol.uuid, lvol.guid, nsid=requested_nsid) @@ -1388,7 +1412,7 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0): exclude_nqns={lvol.nqn}) except SubsystemCapacityError as e: logger.error(str(e)) - return _fail_after_bdev(lvol, rpc_client, str(e)) + return _fail_after_bdev(lvol, rpc_client, str(e), is_primary=is_primary) return add_lvol_on_node(lvol, snode, is_primary=is_primary, secondary_index=secondary_index) else: # A REPLICA add cannot re-claim a slot (its nsid is dictated by the @@ -1415,7 +1439,7 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0): logger.error("Namespace add rejected on %s for %s:%s", snode.get_id()[:8], lvol.get_id(), detail) return _fail_after_bdev( - lvol, rpc_client, "Failed to add bdev to subsystem" + detail) + lvol, rpc_client, "Failed to add bdev to subsystem" + detail, is_primary=is_primary) if is_primary: # Persist the target-assigned nsid; replicas re-add with exactly diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index 8997177fe9..91b18aa9b0 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -636,3 +636,55 @@ def subsystem_get(self, nqn): from simplyblock_core.controllers import lvol_controller assert lvol_controller._resolve_namespaced_subsystem( _NsLvol(), _Boom(), _ProbeNode()) is False + + +class _RollbackRPC: + def __init__(self, ns): + self.ns = list(ns) + self.removed_ns, self.deletes = [], [] + + def subsystem_get(self, nqn): + return {"nqn": nqn, "namespaces": self.ns} + + def nvmf_subsystem_remove_ns(self, nqn, nsid): + self.removed_ns.append(nsid) + self.ns = [n for n in self.ns if n.get("nsid") != nsid] + return True + + def get_bdevs(self, name): + return [{"name": name}] + + def delete_lvol(self, name, sync=False): + self.deletes.append((name, sync)) + return True, None + + +def test_replica_rollback_clears_its_namespace_and_syncs_the_delete(): + """Case 7, run 20260826_221806: a failed replica leg left BOTH its bdev + and its namespace on the peer, because the rollback issued the + leader-gated async delete a non-leader refuses ("Deleting async lvol on + non-leader lvs"). Four such leftovers accumulated on the peer's shared + subsystem, so the primary -- starting clean -- handed nsid 1 to a + different volume and every later fail-over was rejected. The rollback + must remove the namespace it added and delete the bdev synchronously.""" + from simplyblock_core.controllers import lvol_controller as lc + + class _Lvol: + nqn = "nqn.test:lvol:SHARED" + top_bdev = "LVS_1/LVOL_NEW" + bdev_stack = [{"type": "bdev_lvol_clone", "name": "LVS_1/LVOL_NEW"}] + status = "" + def get_id(self): + return "LV_NEW" + def write_to_db(self, *a, **kw): + pass + + rpc = _RollbackRPC([ + {"nsid": 1, "bdev_name": "LVS_1/LVOL_OTHER"}, # someone else's + {"nsid": 5, "bdev_name": "LVS_1/LVOL_NEW"}, # this attempt's + ]) + ok, _msg = lc._fail_after_bdev(_Lvol(), rpc, "boom", is_primary=False) + assert ok is False + assert rpc.removed_ns == [5], "must drop only THIS attempt's namespace" + assert rpc.deletes and all(sync for _n, sync in rpc.deletes), \ + "a replica rollback must use the SYNC delete a non-leader accepts" From fd88c2d2f195120908118a787385dd57a8c17d0f Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 22:51:20 +0200 Subject: [PATCH 077/122] fix: roll back every node that received a fail-over copy, and pass ids Two defects in the same rollback, both silent: 1. A peer failure removed the copy from the PRIMARY only. With three target nodes, a failure on the tertiary left the SECONDARY holding the namespace. The next sibling's primary then auto-assigned an nsid the peer had already given to another volume, and its replica add was rejected -- "wanted nsid=2 ... holds=[(2, )]" (case 7, run 20260826_223631). Divergence between the paths of one shared subsystem is exactly what the primary-assigns/replicas-reuse rule exists to prevent, and the rollback was creating it. 2. It passed the LVol and StorageNode RECORDS to delete_lvol_from_node(lvol_id, node_id), whose "except KeyError: return True" swallowed the type mismatch -- so the rollback reported success while deleting nothing at all. The fail-back clone had the same call. The fail-over clone now tracks every node it placed the copy on and removes it from all of them (sync on the replicas, which is what a non-leader accepts), and both sites pass ids. Co-Authored-By: Claude Fable 5 --- .../controllers/lvol_controller.py | 43 ++++++++++++++++--- .../test_replication_chain_completeness.py | 23 ++++++++++ 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index edb12ba6a1..9caf3fefc6 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -3527,6 +3527,7 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps # Expose the volume on the secondary and tertiary target nodes too (HA), # so connect_lvol returns all client paths. + placed_nodes = [target_node] for peer_id in [target_node.secondary_node_id, target_node.tertiary_node_id]: if not peer_id: continue @@ -3546,12 +3547,31 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps lvol_bdev, error = add_lvol_on_node(new_lvol, peer_node, is_primary=False) if error: logger.error(error) - # remove lvol from primary - ret = delete_lvol_from_node(new_lvol, target_node) - if not ret: - logger.error("") + # Roll back EVERY node that already carries this copy, not just the + # primary. With three target nodes a failure on the tertiary used + # to leave the SECONDARY holding the namespace while the primary + # was cleaned, so the next sibling's primary auto-assigned an nsid + # the peer had already given to someone else -- and its replica add + # was rejected with "wanted nsid=2 ... holds=[(2, )]" + # (case 7, run 20260826_223631). + # + # The ids also matter: this used to pass the LVol and StorageNode + # OBJECTS to delete_lvol_from_node(lvol_id, node_id), whose + # `except KeyError: return True` swallowed the mismatch -- so the + # rollback reported success while deleting nothing at all. + for node in placed_nodes: + try: + if not delete_lvol_from_node( + new_lvol.get_id(), node.get_id(), + sync=node.get_id() != target_node.get_id()): + logger.error("rollback: could not remove %s from %s", + new_lvol.get_id(), node.get_id()[:8]) + except Exception: + logger.exception("rollback: removing %s from %s raised", + new_lvol.get_id(), node.get_id()[:8]) db_controller.release_lvol_ns_slot(new_lvol) return None, error + placed_nodes.append(peer_node) return new_lvol, None @@ -4274,10 +4294,19 @@ def replicate_lvol_on_source_cluster(lvol_id, cluster_id=None, pool_uuid=None): lvol_bdev, error = add_lvol_on_node(new_lvol, secondary_node, is_primary=False) if error: logger.error(error) - # remove lvol from primary - ret = delete_lvol_from_node(new_lvol, source_node) + # IDs, not objects: delete_lvol_from_node(lvol_id, node_id) hits + # `except KeyError: return True` when handed the records, so this + # rollback reported success while deleting nothing -- leaving the + # primary's namespace behind to collide with the next attempt. + try: + ret = delete_lvol_from_node(new_lvol.get_id(), source_node.get_id()) + except Exception: + logger.exception("rollback: removing %s from %s raised", + new_lvol.get_id(), source_node.get_id()[:8]) + ret = False if not ret: - logger.error("") + logger.error("rollback: could not remove %s from %s", + new_lvol.get_id(), source_node.get_id()[:8]) db_controller.release_lvol_ns_slot(new_lvol) return False, error diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index 91b18aa9b0..bdcc5ee2c6 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -688,3 +688,26 @@ def write_to_db(self, *a, **kw): assert rpc.removed_ns == [5], "must drop only THIS attempt's namespace" assert rpc.deletes and all(sync for _n, sync in rpc.deletes), \ "a replica rollback must use the SYNC delete a non-leader accepts" + + +def test_failover_rollback_covers_every_placed_node_with_ids(): + """Case 7, run 20260826_223631: a peer add failure rolled back only the + PRIMARY, so a third target node's failure left the SECONDARY holding the + namespace. The next sibling's primary then auto-assigned an nsid the peer + had already given away, and its replica add was rejected with + 'wanted nsid=2 ... holds=[(2, )]'. Worse, the rollback + passed the LVol/StorageNode OBJECTS to delete_lvol_from_node(lvol_id, + node_id), whose 'except KeyError: return True' swallowed the mismatch -- + so it reported success while deleting nothing.""" + import inspect + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc._create_target_lvol_clone) + assert "placed_nodes" in src, "rollback must track every node that got the copy" + assert "for node in placed_nodes:" in src + assert "new_lvol.get_id(), node.get_id()" in src, "must pass ids, not records" + assert "delete_lvol_from_node(new_lvol, target_node)" not in src + + # the fail-back clone had the same object-vs-id bug + whole = inspect.getsource(lc) + assert "delete_lvol_from_node(new_lvol," not in whole, \ + "no rollback may hand records to an id-taking function" From 6739e1ac2951aac3e657d1f932582873027652f7 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 23:30:49 +0200 Subject: [PATCH 078/122] Claim the fail-over copy's nsid on the target instead of auto-assigning Soak case 7 failed six times on one shape: 20 volumes across 2 shared subsystems, fail-over, and an add_ns rejected on an HA peer -- wanted nsid=1 max_namespaces=10 holds=[(1,531e4060) ... (7,ebdd3e0d)] The peer held the whole group while the copy's primary held none of it, so the primary auto-assigned 1 and the peer already had 1. Two defects: 1. The sibling-affinity pick added earlier ran against the ADVISORY nqn from _resolve_lvol_subsystem. claim_lvol_ns_slot's transaction recounts with concurrent creates visible and may join the lvol to a DIFFERENT subsystem, leaving it in a group whose other members replicate elsewhere. The pick is now re-derived after the claim (_realign_replication_node_after_claim). 2. Auto-assignment on the primary is only safe when the primary sees the whole subsystem. _claim_target_nsid now claims from the union of what every node of the target HA set actually holds and hands the same number to the primary and every replica, so a split group cannot collide even if one is reintroduced by a node that was down at placement time. It returns 0 -- auto-assign, unchanged behaviour -- when the subsystem exists nowhere yet or any node is unreadable, since claiming against a partial view is the bug being fixed. The nsid is NOT carried over from the source cluster: it is local to a subsystem's HA set and clients resolve their paths through connect_lvol. test_failover_target asserted the source's number; it now asserts the claim, and its add_lvol_on_node fake -- which never assigned an nsid at all -- was made faithful to the real primary add. The nsid-consistency fake likewise answered "7" whatever was requested, hiding a claim/persist mismatch. 13 new unit tests for the claim and the realignment; 1895 pass. --- img_hashes.txt | 172 +++++++++++++ .../controllers/lvol_controller.py | 160 +++++++++++- simplyblock_core/test/test_failover_target.py | 16 +- .../test_replication_chain_completeness.py | 17 +- .../test/test_target_nsid_claim.py | 198 +++++++++++++++ tests/unit/test_lvol_nsid_consistency.py | 25 +- xfer_ut.c | 239 ++++++++++++++++++ 7 files changed, 805 insertions(+), 22 deletions(-) create mode 100644 img_hashes.txt create mode 100644 simplyblock_core/test/test_target_nsid_claim.py create mode 100644 xfer_ut.c diff --git a/img_hashes.txt b/img_hashes.txt new file mode 100644 index 0000000000..40effed125 --- /dev/null +++ b/img_hashes.txt @@ -0,0 +1,172 @@ +d41d8cd98f00b204e9800998ecf8427e simplyblock_core/__init__.py +921aad174688b985519e0c72d1d6d0e7 simplyblock_core/cluster_ops.py +eb941f1f1c0fbc49bf6c569fef7d31f8 simplyblock_core/constants.py +d41d8cd98f00b204e9800998ecf8427e simplyblock_core/controllers/__init__.py +a440b45b21c4f14f2e40240dbf2d9bc5 simplyblock_core/controllers/backup_controller.py +eb0275929e5d1dd522763c44b3b40fda simplyblock_core/controllers/backup_events.py +43f023313267a2434a2029deda471f61 simplyblock_core/controllers/cluster_events.py +d41d8cd98f00b204e9800998ecf8427e simplyblock_core/controllers/cluster_expansion/__init__.py +38cba0e6dd4e9374b37c3fe2cd0c6b8f simplyblock_core/controllers/cluster_expansion/executor.py +7e3ed535a62cdcd5e087044502437269 simplyblock_core/controllers/cluster_expansion/orchestrator.py +5388e317669ef6ed861b882f75522e27 simplyblock_core/controllers/cluster_expansion/planner.py +cb317fde8a270cda8bba528d0e4bf286 simplyblock_core/controllers/cluster_expansion/preconditions.py +1dd883da6a0e99bad07ea12cb6ed2727 simplyblock_core/controllers/device_controller.py +49318a96a77b96cd2438b1389a927600 simplyblock_core/controllers/device_events.py +9563d5d608189dc3ee1cc7dc6575d8c8 simplyblock_core/controllers/events_controller.py +96466463b95d74b79cce2a398949956a simplyblock_core/controllers/fdb_backup_controller.py +ef01b95df3f3c501bbd0df35753eeea0 simplyblock_core/controllers/health_controller.py +583833a31731ee88d6c5e4ed9ed8b2f9 simplyblock_core/controllers/host_auth.py +122238a7f90215505a85c9fb4e1d06d7 simplyblock_core/controllers/lvol_controller.py +8ed20e8ada22c39ad2b40dc1aeb91436 simplyblock_core/controllers/lvol_events.py +d7261581a5308b13e1fc830910b4202d simplyblock_core/controllers/mgmt_events.py +4495ba0571fed37306e9d6684e993e87 simplyblock_core/controllers/migration_bdev_ops.py +7a7a80de2a192802cfab2a4d52a619c4 simplyblock_core/controllers/migration_controller.py +faf2454ca99acce7a0b8b12f2a0a1c2e simplyblock_core/controllers/migration_events.py +3329fdc76d009ab22043a4adfdd31162 simplyblock_core/controllers/ops_gate.py +16b973f2a2a44a20ae1d76329572ed33 simplyblock_core/controllers/pool_controller.py +1c79832885997693b642ea125465c224 simplyblock_core/controllers/pool_events.py +10fbc91df11af77ad23a83c9d0dbe66b simplyblock_core/controllers/qos_controller.py +8ef27ba9a004a7f365f7fdc810796cfa simplyblock_core/controllers/replication_policy_controller.py +a4a06dc366b026908f02fc21502792d2 simplyblock_core/controllers/snapshot_controller.py +99f7f70f3a3c0e909aac03802f5006b3 simplyblock_core/controllers/snapshot_events.py +9c08682f202955ed566ed96d9535c829 simplyblock_core/controllers/storage_events.py +441da86657062e418f150da66efa5390 simplyblock_core/controllers/tasks_controller.py +c5cc66cb98bde4768092cdb1a0131ae4 simplyblock_core/controllers/tasks_events.py +b8d69479e568944c6d26da7e9b61bb56 simplyblock_core/controllers/tcp_ports_events.py +91613b33cc8eecfd8e36488c42b1efcb simplyblock_core/db_controller.py +84a808ea97b86654e5547b13472faf48 simplyblock_core/distr_controller.py +212bb9c4ba9198b35be212d5dfd89601 simplyblock_core/exceptions.py +c577aec17bc3aa0428af060e5cbbd8f0 simplyblock_core/fw_api_client.py +3fc3f4fc5f60993d7b138fa9b4adbcad simplyblock_core/jm_raid.py +74e7b278ad56e4adbfc6f5105f23fa70 simplyblock_core/kms/__init__.py +b21dfac624800ae4695edb6dead53a7c simplyblock_core/kms/_base.py +eaa1c679331e65af0a8533856173c486 simplyblock_core/kms/_exceptions.py +707d37b161f048797f4be19654345709 simplyblock_core/kms/_fdb.py +7f917ea95665ee16c0ffd1c08ddbd9c2 simplyblock_core/kms/_hcp.py +fcd90d3a9b7af72c76369eb6fd67e955 simplyblock_core/mgmt_node_ops.py +d41d8cd98f00b204e9800998ecf8427e simplyblock_core/models/__init__.py +f4a6dff300d400be9d9983ea6bcd02b0 simplyblock_core/models/backup.py +1991ef4a4e6f043d90f08232ccd6998a simplyblock_core/models/base_model.py +b7a0b089043fcd1bbc4114cf866f2a98 simplyblock_core/models/cluster.py +41c4c62321b77e7381709d39345be74c simplyblock_core/models/events.py +c1d95ed3eb8531a3736a74ba9d522a16 simplyblock_core/models/hub_cooldown.py +84a5e67573439d80fc8217bf6956e86d simplyblock_core/models/hublvol.py +aaa0f08f3724359abfccac3a6f912651 simplyblock_core/models/iface.py +bad6cffda3d269eac4528a9fbe3bceca simplyblock_core/models/job_schedule.py +fbf8c48e05b78ad0cfadd0d0b6952e3e simplyblock_core/models/lvol_migration.py +93e027ba3737c5d3a66a76c4bef1eb99 simplyblock_core/models/lvol_migration_group.py +7233db81712cd3c02277cf88b2c5509d simplyblock_core/models/lvol_model.py +79a31818d478ecce83c63050ece62065 simplyblock_core/models/lvstore_lock.py +5e8aeda97b87058699292ed0fd3603b9 simplyblock_core/models/mgmt_node.py +b1076bf14319f93df9906a1ff77e9920 simplyblock_core/models/nvme_device.py +16e6498b741406a6ca52e531a2a34611 simplyblock_core/models/pool.py +d350fce28254cf53da607facfdab1cce simplyblock_core/models/port_stat.py +76b86dcb5850954d0b78928a772e4616 simplyblock_core/models/qos.py +82eb6d6570452eaa7052d84dd5a2b47e simplyblock_core/models/replication.py +7e34fc1f2577c045667418687e82796d simplyblock_core/models/restart_lock.py +b17735bfa9fb6099d948ca1ab954a907 simplyblock_core/models/snapshot.py +930d082cbb3d3f2f9076837e38a5283d simplyblock_core/models/stats.py +afadba47b1b755674ecfb2edbae2dafd simplyblock_core/models/storage_node.py +f820ca63541b36fc34dc938a1419da53 simplyblock_core/prom_client.py +9ef4b502175a7d5cc32f6ea82d110dfa simplyblock_core/release_upgrades/__init__.py +cba45f129303eca7295214719b56df4b simplyblock_core/release_upgrades/jc_compression_upgrade.py +ae7b5ff638b16236f39f38ecd52563e5 simplyblock_core/rpc_client.py +fb8278ed7174bd9ca12b44a545b108e6 simplyblock_core/scripts/__init__.py +649b3072b7625d6bbb732a9e66566dc2 simplyblock_core/scripts/collect_logs.py +d41d8cd98f00b204e9800998ecf8427e simplyblock_core/scripts/helpers/__init__.py +455c553a6598cd0178ecb36433f07d10 simplyblock_core/scripts/helpers/nvme_disconnect_by_ip.py +a371a8ebc56236f463fa49c6674faa51 simplyblock_core/scripts/restore_fdb_from_kvfiledump.py +4c20bb18b5f6e04234455dfe8112b3d8 simplyblock_core/services/__init__.py +89ea5203a611d9eb7ba04d1879aa840c simplyblock_core/services/cap_monitor.py +c74e062d9cd028e98503b07fd3a7ee04 simplyblock_core/services/capacity_and_stats_collector.py +9310b22ccdc7d6f5d850e932a6d6098d simplyblock_core/services/device_monitor.py +3b982e4e58d489fc2cc861ee9f5d9e88 simplyblock_core/services/health_check_service.py +8773b45be91a706f4dd50bf6ba14ff87 simplyblock_core/services/hub_controller_manager.py +f22a5aa9946183ec33061a253166b9f0 simplyblock_core/services/lvol_monitor.py +5af944ba3a09848e24984cc560012c00 simplyblock_core/services/lvol_stat_collector.py +e286e508f7f34ea28b6894df29b6819e simplyblock_core/services/main_distr_event_collector.py +db8fd72f53396244adcd18f1e0348655 simplyblock_core/services/mgmt_node_monitor.py +66a30a4a6386a08eb720530341b3ee0e simplyblock_core/services/new_device_discovery.py +1da9113a14a2e5c72438f8174d4d1301 simplyblock_core/services/replication_final_step.py +f49cf6719f484f43c6f543f09a59356d simplyblock_core/services/snapshot_monitor.py +0103fd4d054878e22b450350e72cfd04 simplyblock_core/services/snapshot_replication.py +a8e8f2ac6baaec3a2781cbc774886036 simplyblock_core/services/spdk_http_proxy_server.py +d21aa89347671199e4c4f93db11067e6 simplyblock_core/services/storage_node_monitor.py +f5a0397d343cfc2148eb86192c9e8ebd simplyblock_core/services/tasks_cluster_status.py +659ee98597f9f0f8e9607209a3abebd1 simplyblock_core/services/tasks_runner_backup.py +10a97c1171915bbaa645c649cff88da2 simplyblock_core/services/tasks_runner_backup_merge.py +ed954f1ff465008efb0222c09bb68deb simplyblock_core/services/tasks_runner_batch_migration.py +b4048ca32a99789e6ed76ae16c834985 simplyblock_core/services/tasks_runner_cluster_expand.py +13faa65654c71c55bd8f3a19e34cba15 simplyblock_core/services/tasks_runner_failed_migration.py +4894a2715a481f116ee2f092fe963c8b simplyblock_core/services/tasks_runner_fdb_backup.py +d7f6eb76fb513c62e8a3d5abd5192fa1 simplyblock_core/services/tasks_runner_jc_comp.py +1c1ac93f66e096d52addb6373db50de2 simplyblock_core/services/tasks_runner_lvol_migration.py +9763fc859165deda77a7433f48dc3825 simplyblock_core/services/tasks_runner_migration.py +a45d0d3c7e952a7598ff15c91e90c6af simplyblock_core/services/tasks_runner_new_dev_migration.py +2cd5d1df8cfda73e0eaf1b9e1d96b8b9 simplyblock_core/services/tasks_runner_node_add.py +2b57c422d936551428c28546ce9d7a02 simplyblock_core/services/tasks_runner_node_removal.py +5015f0ed7eb5d465a2e61f0d414a8f19 simplyblock_core/services/tasks_runner_port_allow.py +75434f8a8951e5dabee51f20bd8d8c3a simplyblock_core/services/tasks_runner_replication_final.py +dfafaedfafff3b69802d3f2c9f680390 simplyblock_core/services/tasks_runner_restart.py +6f7c8d54b32d8a3cb60f18eff49a1217 simplyblock_core/services/tasks_runner_sync_lvol_del.py +a3c56c87ab44a01ba71bcb12d778d23e simplyblock_core/settings.py +c0fb4c93ff5b5d32005d63bcdb813d0c simplyblock_core/shell_utils.py +03cda4cb8275cd62cb084da29d8a09af simplyblock_core/snapshot_retention.py +f1bc395dbd5223ae85a7efe1fee9b9cb simplyblock_core/snode_client.py +b12f37e2116565a4801be071c6a2fc10 simplyblock_core/storage_node_ops.py +cc67f35ed45f89fa61d44fc74069898b simplyblock_core/test/conftest.py +4e36ab773d8359225f213cecaf429aab simplyblock_core/test/test_ana_namespace_scope.py +a77e777d1924e7964788ab3e1624b9b3 simplyblock_core/test/test_async_delete_poll.py +93ee14f68659902bb5eb33efabdb7039 simplyblock_core/test/test_backpressure_safety_valve.py +a99d0eb13088fa96da90263687413d29 simplyblock_core/test/test_chain_lock_scope.py +4ebd88bc0d2f073317bebe6398a92023 simplyblock_core/test/test_connect_path_resolution.py +0fdd9f09d496833cbf4450237868243f simplyblock_core/test/test_del_sync_gate_release.py +5b6b39afa4d99b0f3185f7d071178f28 simplyblock_core/test/test_delete_absent_is_done.py +1739de1f19e94c6af8368e0d169be4bf simplyblock_core/test/test_failback_chain_parent.py +b2f945c83409c635074eb2179036e043 simplyblock_core/test/test_failover_clone_race.py +6cd40edbbf823f2d5e8482c3cd39f720 simplyblock_core/test/test_failover_snapshot_selection.py +f202c98493ee5e1e182dd902b40eccb2 simplyblock_core/test/test_failover_target.py +cef1f86b3ef92e3de9a181555edbd587 simplyblock_core/test/test_internal_snapshot_backpressure.py +d394ebf8b8c7e5451817fa4ec07d000d simplyblock_core/test/test_internal_snapshot_scheduler.py +51ab8b315c56ddebb6524922d1f05c03 simplyblock_core/test/test_internal_subsystem_exemption.py +cbc0dd39dd808c076c3b6a0de6e97eea simplyblock_core/test/test_jc_dual_node.py +98a25a8b439113f8730b070e32a68151 simplyblock_core/test/test_lvstore_mutation_lock.py +494ffa54376289a7a3746d003291a785 simplyblock_core/test/test_models.py +468f1f2ee94262aea2a960e1ba79a519 simplyblock_core/test/test_pool_id_or_name.py +8eee2578bf5bffcd5b682212499e8b61 simplyblock_core/test/test_replication_backlog_chain.py +4df442ad73560df92de19c3f02a8499a simplyblock_core/test/test_replication_backlog_destination.py +858ad1c5544632989bf9b3a71ced0019 simplyblock_core/test/test_replication_backlog_reporting.py +87fc1665b2d89826c316e5b9adc6dc06 simplyblock_core/test/test_replication_chain_completeness.py +d6b2259d2fd45d8b672c58dfdcf333d4 simplyblock_core/test/test_replication_chain_predecessor.py +142c27321f4107cb8dcd6f80cebc6d71 simplyblock_core/test/test_replication_commit.py +27c3901c3b17efe712cfeb0441b750ed simplyblock_core/test/test_replication_destination_resolution.py +2aef47c5d33aa862031042362abb4552 simplyblock_core/test/test_replication_failback.py +cbcedc215d05ce9d13e5aa1297da65c4 simplyblock_core/test/test_replication_failback_pool.py +efdcf72be12304a4d6ea97570052b564 simplyblock_core/test/test_replication_final_step.py +ec169938b233d87ed30f5356867b1a02 simplyblock_core/test/test_replication_health.py +428cce0e1f68056e361ec7e7affa4861 simplyblock_core/test/test_replication_leader_follow.py +7fd80e9a3b36f0cbe9f8204d67fccbb7 simplyblock_core/test/test_replication_models.py +674100457e09a93981f8d1bbeca581b4 simplyblock_core/test/test_replication_policies.py +142f851c9f1a973a3b69ad6587ab084f simplyblock_core/test/test_replication_progress.py +c5d56733f8e64b4934a83dfa773dd9e8 simplyblock_core/test/test_replication_relationship_and_delete_source.py +9deb75aea6214d0dc6ad81527332acb2 simplyblock_core/test/test_replication_runner_resilience.py +722b01fc4a5a39f1165d01d365fdccfa simplyblock_core/test/test_replication_source_leader.py +d060d8d46b38d9fca593db7206f0d33a simplyblock_core/test/test_shutdown_task_guard.py +78de0df795baaf32863f4e6916a34967 simplyblock_core/test/test_snapshot_chain_target.py +d6e30188e4eb37f5f54e72c031ca1ccd simplyblock_core/test/test_snapshot_instance_handoff.py +98c1f729b8a91aadcfe5bd15ac5466b3 simplyblock_core/test/test_snapshot_monitor_live_clone.py +7088fd00c8673421cb9d100ec7612b16 simplyblock_core/test/test_snapshot_retention.py +9a95119047fe465944e67c499cd925b3 simplyblock_core/test/test_sync_delete_peer.py +33b8ddf70012155e20620d95b495dfcd simplyblock_core/test/test_tasks_runner_replication_final.py +bc53f292fae277b435d6b1952cdc6daf simplyblock_core/test/test_transfer_hub_heal.py +0c9a3e9dc14315ea372a1f631802c354 simplyblock_core/test/test_utils.py +9e4fa1baf48be1c833f34da643b1a81f simplyblock_core/utils/__init__.py +fc87695c06fb33ca74293dae12f6bb15 simplyblock_core/utils/dial_backoff.py +4d255423494ff9d96f38b2cae3175195 simplyblock_core/utils/helpers.py +70c1bf1532768cc2ff86d061b4b8d17b simplyblock_core/utils/hublvol_reconnect.py +7dcbc009ce235b4df483d38d5b34c13b simplyblock_core/utils/nvme.py +b0b3ac01438e9bbea6b7eeb678f4f52f simplyblock_core/utils/pci.py +9d2effb926da5d0e36be1e86778bbdde simplyblock_core/utils/port_block.py +b45bc54582bfbe1bea24580cf20e8010 simplyblock_core/utils/secrets.py +57af18bc39674a7a16cbb7bc9a8684b5 simplyblock_core/utils/ttl_cache.py +dbf1184940d69c0d5bf7a1f22f000b63 simplyblock_core/workers/cleanup_foundationdb.py diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 9caf3fefc6..911756199b 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -334,6 +334,51 @@ def validate_aes_xts_keys(key1: str, key2: str) -> Tuple[bool, str]: return True, "" +def _sibling_replication_node(lvol, cl, all_lvols=None): + """Target node an already-replicating sibling of ``lvol``'s subsystem uses. + + Namespaced siblings MUST replicate to the same target node. A fail-over + copy keeps the volume's NQN, so every volume of one shared subsystem lands + in the SAME subsystem on the target. When the group is split across two + target primaries, each primary auto-assigns nsids from only its own subset + and the HA peers -- which hold the whole group -- reject the collision + (soak case 7: peer held nsids 1..7 while a split-off primary asked for 1). + + Returns "" when no sibling has a target node yet. + """ + if not (getattr(lvol, "namespaced", False) or lvol.max_namespace_per_subsys > 1): + return "" + for lv in (all_lvols or DBController().get_lvols(cl.get_id())): + if (lv.nqn == lvol.nqn and lv.get_id() != lvol.get_id() + and getattr(lv, "replication_node_id", "")): + return lv.replication_node_id + return "" + + +def _realign_replication_node_after_claim(lvol, cl): + """Re-derive the target node once the AUTHORITATIVE subsystem is known. + + ``_resolve_lvol_subsystem`` is advisory: it may hand out a fresh standalone + nqn while ``claim_lvol_ns_slot``'s transaction -- recounting occupancy with + concurrent creates visible -- then joins the lvol to an EXISTING shared + subsystem instead. The target node was picked against the advisory nqn, so + such an lvol joins a group whose other members replicate elsewhere, and it + is exactly that member whose fail-over copy collides on the peer. Recheck + against the nqn that was actually persisted. + """ + if not getattr(lvol, "replication_node_id", ""): + return + sibling_node_id = _sibling_replication_node(lvol, cl) + if sibling_node_id and sibling_node_id != lvol.replication_node_id: + logger.info( + "LVol %s was placed in subsystem %s by the claim transaction " + "(not the advisory pick); moving its replication node from %s to " + "%s to match the siblings already in that subsystem", + lvol.lvol_name, lvol.nqn, lvol.replication_node_id, sibling_node_id) + lvol.replication_node_id = sibling_node_id + lvol.write_to_db() + + def _resolve_lvol_subsystem(lvol, host_node, cl, namespaced, all_lvols, internal=False): """ADVISORY pre-check of the subsystem pick for a new lvol — fails the @@ -690,13 +735,7 @@ def add_lvol_ha(name, size, host_id_or_name, ha_type, pool_id_or_name, use_comp= # collide when a sibling's nsid is already taken there (soak case 7, # run 20260824_215758: 14 of 20 namespaces failed over, the 15th # died in add_ns). - sibling_node_id = "" - if getattr(lvol, "namespaced", False) or lvol.max_namespace_per_subsys > 1: - for lv in (all_lvols or db_controller.get_lvols(cl.get_id())): - if (lv.nqn == lvol.nqn and lv.get_id() != lvol.get_id() - and getattr(lv, "replication_node_id", "")): - sibling_node_id = lv.replication_node_id - break + sibling_node_id = _sibling_replication_node(lvol, cl, all_lvols) if sibling_node_id: logger.info( f"LVol {lvol.lvol_name} shares subsystem {lvol.nqn} with an " @@ -801,6 +840,8 @@ def add_lvol_ha(name, size, host_id_or_name, ha_type, pool_id_or_name, use_comp= logger.error(str(e)) return False, str(e) + _realign_replication_node_after_claim(lvol, cl) + if ha_type == "single": if host_node.status == StorageNode.STATUS_ONLINE: lvol_bdev, error = add_lvol_on_node(lvol, host_node) @@ -1385,7 +1426,11 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0): # nsid (ns_id == 0, e.g. a drain-queued registration firing early) # must fail loudly instead of guessing. if is_primary: - requested_nsid = None + # 0/unset means "let SPDK auto-assign", which is what an ordinary + # create wants. A fail-over copy instead arrives with an nsid the + # CONTROL PLANE claimed across the whole target HA set, and the + # primary must use exactly that one (see _claim_target_nsid). + requested_nsid = lvol.ns_id or None else: if not lvol.ns_id: return _fail_after_bdev( @@ -1445,6 +1490,14 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0): # Persist the target-assigned nsid; replicas re-add with exactly # this value, so it must never be overwritten by a replica's # (identical) response. + if requested_nsid and int(ret) != int(requested_nsid): + # The node accepted the add but at a DIFFERENT number than the + # control plane claimed. The replicas would then be told the + # claimed one and collide. Trust what the node actually did. + logger.warning( + "Node %s placed %s at nsid %s, not the claimed %s; the " + "replicas will follow the node", + snode.get_id(), lvol.get_id(), ret, requested_nsid) lvol.ns_id = int(ret) if not is_primary: @@ -3464,9 +3517,81 @@ def replication_stop(lvol_id, delete=False, from_policy=False): return True +def _claim_target_nsid(db_controller, new_lvol, target_node): + """Pick the nsid a fail-over copy takes on the target HA set, or 0. + + The primary used to auto-assign, which is only safe when the primary sees + the whole shared subsystem. It does not when the group is split across two + target primaries: each counts only its own subset, hands out an nsid the + HA PEER already gave to a sibling, and the peer's add_ns is rejected + ("wanted nsid=1 ... holds=[(1, ) ... (7, ...)]", soak case 7). + Splitting is prevented at create time now, but a node that was down when + the group was placed can still reintroduce it, so do not depend on it: + claim the nsid from the UNION of what every node of the target HA set + actually holds, and give the same number to the primary and the replicas. + + The nsid does NOT have to match the source cluster's -- clients resolve + their paths through connect_lvol -- it only has to be consistent across + the paths of THIS subsystem. + + Returns 0 when the subsystem exists nowhere yet (nothing to collide with, + so ordinary auto-assignment is correct). + """ + node_ids = [target_node.get_id()] + for peer_id in [target_node.secondary_node_id, target_node.tertiary_node_id]: + if peer_id and peer_id not in node_ids: + node_ids.append(peer_id) + + occupied, seen_subsystem, max_ns = set(), False, 0 + for node_id in node_ids: + try: + node = db_controller.get_storage_node_by_id(node_id) + except KeyError: + continue + if node.status != StorageNode.STATUS_ONLINE: + continue + try: + subsystem = node.rpc_client().subsystem_get(new_lvol.nqn) + except Exception as e: + # Unreadable node: claiming against a partial view is exactly the + # bug being fixed, so fall back to auto-assignment rather than + # inventing a number that may already be taken there. + logger.warning("Cannot read subsystem %s on node %s (%s); leaving " + "the nsid to auto-assignment", new_lvol.nqn, node_id, e) + return 0 + if not subsystem: + continue + seen_subsystem = True + max_ns = max(max_ns, subsystem.get("max_namespaces") or 0) + for ns in (subsystem.get("namespaces") or []): + # A namespace belonging to THIS copy is evicted right before the + # add, so its slot is free to reuse. + if ns.get("uuid") == new_lvol.uuid: + continue + if ns.get("nsid"): + occupied.add(int(ns["nsid"])) + + if not seen_subsystem: + return 0 + + limit = max_ns or (new_lvol.max_namespace_per_subsys or 0) or (max(occupied) + 1) + for nsid in range(1, limit + 1): + if nsid not in occupied: + logger.info("Claimed nsid %d for fail-over copy %s in subsystem %s " + "(occupied on the target HA set: %s)", + nsid, new_lvol.get_id(), new_lvol.nqn, sorted(occupied)) + return nsid + # Full: let the add fail with the node's own diagnostic rather than + # silently picking a colliding number here. + logger.error("Subsystem %s is full on the target HA set (%d namespaces); " + "no nsid to claim for %s", + new_lvol.nqn, len(occupied), new_lvol.get_id()) + return 0 + + def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snapshot): """Create a writable clone of *lvol* on *target_node* (primary + online HA - peers) from *snapshot*, preserving the original NQN/ns_id. + peers) from *snapshot*, preserving the original NQN. Shared by fail-over (replicate_lvol_on_target_cluster) and migration-commit (replication_commit). Returns (new_lvol, error). The new lvol is left in @@ -3493,10 +3618,13 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps new_lvol.top_bdev = f"{new_lvol.lvs_name}/{new_lvol.lvol_bdev}" new_lvol.snapshot_name = snapshot.snap_bdev new_lvol.status = LVol.STATUS_IN_CREATION - # Preserve the ORIGINAL subsystem NQN and namespace id: the client must - # reconnect to the SAME NQN/NS on the target cluster — only the IP/port - # differ. new_lvol is a deep copy of lvol, so nqn/ns_id are already - # identical; do NOT rewrite the NQN with the target cluster's prefix. + # Preserve the ORIGINAL subsystem NQN: the client must reconnect to the + # SAME NQN on the target cluster — only the IP/port differ. new_lvol is a + # deep copy of lvol, so the nqn is already identical; do NOT rewrite it + # with the target cluster's prefix. The NSID is a different matter: it is + # local to a subsystem's own HA set, clients resolve their paths through + # connect_lvol, and the source's number may already be taken on the + # target — so it is re-claimed below rather than carried over. new_lvol.bdev_stack = [ { @@ -3512,6 +3640,12 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps if new_lvol.crypto_bdev: new_lvol.bdev_stack.append({"type": "crypto"}) + # The deep copy carries the SOURCE cluster's nsid, which means nothing on + # the target. Replace it with one claimed across the whole target HA set + # (0 = nothing there yet, auto-assign) BEFORE the record is written, so the + # primary and every replica add the namespace at the same number. + new_lvol.ns_id = _claim_target_nsid(db_controller, new_lvol, target_node) + new_lvol.write_to_db(db_controller.kv_store) _evict_stale_namespace(new_lvol, target_node) diff --git a/simplyblock_core/test/test_failover_target.py b/simplyblock_core/test/test_failover_target.py index 74a17d0787..1e8188cf4c 100644 --- a/simplyblock_core/test/test_failover_target.py +++ b/simplyblock_core/test/test_failover_target.py @@ -124,6 +124,11 @@ def _rep_write(self, kv=None): def _add_lvol_on_node(new_lvol, node, is_primary=True, **kw): add_calls.append((node.get_id(), is_primary)) + if is_primary: + # Faithful to the real primary add, which persists the nsid the + # target actually used: the control plane's claim when there was + # one, the target's own pick (lowest free) otherwise. + new_lvol.ns_id = new_lvol.ns_id or 1 return ({"uuid": "BDEV-UUID", "driver_specific": {"lvol": {"blobid": 123}}}, None) monkeypatch.setattr(lvol_controller, "add_lvol_on_node", _add_lvol_on_node) @@ -159,16 +164,21 @@ def test_failover_preserves_nqn_ns_and_returns_paths(monkeypatch, patched): result = lvol_controller.replicate_lvol_on_target_cluster("LV1") - # Same NQN + namespace as the original volume. + # Same NQN as the original volume -- the client reconnects to the SAME + # subsystem, only the IP/port differ. The nsid is NOT carried over: it is + # claimed on the target HA set (nothing there yet here, so the target + # picks 1). Cross-cluster nsid equality is not required -- clients resolve + # their paths through connect_lvol -- and insisting on the source's number + # is what collided with a sibling already holding it (soak case 7). assert result["nqn"] == "nqn.orig:lvol:LV1" - assert result["ns_id"] == 7 + assert result["ns_id"] == 1 assert len(result["connection_strings"]) == 1 rep = patched["rep"] assert rep.state == LVolReplication.STATE_FAILED_OVER assert rep.direction == LVolReplication.DIRECTION_TO_TARGET assert rep.target_nqn == "nqn.orig:lvol:LV1" - assert rep.target_ns_id == 7 + assert rep.target_ns_id == 1 assert rep.source_cluster_id == "CL_src" assert rep.target_cluster_id == "CL_tgt" # Source volume flipped to non-source after fail-over. diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index bdcc5ee2c6..be5fb4ac0f 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -471,11 +471,22 @@ def test_namespaced_siblings_replicate_to_the_same_target_node(): import inspect from simplyblock_core.controllers import lvol_controller as lc src = inspect.getsource(lc.add_lvol_ha) - assert "sibling_node_id" in src, "namespaced siblings must share a replication node" + assert "_sibling_replication_node" in src, "namespaced siblings must share a replication node" pick = src.index("_get_next_3_nodes(replication_cluster_id") - check = src.index("sibling_node_id") + check = src.index("_sibling_replication_node") assert check < pick, "the sibling lookup must precede the capacity-based pick" - assert "lv.nqn == lvol.nqn" in src, "siblings are identified by shared NQN" + + # ...and it must run AGAIN after claim_lvol_ns_slot. That transaction is + # what authoritatively decides the subsystem: the earlier lookup only saw + # the ADVISORY pick from _resolve_lvol_subsystem, so a volume the + # transaction rehomed into an existing shared subsystem would otherwise + # keep a target node chosen for a subsystem it is no longer in -- which is + # how the group was still split in run 20260826_230358 (peer held nsids + # 1..7, the split-off primary asked for 1). + claim = src.index("claim_lvol_ns_slot") + realign = src.index("_realign_replication_node_after_claim") + assert claim < realign, "the target node must be re-derived from the subsystem the CLAIM chose" + assert "lv.nqn == lvol.nqn" in inspect.getsource(lc._sibling_replication_node), "siblings are identified by shared NQN" def test_clone_register_confirms_the_bdev_before_add_ns(): diff --git a/simplyblock_core/test/test_target_nsid_claim.py b/simplyblock_core/test/test_target_nsid_claim.py new file mode 100644 index 0000000000..6fa6a8515f --- /dev/null +++ b/simplyblock_core/test/test_target_nsid_claim.py @@ -0,0 +1,198 @@ +"""A fail-over copy's nsid is claimed by the CONTROL PLANE, not auto-assigned. + +Soak case 7 failed six times on the same shape. Twenty volumes share two +subsystems; their fail-over copies must all land in ONE subsystem per group on +the target cluster. When the group is split across two target primaries, each +primary counts only the namespaces it holds itself, hands out an nsid the HA +PEER already gave to a sibling, and the peer's add_ns is rejected: + + Namespace add rejected ... [node a3220281 wanted nsid=1 max_namespaces=10 + holds=[(1, 531e4060-...), (2, b63ed6f1-...), ... (7, ebdd3e0d-...)]] + +Two independent defects produced that split, and both are covered here: + + * ``_resolve_lvol_subsystem`` is only ADVISORY -- ``claim_lvol_ns_slot``'s + transaction may join the lvol to a different subsystem than the one the + target node was picked against, so the pick is re-derived afterwards. + * the nsid itself is claimed from the UNION of what every node of the target + HA set holds, so even a split group cannot collide. +""" +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core.controllers import lvol_controller +from simplyblock_core.models.storage_node import StorageNode + +NQN = "nqn.2023-02.io.simplyblock:cl:lvol:shared" + + +def _lvol(uuid, nqn=NQN, repl_node="", max_ns=10): + lv = MagicMock(name="lvol-" + uuid) + lv.uuid = uuid + lv.nqn = nqn + lv.get_id.return_value = uuid + lv.replication_node_id = repl_node + lv.max_namespace_per_subsys = max_ns + lv.namespaced = True + lv.lvol_name = uuid + return lv + + +def _node(node_id, namespaces=None, exists=True, status=StorageNode.STATUS_ONLINE, + max_ns=10, raises=False): + n = MagicMock(name="node-" + node_id) + n.get_id.return_value = node_id + n.status = status + n.secondary_node_id = "" + n.tertiary_node_id = "" + rpc = MagicMock() + if raises: + rpc.subsystem_get.side_effect = RuntimeError("connection refused") + else: + rpc.subsystem_get.return_value = ( + {"nqn": NQN, "max_namespaces": max_ns, + "namespaces": namespaces or []} if exists else None) + n.rpc_client.return_value = rpc + return n + + +def _db(nodes): + db = MagicMock() + by_id = {n.get_id.return_value: n for n in nodes} + + def _get(node_id): + if node_id not in by_id: + raise KeyError(node_id) + return by_id[node_id] + + db.get_storage_node_by_id.side_effect = _get + return db + + +class TestClaimTargetNsid(unittest.TestCase): + + def test_absent_subsystem_leaves_auto_assignment(self): + primary = _node("P", exists=False) + nsid = lvol_controller._claim_target_nsid( + _db([primary]), _lvol("copy-1"), primary) + self.assertEqual(nsid, 0, "nothing to collide with -- SPDK assigns") + + def test_claims_across_the_union_not_just_the_primary(self): + """The exact case-7 shape: peer holds 1..7, primary holds none.""" + peer = _node("Q", namespaces=[ + {"nsid": i, "uuid": "sibling-%d" % i} for i in range(1, 8)]) + primary = _node("P", namespaces=[]) + primary.secondary_node_id = "Q" + nsid = lvol_controller._claim_target_nsid( + _db([primary, peer]), _lvol("copy-8"), primary) + self.assertEqual( + nsid, 8, + "the primary holds nothing but the peer holds 1..7; auto-assigning " + "1 here is what the peer rejected") + + def test_reuses_the_lowest_free_slot(self): + peer = _node("Q", namespaces=[{"nsid": 1, "uuid": "a"}, + {"nsid": 3, "uuid": "b"}]) + primary = _node("P", namespaces=[{"nsid": 1, "uuid": "a"}]) + primary.secondary_node_id = "Q" + self.assertEqual( + lvol_controller._claim_target_nsid( + _db([primary, peer]), _lvol("copy-x"), primary), 2) + + def test_own_namespace_is_not_an_obstacle(self): + """A retry evicts this copy's own namespace, so its slot is free.""" + primary = _node("P", namespaces=[{"nsid": 1, "uuid": "copy-me"}, + {"nsid": 2, "uuid": "other"}]) + self.assertEqual( + lvol_controller._claim_target_nsid( + _db([primary]), _lvol("copy-me"), primary), 1) + + def test_offline_peer_does_not_shrink_the_claim(self): + """An offline peer holds no namespaces the add can collide with.""" + peer = _node("Q", namespaces=[{"nsid": 1, "uuid": "a"}], + status=StorageNode.STATUS_OFFLINE) + primary = _node("P", namespaces=[{"nsid": 1, "uuid": "a"}, + {"nsid": 2, "uuid": "b"}]) + primary.secondary_node_id = "Q" + self.assertEqual( + lvol_controller._claim_target_nsid( + _db([primary, peer]), _lvol("copy-y"), primary), 3) + + def test_unreadable_node_falls_back_rather_than_guessing(self): + peer = _node("Q", raises=True) + primary = _node("P", namespaces=[{"nsid": 1, "uuid": "a"}]) + primary.secondary_node_id = "Q" + self.assertEqual( + lvol_controller._claim_target_nsid( + _db([primary, peer]), _lvol("copy-z"), primary), 0, + "claiming against a partial view is the bug, not the fix") + + def test_full_subsystem_defers_to_the_node_diagnostic(self): + primary = _node("P", max_ns=3, namespaces=[ + {"nsid": i, "uuid": "s%d" % i} for i in range(1, 4)]) + self.assertEqual( + lvol_controller._claim_target_nsid( + _db([primary]), _lvol("copy-full"), primary), 0) + + +class TestSiblingAffinity(unittest.TestCase): + + def test_sibling_in_the_same_subsystem_sets_the_target(self): + cl = MagicMock() + lv = _lvol("new") + siblings = [_lvol("old", repl_node="T1"), + _lvol("other", nqn="nqn:x", repl_node="T2")] + self.assertEqual( + lvol_controller._sibling_replication_node(lv, cl, siblings), "T1") + + def test_other_subsystems_are_not_siblings(self): + cl = MagicMock() + lv = _lvol("new") + others = [_lvol("elsewhere", nqn="nqn:other", repl_node="T2")] + self.assertEqual( + lvol_controller._sibling_replication_node(lv, cl, others), "") + + def test_non_namespaced_lvol_has_no_affinity(self): + cl = MagicMock() + lv = _lvol("new", max_ns=1) + lv.namespaced = False + self.assertEqual( + lvol_controller._sibling_replication_node( + lv, cl, [_lvol("old", repl_node="T1")]), "") + + +class TestRealignAfterClaim(unittest.TestCase): + """The advisory pick and the transaction can disagree about the subsystem.""" + + def setUp(self): + self.cl = MagicMock() + patcher = patch.object(lvol_controller, "DBController") + self.db = patcher.start().return_value + self.addCleanup(patcher.stop) + + def test_moves_to_the_siblings_target_when_the_claim_rehomed_it(self): + lv = _lvol("new", repl_node="T2") + self.db.get_lvols.return_value = [_lvol("old", repl_node="T1")] + lvol_controller._realign_replication_node_after_claim(lv, self.cl) + self.assertEqual(lv.replication_node_id, "T1", + "a volume the transaction put in this subsystem must " + "replicate where the subsystem already replicates") + lv.write_to_db.assert_called_once() + + def test_no_write_when_the_pick_already_agrees(self): + lv = _lvol("new", repl_node="T1") + self.db.get_lvols.return_value = [_lvol("old", repl_node="T1")] + lvol_controller._realign_replication_node_after_claim(lv, self.cl) + self.assertEqual(lv.replication_node_id, "T1") + lv.write_to_db.assert_not_called() + + def test_non_replicated_lvol_is_untouched(self): + lv = _lvol("new", repl_node="") + self.db.get_lvols.return_value = [_lvol("old", repl_node="T1")] + lvol_controller._realign_replication_node_after_claim(lv, self.cl) + self.assertEqual(lv.replication_node_id, "") + lv.write_to_db.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_lvol_nsid_consistency.py b/tests/unit/test_lvol_nsid_consistency.py index f825bfec01..3ac931c510 100644 --- a/tests/unit/test_lvol_nsid_consistency.py +++ b/tests/unit/test_lvol_nsid_consistency.py @@ -29,8 +29,14 @@ class _Base(unittest.TestCase): def setUp(self): self.rpc = MagicMock(name="rpc") - self.rpc.nvmf_subsystem_add_ns2.return_value = ("7", None) - self.rpc.nvmf_subsystem_add_ns.return_value = "7" + # SPDK echoes the nsid it actually used: the requested one when the + # caller asked for a specific number, its own pick otherwise. A fake + # that always answers "7" hides a mismatch between the claim and the + # persisted value. + self.rpc.nvmf_subsystem_add_ns2.side_effect = ( + lambda *a, **kw: (str(kw.get("nsid") or 7), None)) + self.rpc.nvmf_subsystem_add_ns.side_effect = ( + lambda *a, **kw: str(kw.get("nsid") or 7)) self.rpc.get_bdevs.return_value = [ {"uuid": "lvol-bdev-uuid", "driver_specific": {"lvol": {"blobid": 33}}}] @@ -63,7 +69,7 @@ def setUp(self): patch.object(lvol_controller, "DBController"), patch.object( lvol_controller, "_fail_after_bdev", - side_effect=lambda lvol, rpc, msg: (False, msg)), + side_effect=lambda lvol, rpc, msg, is_primary=True: (False, msg)), patch( "simplyblock_core.controllers.migration_controller" ".get_active_migration_for_nqn", @@ -87,6 +93,19 @@ def test_primary_autoassigns_and_persists(self): self.assertEqual(self.lvol.ns_id, 7, "the assigned nsid must be persisted for the replicas") + def test_primary_honours_a_control_plane_claim(self): + """A fail-over copy arrives with an nsid claimed across the target HA + set; the primary must add it at exactly that number instead of + auto-assigning from its own partial view (soak case 7).""" + self.lvol.ns_id = 8 + ret, err = lvol_controller.add_lvol_on_node(self.lvol, self.snode) + self.assertIsNone(err) + kwargs = self.rpc.nvmf_subsystem_add_ns2.call_args.kwargs + self.assertEqual(kwargs.get("nsid"), 8, + "the primary must use the control-plane claim") + self.assertEqual(self.lvol.ns_id, 8, + "the claim must survive the add for the replicas") + def test_replica_reuses_primary_assigned_nsid(self): self.lvol.ns_id = 7 ret, err = lvol_controller.add_lvol_on_node( diff --git a/xfer_ut.c b/xfer_ut.c new file mode 100644 index 0000000000..ee993540c7 --- /dev/null +++ b/xfer_ut.c @@ -0,0 +1,239 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (C) 2026 Simplyblock. + * + * Unit tests for the transfer dispatch path (xfer_replication + + * helper_xfer_poller). Written for the fix that removed the one-item-per- + * poller-tick serialization: the dispatcher must fill the whole in-flight + * window in one pass, and the helper must drain its ready ring. + * + * Reuses lvol_ut.c wholesale (its stub set is what makes lvol.c link); + * its main() is renamed away and only the xfer suite is registered here, + * so this binary is independent of the state of the legacy suites. + */ +#define main lvol_ut_main_disabled +#include "../lvol.c/lvol_ut.c" +#undef main + +static struct spdk_lvol g_xfer_lvol; +static struct spdk_lvol_store g_xfer_lvs; + +static struct spdk_lvs_xfer * +make_xfer(int cluster_batch, uint32_t num_clusters, const int *allocated) +{ + struct spdk_lvs_xfer *xfer = calloc(1, sizeof(*xfer)); + SPDK_CU_ASSERT_FATAL(xfer != NULL); + + memset(&g_xfer_lvol, 0, sizeof(g_xfer_lvol)); + memset(&g_xfer_lvs, 0, sizeof(g_xfer_lvs)); + g_xfer_lvol.lvol_store = &g_xfer_lvs; + snprintf(g_xfer_lvol.name, sizeof(g_xfer_lvol.name), "xfer_ut_lvol"); + + xfer->lvol = &g_xfer_lvol; + xfer->type = XFER_REPLICATE_SNAPSHOT; + xfer->state = XFER_STATE_TRANSFER_CLUSTERS; + xfer->final_step = false; + xfer->cluster_batch = cluster_batch; + xfer->page_size = 16; + xfer->page_per_cluster = 4; + xfer->num_clusters = num_clusters; + xfer->clusters = calloc(num_clusters, sizeof(uint64_t)); + SPDK_CU_ASSERT_FATAL(xfer->clusters != NULL); + for (uint32_t i = 0; i < num_clusters; i++) { + xfer->clusters[i] = allocated[i] ? 0xABCD0000 + i : 0; + } + + xfer->free_ring = spdk_ring_create(SPDK_RING_TYPE_MP_MC, cluster_batch, + SPDK_ENV_NUMA_ID_ANY); + xfer->ready_ring = spdk_ring_create(SPDK_RING_TYPE_MP_MC, cluster_batch, + SPDK_ENV_NUMA_ID_ANY); + SPDK_CU_ASSERT_FATAL(xfer->free_ring != NULL && xfer->ready_ring != NULL); + + xfer->reqs = calloc(cluster_batch, sizeof(*xfer->reqs)); + SPDK_CU_ASSERT_FATAL(xfer->reqs != NULL); + for (int i = 0; i < cluster_batch; i++) { + xfer->reqs[i].payload = calloc(1, xfer->page_size * xfer->page_per_cluster); + SPDK_CU_ASSERT_FATAL(xfer->reqs[i].payload != NULL); + xfer->reqs[i].xfer = xfer; + xfer->reqs[i].type = xfer->type; + } + xfer_fill_queue(xfer, cluster_batch); + xfer->timeout = spdk_get_ticks(); + return xfer; +} + +static void +free_xfer(struct spdk_lvs_xfer *xfer) +{ + for (int i = 0; i < xfer->cluster_batch; i++) { + free(xfer->reqs[i].payload); + } + free(xfer->reqs); + spdk_ring_free(xfer->free_ring); + spdk_ring_free(xfer->ready_ring); + free(xfer->clusters); + free(xfer); +} + +static uint32_t +drain_ready(struct spdk_lvs_xfer *xfer, uint64_t *offsets, uint32_t max) +{ + struct spdk_lvs_xfer_req *req; + uint32_t n = 0; + + while (n < max && spdk_ring_dequeue(xfer->ready_ring, (void **)&req, 1) == 1) { + if (offsets != NULL) { + offsets[n] = req->offset; + } + n++; + } + return n; +} + +/* The fix itself: one call must fill the whole window, not one cluster. */ +static void +xfer_fills_whole_window(void) +{ + const int alloc[20] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + struct spdk_lvs_xfer *xfer = make_xfer(8, 20, alloc); + + int count = xfer_replication(xfer); + + CU_ASSERT(count == 8); + CU_ASSERT(xfer->outstanding_io == 8); + CU_ASSERT(xfer->idx == 8); + CU_ASSERT(xfer->hold_idx == 8); + CU_ASSERT(drain_ready(xfer, NULL, 16) == 8); + + free_xfer(xfer); +} + +/* A window larger than the remaining work stops at the work, not the window. */ +static void +xfer_stops_at_last_cluster(void) +{ + const int alloc[5] = {1, 1, 1, 0, 0}; + struct spdk_lvs_xfer *xfer = make_xfer(8, 5, alloc); + + int count = xfer_replication(xfer); + + CU_ASSERT(count == 3); + CU_ASSERT(xfer->outstanding_io == 3); + CU_ASSERT(drain_ready(xfer, NULL, 16) == 3); + + free_xfer(xfer); +} + +/* Sparse maps: unallocated clusters are skipped, offsets match the map. */ +static void +xfer_skips_unallocated_clusters(void) +{ + const int alloc[10] = {1, 0, 0, 1, 0, 1, 0, 0, 0, 1}; + struct spdk_lvs_xfer *xfer = make_xfer(8, 10, alloc); + uint64_t offsets[8] = {0}; + + int count = xfer_replication(xfer); + + CU_ASSERT(count == 4); + CU_ASSERT(drain_ready(xfer, offsets, 8) == 4); + CU_ASSERT(offsets[0] == 0 * xfer->page_per_cluster); + CU_ASSERT(offsets[1] == 3 * xfer->page_per_cluster); + CU_ASSERT(offsets[2] == 5 * xfer->page_per_cluster); + CU_ASSERT(offsets[3] == 9 * xfer->page_per_cluster); + + free_xfer(xfer); +} + +/* Completions recycle requests: the NEXT pass continues where the map left + * off, and the batch transition still fires when everything succeeded. */ +static void +xfer_second_pass_continues_and_completes(void) +{ + const int alloc[6] = {1, 1, 1, 1, 1, 1}; + struct spdk_lvs_xfer *xfer = make_xfer(4, 6, alloc); + struct spdk_lvs_xfer_req *req; + + CU_ASSERT(xfer_replication(xfer) == 4); + CU_ASSERT(xfer->hold_idx == 4); + + /* complete the four in-flight requests the way local/remote completion + * does: status DONE, back on the free ring */ + for (int i = 0; i < 4; i++) { + CU_ASSERT(spdk_ring_dequeue(xfer->ready_ring, (void **)&req, 1) == 1); + req->status = XFER_REQ_STATUS_DONE; + xfer->success_cnt++; + CU_ASSERT(spdk_ring_enqueue(xfer->free_ring, (void **)&req, 1, NULL) == 1); + } + + CU_ASSERT(xfer_replication(xfer) == 2); + CU_ASSERT(xfer->idx == 6); + CU_ASSERT(xfer->hold_idx == 6); + + /* complete the tail, then the state machine must leave the transfer */ + for (int i = 0; i < 2; i++) { + CU_ASSERT(spdk_ring_dequeue(xfer->ready_ring, (void **)&req, 1) == 1); + req->status = XFER_REQ_STATUS_DONE; + xfer->success_cnt++; + CU_ASSERT(spdk_ring_enqueue(xfer->free_ring, (void **)&req, 1, NULL) == 1); + } + xfer_replication(xfer); + CU_ASSERT(xfer->state == XFER_STATE_DONE); + + free_xfer(xfer); +} + +/* The helper must drain its ready ring, not take one request per tick. */ +static void +helper_drains_ready_ring(void) +{ + const int alloc[8] = {1, 1, 1, 1, 1, 1, 1, 1}; + struct spdk_lvs_xfer *xfer = make_xfer(8, 8, alloc); + struct spdk_lvs_poll_group lpg; + struct remote_lvol_info rmt; + + CU_ASSERT(xfer_replication(xfer) == 8); + + memset(&lpg, 0, sizeof(lpg)); + TAILQ_INIT(&lpg.rmt_lvols); + memset(&rmt, 0, sizeof(rmt)); + rmt.status = true; + rmt.type = XFER_REPLICATE_SNAPSHOT; + rmt.desc = (struct spdk_bdev_desc *)0x1; /* only null-checked */ + rmt.channel = (struct spdk_io_channel *)0x1; /* only null-checked */ + rmt.md_channel = (struct spdk_io_channel *)0x1; + rmt.ready_ring = xfer->ready_ring; + rmt.free_ring = xfer->free_ring; + TAILQ_INSERT_TAIL(&lpg.rmt_lvols, &rmt, entry); + + helper_xfer_poller(&lpg); + + CU_ASSERT(rmt.outstanding_io == 8); + CU_ASSERT(drain_ready(xfer, NULL, 8) == 0); /* ring fully drained */ + + free_xfer(xfer); +} + +int +main(int argc, char **argv) +{ + CU_pSuite suite = NULL; + unsigned int num_failures; + + CU_initialize_registry(); + + suite = CU_add_suite("lvol_xfer", NULL, NULL); + CU_ADD_TEST(suite, xfer_fills_whole_window); + CU_ADD_TEST(suite, xfer_stops_at_last_cluster); + CU_ADD_TEST(suite, xfer_skips_unallocated_clusters); + CU_ADD_TEST(suite, xfer_second_pass_continues_and_completes); + CU_ADD_TEST(suite, helper_drains_ready_ring); + + allocate_threads(1); + set_thread(0); + + num_failures = spdk_ut_run_tests(argc, argv, NULL); + + free_threads(); + CU_cleanup_registry(); + return num_failures; +} From 904d9a652f61dc603c4f1ccd52e38c9508a496a2 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 26 Aug 2026 23:56:12 +0200 Subject: [PATCH 079/122] Keep a shared subsystem on one target primary from BOTH entry points Run 20260826_233417: the nsid claim worked -- all ten copies got distinct nsids 1..10 -- yet case 7 still failed, because the ten copies of one subsystem landed on THREE target primaries: a3220281: 1,2,3,6 50037447: 1,2,4,5,7,8,9,10 5198fb03: 3..10 No node advertised the whole subsystem. A shared NQN whose paths expose different namespace sets is incoherent to the client kernel, which showed 8 of 10 namespaces on one client and 0 on the other. The sibling rule was only in add_lvol_ha. `volume add --replication-policy` does not go through it: replication_start attaches the policy and picked the target node purely by capacity. It now consults _sibling_replication_node first, and keeps the subsystem whole even when that node is the snapshot origin it would rather avoid -- placement is an optimisation, a coherent subsystem is not. _create_target_lvol_clone re-checks at creation time as the last line of defence: the target node was chosen when the policy was attached, so it can be stale by the time the copy is built. If the subsystem already lives somewhere else in that cluster the copy follows it, and if that home node is offline the copy fails loudly rather than silently splitting the subsystem. Two test fakes were missing state their real counterparts have (StorageNode .cluster_id, DBController.get_lvols) and were completed rather than worked around. 1900 pass. --- .../controllers/lvol_controller.py | 85 +++++++++++++++++-- .../test_replication_chain_completeness.py | 25 ++++++ .../test/test_target_nsid_claim.py | 62 ++++++++++++++ 3 files changed, 165 insertions(+), 7 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 911756199b..18188945dd 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -3283,13 +3283,37 @@ def replication_start(lvol_id, replication_cluster_id=None, mode=None, interval_ if not replication_cluster_id: logger.error(f"Cluster: {snode.cluster_id} not replicated") return False - random_nodes = _get_next_3_nodes(replication_cluster_id, lvol.size) - for r_node in random_nodes: - if r_node.get_id() not in excluded_nodes: - logger.info(f"Replicating on node: {r_node.get_id()}") - lvol.replication_node_id = r_node.get_id() - lvol.write_to_db() - break + # Volumes sharing a subsystem MUST replicate to the same target node + # (see _sibling_replication_node). add_lvol_ha enforces that for a + # volume created with a replication cluster; this is the OTHER entry + # point -- attaching a policy -- and it used to pick purely by + # capacity, which is how soak case 7 kept splitting one 10-namespace + # subsystem across three target primaries (run 20260826_233417: nsids + # 1,2,3,6 on one node, 1,2,4,5,7..10 on a second, 3..10 on a third). + # No node then advertised the whole subsystem, and the client's kernel + # showed 0 of the 10 namespaces on one of its paths. + sibling_node_id = _sibling_replication_node(lvol, cluster) + if sibling_node_id and sibling_node_id in excluded_nodes: + # Keeping a subsystem whole outranks the clone/origin placement + # preference, which is only an optimisation. + logger.warning( + "LVol %s must replicate to %s to keep subsystem %s whole, " + "though that node is the origin of its snapshot", + lvol.get_id(), sibling_node_id, lvol.nqn) + if sibling_node_id: + logger.info( + "Replicating on node %s: it is where subsystem %s already " + "replicates", sibling_node_id, lvol.nqn) + lvol.replication_node_id = sibling_node_id + lvol.write_to_db() + else: + random_nodes = _get_next_3_nodes(replication_cluster_id, lvol.size) + for r_node in random_nodes: + if r_node.get_id() not in excluded_nodes: + logger.info(f"Replicating on node: {r_node.get_id()}") + lvol.replication_node_id = r_node.get_id() + lvol.write_to_db() + break if not lvol.replication_node_id: logger.error(f"Replication node not found for lvol: {lvol.get_id()}") return False @@ -3589,6 +3613,27 @@ def _claim_target_nsid(db_controller, new_lvol, target_node): return 0 +def _subsystem_home_node(db_controller, nqn, cluster_id): + """Node in *cluster_id* that already hosts copies of subsystem *nqn*, or "". + + Every path of one shared subsystem must advertise the SAME namespaces, so + all of its volumes have to live on one primary and its HA peers. Whichever + node got there first owns the subsystem for that cluster. + """ + for lv in db_controller.get_lvols(): + if lv.nqn != nqn or lv.status == LVol.STATUS_IN_DELETION: + continue + if getattr(lv, "deleted", False) or not lv.node_id: + continue + try: + node = db_controller.get_storage_node_by_id(lv.node_id) + except KeyError: + continue + if node.cluster_id == cluster_id: + return lv.node_id + return "" + + def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snapshot): """Create a writable clone of *lvol* on *target_node* (primary + online HA peers) from *snapshot*, preserving the original NQN. @@ -3598,6 +3643,32 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps STATUS_IN_CREATION with lvol_uuid/blobid populated; the caller is responsible for setting STATUS_ONLINE and any replication bookkeeping. """ + # Last line of defence for the one-subsystem-one-primary invariant. The + # target node was chosen when the policy was attached, long before this + # copy is created; if anything put a sibling somewhere else in the + # meantime, following the stale pick would build a subsystem whose paths + # expose different namespace sets -- which clients resolve by showing + # NONE of them. Placement is negotiable, a coherent subsystem is not. + home = _subsystem_home_node(db_controller, lvol.nqn, target_node.cluster_id) + if home and home != target_node.get_id(): + try: + home_node = db_controller.get_storage_node_by_id(home) + except KeyError: + home_node = None + if home_node and home_node.status == StorageNode.STATUS_ONLINE: + logger.warning( + "Subsystem %s already lives on node %s in cluster %s; placing " + "the copy of %s there instead of %s to keep it whole", + lvol.nqn, home, target_node.cluster_id, lvol.get_id(), + target_node.get_id()) + target_node = home_node + else: + return None, ( + f"Subsystem {lvol.nqn} already has copies on node {home}, " + f"which is not online; placing this copy on " + f"{target_node.get_id()} would split the subsystem across " + f"primaries and hide its namespaces from clients") + new_lvol = copy.deepcopy(lvol) new_lvol.uuid = str(uuid.uuid4()) new_lvol.create_dt = str(datetime.now()) diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index be5fb4ac0f..fdfd459785 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -394,6 +394,7 @@ def __init__(self, nid, secondary="", tertiary=""): self._id, self.secondary_node_id, self.tertiary_node_id = nid, secondary, tertiary self.lvstore = "LVS_1" self.status = lc.StorageNode.STATUS_ONLINE + self.cluster_id = "CL_tgt" def get_id(self): return self._id @@ -406,6 +407,10 @@ def get_storage_node_by_id(self, nid): return {"P": primary, "S": peer}[nid] def release_lvol_ns_slot(self, lvol): pass + def get_lvols(self): + # No copy of this subsystem exists on the target yet, so the + # one-subsystem-one-primary guard has nothing to redirect to. + return [] class _Lvol: uuid = "ORIG"; nqn = "nqn.test:lvol:ORIG"; ns_id = 7 @@ -722,3 +727,23 @@ def test_failover_rollback_covers_every_placed_node_with_ids(): whole = inspect.getsource(lc) assert "delete_lvol_from_node(new_lvol," not in whole, \ "no rollback may hand records to an id-taking function" + + +def test_policy_attach_also_keeps_a_subsystem_on_one_target_node(): + """Run 20260826_233417: add_lvol_ha had the sibling rule but + replication_start -- the path a `volume add --replication-policy` takes -- + picked the target node purely by capacity. One 10-namespace subsystem + ended up on THREE target primaries (nsids 1,2,3,6 / 1,2,4,5,7-10 / + 3-10); no node advertised the whole set and a client saw 0 of the 10. + + Both entry points must consult the sibling rule, and _create_target_lvol_clone + re-checks it at creation time as the last line of defence.""" + import inspect + from simplyblock_core.controllers import lvol_controller as lc + + src = inspect.getsource(lc.replication_start) + assert "_sibling_replication_node" in src, "attaching a policy must honour the shared-subsystem rule too" + assert src.index("_sibling_replication_node") < src.index("_get_next_3_nodes"), "the sibling lookup must precede the capacity-based pick" + + clone = inspect.getsource(lc._create_target_lvol_clone) + assert "_subsystem_home_node" in clone, "the copy must not be built on a node that splits the subsystem" diff --git a/simplyblock_core/test/test_target_nsid_claim.py b/simplyblock_core/test/test_target_nsid_claim.py index 6fa6a8515f..51a310119a 100644 --- a/simplyblock_core/test/test_target_nsid_claim.py +++ b/simplyblock_core/test/test_target_nsid_claim.py @@ -196,3 +196,65 @@ def test_non_replicated_lvol_is_untouched(self): if __name__ == "__main__": unittest.main() + + +class TestSubsystemHomeNode(unittest.TestCase): + """One shared subsystem lives on exactly one primary per cluster.""" + + def _db_with(self, lvols, nodes): + db = MagicMock() + db.get_lvols.return_value = lvols + by_id = {n.get_id.return_value: n for n in nodes} + + def _get(node_id): + if node_id not in by_id: + raise KeyError(node_id) + return by_id[node_id] + + db.get_storage_node_by_id.side_effect = _get + return db + + def _copy(self, uuid, node_id, nqn=NQN, status="online", deleted=False): + lv = MagicMock() + lv.uuid = uuid + lv.get_id.return_value = uuid + lv.nqn = nqn + lv.node_id = node_id + lv.status = status + lv.deleted = deleted + return lv + + def _node(self, node_id, cluster_id): + n = MagicMock() + n.get_id.return_value = node_id + n.cluster_id = cluster_id + return n + + def test_finds_the_node_already_hosting_the_subsystem(self): + db = self._db_with( + [self._copy("c1", "N1")], + [self._node("N1", "CL_tgt")]) + self.assertEqual( + lvol_controller._subsystem_home_node(db, NQN, "CL_tgt"), "N1") + + def test_ignores_copies_in_a_different_cluster(self): + db = self._db_with( + [self._copy("c1", "N_src")], + [self._node("N_src", "CL_src")]) + self.assertEqual( + lvol_controller._subsystem_home_node(db, NQN, "CL_tgt"), "") + + def test_ignores_other_subsystems(self): + db = self._db_with( + [self._copy("c1", "N1", nqn="nqn:other")], + [self._node("N1", "CL_tgt")]) + self.assertEqual( + lvol_controller._subsystem_home_node(db, NQN, "CL_tgt"), "") + + def test_a_volume_being_deleted_does_not_own_the_subsystem(self): + from simplyblock_core.models.lvol_model import LVol + db = self._db_with( + [self._copy("c1", "N1", status=LVol.STATUS_IN_DELETION)], + [self._node("N1", "CL_tgt")]) + self.assertEqual( + lvol_controller._subsystem_home_node(db, NQN, "CL_tgt"), "") From 42d2660b04361a23fed4df57ff005cc51d5dc3f1 Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 27 Aug 2026 00:37:32 +0200 Subject: [PATCH 080/122] Fail-back: retire the superseded original before the returning volume Run 20260826_235940 failed over all 20 volumes correctly -- both HA pairs exposed identical namespace sets, clients saw 10/10 -- then wedged on fail-back with all 20 cutover tasks suspended on: Subsystem ...e3f4f160 is full on the target HA set (10 namespaces); no nsid to claim for 9b78b3f7 After a fail-over the ORIGINAL volume stays on its cluster still holding its namespace. The fail-back returns into that same subsystem, and a shared subsystem is sized to its group (max_namespaces=10 for ten volumes), so there is no slot for the returning volume. Dedicated subsystems hid this for three years of cases 3/5/8: sized 1 they were equally full, but each fail-back minted a fresh per-volume NQN and never came back to an occupied one. The original is now deleted before the returning volume is built, which is what frees the slot. Its SNAPSHOTS are deliberately left alone: the newest one still present on that cluster is the common base the fail-back clones and transfers a delta against -- that is what makes a fail-back an online migration rather than a full copy. delete_lvol only removes a parent snapshot that was already soft-deleted, so the base survives. The volume being superseded is resolved through the LVolReplication record (source_lvol of the record naming this volume as target), not by matching the uuid at the tail of the NQN: with a shared subsystem all ten volumes carry the SAME NQN, so that match is ambiguous. A first fail-over has no such record and retires nothing. Three more test fakes were missing get_lvol_replication_objects, which the real DBController has; completed rather than worked around. 1905 pass. --- .../controllers/lvol_controller.py | 63 +++++++++++++ simplyblock_core/test/test_failover_target.py | 6 ++ .../test_replication_chain_completeness.py | 4 + .../test/test_target_nsid_claim.py | 93 +++++++++++++++++++ 4 files changed, 166 insertions(+) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 18188945dd..889aaa88a8 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -3613,6 +3613,61 @@ def _claim_target_nsid(db_controller, new_lvol, target_node): return 0 +def _retire_superseded_original(db_controller, lvol, dest_cluster_id): + """Delete the volume a fail-back replaces. Returns (ok, error). + + After a fail-over the ORIGINAL volume stays on its cluster, still holding + its namespace. Failing back builds the returning volume in the SAME + subsystem, so unless the original goes first there is no slot for it: a + shared subsystem is sized to its group (max_namespaces=10 for ten + volumes) and the add fails outright -- "Subsystem ... is full on the + target HA set", which suspended all 20 cutover tasks in soak case 7 run + 20260826_235940. Dedicated subsystems hid this: sized 1 they were equally + full, but each fail-back minted a fresh per-volume NQN and so never came + back to an occupied one. + + The original's SNAPSHOTS are deliberately left alone. The newest one still + present on that cluster is the common base the fail-back clones and + transfers a delta against -- that is what makes a fail-back an online + migration rather than a full copy. + + A no-op unless *lvol* is itself a fail-over copy whose original still + lives on the destination cluster, so a first fail-over deletes nothing. + """ + original = None + for rep in db_controller.get_lvol_replication_objects(): + target = getattr(rep, "target_lvol", None) + if target and target.get_id() == lvol.get_id() and rep.source_lvol: + original = rep.source_lvol # keep the LAST match: the + # most recent fail-over wins + if not original: + return True, "" + try: + current = db_controller.get_lvol_by_id(original.get_id()) + except KeyError: + return True, "" # already gone + if current.status in (LVol.STATUS_DELETED, LVol.STATUS_IN_DELETION): + return True, "" + try: + node = db_controller.get_storage_node_by_id(current.node_id) + except KeyError: + return True, "" + if node.cluster_id != dest_cluster_id: + return True, "" # not in our way + + logger.info( + "Fail-back: deleting the superseded original %s (nsid %s of subsystem " + "%s) on node %s so the returning volume has a namespace slot", + current.get_id(), current.ns_id, current.nqn, current.node_id[:8]) + try: + delete_lvol(current) + except Exception as e: + return False, ( + f"Fail-back cannot free the namespace of the superseded original " + f"{current.get_id()} in subsystem {current.nqn}: {e}") + return True, "" + + def _subsystem_home_node(db_controller, nqn, cluster_id): """Node in *cluster_id* that already hosts copies of subsystem *nqn*, or "". @@ -3643,6 +3698,14 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps STATUS_IN_CREATION with lvol_uuid/blobid populated; the caller is responsible for setting STATUS_ONLINE and any replication bookkeeping. """ + # A fail-back returns into the subsystem the original still occupies, so + # the original has to go first (its snapshots stay: they are the delta + # base). A first fail-over has no original here and this does nothing. + ok, err = _retire_superseded_original(db_controller, lvol, + target_node.cluster_id) + if not ok: + return None, err + # Last line of defence for the one-subsystem-one-primary invariant. The # target node was chosen when the policy was attached, long before this # copy is created; if anything put a sibling somewhere else in the diff --git a/simplyblock_core/test/test_failover_target.py b/simplyblock_core/test/test_failover_target.py index 1e8188cf4c..53230f6977 100644 --- a/simplyblock_core/test/test_failover_target.py +++ b/simplyblock_core/test/test_failover_target.py @@ -79,6 +79,12 @@ def __init__(self, nodes, clusters, existing_lvols=None): def get_lvol_by_id(self, lid): return _src_lvol() + def get_lvol_replication_objects(self): + # Real DBController exposes this; the fail-back retirement asks it + # whether this volume is a copy of something. In these fail-OVER + # tests it is not, so there is nothing to retire. + return [] + def get_storage_node_by_id(self, nid): return self._nodes[nid] diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index fdfd459785..bfb9de1adb 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -411,6 +411,10 @@ def get_lvols(self): # No copy of this subsystem exists on the target yet, so the # one-subsystem-one-primary guard has nothing to redirect to. return [] + def get_lvol_replication_objects(self): + # This volume is not a fail-over copy, so no original is + # superseded and nothing gets retired. + return [] class _Lvol: uuid = "ORIG"; nqn = "nqn.test:lvol:ORIG"; ns_id = 7 diff --git a/simplyblock_core/test/test_target_nsid_claim.py b/simplyblock_core/test/test_target_nsid_claim.py index 51a310119a..23fa109f3c 100644 --- a/simplyblock_core/test/test_target_nsid_claim.py +++ b/simplyblock_core/test/test_target_nsid_claim.py @@ -258,3 +258,96 @@ def test_a_volume_being_deleted_does_not_own_the_subsystem(self): [self._node("N1", "CL_tgt")]) self.assertEqual( lvol_controller._subsystem_home_node(db, NQN, "CL_tgt"), "") + + +class TestRetireSupersededOriginal(unittest.TestCase): + """A fail-back must free the slot the original still holds.""" + + def setUp(self): + from simplyblock_core.models.lvol_model import LVol + self.LVol = LVol + patcher = patch.object(lvol_controller, "delete_lvol") + self.delete = patcher.start() + self.addCleanup(patcher.stop) + + def _rep(self, source, target): + r = MagicMock() + r.source_lvol = source + r.target_lvol = target + return r + + def _vol(self, uuid, node_id="N1", status="online"): + lv = MagicMock() + lv.uuid = uuid + lv.get_id.return_value = uuid + lv.node_id = node_id + lv.status = status + lv.nqn = NQN + lv.ns_id = 3 + return lv + + def _db(self, reps, lvols, node_cluster): + db = MagicMock() + db.get_lvol_replication_objects.return_value = reps + by_id = {lv.get_id.return_value: lv for lv in lvols} + + def _get_lvol(uid): + if uid not in by_id: + raise KeyError(uid) + return by_id[uid] + + def _get_node(nid): + if nid not in node_cluster: + raise KeyError(nid) + n = MagicMock() + n.cluster_id = node_cluster[nid] + return n + + db.get_lvol_by_id.side_effect = _get_lvol + db.get_storage_node_by_id.side_effect = _get_node + return db + + def test_first_failover_deletes_nothing(self): + """No record names this volume as a copy, so it IS the original.""" + lvol = self._vol("orig") + db = self._db([], [lvol], {"N1": "CL_src"}) + ok, err = lvol_controller._retire_superseded_original(db, lvol, "CL_tgt") + self.assertTrue(ok) + self.assertEqual(err, "") + self.delete.assert_not_called() + + def test_failback_deletes_the_original_on_the_destination(self): + original, copy = self._vol("orig"), self._vol("copy", node_id="N2") + db = self._db([self._rep(original, copy)], [original, copy], + {"N1": "CL_src", "N2": "CL_tgt"}) + ok, err = lvol_controller._retire_superseded_original(db, copy, "CL_src") + self.assertTrue(ok, err) + self.delete.assert_called_once_with(original) + + def test_an_original_on_another_cluster_is_not_in_the_way(self): + original, copy = self._vol("orig"), self._vol("copy", node_id="N2") + db = self._db([self._rep(original, copy)], [original, copy], + {"N1": "CL_src", "N2": "CL_tgt"}) + ok, _ = lvol_controller._retire_superseded_original(db, copy, "CL_third") + self.assertTrue(ok) + self.delete.assert_not_called() + + def test_already_deleting_original_is_left_alone(self): + original = self._vol("orig", status=self.LVol.STATUS_IN_DELETION) + copy = self._vol("copy", node_id="N2") + db = self._db([self._rep(original, copy)], [original, copy], + {"N1": "CL_src", "N2": "CL_tgt"}) + ok, _ = lvol_controller._retire_superseded_original(db, copy, "CL_src") + self.assertTrue(ok) + self.delete.assert_not_called() + + def test_a_failed_delete_aborts_the_failback_with_a_clear_reason(self): + original, copy = self._vol("orig"), self._vol("copy", node_id="N2") + db = self._db([self._rep(original, copy)], [original, copy], + {"N1": "CL_src", "N2": "CL_tgt"}) + self.delete.side_effect = RuntimeError("lvstore restart in progress") + ok, err = lvol_controller._retire_superseded_original(db, copy, "CL_src") + self.assertFalse(ok) + self.assertIn("lvstore restart in progress", err) + self.assertIn("namespace", err, + "the message must say WHY the fail-back cannot proceed") From 1d1b7067536516eada0e1d702a584ec11d14c79c Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 27 Aug 2026 10:44:25 +0200 Subject: [PATCH 081/122] Redact the GitHub token before it reaches a log line repl_soak.py echoes every command it runs, and the checkout step's URL carries a personal access token (https://x-access-token:@github.com/...). It was printed in full into the run log and into every lab log copied off the mgmt node -- logs that get pasted into tickets and chat and then sit on the instance for its lifetime. Redacting in log() rather than at the call site: a future command that happens to carry a credential is redacted by default instead of depending on whoever adds it to remember. The failure path prints the command and the captured streams too, so those go through it as well. The token that was already exposed needs rotating; this only stops the leak. --- scripts/repl_soak.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/repl_soak.py b/scripts/repl_soak.py index 5d7ae46b39..b5286ec82f 100644 --- a/scripts/repl_soak.py +++ b/scripts/repl_soak.py @@ -37,8 +37,19 @@ "-o", "ConnectTimeout=30", "-i", KEY] +# Any credential that reaches a log line is leaked: these logs are pasted into +# tickets and chat, and the lab copies keep them on disk for the life of the +# instance. The fetch URL carries a GitHub token, so redact before printing +# rather than trusting each call site to remember. +_SECRET_RE = re.compile(r"(x-access-token:)[^@\s]+(@)") + + +def redact(msg): + return _SECRET_RE.sub(r"\1***\2", str(msg)) + + def log(msg): - print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) + print(f"[{time.strftime('%H:%M:%S')}] {redact(msg)}", flush=True) def sh(cmd, cwd=None, env=None, check=True, capture=False): @@ -47,8 +58,8 @@ def sh(cmd, cwd=None, env=None, check=True, capture=False): text=True, capture_output=capture) if check and r.returncode != 0: if capture: - print(r.stdout[-2000:], r.stderr[-2000:]) - raise SystemExit(f"step failed (rc={r.returncode}): {cmd}") + print(redact(r.stdout[-2000:]), redact(r.stderr[-2000:])) + raise SystemExit(f"step failed (rc={r.returncode}): {redact(cmd)}") return r From 995c2221d3a78617805fab2635b562803984cf86 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 27 Aug 2026 13:19:21 +0100 Subject: [PATCH 082/122] fix: restore do_replicate=True and replication config after failback UUID swap --- .../tasks_runner_replication_final.py | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index bec1f6b3a6..f0b2756b5c 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -134,20 +134,42 @@ def _swap_failback_lvol_uuid(rep, failback_source_id): stale_uuid = new_lvol.get_id() - # Copy replication config from the original source before removing its record. + # The failover's _finalize() cleared do_replicate / replication_interval_min / + # replication_policy_id on the original cluster-1 source to stop it replicating. + # Those cleared values must NOT be propagated here; instead restore them from + # the failback source (cluster-2 volume) whose DB record is still intact at + # this point — _finalize() clears its fields in a later step. + failback_src_interval = 0 + failback_src_policy_id = "" + try: + failback_src = db.get_lvol_by_id(rep.source_lvol.get_id()) + failback_src_interval = failback_src.replication_interval_min + failback_src_policy_id = failback_src.replication_policy_id + except Exception as exc: + logger.warning( + "failback UUID swap: could not read failback source %s for interval/policy: %s", + rep.source_lvol.get_id(), exc) + + # Copy replication_node_id / replication_mode from the original source — + # these fields were NOT cleared by the failover's _finalize(), so they still + # point at the correct cluster-2 target node. try: old_lvol = db.get_lvol_by_id(failback_source_id) - new_lvol.do_replicate = old_lvol.do_replicate new_lvol.replication_node_id = old_lvol.replication_node_id new_lvol.replication_mode = old_lvol.replication_mode - new_lvol.replication_interval_min = old_lvol.replication_interval_min - new_lvol.replication_policy_id = old_lvol.replication_policy_id old_lvol.remove(db.kv_store) except KeyError: logger.warning( "failback UUID swap: original source lvol %s already absent from DB", failback_source_id) + # Explicitly re-enable replication on the restored volume. The original + # source had do_replicate cleared during failover; failback means it is the + # active source again and should resume its cadence. + new_lvol.do_replicate = True + new_lvol.replication_interval_min = failback_src_interval + new_lvol.replication_policy_id = failback_src_policy_id + # Write the clone under the original source UUID. new_lvol.uuid = failback_source_id new_lvol.write_to_db(db.kv_store) From 4fa1950569236231be788b4f2234d52ba8598b50 Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 27 Aug 2026 16:50:10 +0200 Subject: [PATCH 083/122] Converge the cutover delta before freezing, and give the lvstore to it Case 10 (online migration under heavy IO, run 20260827_110415) measured the client-observed freeze at avg 40.4s, max 71.9s, with fio logging 8 errors. The freeze is bdev_lvol_transfer_final_step, which copies everything written since the cutover clone's base snapshot -- so the freeze is as long as the write window in front of it. Four things made that window enormous. 1. SHRINK_ROUNDS was a fixed 2: a count, not a convergence criterion. Under load it simply stopped while the delta was still large. Rounds now repeat until one transfers within REPL_CUTOVER_CONVERGE_TARGET_SEC (2s), bounded by REPL_CUTOVER_MAX_SHRINK_ROUNDS -- a volume written faster than it replicates freezes anyway, and says so, rather than looping forever. 2. Each round returned to the task scheduler, so TASK_EXEC_INTERVAL_SEC (10s) of fresh writes joined every round -- a floor no number of rounds could beat. The loop now polls at 200ms and takes the next snapshot immediately, before any yielding decision, because that IS the mechanism: a round must carry only what was written while the previous one transferred. The inline window scales with the last round (the runner is single-threaded, so a flat budget would stall other volumes' cutovers): short rounds stay inline, slow ones yield, and yielding is free there because the freeze is far away. 3. The operator preconnect gate sat BETWEEN the base snapshot and the freeze, so every second of it was a second the frozen step had to copy. With no operator the 120s fallback fired 34 times in that run. It is now opt-in via REPL_CUTOVER_PROCEED_REQUIRED (default off). Deployments whose operator posts cutover-proceed set it and keep that cost until the clone's base can be advanced after the signal -- noted, not yet built. 4. Nothing stopped the other volumes on the same lvstore from replicating through a cutover, stretching every round. Two priorities now share an lvstore: a volume in final cutover owns it for the convergence rounds AND the freeze, and consistency groups outrank loose volumes -- a group's members transfer in parallel with each other while everything else waits, which serializes groups against each other instead of interleaving them. A group cuts over as a group: members are exempt from a sibling's claim. 2037 pass. --- simplyblock_core/constants.py | 30 ++ .../services/snapshot_replication.py | 91 +++++ .../tasks_runner_replication_final.py | 205 +++++++++-- .../test/test_cutover_convergence.py | 345 ++++++++++++++++++ .../test_tasks_runner_replication_final.py | 75 +++- 5 files changed, 700 insertions(+), 46 deletions(-) create mode 100644 simplyblock_core/test/test_cutover_convergence.py diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index b606e9aa81..e8ce1eef2d 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -374,6 +374,36 @@ def get_config_var(name, default=None): # if the operator is unavailable. Cutover proceeds regardless after this many seconds. REPL_CUTOVER_PROCEED_TIMEOUT_SEC = 120 +# --- cutover delta convergence ------------------------------------------- +# The IO freeze copies everything written since the last replicated snapshot, +# so the cutover converges the delta FIRST: take a snapshot, transfer it, and +# immediately take the next, until a round transfers in "low seconds". A fixed +# two rounds (the previous behaviour) does not converge under load -- it just +# stops. +REPL_CUTOVER_CONVERGE_TARGET_SEC = 2.0 +# Safety bound: a volume written faster than it replicates never converges, so +# stop and freeze rather than looping forever. +REPL_CUTOVER_MAX_SHRINK_ROUNDS = 12 +# Rounds must follow each other within MILLISECONDS. Returning to the task +# scheduler between them costs TASK_EXEC_INTERVAL_SEC (10s) of fresh writes +# each time, which puts a floor under the delta no number of rounds can beat. +REPL_CUTOVER_POLL_INTERVAL_SEC = 0.2 +# How long a single runner pass may stay inside the convergence loop. +REPL_CUTOVER_CONVERGE_BUDGET_SEC = 60 +# Always worth polling inline for at least this long: a round that finishes +# just after the pass is handed back costs a full TASK_EXEC_INTERVAL_SEC of +# writes in the next round. +REPL_CUTOVER_MIN_INLINE_SEC = 5 + +# Whether to block the cutover on the operator's preconnect signal. The wait +# sits BETWEEN the cutover clone's base snapshot and the freeze, so every +# second of it is a second of writes the frozen final step must copy: with no +# operator present the 120s fallback timeout fired 34 times in one soak run and +# fed the 25-72s freezes. Deployments whose operator posts +# .../replication/cutover-proceed set this True and accept that cost until the +# clone's base can be advanced after the signal. +REPL_CUTOVER_PROCEED_REQUIRED = False + SPDK_PROXY_MULTI_THREADING_ENABLED=True SPDK_PROXY_TIMEOUT=60*5 LVOL_NVME_CONNECT_RECONNECT_DELAY=2 diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index 03c156c27d..696e496bcc 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -127,9 +127,100 @@ def _unreplicated_local_ancestor(snode, snapshot, replicate_to_source): return ("blocked", None, "chain deeper than 64 blobs") +def _group_id_for_lvol(lvol): + """The consistency group *lvol* belongs to, or "". + + A group is owned by a replication policy and pinned to one node/LVS. + """ + policy_id = getattr(lvol, "replication_policy_id", "") + if not policy_id: + return "" + try: + group = db.get_consistency_group_for_policy(policy_id) + except Exception as e: # noqa: BLE001 + logger.warning("Could not resolve the consistency group of %s: %s", + lvol.get_id(), e) + return "" + return group.get_id() if group else "" + + +def _lvs_transfer_hold(task, snapshot): + """Why this transfer must wait, or "" when it may start now. + + Two priorities share one lvstore's bandwidth: + + 1. A volume in its FINAL CUTOVER owns the lvstore. Its convergence rounds + decide how long client IO freezes -- every second another volume steals + from a round is a second of writes the frozen final step must copy -- + so nothing else on that lvstore transfers meanwhile. Members of the same + consistency group are exempt: the group cuts over together. + + 2. Consistency groups outrank loose volumes. A group's members transfer in + PARALLEL with each other (their snapshots belong to one generation and + are only useful together), and everything else on the lvstore waits, so + groups are effectively serialized against each other rather than + interleaved. + """ + own_lvol = getattr(snapshot, "lvol", None) + lvs_name = getattr(own_lvol, "lvs_name", "") if own_lvol else "" + if not lvs_name: + return "" + own_id = own_lvol.get_id() + own_group = _group_id_for_lvol(own_lvol) + + tasks = db.get_job_tasks(task.cluster_id) + + # --- priority 1: a cutover in progress on this lvstore ----------------- + for t in tasks: + if t.function_name != JobSchedule.FN_REPLICATION_FINAL: + continue + if t.status == JobSchedule.STATUS_DONE or t.canceled: + continue + params = t.function_params or {} + if params.get("cutover_lvs") != lvs_name: + continue + holder = params.get("lvol_id") + if not holder or holder == own_id: + return "" # our own cutover: keep moving + holder_group = params.get("cutover_group") or "" + if holder_group and holder_group == own_group: + return "" # same group: cut over together + return (f"lvol {holder[:8]} is in final cutover on lvstore {lvs_name}") + + # --- priority 2: a consistency group is transferring on this lvstore --- + for t in tasks: + if t.function_name != JobSchedule.FN_SNAPSHOT_REPLICATION: + continue + if t.status != JobSchedule.STATUS_RUNNING or t.get_id() == task.get_id(): + continue + other_snap_id = (t.function_params or {}).get("snapshot_id") + if not other_snap_id: + continue + try: + other_lvol = db.get_snapshot_by_id(other_snap_id).lvol + except KeyError: + continue + if getattr(other_lvol, "lvs_name", "") != lvs_name: + continue + other_group = _group_id_for_lvol(other_lvol) + if other_group and other_group != own_group: + return (f"consistency group {other_group.split('/')[-1][:8]} is " + f"transferring on lvstore {lvs_name}") + + return "" + + def process_snap_replicate_start(task, snapshot): # 1 create lvol on remote node logger.info("Starting snapshot replication task") + + hold = _lvs_transfer_hold(task, snapshot) + if hold: + # Not a failure and not a retry: come back when the lvstore frees up. + task.function_result = f"held: {hold}" + task.write_to_db() + logger.info("Holding replication of %s: %s", snapshot.get_id(), hold) + return False # Drive the transfer from whichever member of the SOURCE lvstore leads it # now — the snapshot exists on every member, so an outage of the recorded # primary must not stop replication (see _source_leader_node). diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 761002a90d..2a3820dfda 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -37,6 +37,32 @@ db = db_controller.DBController() +def _group_id_for_lvol(lvol): + """The consistency group *lvol* belongs to, or "". + + A group is owned by a replication policy and pinned to one node/LVS, so a + volume's group is the group of its policy. + """ + policy_id = getattr(lvol, "replication_policy_id", "") + if not policy_id: + return "" + try: + group = db.get_consistency_group_for_policy(policy_id) + except Exception as e: # noqa: BLE001 + logger.warning("Could not resolve the consistency group of %s: %s", + lvol.get_id(), e) + return "" + return group.get_id() if group else "" + + +def _release_lvs_claim(task): + """Let other volumes on this LVS replicate again.""" + released = task.function_params.pop("cutover_lvs", None) is not None + task.function_params.pop("cutover_group", None) + if released: + task.write_to_db(db.kv_store) + + def _finalize(task, ok, err): if ok: replication_id = task.function_params.get("replication_id") @@ -97,11 +123,15 @@ def _finalize(task, ok, err): # reported loudly but does not un-succeed the task. logger.error(f"Source volume {src_lvol_id} could not be " f"deleted after the cutover: {e}") + _release_lvs_claim(task) return True task.function_result = err or "cutover failed, retrying" task.status = JobSchedule.STATUS_SUSPENDED task.retry += 1 + # A retry re-claims the LVS on its next pass; holding the claim across the + # wait would stall every other volume's replication for nothing. + _release_lvs_claim(task) task.write_to_db(db.kv_store) return False @@ -145,6 +175,18 @@ def task_runner(task: JobSchedule): if task.status in [JobSchedule.STATUS_NEW, JobSchedule.STATUS_SUSPENDED, JobSchedule.STATUS_RUNNING]: task.status = JobSchedule.STATUS_RUNNING task.function_params.setdefault("start_time", int(time.time())) + # Claim the source LVS for the whole cutover -- the convergence rounds + # AND the freeze. Other volumes' snapshot transfers on this LVS queue + # behind it (see snapshot_replication._lvs_locked_by_cutover): they + # compete for the same lvstore and hub bandwidth, and every second they + # steal from a convergence round is a second of writes that lands in + # the freeze. + task.function_params["cutover_lvs"] = getattr(lvol, "lvs_name", "") + # A consistency group cuts over AS A GROUP: its members' transfers and + # cutovers run alongside each other, and only volumes outside the group + # are held. Recording the group on the claim is what lets the + # replication side tell a sibling member from an unrelated volume. + task.function_params["cutover_group"] = _group_id_for_lvol(lvol) task.write_to_db(db.kv_store) # ---- SHRINK PHASE ----------------------------------------------- # @@ -170,8 +212,13 @@ def task_runner(task: JobSchedule): # (operator calls POST .../replication/cutover-proceed after its preconnect # Job succeeds). REPL_CUTOVER_PROCEED_TIMEOUT_SEC is the safety fallback # so cutover proceeds even if the operator is unavailable. + # Every second spent here is a second of writes the FROZEN final step + # must copy, because the cutover clone's base snapshot was taken before + # it. With no operator to signal, the 120s fallback fired 34 times in + # soak run 20260827_110415 and produced 25-72s freezes. Only wait when + # the deployment actually has an operator posting cutover-proceed. replication_id = params.get("replication_id") - if replication_id: + if replication_id and constants.REPL_CUTOVER_PROCEED_REQUIRED: try: rep = db.get_lvol_replication_by_id(replication_id) if not rep.cutover_proceed: @@ -203,48 +250,138 @@ def task_runner(task: JobSchedule): return True -SHRINK_ROUNDS = 2 -def _shrink_step(task, lvol): - """Advance the delta-shrink state machine one step. - - Returns (done, error): done=True when all rounds are replicated and the - cutover may start IMMEDIATELY; error aborts the task. - """ +def _take_shrink_snapshot(task, lvol): + """Snapshot the source and record it as the round in flight.""" + from simplyblock_core.controllers import snapshot_controller params = task.function_params - if int(time.time()) > params.get("shrink_deadline", 0): - return False, "shrink phase timed out waiting for replication" + new_snap, err = snapshot_controller.add( + lvol.get_id(), f"repl_commit_{uuid_lib.uuid4()}", + snap_type=SnapShot.TYPE_INTERNAL) + if err: + return None, f"shrink round {params.get('shrink_round', 0) + 1} snapshot failed: {err}" + params["shrink_round"] = params.get("shrink_round", 0) + 1 + params["shrink_snap_id"] = new_snap + params["shrink_started_at"] = time.time() + return new_snap, None + - snap_id = params["shrink_snap_id"] +def _shrink_round_done(snap_id): + """True once the round's snapshot is replicated AND chained on the target. + + target_replicated_snap_uuid is set at replicate-finish, after the target + copy is chained and converted -- replicated and usable in one signal. + """ try: - snap = db.get_snapshot_by_id(snap_id) + return bool(db.get_snapshot_by_id(snap_id).target_replicated_snap_uuid) except KeyError: - return False, f"shrink snapshot {snap_id} disappeared" + return None # disappeared - # target_replicated_snap_uuid is set at replicate-finish AFTER the target - # copy is chained and converted — replicated AND converted in one signal. - if not snap.target_replicated_snap_uuid: - task.function_result = (f"shrink round {params['shrink_round']}: waiting " - f"for {snap_id[:8]} to replicate") - return False, None - if params["shrink_round"] >= SHRINK_ROUNDS: - return True, None +def _inline_window(last_round_secs): + """How long a pass may poll inline, given the last round's transfer time. - # Round replicated — IMMEDIATELY take the next snapshot: its delta covers - # only the wait window of the previous round. - from simplyblock_core.controllers import snapshot_controller - new_snap, err = snapshot_controller.add( - lvol.get_id(), f"repl_commit_{uuid_lib.uuid4()}", - snap_type=SnapShot.TYPE_INTERNAL) - if err: - return False, f"shrink round {params['shrink_round'] + 1} snapshot failed: {err}" - params["shrink_round"] += 1 - params["shrink_snap_id"] = new_snap - task.function_result = f"shrink round {params['shrink_round']}: snapshot taken" - task.write_to_db(db.kv_store) - return False, None + The runner is single-threaded, so a flat multi-minute budget would stall + every other volume's cutover behind this one. Scaling to the last round + keeps the loop inline exactly where it matters -- near convergence, where + the next snapshot must follow within milliseconds -- and yields early when + rounds are still long, which is where yielding costs nothing because the + freeze is far away regardless. + """ + return min(constants.REPL_CUTOVER_CONVERGE_BUDGET_SEC, + max(constants.REPL_CUTOVER_MIN_INLINE_SEC, last_round_secs * 3)) + + +def _shrink_step(task, lvol): + """Converge the delta, then hand straight over to the cutover. + + Returns (done, error). done=True means the delta is as small as it is going + to get and the freeze may start IMMEDIATELY. + + This loop deliberately does NOT return to the task scheduler between + rounds. Each return costs TASK_EXEC_INTERVAL_SEC (10s) before the next + pass, and every one of those seconds is written by the client and lands in + the next round -- a floor on the delta that more rounds cannot lower. Here + a round ends and the next snapshot is taken within + REPL_CUTOVER_POLL_INTERVAL_SEC, so a round only carries the writes made + during the previous round's transfer. + """ + params = task.function_params + deadline = params.get("shrink_deadline", 0) + # How long this pass may poll inline before handing the runner back. The + # runner is single-threaded, so a flat multi-minute budget would stall + # every OTHER volume's cutover behind this one. Scale it to how long the + # last round actually took: near convergence the rounds are seconds and we + # stay inline (which is the whole point -- the next snapshot must follow + # within milliseconds), while a slow early round yields quickly, and + # yielding costs nothing there because we are far from the freeze anyway. + budget_end = time.time() + _inline_window( + (params.get("shrink_round_times") or [0])[-1]) + + while True: + if int(time.time()) > deadline: + return False, "shrink phase timed out waiting for replication" + + snap_id = params["shrink_snap_id"] + done = _shrink_round_done(snap_id) + if done is None: + return False, f"shrink snapshot {snap_id} disappeared" + + if not done: + if time.time() >= budget_end: + # Give the pass back so the runner can service other tasks; + # the round is still in flight and resumes on the next pass. + task.function_result = (f"shrink round {params['shrink_round']}: waiting " + f"for {snap_id[:8]} to replicate") + return False, None + time.sleep(constants.REPL_CUTOVER_POLL_INTERVAL_SEC) + continue + + elapsed = time.time() - params.get("shrink_started_at", time.time()) + params.setdefault("shrink_round_times", []).append(round(elapsed, 2)) + logger.info("cutover convergence: lvol=%s round %d transferred in %.2fs", + lvol.get_id(), params["shrink_round"], elapsed) + + # Converged: this round's delta -- the writes made during the previous + # round -- moved in low seconds, so the freeze that copies the next + # such window will be about as short. + if elapsed <= constants.REPL_CUTOVER_CONVERGE_TARGET_SEC: + task.function_result = (f"converged in {params['shrink_round']} rounds " + f"(last {elapsed:.2f}s)") + return True, None + + if params["shrink_round"] >= constants.REPL_CUTOVER_MAX_SHRINK_ROUNDS: + # Written faster than it replicates. Freezing now is still the best + # move -- the freeze at least stops the writes -- but say so. + logger.warning( + "cutover convergence: lvol=%s did not converge in %d rounds " + "(last round %.2fs > %.2fs target); freezing anyway", + lvol.get_id(), params["shrink_round"], elapsed, + constants.REPL_CUTOVER_CONVERGE_TARGET_SEC) + task.function_result = (f"not converged after {params['shrink_round']} " + f"rounds (last {elapsed:.2f}s)") + return True, None + + # IMMEDIATELY take the next snapshot. This is the whole mechanism: the + # next round carries only what was written while this one transferred, + # so waiting here -- for the scheduler or for anything else -- puts + # those seconds straight into the freeze. It happens before any + # yielding decision for exactly that reason. + _, err = _take_shrink_snapshot(task, lvol) + if err: + return False, err + task.write_to_db(db.kv_store) + + # Re-arm the inline window from what this round just measured: rounds + # near convergence are short and stay inline, a slow one hands the + # runner back so other volumes' cutovers are not stuck behind it. + budget_end = time.time() + _inline_window(elapsed) + if time.time() >= budget_end: + task.function_result = (f"shrink round {params['shrink_round'] - 1} done " + f"({elapsed:.2f}s); continuing next pass") + task.write_to_db(db.kv_store) + return False, None def _prepare_cutover(task, lvol, src_node, tgt_node): diff --git a/simplyblock_core/test/test_cutover_convergence.py b/simplyblock_core/test/test_cutover_convergence.py new file mode 100644 index 0000000000..c4170b1146 --- /dev/null +++ b/simplyblock_core/test/test_cutover_convergence.py @@ -0,0 +1,345 @@ +"""The cutover must converge the delta before it freezes client IO. + +Soak run 20260827_110415, case 10 (online migration under heavy IO) measured +the client-observed freeze at avg 40.4s, max 71.9s, with fio logging 8 errors. +The freeze is bdev_lvol_transfer_final_step, which copies everything written +since the cutover clone's base snapshot -- so the freeze lasts as long as the +write window that precedes it. + +The intended sequence is: snapshot -> transfer -> IMMEDIATELY (milliseconds) +the next snapshot -> repeat until a round transfers in low seconds -> +IMMEDIATELY the final lvol transfer. Three things broke it: + + * SHRINK_ROUNDS was a fixed 2 -- a count, not a convergence criterion, so + under load it simply stopped while the delta was still large; + * each round returned to the task scheduler, costing TASK_EXEC_INTERVAL_SEC + (10s) of fresh writes per round -- a floor no number of rounds can beat; + * the operator preconnect gate sat BETWEEN the base snapshot and the freeze, + and with no operator its 120s fallback fired 34 times in that run. +""" +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import constants +from simplyblock_core.services import tasks_runner_replication_final as runner + + +class _Task: + def __init__(self, **params): + self.function_params = dict(params) + self.function_result = "" + self.status = "" + self.retry = 0 + self.cluster_id = "CL" + + def write_to_db(self, *a, **kw): + pass + + +def _lvol(uuid="LV1", lvs="LVS_1"): + lv = MagicMock() + lv.get_id.return_value = uuid + lv.uuid = uuid + lv.lvs_name = lvs + return lv + + +class _Clock: + """A clock that only moves when the code under test waits. + + The convergence loop measures a round as (now - shrink_started_at), so a + fake that returns the same instant for both makes every round look + instantaneous and the test proves nothing. + """ + + def __init__(self, now=1000.0): + self.now = now + + def __call__(self): + return self.now + + def sleep(self, seconds): + self.now += seconds + + +class TestConvergence(unittest.TestCase): + """_shrink_step loops until a round is fast, without leaving the pass.""" + + def _run(self, round_times, max_rounds=None): + """Drive _shrink_step with round i taking round_times[i] seconds.""" + clock = _Clock() + task = _Task(shrink_snap_id="S0", shrink_round=1, + shrink_deadline=10 ** 9, lvol_id="LV1") + task.function_params["shrink_started_at"] = clock.now + state = {"i": 0} + taken = [] + + def _done(snap_id): + # replicated once this round's transfer time has actually passed + started = task.function_params["shrink_started_at"] + idx = min(state["i"], len(round_times) - 1) + return (clock.now - started) >= round_times[idx] + + def _take(task_, lvol_): + state["i"] += 1 + taken.append(state["i"]) + task_.function_params["shrink_round"] += 1 + task_.function_params["shrink_snap_id"] = "S%d" % state["i"] + task_.function_params["shrink_started_at"] = clock.now + return "S%d" % state["i"], None + + patches = [ + patch.object(runner.time, "time", clock), + patch.object(runner.time, "sleep", clock.sleep), + patch.object(runner, "_shrink_round_done", side_effect=_done), + patch.object(runner, "_take_shrink_snapshot", side_effect=_take), + ] + if max_rounds is not None: + patches.append( + patch.object(constants, "REPL_CUTOVER_MAX_SHRINK_ROUNDS", max_rounds)) + for p in patches: + p.start() + self.addCleanup(p.stop) + + done, err = runner._shrink_step(task, _lvol()) + return done, err, task, taken, clock + + def test_a_fast_round_converges_and_hands_over(self): + """Round transferred inside the target -> freeze immediately.""" + done, err, task, taken, _ = self._run([0.5]) + self.assertTrue(done) + self.assertIsNone(err) + self.assertIn("converged", task.function_result) + self.assertEqual(taken, [], "a fast first round needs no further rounds") + + def test_a_slow_round_takes_another_snapshot_without_leaving_the_pass(self): + """The whole point: rounds follow each other in milliseconds.""" + done, err, task, taken, clock = self._run([3.0, 3.0, 0.4]) + self.assertTrue(done) + self.assertIsNone(err) + self.assertEqual(taken, [1, 2], + "each slow round must be followed immediately by the next") + self.assertIn("converged", task.function_result) + # ~6.4s of transfers, not 6.4 + 3 x TASK_EXEC_INTERVAL_SEC of + # rescheduling: the delta a round carries is the previous round's + # transfer, nothing more. + self.assertLess(clock.now - 1000.0, 7.0) + + def test_it_gives_up_after_the_round_cap_and_freezes_anyway(self): + """Written faster than it replicates: freeze rather than loop forever.""" + done, err, task, taken, _ = self._run([3.0] * 20, max_rounds=3) + self.assertTrue(done, "the cap must hand over, not fail the cutover") + self.assertIsNone(err) + self.assertIn("not converged", task.function_result) + + def test_a_vanished_snapshot_is_an_error(self): + # This one runs on the real clock, so the deadline has to be a real + # future epoch -- 10**9 is 2001 and would trip the timeout instead. + task = _Task(shrink_snap_id="S0", shrink_round=1, + shrink_deadline=10 ** 12, lvol_id="LV1") + with patch.object(runner, "_shrink_round_done", return_value=None): + done, err = runner._shrink_step(task, _lvol()) + self.assertFalse(done) + self.assertIn("disappeared", err) + + def test_the_deadline_still_bounds_the_phase(self): + task = _Task(shrink_snap_id="S0", shrink_round=1, + shrink_deadline=0, lvol_id="LV1") + with patch.object(runner, "_shrink_round_done", return_value=False): + done, err = runner._shrink_step(task, _lvol()) + self.assertFalse(done) + self.assertIn("timed out", err) + + def test_it_yields_the_pass_when_the_budget_runs_out(self): + """A very slow transfer must not hog the runner forever.""" + clock = _Clock() + task = _Task(shrink_snap_id="S0", shrink_round=1, + shrink_deadline=10 ** 9, lvol_id="LV1") + task.function_params["shrink_started_at"] = clock.now + with patch.object(runner.time, "time", clock), \ + patch.object(runner.time, "sleep", clock.sleep), \ + patch.object(runner, "_shrink_round_done", return_value=False), \ + patch.object(constants, "REPL_CUTOVER_CONVERGE_BUDGET_SEC", 5): + done, err = runner._shrink_step(task, _lvol()) + self.assertFalse(done) + self.assertIsNone(err, "yielding the pass is not a failure") + self.assertIn("waiting", task.function_result) + + +class TestProceedGate(unittest.TestCase): + """The preconnect wait must be opt-in: it costs freeze time.""" + + def test_disabled_by_default(self): + self.assertFalse( + constants.REPL_CUTOVER_PROCEED_REQUIRED, + "waiting for a signal nobody sends put 120s of writes into the " + "frozen final step") + + def test_the_wait_is_guarded_by_the_flag(self): + import inspect + src = inspect.getsource(runner.task_runner) + self.assertIn("constants.REPL_CUTOVER_PROCEED_REQUIRED", src) + self.assertLess( + src.index("REPL_CUTOVER_PROCEED_REQUIRED"), + src.index("run_cutover"), + "the gate must be evaluated before the freeze") + + +if __name__ == "__main__": + unittest.main() + + +class TestLvsAdmission(unittest.TestCase): + """One lvstore's bandwidth, two priorities. + + Case 10 ran 9 volumes over shared lvstores and every one kept replicating + while another was trying to converge, stretching each round -- and every + second a round is stretched is a second of writes that lands in the frozen + final step. + """ + + def setUp(self): + from simplyblock_core.services import snapshot_replication as sr + self.sr = sr + patcher = patch.object(sr, "db") + self.db = patcher.start() + self.addCleanup(patcher.stop) + self.groups = {} # lvol id -> group id + gp = patch.object(sr, "_group_id_for_lvol", + side_effect=lambda lv: self.groups.get(lv.get_id(), "")) + gp.start() + self.addCleanup(gp.stop) + + # -- fixtures --------------------------------------------------------- + def _cutover_task(self, lvol_id, lvs, group="", status="running", + canceled=False): + from simplyblock_core.models.job_schedule import JobSchedule + t = MagicMock() + t.function_name = JobSchedule.FN_REPLICATION_FINAL + t.status = status + t.canceled = canceled + t.function_params = {"lvol_id": lvol_id, "cutover_lvs": lvs, + "cutover_group": group} + return t + + def _transfer_task(self, snap_id, task_id="T_other", status=None): + from simplyblock_core.models.job_schedule import JobSchedule + t = MagicMock() + t.function_name = JobSchedule.FN_SNAPSHOT_REPLICATION + t.status = status or JobSchedule.STATUS_RUNNING + t.canceled = False + t.get_id.return_value = task_id + t.function_params = {"snapshot_id": snap_id} + return t + + def _lv(self, lvol_id, lvs="LVS_1"): + lv = MagicMock() + lv.get_id.return_value = lvol_id + lv.lvs_name = lvs + return lv + + def _snapshot(self, lvol_id="LV_other", lvs="LVS_1"): + snap = MagicMock() + snap.lvol = self._lv(lvol_id, lvs) + return snap + + def _task(self, task_id="T_me"): + t = MagicMock() + t.cluster_id = "CL" + t.get_id.return_value = task_id + return t + + # -- priority 1: a cutover owns its lvstore --------------------------- + def test_another_volumes_cutover_holds_this_lvstore(self): + self.db.get_job_tasks.return_value = [ + self._cutover_task("LV_cutting", "LVS_1")] + self.assertIn("final cutover", + self.sr._lvs_transfer_hold(self._task(), self._snapshot())) + + def test_the_volume_in_cutover_may_still_replicate(self): + """Its convergence snapshots are exactly what must keep moving.""" + self.db.get_job_tasks.return_value = [ + self._cutover_task("LV_cutting", "LVS_1")] + self.assertEqual( + self.sr._lvs_transfer_hold( + self._task(), self._snapshot(lvol_id="LV_cutting")), "") + + def test_a_group_member_is_not_held_by_its_groups_cutover(self): + """A consistency group cuts over as a group, not one member at a time.""" + self.groups = {"LV_cutting": "CL/G1", "LV_sibling": "CL/G1"} + self.db.get_job_tasks.return_value = [ + self._cutover_task("LV_cutting", "LVS_1", group="CL/G1")] + self.assertEqual( + self.sr._lvs_transfer_hold( + self._task(), self._snapshot(lvol_id="LV_sibling")), "") + + def test_a_volume_outside_the_group_is_still_held_by_its_cutover(self): + self.groups = {"LV_cutting": "CL/G1", "LV_loose": ""} + self.db.get_job_tasks.return_value = [ + self._cutover_task("LV_cutting", "LVS_1", group="CL/G1")] + self.assertIn("final cutover", self.sr._lvs_transfer_hold( + self._task(), self._snapshot(lvol_id="LV_loose"))) + + def test_a_cutover_on_a_different_lvstore_does_not_hold_us(self): + self.db.get_job_tasks.return_value = [ + self._cutover_task("LV_cutting", "LVS_9")] + self.assertEqual( + self.sr._lvs_transfer_hold(self._task(), self._snapshot()), "") + + def test_a_finished_or_cancelled_cutover_holds_nothing(self): + from simplyblock_core.models.job_schedule import JobSchedule + for task in (self._cutover_task("LV_x", "LVS_1", + status=JobSchedule.STATUS_DONE), + self._cutover_task("LV_x", "LVS_1", canceled=True)): + self.db.get_job_tasks.return_value = [task] + self.assertEqual( + self.sr._lvs_transfer_hold(self._task(), self._snapshot()), "") + + def test_a_task_that_has_not_claimed_an_lvs_holds_nothing(self): + """Before the cutover starts its rounds there is nothing to protect.""" + t = self._cutover_task("LV_cutting", "LVS_1") + del t.function_params["cutover_lvs"] + self.db.get_job_tasks.return_value = [t] + self.assertEqual( + self.sr._lvs_transfer_hold(self._task(), self._snapshot()), "") + + # -- priority 2: groups outrank loose volumes, and serialize ----------- + def test_a_transferring_group_holds_a_volume_from_another_group(self): + self.groups = {"LV_a": "CL/G1", "LV_b": "CL/G2"} + self.db.get_job_tasks.return_value = [self._transfer_task("S_a")] + self.db.get_snapshot_by_id.return_value = MagicMock(lvol=self._lv("LV_a")) + self.assertIn("consistency group", self.sr._lvs_transfer_hold( + self._task(), self._snapshot(lvol_id="LV_b"))) + + def test_members_of_the_same_group_transfer_in_parallel(self): + self.groups = {"LV_a": "CL/G1", "LV_b": "CL/G1"} + self.db.get_job_tasks.return_value = [self._transfer_task("S_a")] + self.db.get_snapshot_by_id.return_value = MagicMock(lvol=self._lv("LV_a")) + self.assertEqual( + self.sr._lvs_transfer_hold(self._task(), self._snapshot(lvol_id="LV_b")), + "") + + def test_a_transferring_group_outranks_a_loose_volume(self): + self.groups = {"LV_a": "CL/G1", "LV_loose": ""} + self.db.get_job_tasks.return_value = [self._transfer_task("S_a")] + self.db.get_snapshot_by_id.return_value = MagicMock(lvol=self._lv("LV_a")) + self.assertIn("consistency group", self.sr._lvs_transfer_hold( + self._task(), self._snapshot(lvol_id="LV_loose"))) + + def test_loose_volumes_still_transfer_in_parallel_with_each_other(self): + """No group involved: unchanged behaviour, no new serialization.""" + self.groups = {"LV_a": "", "LV_b": ""} + self.db.get_job_tasks.return_value = [self._transfer_task("S_a")] + self.db.get_snapshot_by_id.return_value = MagicMock(lvol=self._lv("LV_a")) + self.assertEqual( + self.sr._lvs_transfer_hold(self._task(), self._snapshot(lvol_id="LV_b")), + "") + + def test_a_group_transferring_on_another_lvstore_does_not_hold_us(self): + self.groups = {"LV_a": "CL/G1", "LV_b": "CL/G2"} + self.db.get_job_tasks.return_value = [self._transfer_task("S_a")] + self.db.get_snapshot_by_id.return_value = MagicMock( + lvol=self._lv("LV_a", lvs="LVS_9")) + self.assertEqual( + self.sr._lvs_transfer_hold(self._task(), self._snapshot(lvol_id="LV_b")), + "") diff --git a/simplyblock_core/test/test_tasks_runner_replication_final.py b/simplyblock_core/test/test_tasks_runner_replication_final.py index 61e53bac69..b678f28cee 100644 --- a/simplyblock_core/test/test_tasks_runner_replication_final.py +++ b/simplyblock_core/test/test_tasks_runner_replication_final.py @@ -180,6 +180,16 @@ def _mk(monkeypatch, snaps, params): def test_shrink_waits_until_replicated(monkeypatch): + """An unreplicated round yields the pass instead of failing. + + It now POLLS for a short window first (a round that lands just after the + pass is handed back would otherwise cost a full TASK_EXEC_INTERVAL_SEC of + writes in the next round), so squeeze the inline window to nothing to keep + the test instant. + """ + from simplyblock_core import constants + monkeypatch.setattr(constants, "REPL_CUTOVER_MIN_INLINE_SEC", 0) + monkeypatch.setattr(constants, "REPL_CUTOVER_CONVERGE_BUDGET_SEC", 0) runner, task = _mk(monkeypatch, {"S1": _ShrinkSnap(replicated=False)}, {"shrink_round": 1, "shrink_snap_id": "S1", "shrink_deadline": 2**60}) @@ -188,36 +198,77 @@ def test_shrink_waits_until_replicated(monkeypatch): assert "waiting" in task.function_result +def test_a_fast_round_converges_instead_of_taking_another(monkeypatch): + """The criterion is transfer TIME, not a round count. + + A round that replicated within REPL_CUTOVER_CONVERGE_TARGET_SEC means the + next such window -- the one the freeze copies -- is about as small, so the + cutover starts immediately rather than taking more snapshots for nothing. + """ + runner, task = _mk(monkeypatch, {"S1": _ShrinkSnap(replicated=True)}, + {"shrink_round": 1, "shrink_snap_id": "S1", + "shrink_deadline": 2**60, + "shrink_started_at": __import__("time").time()}) + taken = [] + + def _add(lid, name, snap_type="user"): + taken.append((lid, snap_type)) + return "S2", None + import simplyblock_core.controllers.snapshot_controller as sc + monkeypatch.setattr(sc, "add", _add) + + done, err = runner._shrink_step(task, _ShrinkLvol()) + assert (done, err) == (True, None) + assert taken == [], "a converged round must not take another snapshot" + assert "converged" in task.function_result + + def test_shrink_takes_next_snapshot_immediately(monkeypatch): + """A SLOW round is followed by the next one straight away. + + The delta the next round carries is only what was written during this + round's transfer -- which is the whole mechanism by which the freeze gets + shorter. + """ + import time as _time + from simplyblock_core import constants runner, task = _mk(monkeypatch, {"S1": _ShrinkSnap(replicated=True)}, {"shrink_round": 1, "shrink_snap_id": "S1", - "shrink_deadline": 2**60}) + "shrink_deadline": 2**60, + # started long enough ago to be well over the target + "shrink_started_at": _time.time() - 60}) taken = [] def _add(lid, name, snap_type="user"): taken.append((lid, snap_type)) + # the second round reports as still in flight, so the loop yields + runner.db._snaps["S2"] = _ShrinkSnap(replicated=False) return "S2", None import simplyblock_core.controllers.snapshot_controller as sc monkeypatch.setattr(sc, "add", _add) + monkeypatch.setattr(constants, "REPL_CUTOVER_MIN_INLINE_SEC", 0) + monkeypatch.setattr(constants, "REPL_CUTOVER_CONVERGE_BUDGET_SEC", 0) done, err = runner._shrink_step(task, _ShrinkLvol()) assert (done, err) == (False, None) - assert taken == [("LV1", "internal")] or taken[0][0] == "LV1" + assert taken and taken[0][0] == "LV1" assert task.function_params["shrink_round"] == 2 assert task.function_params["shrink_snap_id"] == "S2" -def test_shrink_completes_after_last_round(monkeypatch): - runner, task = _mk(monkeypatch, {"S2": _ShrinkSnap(replicated=True)}, - {"shrink_round": runner_rounds(), "shrink_snap_id": "S2", - "shrink_deadline": 2**60}) +def test_shrink_hands_over_when_it_cannot_converge(monkeypatch): + """Written faster than it replicates: freeze anyway, but say so.""" + import time as _time + from simplyblock_core import constants + monkeypatch.setattr(constants, "REPL_CUTOVER_MAX_SHRINK_ROUNDS", 3) + runner, task = _mk(monkeypatch, {"S1": _ShrinkSnap(replicated=True)}, + {"shrink_round": 3, "shrink_snap_id": "S1", + "shrink_deadline": 2**60, + "shrink_started_at": _time.time() - 60}) done, err = runner._shrink_step(task, _ShrinkLvol()) - assert (done, err) == (True, None), "cutover must start IMMEDIATELY after the last round" - - -def runner_rounds(): - import simplyblock_core.services.tasks_runner_replication_final as runner - return runner.SHRINK_ROUNDS + assert (done, err) == (True, None), \ + "the round cap must hand over to the freeze, not fail the cutover" + assert "not converged" in task.function_result def test_shrink_deadline_aborts(monkeypatch): From f487978fa9f15c3f95396e0c78f65de41a60f7e8 Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 27 Aug 2026 17:05:50 +0200 Subject: [PATCH 084/122] test: verify client cleanup, and give the chaos cases a cadence they can hold Two harness defects from run 20260827_110415. Case 11 died on "mount: /dev/nvme0n1 already mounted or mount point busy" and never tested retention at all -- it tested case 10's debris. That case's cutover froze client IO for 25-72s, its 13 unmounts hit their 15s timeout, and the `umount -l` fallback only DETACHES the path: the mount stays live until the hung IO drains, so the device was still mounted when the next case connected. Every command in the cleanup path was best-effort, so nothing noticed. Cleanup now verifies its own result against /proc/mounts (the only source that does not believe a lazy unmount's exit code) and escalates: kill the holders with fuser -km first -- umount -l on a mount someone still has open never completes -- then unmount, disconnect, and re-read, up to three rounds. A case refuses to start on a client that will not come clean, so contamination is reported where it happens instead of failing the next case on something unrelated, and cleanup_client warns loudly when it leaves debris. Mounting also checks the device is not already mounted, which turns a bare rc=32 into the actual reason. Case 13 was killed after 50 minutes without running a single chaos round: its 20 policies x 2 volumes inherited the default 1-minute cadence, which asks for 40 transfers a minute. The lag settled in a stable 313-383s band with outstanding pinned at 35-40 against a 180s gate -- not diverging, but never converging, so it would have burned the full 2h timeout. These cases now use a 10-minute cadence (CHAOS_PHASE_INTERVAL_MIN) with the gate derived from it, and print both so a stuck setup is diagnosable from the log alone. 2037 pass. --- scripts/test_async_replication.py | 111 ++++++++++++++++++++++++++++-- 1 file changed, 107 insertions(+), 4 deletions(-) diff --git a/scripts/test_async_replication.py b/scripts/test_async_replication.py index a110af1dd8..8453073e69 100644 --- a/scripts/test_async_replication.py +++ b/scripts/test_async_replication.py @@ -441,6 +441,15 @@ def connect_and_mount(client_ip, key_path, mgmt_ip, lvols, fmt=True, mount_base= mnt = f"{mount_base}{idx}" if fmt: run(client_ip, key_path, f"sudo mkfs.xfs -f {dev}") + held = run(client_ip, key_path, + f"grep -E '^{dev} ' /proc/mounts | head -1 || true", + check=False, quiet=True, timeout=60).strip() + if held: + raise RuntimeError( + f"{dev} is already mounted ({held}) before mounting {lv} at " + f"{mnt}: a previous case's mount is still live on this device. " + f"mount would fail with a bare 'already mounted or mount point " + f"busy' (case 11, run 20260827_110415).") run(client_ip, key_path, f"sudo mkdir -p {mnt} && sudo mount {dev} {mnt}") mounts.append({"lvol": lv, "nqn": conn["nqn"], "dev": dev, "mount": mnt}) print(f" vol {lv} -> {dev} @ {mnt}") @@ -532,6 +541,78 @@ def stop_fio(client_ip, key_path): time.sleep(3) +def client_dirt(client_ip, key_path): + """What is still mounted or connected on the client. Empty dict = clean. + + Reads /proc/mounts rather than trusting umount's exit code: a lazy unmount + reports success and leaves the mount live until its IO drains, which is + exactly how a finished case hands its devices to the next one. + """ + mounts = run(client_ip, key_path, + "grep -oE '/mnt/repl[^ ]*' /proc/mounts 2>/dev/null | sort -u || true", + check=False, quiet=True, timeout=60) + subsys = run(client_ip, key_path, + "sudo nvme list-subsys 2>/dev/null " + "| grep -oE 'nqn\\.2023-02\\.io\\.simplyblock:[^ ,]+' | sort -u || true", + check=False, quiet=True, timeout=90) + dirt = {} + if mounts.split(): + dirt["mounts"] = mounts.split() + if subsys.split(): + dirt["subsystems"] = subsys.split() + return dirt + + +def force_client_clean(client_ip, key_path, rounds=3): + """Unmount and disconnect everything, and keep at it until it is gone. + + Each round kills whatever holds the mount (hung fio keeps a lazy unmount + pinned forever), unmounts, disconnects every simplyblock subsystem, then + re-reads /proc/mounts. Returns the remaining dirt, empty when clean. + """ + dirt = client_dirt(client_ip, key_path) + for attempt in range(rounds): + if not dirt: + return {} + if attempt: + print(f" [{client_ip}] client still dirty ({dirt}); escalating " + f"(round {attempt + 1}/{rounds})") + run(client_ip, key_path, "sudo pkill -x fio || true", check=False, timeout=60) + # Kill the holders first: umount -l on a mount someone still has open + # never completes, and the device stays mounted underneath. + run(client_ip, key_path, + "for m in $(grep -oE '/mnt/repl[^ ]*' /proc/mounts 2>/dev/null | sort -u); do " + "sudo timeout 20 fuser -km \"$m\" 2>/dev/null || true; " + "sudo timeout 15 umount \"$m\" 2>/dev/null " + "|| sudo timeout 15 umount -f \"$m\" 2>/dev/null " + "|| sudo timeout 15 umount -l \"$m\" 2>/dev/null || true; done", + check=False, timeout=300) + run(client_ip, key_path, + "for n in $(sudo nvme list-subsys 2>/dev/null " + "| grep -oE 'nqn\\.2023-02\\.io\\.simplyblock:[^ ,]+' | sort -u); do " + "sudo timeout 20 nvme disconnect -n \"$n\" >/dev/null 2>&1 || true; done", + check=False, timeout=300) + # A lazy unmount finishes asynchronously once its holders are gone. + for _ in range(10): + time.sleep(3) + dirt = client_dirt(client_ip, key_path) + if not dirt: + return {} + return dirt + + +def assert_client_clean(client_ip, key_path, where): + """Refuse to run *where* on a client another case left dirty.""" + dirt = force_client_clean(client_ip, key_path) + if dirt: + raise RuntimeError( + f"{where}: client {client_ip} could not be returned to a clean " + f"state -- still {dirt}. Whatever ran before it left mounts or " + f"controllers behind (a cutover freeze outlasting the 15s unmount " + f"timeout does exactly this), and starting here would test that " + f"debris instead of the case.") + + def cleanup_client(client_ip, key_path, mounts): # Unmount before disconnecting, with a lazy fallback: a plain (or forced) # unmount fails once the transport is dead, and disconnecting underneath a @@ -548,6 +629,14 @@ def cleanup_client(client_ip, key_path, mounts): run(client_ip, key_path, f"sudo timeout 30 nvme disconnect -n {m['nqn']} 2>/dev/null || true", check=False, timeout=90) + # Verify, and escalate rather than hand the next case a live mount. Not + # fatal here -- the case that owns these mounts has already done its work + # and its verdict should stand -- but loud, because this is where the + # contamination starts and the NEXT case is where it gets blamed. + dirt = force_client_clean(client_ip, key_path) + if dirt: + print(f" [{client_ip}] WARNING: cleanup left {dirt} behind; the next " + f"case will refuse to start until this clears") def prepare_mount_points(client_ip, key_path): @@ -576,6 +665,10 @@ def prepare_mount_points(client_ip, key_path): "| grep -oE 'nqn\\.2023-02\\.io\\.simplyblock:[^ ,]+'); do " "sudo timeout 20 nvme disconnect -n \"$n\" >/dev/null 2>&1 || true; done", check=False, timeout=300) + # The commands above are best-effort by design; this is the part that + # decides whether we may proceed. A lazy unmount reports success while the + # mount is still live, so the only trustworthy check is /proc/mounts. + assert_client_clean(client_ip, key_path, "prepare_mount_points") # --------------------------------------------------------------------------- # @@ -2644,6 +2737,12 @@ def test_case_12(meta): # ends in one of two accepted verdicts (completed despite the kill / failed # then succeeded on retry after recovery) and any third outcome fails the case. CHAOS_PHASE_ROUNDS = int(os.environ.get("CHAOS_PHASE_ROUNDS", "20")) +# 20 policies x 2 volumes = 40 volumes replicating at once. At the default +# 1-minute cadence that is 40 transfers a minute and the cluster steady-states +# at ~6 minutes of lag -- above the 3-period gate, so the setup phase never +# completes (run 20260827_110415, killed after 50 minutes without a round). +# The cadence has to be one this volume count can hold. +CHAOS_PHASE_INTERVAL_MIN = int(os.environ.get("CHAOS_PHASE_INTERVAL_MIN", "10")) CHAOS_PHASE_VOLS_PER_POLICY = int(os.environ.get("CHAOS_PHASE_VOLS_PER_POLICY", "2")) CHAOS_PHASE_VOL_SIZE = os.environ.get("CHAOS_PHASE_VOL_SIZE", "10G") #: cap for the randomized kill delay when a phase turns out to be slow @@ -2765,8 +2864,10 @@ def _chaos_phase_case(meta, phase, title): # The cutover under test in case 15 is the FINAL MIGRATION STEP: a # migration-mode policy committed on the SOURCE volumes. mode = "migration" if phase == "commit" else "failover" - print(" seed=%d rounds=%d vols/policy=%d mode=%s phase=%s" - % (seed, CHAOS_PHASE_ROUNDS, CHAOS_PHASE_VOLS_PER_POLICY, mode, phase)) + print(" seed=%d rounds=%d vols/policy=%d mode=%s phase=%s cadence=%dmin " + "lag_gate=%ds" + % (seed, CHAOS_PHASE_ROUNDS, CHAOS_PHASE_VOLS_PER_POLICY, mode, phase, + CHAOS_PHASE_INTERVAL_MIN, lag_gate_for(CHAOS_PHASE_INTERVAL_MIN))) prepare_mount_points(client_ip, key_path) delete_test_volumes(mgmt_ip, key_path, _all_test_pools(meta)) @@ -2777,7 +2878,8 @@ def _chaos_phase_case(meta, phase, title): policy = set_cluster_replication( mgmt_ip, key_path, src_uuid, tgt_uuid, pool_uuid_of(mgmt_ip, key_path, tgt["pool"]), mode=mode, - policy_name="pol_chaos_%s_%02d" % (phase, r)) + policy_name="pol_chaos_%s_%02d" % (phase, r), + interval_min=CHAOS_PHASE_INTERVAL_MIN) vols = [] for v in range(CHAOS_PHASE_VOLS_PER_POLICY): name = "replvol%02d_%d" % (r, v) @@ -2794,7 +2896,8 @@ def _chaos_phase_case(meta, phase, title): baseline = write_baseline(client_ip, key_path, mounts) baseline_ts = time.time() cleanup_client(client_ip, key_path, mounts) - wait_replication_caught_up(mgmt_ip, key_path, all_lvols, timeout=7200) + wait_replication_caught_up(mgmt_ip, key_path, all_lvols, timeout=7200, + max_lag=lag_gate_for(CHAOS_PHASE_INTERVAL_MIN)) wait_data_replicated(mgmt_ip, key_path, all_lvols, baseline_ts, timeout=7200) print(" all %d volumes replicated; starting the rounds" % len(all_lvols)) From 2e55ad28f02ff36e35230b6bb5a9612bade45142 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 27 Aug 2026 16:51:38 +0100 Subject: [PATCH 085/122] fix: extend set_cutover_proceed to signal failback cutovers via target_lvol lookup --- .../replication_policy_controller.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/simplyblock_core/controllers/replication_policy_controller.py b/simplyblock_core/controllers/replication_policy_controller.py index f1e97db912..7ee30ec3be 100644 --- a/simplyblock_core/controllers/replication_policy_controller.py +++ b/simplyblock_core/controllers/replication_policy_controller.py @@ -360,16 +360,30 @@ def _failover_volumes(volumes, what): def set_cutover_proceed(lvol_id): - """Signal that the operator has connected the target NVMe paths. + """Signal that the operator has connected the NVMe paths. - Finds the cutover_pending LVolReplication for *lvol_id* (source side) and + Finds the cutover_pending LVolReplication for *lvol_id* — either as the + source (migration direction) or as the target (failback direction) — and sets cutover_proceed = True so the task runner advances past the wait. + During failback the replication direction is reversed: the original source + volume becomes the TARGET of the reverse replication, so _active_relationship + (which searches by source) would miss it. The fallback search by target_lvol + handles this case without changing the API surface. + Returns the replication ID on success, raises KeyError when no matching cutover_pending record is found. """ rep = _active_relationship(lvol_id) if rep is None or rep.state != LVolReplication.STATE_CUTOVER_PENDING: + # Failback path: lvol_id is the target of the reverse replication. + rep = None + for r in reversed(db.get_lvol_replication_objects()): + if (r.target_lvol and r.target_lvol.get_id() == lvol_id + and r.state == LVolReplication.STATE_CUTOVER_PENDING): + rep = r + break + if rep is None: raise KeyError( f"No cutover_pending replication found for volume {lvol_id}") rep.cutover_proceed = True From 51ccb7a1d627e33ce332f8a634914acd4b703430 Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 27 Aug 2026 18:40:45 +0200 Subject: [PATCH 086/122] Stamp round 1, so the cutover actually converges Run 20260827_172734 deployed the convergence loop and the freeze did not move: 9-55s server-side, 20-77s client-observed. The loop was running and logging -- and every line read cutover convergence: lvol=... round 1 transferred in -0.00s Negative zero, twelve times out of twelve. replication_commit creates the cutover task with shrink_round=1 and shrink_snap_id but never stamped shrink_started_at, and the loop defaulted the missing value to now: elapsed = time.time() - params.get("shrink_started_at", time.time()) so round 1 always measured as instant, always compared under the 2s target, and always declared convergence. Exactly one round ran and the freeze then copied everything written since that round's snapshot was TAKEN -- including the whole time it spent replicating, which is minutes under load. That is the entire freeze. The task now carries the stamp, and an UNMEASURED round is treated as not converged rather than as instant: a missing measurement must never read as "the delta is small". The other three fixes did work in that run -- the per-LVS exclusion logged 320 holds ("lvol b12f0312 is in final cutover on lvstore ..."), and no cutover waited on the operator gate. Every earlier test supplied shrink_started_at by hand, which is exactly why they passed while the real creation path was broken; the new ones drive the controller's own param block and the unmeasured-round path. 2039 pass. --- .../controllers/lvol_controller.py | 6 +++ .../tasks_runner_replication_final.py | 23 +++++++-- .../test/test_cutover_convergence.py | 49 +++++++++++++++++++ 3 files changed, 73 insertions(+), 5 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index b3b8e3dc8e..a61c3b2d3b 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -4421,6 +4421,12 @@ def replication_commit(lvol_id, delete_source=False): "final_state": LVolReplication.STATE_CUTOVER_DONE, "shrink_round": 1, "shrink_snap_id": snap_uuid, + # When this round started transferring. The convergence loop + # measures each round against it to decide whether the delta is + # small enough to freeze; without it round 1 measures as 0.00s and + # "converges" instantly, which is how the freeze stayed at 9-55s + # with the loop deployed (run 20260827_172734). + "shrink_started_at": time.time(), "shrink_deadline": int(time.time()) + constants.REPL_CUTOVER_SHRINK_TIMEOUT_SEC, # Migration semantics on request: retire the source volume once # the cutover state is durable (see _finalize in the final runner). diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 2a3820dfda..a531480336 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -338,10 +338,22 @@ def _shrink_step(task, lvol): time.sleep(constants.REPL_CUTOVER_POLL_INTERVAL_SEC) continue - elapsed = time.time() - params.get("shrink_started_at", time.time()) - params.setdefault("shrink_round_times", []).append(round(elapsed, 2)) - logger.info("cutover convergence: lvol=%s round %d transferred in %.2fs", - lvol.get_id(), params["shrink_round"], elapsed) + started_at = params.get("shrink_started_at") + if started_at is None: + # Unmeasurable round (an older task, or one enqueued without the + # stamp). Treat it as NOT converged rather than as instant: a + # missing measurement must never be read as "the delta is small", + # which is precisely the mistake that kept the freeze at 9-55s. + elapsed = float("inf") + logger.warning( + "cutover convergence: lvol=%s round %d has no start stamp; " + "taking another round rather than assuming it was fast", + lvol.get_id(), params["shrink_round"]) + else: + elapsed = time.time() - started_at + params.setdefault("shrink_round_times", []).append(round(elapsed, 2)) + logger.info("cutover convergence: lvol=%s round %d transferred in %.2fs", + lvol.get_id(), params["shrink_round"], elapsed) # Converged: this round's delta -- the writes made during the previous # round -- moved in low seconds, so the freeze that copies the next @@ -376,7 +388,8 @@ def _shrink_step(task, lvol): # Re-arm the inline window from what this round just measured: rounds # near convergence are short and stay inline, a slow one hands the # runner back so other volumes' cutovers are not stuck behind it. - budget_end = time.time() + _inline_window(elapsed) + budget_end = time.time() + _inline_window( + elapsed if elapsed != float("inf") else 0) if time.time() >= budget_end: task.function_result = (f"shrink round {params['shrink_round'] - 1} done " f"({elapsed:.2f}s); continuing next pass") diff --git a/simplyblock_core/test/test_cutover_convergence.py b/simplyblock_core/test/test_cutover_convergence.py index c4170b1146..c872f36a1c 100644 --- a/simplyblock_core/test/test_cutover_convergence.py +++ b/simplyblock_core/test/test_cutover_convergence.py @@ -343,3 +343,52 @@ def test_a_group_transferring_on_another_lvstore_does_not_hold_us(self): self.assertEqual( self.sr._lvs_transfer_hold(self._task(), self._snapshot(lvol_id="LV_b")), "") + + +class TestRoundOneIsMeasured(unittest.TestCase): + """The regression that let the freeze survive the convergence loop. + + replicate/commit creates the cutover task itself, and its params are the + ONLY ones the loop ever sees for round 1. Every earlier test supplied + shrink_started_at by hand, so none of them noticed the controller did not: + the round then measured as 0.00s, counted as converged, and the freeze + copied the whole delta (run 20260827_172734, 9-55s server-side). + """ + + def test_the_controller_stamps_the_start_of_round_one(self): + import inspect + from simplyblock_core.controllers import lvol_controller as lc + src = inspect.getsource(lc.replication_commit) + self.assertIn('"shrink_started_at"', src, + "round 1 must carry the stamp the loop measures against") + self.assertLess(src.index('"shrink_round": 1'), + src.index('"shrink_deadline"'), + "sanity: this is the cutover task's param block") + + def test_an_unmeasured_round_is_not_treated_as_converged(self): + """Belt and braces for tasks enqueued without the stamp.""" + clock = _Clock() + task = _Task(shrink_snap_id="S0", shrink_round=1, + shrink_deadline=10 ** 9, lvol_id="LV1") + # deliberately NO shrink_started_at + taken = [] + + def _take(task_, lvol_): + taken.append(task_.function_params["shrink_round"]) + task_.function_params["shrink_round"] += 1 + task_.function_params["shrink_snap_id"] = "S1" + task_.function_params["shrink_started_at"] = clock.now + return "S1", None + + with patch.object(runner.time, "time", clock), \ + patch.object(runner.time, "sleep", clock.sleep), \ + patch.object(runner, "_shrink_round_done", return_value=True), \ + patch.object(runner, "_take_shrink_snapshot", side_effect=_take), \ + patch.object(constants, "REPL_CUTOVER_MIN_INLINE_SEC", 0), \ + patch.object(constants, "REPL_CUTOVER_CONVERGE_BUDGET_SEC", 0): + done, err = runner._shrink_step(task, _lvol()) + + self.assertFalse(done, "an unmeasured round must not end the shrink phase") + self.assertIsNone(err) + self.assertEqual(taken, [1], + "it must take another round instead of freezing") From 51c5670d60ac60d694973839c4d27f6b5a2d48ac Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 27 Aug 2026 19:22:55 +0200 Subject: [PATCH 087/122] Queue cutovers per lvstore instead of starving all but one Run 20260827_185009 stalled with the convergence finally working. All 20 volumes entered their cutover together and every one wrote cutover_lvs, because the claim was a marker with no mutual exclusion. The replication side then picked the arbitrary first claim as the owner and held everyone else -- including the other nine cutovers' own shrink snapshots. Per lvstore, one volume progressed and nine sat at "round 1: waiting to replicate": ACTIVE LVS CLAIMS: 20 (LVS_10 x10, LVS_13 x10) currently HELD transfers: 53, snapshot tasks stuck in `new`: 75 and in the run before it, queued cutovers ran their shrink deadline down until 17 of them died of "max retry reached (8/8)". The exclusion is what was asked for; the missing half is that a cutover which cannot have the lvstore must WAIT rather than start. It now checks for an existing owner before claiming: the earliest active claim wins (deterministic, so two racing tasks agree instead of each seeing the other), and a loser suspends with its shrink phase untouched, no retry burned, and its deadline pushed out -- queueing is not a failure and must not time out. A consistency group is exempt in the direction that matters: a sibling of the owner's group joins it rather than queueing, because a group cuts over together. 2046 pass, including a behavioural test that fails if a queued task starts its shrink phase, burns a retry, or lets its deadline run down. --- .../tasks_runner_replication_final.py | 63 ++++++- .../test/test_cutover_convergence.py | 159 ++++++++++++++++++ 2 files changed, 216 insertions(+), 6 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index a531480336..7b381f2232 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -37,6 +37,31 @@ db = db_controller.DBController() +def _lvs_cutover_owner(task, lvs_name): + """The task that already owns *lvs_name* for a cutover, or None. + + Deterministic: the earliest-created active claim wins, so two tasks racing + the same lvstore agree on who owns it instead of each seeing the other. + """ + if not lvs_name: + return None + owners = [] + for other in db.get_job_tasks(task.cluster_id): + if other.function_name != JobSchedule.FN_REPLICATION_FINAL: + continue + if other.get_id() == task.get_id(): + continue + if other.status == JobSchedule.STATUS_DONE or other.canceled: + continue + if (other.function_params or {}).get("cutover_lvs") != lvs_name: + continue + owners.append(other) + if not owners: + return None + owners.sort(key=lambda t: (str(getattr(t, "create_dt", "")), t.get_id())) + return owners[0] + + def _group_id_for_lvol(lvol): """The consistency group *lvol* belongs to, or "". @@ -181,12 +206,38 @@ def task_runner(task: JobSchedule): # compete for the same lvstore and hub bandwidth, and every second they # steal from a convergence round is a second of writes that lands in # the freeze. - task.function_params["cutover_lvs"] = getattr(lvol, "lvs_name", "") - # A consistency group cuts over AS A GROUP: its members' transfers and - # cutovers run alongside each other, and only volumes outside the group - # are held. Recording the group on the claim is what lets the - # replication side tell a sibling member from an unrelated volume. - task.function_params["cutover_group"] = _group_id_for_lvol(lvol) + lvs_name = getattr(lvol, "lvs_name", "") + own_group = _group_id_for_lvol(lvol) + owner = _lvs_cutover_owner(task, lvs_name) + if owner is not None: + owner_group = (owner.function_params or {}).get("cutover_group") or "" + owner_lvol = str((owner.function_params or {}).get("lvol_id")) + # A consistency group cuts over AS A GROUP, so a sibling member + # joins the owner rather than queueing behind it. + if not (own_group and own_group == owner_group): + # WAIT, do not start. Beginning the shrink phase here would + # take snapshots that cannot replicate (the owner holds the + # lvstore), and their deadline would run down until the task + # died of max retries -- which is exactly what happened to 17 + # tasks in run 20260827_185009. + task.function_params["shrink_deadline"] = ( + int(time.time()) + constants.REPL_CUTOVER_SHRINK_TIMEOUT_SEC) + task.function_result = ( + f"queued for lvstore {lvs_name} behind {owner_lvol[:8]}") + task.status = JobSchedule.STATUS_SUSPENDED + task.write_to_db(db.kv_store) + return False # no retry burned: this is a queue, not a failure + + # Claim the source LVS for the whole cutover -- the convergence rounds + # AND the freeze. Other volumes' snapshot transfers on this LVS queue + # behind it (see snapshot_replication._lvs_transfer_hold): they compete + # for the same lvstore and hub bandwidth, and every second they steal + # from a convergence round is a second of writes that lands in the + # freeze. + task.function_params["cutover_lvs"] = lvs_name + # Recording the group is what lets the replication side tell a sibling + # member from an unrelated volume. + task.function_params["cutover_group"] = own_group task.write_to_db(db.kv_store) # ---- SHRINK PHASE ----------------------------------------------- # diff --git a/simplyblock_core/test/test_cutover_convergence.py b/simplyblock_core/test/test_cutover_convergence.py index c872f36a1c..1a4c598fe0 100644 --- a/simplyblock_core/test/test_cutover_convergence.py +++ b/simplyblock_core/test/test_cutover_convergence.py @@ -392,3 +392,162 @@ def _take(task_, lvol_): self.assertIsNone(err) self.assertEqual(taken, [1], "it must take another round instead of freezing") + + +class TestCutoverQueue(unittest.TestCase): + """One cutover per lvstore, and the losers queue instead of starving. + + Run 20260827_185009: all 20 volumes entered their cutover together and each + wrote cutover_lvs, so the claim was a marker with no exclusion. The + replication side then held everyone but an arbitrary winner -- including + the other cutovers' own shrink snapshots -- and 9 of 10 volumes per + lvstore sat at "round 1: waiting to replicate" until their deadline killed + them (17 x "max retry reached"). + """ + + def setUp(self): + patcher = patch.object(runner, "db") + self.db = patcher.start() + self.addCleanup(patcher.stop) + self.db.kv_store = "KV" + gp = patch.object(runner, "_group_id_for_lvol", return_value="") + self.group_of = gp.start() + self.addCleanup(gp.stop) + + def _task(self, task_id, lvs=None, group="", status="running", + canceled=False, created="2026-01-01"): + from simplyblock_core.models.job_schedule import JobSchedule + t = MagicMock() + t.function_name = JobSchedule.FN_REPLICATION_FINAL + t.get_id.return_value = task_id + t.status = status + t.canceled = canceled + t.create_dt = created + t.function_params = {"lvol_id": "LV_" + task_id} + if lvs: + t.function_params["cutover_lvs"] = lvs + t.function_params["cutover_group"] = group + return t + + def test_no_owner_means_the_lvstore_is_free(self): + me = self._task("T1") + self.db.get_job_tasks.return_value = [me] + self.assertIsNone(runner._lvs_cutover_owner(me, "LVS_1")) + + def test_an_active_claim_owns_the_lvstore(self): + me, other = self._task("T1"), self._task("T2", lvs="LVS_1") + self.db.get_job_tasks.return_value = [me, other] + owner = runner._lvs_cutover_owner(me, "LVS_1") + self.assertIsNotNone(owner) + self.assertEqual(owner.get_id(), "T2") + + def test_a_finished_or_cancelled_cutover_owns_nothing(self): + from simplyblock_core.models.job_schedule import JobSchedule + me = self._task("T1") + for dead in (self._task("T2", lvs="LVS_1", + status=JobSchedule.STATUS_DONE), + self._task("T3", lvs="LVS_1", canceled=True)): + self.db.get_job_tasks.return_value = [me, dead] + self.assertIsNone(runner._lvs_cutover_owner(me, "LVS_1"), + "a dead task must not hold the lvstore forever") + + def test_the_earliest_claim_wins_deterministically(self): + """Two tasks racing must agree on the winner, not each see the other.""" + me = self._task("T1") + early = self._task("T2", lvs="LVS_1", created="2026-01-01") + late = self._task("T3", lvs="LVS_1", created="2026-06-01") + self.db.get_job_tasks.return_value = [me, late, early] + self.assertEqual(runner._lvs_cutover_owner(me, "LVS_1").get_id(), "T2") + + def test_a_claim_on_another_lvstore_is_irrelevant(self): + me, other = self._task("T1"), self._task("T2", lvs="LVS_9") + self.db.get_job_tasks.return_value = [me, other] + self.assertIsNone(runner._lvs_cutover_owner(me, "LVS_1")) + + +class TestQueuedCutoverDoesNotStarve(unittest.TestCase): + """A cutover that cannot have the lvstore waits without cost.""" + + def setUp(self): + from simplyblock_core.models.job_schedule import JobSchedule + from simplyblock_core.models.storage_node import StorageNode + self.JobSchedule = JobSchedule + patcher = patch.object(runner, "db") + self.db = patcher.start() + self.addCleanup(patcher.stop) + self.db.kv_store = "KV" + + lvol = MagicMock() + lvol.get_id.return_value = "LV_me" + lvol.lvs_name = "LVS_1" + self.db.get_lvol_by_id.return_value = lvol + + node = MagicMock() + node.status = StorageNode.STATUS_ONLINE + node.get_id.return_value = "N1" + node.cluster_id = "CL" + self.db.get_storage_node_by_id.return_value = node + + gp = patch.object(runner, "_group_id_for_lvol", return_value="") + gp.start() + self.addCleanup(gp.stop) + # If the shrink phase ran, the test would see it here. + sp = patch.object(runner, "_shrink_step", + side_effect=AssertionError( + "a queued cutover must not start its shrink phase")) + sp.start() + self.addCleanup(sp.stop) + + def _me(self): + t = MagicMock() + t.function_name = self.JobSchedule.FN_REPLICATION_FINAL + t.get_id.return_value = "T_me" + t.cluster_id = "CL" + t.status = self.JobSchedule.STATUS_NEW + t.canceled = False + t.retry = 0 + t.max_retry = 8 + t.create_dt = "2026-06-01" + t.function_params = { + "lvol_id": "LV_me", "src_node_id": "N1", "tgt_node_id": "N2", + "shrink_round": 1, "shrink_snap_id": "S1", + "shrink_deadline": 1, # already expired + } + return t + + def _owner(self): + t = MagicMock() + t.function_name = self.JobSchedule.FN_REPLICATION_FINAL + t.get_id.return_value = "T_owner" + t.status = self.JobSchedule.STATUS_RUNNING + t.canceled = False + t.create_dt = "2026-01-01" + t.function_params = {"lvol_id": "LV_owner", "cutover_lvs": "LVS_1", + "cutover_group": ""} + return t + + def test_it_queues_without_burning_a_retry_or_its_deadline(self): + me, owner = self._me(), self._owner() + self.db.get_job_tasks.return_value = [me, owner] + + result = runner.task_runner(me) + + self.assertFalse(result) + self.assertEqual(me.status, self.JobSchedule.STATUS_SUSPENDED) + self.assertEqual(me.retry, 0, "queueing is not a failure") + self.assertIn("queued for lvstore", me.function_result) + self.assertGreater( + me.function_params["shrink_deadline"], 10 ** 9, + "the deadline must be pushed out while queued, or the task dies of " + "max retries waiting for a lock it cannot win") + self.assertNotIn("cutover_lvs", me.function_params, + "a queued task must not also claim the lvstore") + + def test_a_group_sibling_joins_the_owner_instead_of_queueing(self): + me, owner = self._me(), self._owner() + owner.function_params["cutover_group"] = "CL/G1" + with patch.object(runner, "_group_id_for_lvol", return_value="CL/G1"): + self.db.get_job_tasks.return_value = [me, owner] + # It proceeds into the shrink phase, which this fixture makes raise. + with self.assertRaises(AssertionError): + runner.task_runner(me) From d5f058b837f16cc7d3714efdcf9838a7b27e6720 Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 27 Aug 2026 20:55:47 +0200 Subject: [PATCH 088/122] Make a dying cutover say why Run 20260827_194551: all 20 fail-back cutovers ended as "max retry reached (8/8)" -- 160 failed attempts that produced not one log line, so three investigations of that run could not name the failing branch. _finalize's failure path only wrote function_result, and the max-retry branch then overwrote it, destroying the last trace of the cause. Each failed attempt is now logged with the lvol and the attempt number, the reason is kept in last_error where the max-retry branch cannot clobber it, and giving up reports "max retry reached (8) after: " instead of the symptom alone. The fast-retry branch (target node not online) says which node and what state, since that one can burn all eight attempts in 80 seconds. No behaviour change beyond observability -- the run has to be repeated to get the cause, and this is what makes the repeat worth anything. --- .../tasks_runner_replication_final.py | 23 +++++++++- .../test/test_cutover_convergence.py | 45 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 7b381f2232..904350d3c3 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -152,6 +152,13 @@ def _finalize(task, ok, err): return True task.function_result = err or "cutover failed, retrying" + # Keep the reason where the max-retry branch cannot overwrite it, and say + # it out loud: a task that quietly retries to death costs a whole lab run + # to diagnose (run 20260827_194551 -- 20 tasks, 160 attempts, no log line). + task.function_params["last_error"] = task.function_result + logger.warning("cutover attempt %d/%d failed for lvol %s: %s", + task.retry + 1, task.max_retry, + task.function_params.get("lvol_id"), task.function_result) task.status = JobSchedule.STATUS_SUSPENDED task.retry += 1 # A retry re-claims the LVS on its next pass; holding the claim across the @@ -168,7 +175,17 @@ def task_runner(task: JobSchedule): return _finalize(task, False, "missing lvol_id in task params") if task.retry >= task.max_retry or task.canceled is True: - task.function_result = "task cancelled" if task.canceled else "max retry reached" + if task.canceled: + task.function_result = "task cancelled" + else: + # Carry the last real error: "max retry reached" on its own names + # a symptom and hides the cause. + last = task.function_params.get("last_error") + task.function_result = (f"max retry reached ({task.max_retry}) after: {last}" + if last else "max retry reached") + logger.error("cutover gave up on lvol %s after %d attempts: %s", + task.function_params.get("lvol_id"), task.max_retry, + last or "reason not recorded") task.status = JobSchedule.STATUS_DONE task.write_to_db(db.kv_store) return True @@ -188,6 +205,10 @@ def task_runner(task: JobSchedule): pass if tgt_node.status != StorageNode.STATUS_ONLINE: + logger.warning("cutover for lvol %s waiting: target node %s is %s", + params.get("lvol_id"), tgt_node.get_id(), tgt_node.status) + task.function_params["last_error"] = ( + f"target node {tgt_node.get_id()[:8]} is {tgt_node.status}") task.function_result = "target node not online, retrying" task.status = JobSchedule.STATUS_SUSPENDED task.retry += 1 diff --git a/simplyblock_core/test/test_cutover_convergence.py b/simplyblock_core/test/test_cutover_convergence.py index 1a4c598fe0..c02a12aeac 100644 --- a/simplyblock_core/test/test_cutover_convergence.py +++ b/simplyblock_core/test/test_cutover_convergence.py @@ -551,3 +551,48 @@ def test_a_group_sibling_joins_the_owner_instead_of_queueing(self): # It proceeds into the shrink phase, which this fixture makes raise. with self.assertRaises(AssertionError): runner.task_runner(me) + + +class TestCutoverFailuresAreVisible(unittest.TestCase): + """160 failed attempts must not produce zero log lines. + + Run 20260827_194551: every fail-back cutover ended as "max retry reached + (8/8)" with nothing logged and the cause overwritten, so three separate + investigations could not name the failing branch. + """ + + def setUp(self): + patcher = patch.object(runner, "db") + self.db = patcher.start() + self.addCleanup(patcher.stop) + self.db.kv_store = "KV" + + def _task(self, retry=0): + from simplyblock_core.models.job_schedule import JobSchedule + t = _Task(lvol_id="LV1") + t.status = JobSchedule.STATUS_RUNNING + t.retry = retry + t.max_retry = 8 + t.canceled = False + return t + + def test_a_failed_attempt_is_logged_and_remembered(self): + task = self._task() + with self.assertLogs(runner.logger, level="WARNING") as logs: + runner._finalize(task, False, "target subsystem is full") + self.assertIn("target subsystem is full", "\n".join(logs.output)) + self.assertEqual(task.function_params["last_error"], + "target subsystem is full") + + def test_giving_up_reports_the_cause_not_just_the_symptom(self): + from simplyblock_core.models.job_schedule import JobSchedule + task = self._task(retry=8) + task.function_params["last_error"] = "target subsystem is full" + task.function_params.update({"src_node_id": "N1", "tgt_node_id": "N2"}) + with self.assertLogs(runner.logger, level="ERROR") as logs: + runner.task_runner(task) + self.assertEqual(task.status, JobSchedule.STATUS_DONE) + self.assertIn("target subsystem is full", task.function_result, + "'max retry reached' alone names a symptom and hides the " + "cause") + self.assertIn("target subsystem is full", "\n".join(logs.output)) From ae7c5bf8152ea77c56d8fbe855a8e6aaac5e5a1e Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 27 Aug 2026 20:35:20 +0100 Subject: [PATCH 089/122] run replication-final tasks in parallel threads to eliminate serial shrink-round delays --- .../tasks_runner_replication_final.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index f0b2756b5c..65b20d7096 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -21,6 +21,7 @@ the cutover phase has prepared them) """ import time +import threading import uuid as uuid_lib from datetime import datetime @@ -399,6 +400,13 @@ def _prepare_cutover(task, lvol, src_node, tgt_node): return None +def _run_task_safe(task): + try: + task_runner(task) + except Exception as e: + logger.error(f"replication-final task {task.uuid} failed: {e}", exc_info=True) + + def main(): logger.info("Starting replication-final tasks runner...") while True: @@ -408,6 +416,7 @@ def main(): logger.error(f"Failed to get clusters: {e}") time.sleep(3) continue + threads = [] for cl in clusters: for task in db.get_job_tasks(cl.get_id(), reverse=False): if task.function_name != JobSchedule.FN_REPLICATION_FINAL: @@ -415,13 +424,11 @@ def main(): if task.status == JobSchedule.STATUS_DONE: continue task = db.get_task_by_id(task.uuid) - try: - res = task_runner(task) - except Exception as e: - logger.error(f"replication-final task {task.uuid} failed: {e}", exc_info=True) - res = False - if not res: - time.sleep(3) + t = threading.Thread(target=_run_task_safe, args=(task,), daemon=True) + threads.append(t) + t.start() + for t in threads: + t.join() time.sleep(constants.TASK_EXEC_INTERVAL_SEC) From 15edcc91f6459930799d04db9f63ad9a27175ca1 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 27 Aug 2026 20:59:30 +0100 Subject: [PATCH 090/122] fix flaky test: use ordinal sum instead of hash() for mgmt_ip to avoid PYTHONHASHSEED-dependent collisions --- tests/unit/test_node_online_device_readmit.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_node_online_device_readmit.py b/tests/unit/test_node_online_device_readmit.py index cc99e84870..790b699eaa 100644 --- a/tests/unit/test_node_online_device_readmit.py +++ b/tests/unit/test_node_online_device_readmit.py @@ -59,7 +59,10 @@ def _node(uuid, status): # Distinct per-node mgmt_ip: get_next_cluster_status dedups affected # nodes by physical host (mgmt_ip); leaving the model default ("") on # every node collapses all affected nodes into one and undercounts. - n.mgmt_ip = f"10.99.0.{abs(hash(uuid)) % 250 + 1}" + # Use sum-of-ordinals rather than hash() — Python randomizes hash() per + # process (PYTHONHASHSEED), so two UUIDs can collide under some seeds and + # make the dedup return a count of 1 instead of 2, flipping the verdict. + n.mgmt_ip = f"10.99.0.{(sum(ord(c) for c in uuid) % 250) + 1}" return n From ea4440e3a365afd97edc55dfcf644789972612d3 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 27 Aug 2026 22:03:14 +0100 Subject: [PATCH 091/122] failback: delete stale failed_over LVolReplication after UUID swap to stop operator seeing perpetual failed_over --- .../services/tasks_runner_replication_final.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 65b20d7096..b392c9b483 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -54,6 +54,20 @@ def _finalize(task, ok, err): failback_source_id = task.function_params.get("failback_source_lvol_id") if failback_source_id and rep is not None: _swap_failback_lvol_uuid(rep, failback_source_id) + # Remove the stale failed_over LVolReplication that predates the + # failback. Without this the operator's get_relationship query keeps + # finding the old record and reports failed_over indefinitely even + # though IO has already returned to the original source cluster. + prior_replication_id = task.function_params.get("failback_prior_replication_id") + if prior_replication_id: + try: + prior_rep = db.get_lvol_replication_by_id(prior_replication_id) + prior_rep.remove(db.kv_store) + logger.info( + "failback: removed stale failed_over replication record %s", + prior_replication_id) + except KeyError: + pass task.function_result = "cutover done" task.status = JobSchedule.STATUS_DONE task.function_params["end_time"] = int(time.time()) @@ -384,6 +398,7 @@ def _prepare_cutover(task, lvol, src_node, tgt_node): and prior.source_cluster_id == tgt_node.cluster_id and prior.source_lvol): task.function_params["failback_source_lvol_id"] = prior.source_lvol.get_id() + task.function_params["failback_prior_replication_id"] = prior.get_id() logger.info( "failback cutover detected: original source UUID %s will be " "preserved on new clone %s after cutover", From d97daeafc3223bff2fe47a0096b1eb6ce7368fff Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 28 Aug 2026 00:30:19 +0200 Subject: [PATCH 092/122] Instrument the replication pipeline so its delays are separable Seven case-7 investigations could not answer one question: of the 588s a convergence round takes, how much is DATA TRANSFER and how much is orchestration? Round duration was the only number available, and it spans the snapshot, the landing-volume create, the hub attach, the transfer, the detach, add_clone and convert on two nodes, DB writes, and up to TASK_EXEC_INTERVAL_SEC of task-runner latency per state change. Every hardware-level theory tested against that number -- qpair fair share, poll groups, 2 MiB granularity, chain depth -- came out an order of magnitude off, because the number is not a throughput. New simplyblock_core/xfer_timing.py emits one parseable line per phase, each carrying its own epoch stamp (container clocks are skewed from the host's, so `docker service logs -t` ordering cannot be trusted across services): XFER-TIMING t=... phase=transfer_complete lvol=1c8874f3 snap=a0f48bf5 \ round=2 ms=1843.2 bytes=33554432 mbps=18.2 ok=1 Instrumented, in pipeline order: take_shrink_snapshot, landing_volume_create, hub_attach, transfer_submit, transfer_running (with offset -- the only direct read on throughput), transfer_complete (with bytes), hub_detach, chain_add_clone and chain_convert per node, replicate_finish, round_total, round_gap_to_next_snapshot, and task_pass per runner pass so scheduler latency is visible. The freeze is broken down separately: fence_source, final_step_transfer, final_peer_add_clone, enable_target_paths, and freeze_total measured fence -> paths-live, which is the window that has to fit inside the client's 8s fast_io_fail_tmo. Harness: fio's aggregate bandwidth has been written to /tmp/fio_repl.log all along under --status-interval=15 and never read -- which is why three analyses GUESSED the client write rate and were 5x low. fio_bandwidth() now reports it at phase boundaries, and collect_xfer_timing() pulls the CP lines off the services, including when the fail-back FAILS, which is exactly when the breakdown is needed. scripts/xfer_timing_report.py turns a collected dump into the breakdown, with an UNACCOUNTED line: if round_total dwarfs the sum of its measured parts, the missing time is somewhere nobody is looking, and that gap is the finding. Validated against a synthetic dump rather than discovered on the lab. Pure instrumentation, no behaviour change. Two test fakes were missing methods their real counterparts have (LVol.get_id) and were completed. 2048 pass. --- scripts/test_async_replication.py | 59 +++++ scripts/xfer_timing_report.py | 215 ++++++++++++++++++ .../services/replication_final_step.py | 36 ++- .../services/snapshot_replication.py | 53 ++++- .../tasks_runner_replication_final.py | 43 +++- .../test/test_replication_final_step.py | 5 + simplyblock_core/xfer_timing.py | 123 ++++++++++ 7 files changed, 511 insertions(+), 23 deletions(-) create mode 100644 scripts/xfer_timing_report.py create mode 100644 simplyblock_core/xfer_timing.py diff --git a/scripts/test_async_replication.py b/scripts/test_async_replication.py index 8453073e69..49299b1649 100644 --- a/scripts/test_async_replication.py +++ b/scripts/test_async_replication.py @@ -507,6 +507,57 @@ def write_fio_jobfile(client_ip, key_path, mounts, return jobfile +def fio_bandwidth(client_ip, key_path, label=""): + """Print fio's own aggregate bandwidth, and return (read_mbps, write_mbps). + + fio has been writing this all along under --status-interval=15; nothing + read it, so every earlier analysis inferred the client rate from round + durations instead (and was 5x out). + """ + out = run(client_ip, key_path, + "grep -aE '^ *(READ|WRITE): bw=' %s 2>/dev/null | tail -4 || true" + % FIO_LOG, check=False, quiet=True) + rd = wr = None + for line in (out or "").splitlines(): + m = re.search(r"\((\d+(?:\.\d+)?)([kMG]?B)/s\)", line) + if not m: + continue + val = float(m.group(1)) + unit = m.group(2) + mbps = val / 1000.0 if unit == "kB" else (val * 1000.0 if unit == "GB" else val) + if line.strip().startswith("READ"): + rd = mbps + elif line.strip().startswith("WRITE"): + wr = mbps + if rd is not None or wr is not None: + print(" [fio %s] %s read %s MB/s, write %s MB/s" + % (label, client_ip, + "%.0f" % rd if rd is not None else "?", + "%.0f" % wr if wr is not None else "?")) + else: + print(" [fio %s] %s no aggregate lines yet" % (label, client_ip)) + return rd, wr + + +def collect_xfer_timing(mgmt_ip, key_path, label): + """Pull XFER-TIMING lines off the CP services into one file on the mgmt node. + + Container clocks are skewed from the host's, so every line carries its own + epoch stamp and we sort on that rather than on docker's timestamps. + """ + dest = "~/xfer_timing_%s.log" % label + services = ("app_TasksRunnerReplicationFinal app_SnapshotReplication " + "app_SnapshotMonitor app_LVolMonitor") + cmd = ("rm -f %s; for S in %s; do " + "sudo docker service logs $S 2>&1 | grep -a XFER-TIMING >> %s || true; " + "done; sort -t= -k2 -n %s -o %s 2>/dev/null || true; wc -l < %s" + % (dest, services, dest, dest, dest, dest)) + out = run(mgmt_ip, key_path, cmd, check=False, quiet=True, timeout=600) + count = (out or "").strip().splitlines()[-1] if (out or "").strip() else "0" + print(" [timing %s] collected %s XFER-TIMING lines -> %s" % (label, count, dest)) + return dest + + def start_fio(client_ip, key_path, jobfile): print("Starting continuous fio load...") run(client_ip, key_path, @@ -1910,8 +1961,13 @@ def test_case_7(meta): start_fio(ip, key_path, write_fio_jobfile(ip, key_path, mounts_by_client[ip], size="1G")) ns_gate = lag_gate_for(NS_INTERVAL_MIN) + for ip in assign: + fio_bandwidth(ip, key_path, "steady-state") wait_replication_caught_up(mgmt_ip, key_path, lvols, timeout=3600, max_lag=ns_gate) wait_data_replicated(mgmt_ip, key_path, lvols, baseline_ts, timeout=3600) + for ip in assign: + fio_bandwidth(ip, key_path, "pre-failover") + collect_xfer_timing(mgmt_ip, key_path, "case7_pre_failover") print("Killing the source cluster (both nodes)...") for ip in src["storage_public_ips"][:2]: @@ -1981,7 +2037,10 @@ def test_case_7(meta): if done == len(tgt_lvols): break time.sleep(15) + collect_xfer_timing(mgmt_ip, key_path, "case7_failback") if done != len(tgt_lvols): + # The breakdown matters MOST here: a stalled fail-back is the case we + # have failed to explain seven times. raise RuntimeError(f"FAIL: only {done}/{len(tgt_lvols)} fail-back cutovers completed") back = failed_over_targets(mgmt_ip, key_path, tgt_lvols) diff --git a/scripts/xfer_timing_report.py b/scripts/xfer_timing_report.py new file mode 100644 index 0000000000..5b0395add4 --- /dev/null +++ b/scripts/xfer_timing_report.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Turn XFER-TIMING lines into the breakdown the soak could not produce. + +The question this exists to answer: of the time a convergence round takes, how +much is DATA TRANSFER and how much is orchestration? Round duration was +previously the only number available, and it spans the landing-volume create, +the hub attach, the transfer, the detach, add_clone and convert on two nodes, +DB writes, and up to TASK_EXEC_INTERVAL_SEC of task-runner latency per state +change. Every hardware-level theory we tested against that number came out an +order of magnitude off, because it is not a throughput. + +Usage: + python scripts/xfer_timing_report.py [--csv out.csv] + +Input is anything containing XFER-TIMING lines (a `docker service logs` dump is +fine). Lines look like: + + XFER-TIMING t=1787868791.244 phase=transfer_complete lvol=1c8874f3 \ + snap=a0f48bf5 round=2 ms=1843.2 bytes=33554432 mbps=18.2 ok=1 +""" +import argparse +import re +import sys +from collections import defaultdict, OrderedDict + +LINE = re.compile(r"XFER-TIMING\s+(.*)$") +KV = re.compile(r"(\w+)=(\S+)") + +# pipeline order for display; anything unlisted is appended +ROUND_PHASES = [ + "take_shrink_snapshot", "landing_volume_create", "hub_attach", + "transfer_submit", "transfer_complete", "hub_detach", + "chain_add_clone", "chain_convert", "round_total", + "round_gap_to_next_snapshot", +] +FREEZE_PHASES = [ + "final_hub_attach", "fence_source", "final_step_transfer", + "final_peer_add_clone", "enable_target_paths", "freeze_total", +] +# what counts as moving data, as opposed to arranging for data to move +TRANSFER_PHASES = {"transfer_complete", "final_step_transfer"} + + +def parse(path): + events = [] + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for raw in fh: + m = LINE.search(raw) + if not m: + continue + rec = {} + for k, v in KV.findall(m.group(1)): + if k in ("t", "ms", "mbps"): + try: + rec[k] = float(v) + except ValueError: + rec[k] = None + elif k in ("round", "bytes", "offset"): + try: + rec[k] = int(v) + except ValueError: + rec[k] = None + else: + rec[k] = v + if "phase" in rec: + events.append(rec) + events.sort(key=lambda r: r.get("t") or 0) + return events + + +def transfer_rate_from_progress(events): + """MB/s per (lvol, snap) inferred from transfer_running offsets. + + This is the one direct read on throughput: `offset` is bytes moved, so the + slope between the first and last sample is the real rate. + """ + runs = defaultdict(list) + for e in events: + if e["phase"] == "transfer_running" and e.get("offset") is not None: + runs[(e.get("lvol"), e.get("snap"))].append((e["t"], e["offset"])) + out = {} + for key, samples in runs.items(): + if len(samples) < 2: + continue + samples.sort() + (t0, o0), (t1, o1) = samples[0], samples[-1] + dt, do = t1 - t0, o1 - o0 + if dt > 0 and do > 0: + out[key] = (do / 1e6 / dt, do, dt, len(samples)) + return out + + +def report(events, csv_path=None): + if not events: + print("no XFER-TIMING lines found -- was the instrumented build deployed?") + return 1 + + span = events[-1]["t"] - events[0]["t"] + print("%d timing events over %.1fs\n" % (len(events), span)) + + # ---- per-phase totals ------------------------------------------------- + agg = defaultdict(lambda: [0, 0.0, 0.0]) # count, total_ms, max_ms + for e in events: + if e.get("ms") is None: + continue + a = agg[e["phase"]] + a[0] += 1 + a[1] += e["ms"] + a[2] = max(a[2], e["ms"]) + + ordered = [p for p in ROUND_PHASES + FREEZE_PHASES if p in agg] + ordered += [p for p in sorted(agg) if p not in ordered] + + print("%-28s %6s %12s %10s %10s" % ("phase", "n", "total_s", "mean_ms", "max_ms")) + print("-" * 70) + for p in ordered: + n, tot, mx = agg[p] + print("%-28s %6d %12.1f %10.1f %10.1f" + % (p, n, tot / 1000.0, tot / n, mx)) + + # ---- the split that matters ------------------------------------------- + # round_total/freeze_total are envelopes; don't double-count them. + envelopes = {"round_total", "freeze_total"} + moved = sum(agg[p][1] for p in agg if p in TRANSFER_PHASES) + arranged = sum(agg[p][1] for p in agg + if p not in TRANSFER_PHASES and p not in envelopes) + if moved + arranged > 0: + pct = 100.0 * moved / (moved + arranged) + print("\nDATA MOVEMENT %8.1fs (%.1f%%)" % (moved / 1000.0, pct)) + print("ORCHESTRATION %8.1fs (%.1f%%)" % (arranged / 1000.0, 100 - pct)) + if pct < 25: + print(" -> the pipeline is dominated by orchestration, not throughput;") + print(" tuning the transfer path cannot fix this.") + + # ---- what the instrumentation cannot explain ------------------------- + # If round_total dwarfs the sum of its measured parts, the missing time is + # somewhere we are not looking -- and that gap is the finding, not a + # rounding error. + inner = [p for p in ROUND_PHASES + if p not in ("round_total", "round_gap_to_next_snapshot")] + measured = sum(agg[p][1] for p in inner if p in agg) + envelope = agg.get("round_total", [0, 0.0, 0.0])[1] + if envelope > 0: + unaccounted = envelope - measured + pct = 100.0 * unaccounted / envelope + print("") + print("round envelopes %8.1fs" % (envelope / 1000.0)) + print("measured phases %8.1fs" % (measured / 1000.0)) + print("UNACCOUNTED %8.1fs (%.1f%% of the envelope)" + % (unaccounted / 1000.0, pct)) + if pct > 30: + print(" -> most of a round is NOT in any instrumented phase.") + print(" Look at task_pass spacing first (scheduler latency),") + print(" then add phases where the gap actually is.") + + # ---- measured transfer throughput ------------------------------------ + rates = transfer_rate_from_progress(events) + if rates: + print("\nmeasured transfer throughput (from transfer_running offsets):") + print("%-12s %-10s %10s %12s %8s" % ("lvol", "snap", "MB/s", "bytes", "samples")) + for (lvol, snap), (mbps, nbytes, dt, n) in sorted( + rates.items(), key=lambda kv: -kv[1][0]): + print("%-12s %-10s %10.1f %12d %8d" % (lvol, snap, mbps, nbytes, n)) + else: + print("\nno transfer_running samples with offsets -- cannot measure") + print("throughput directly; only envelope durations are available.") + + # ---- the freeze, per volume ------------------------------------------ + freezes = [e for e in events if e["phase"] == "freeze_total"] + if freezes: + print("\nclient-visible freeze windows (fence -> paths live):") + for e in sorted(freezes, key=lambda r: -(r.get("ms") or 0)): + flag = " <-- OVER the 8s fast_io_fail_tmo" if (e.get("ms") or 0) > 8000 else "" + print(" lvol=%-10s %8.2fs%s" % (e.get("lvol"), (e["ms"] or 0) / 1000.0, flag)) + + # ---- task-runner latency -------------------------------------------- + passes = defaultdict(list) + for e in events: + if e["phase"] == "task_pass": + passes[e.get("lvol")].append(e["t"]) + if passes: + gaps = [] + for _lvol, times in passes.items(): + times.sort() + gaps += [b - a for a, b in zip(times, times[1:])] + if gaps: + gaps.sort() + print("\ntask-runner pass spacing: n=%d median=%.1fs max=%.1fs" + % (len(gaps), gaps[len(gaps) // 2], gaps[-1])) + print(" (each state change of a cutover costs about one of these)") + + if csv_path: + import csv + cols = ["t", "phase", "lvol", "snap", "round", "ms", "bytes", "mbps", + "offset", "state", "node", "ok"] + with open(csv_path, "w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=cols, extrasaction="ignore") + w.writeheader() + for e in events: + w.writerow(e) + print("\nwrote %s" % csv_path) + return 0 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("logfile") + ap.add_argument("--csv", default=None, help="also write the raw events as CSV") + args = ap.parse_args() + return report(parse(args.logfile), args.csv) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/simplyblock_core/services/replication_final_step.py b/simplyblock_core/services/replication_final_step.py index 907c054f28..2239a2410d 100644 --- a/simplyblock_core/services/replication_final_step.py +++ b/simplyblock_core/services/replication_final_step.py @@ -13,6 +13,7 @@ choreography is the simple "no-overlap" case (target primary → optimized, other target paths → non_optimized, all source paths → inaccessible). """ +from simplyblock_core import xfer_timing from simplyblock_core import db_controller, utils from simplyblock_core.models.storage_node import StorageNode @@ -240,26 +241,40 @@ def run_cutover(src_node, tgt_node, lvol, tgt_lvol_composite, tgt_map_id, # ENODEV (-19) and left the volume stuck in cutover_pending forever, while # snapshot replication (which passes the n1 bdev) kept working. This matches # tasks_runner_lvol_migration, which uses the second element for the same RPC. - _ctrl_name, hub_bdev, err = ensure_hub_attached(src_rpc, tgt_node) + with xfer_timing.phase("final_hub_attach", lvol=lvol.get_id()): + _ctrl_name, hub_bdev, err = ensure_hub_attached(src_rpc, tgt_node) if err: return False, err # Fence the source FIRST: from here on the source cannot take IO by any # means; the delta the freeze copies is definitively final. - fence_source_paths(src_node, src_node.lvstore, lvol.nqn, lvol.ns_id) + # + # The client-visible freeze begins at this call and ends at + # enable_target_paths, so freeze_total below is the number that has to fit + # inside the client's fast_io_fail_tmo (8s in the soak). + _freeze_started = xfer_timing.now() + xfer_timing.stamp("freeze_begin", lvol=lvol.get_id(), nqn=lvol.nqn, + nsid=lvol.ns_id) + with xfer_timing.phase("fence_source", lvol=lvol.get_id()): + fence_source_paths(src_node, src_node.lvstore, lvol.nqn, lvol.ns_id) logger.info( f"[IO-FREEZE] bdev_lvol_transfer_final_step starting: lvol={lvol.uuid} " f"src={src_lvol_composite} tgt_snap={tgt_snap_composite} " f"gateway={hub_bdev} op={operation}") - ret = src_rpc.bdev_lvol_transfer_final_step( - src_lvol_composite, tgt_map_id, tgt_snap_composite, - _FINAL_STEP_BATCH, hub_bdev, operation) + with xfer_timing.phase("final_step_transfer", lvol=lvol.get_id(), + batch=_FINAL_STEP_BATCH): + ret = src_rpc.bdev_lvol_transfer_final_step( + src_lvol_composite, tgt_map_id, tgt_snap_composite, + _FINAL_STEP_BATCH, hub_bdev, operation) if ret is None: # The freeze failed with the source fenced: restore the source paths so # the client resumes there (nothing moved; source is still authoritative) # rather than leaving the volume dark until a retry succeeds. - restore_source_paths(src_node, src_node.lvstore, lvol.nqn, lvol.ns_id) + with xfer_timing.phase("restore_source_paths", lvol=lvol.get_id()): + restore_source_paths(src_node, src_node.lvstore, lvol.nqn, lvol.ns_id) + xfer_timing.gap("freeze_total", _freeze_started, lvol=lvol.get_id(), + aborted=1) return False, "bdev_lvol_transfer_final_step failed" logger.info(f"[IO-RESUME] final step Done: lvol={lvol.uuid} io now live on target") @@ -267,10 +282,17 @@ def run_cutover(src_node, tgt_node, lvol, tgt_lvol_composite, tgt_map_id, # bdev_lvol_transfer_final_step handles the primary internally; peers need an # explicit add_clone. Non-fatal — a missing peer link self-heals on rejoin. for peer in _online_peers(tgt_node): + with xfer_timing.phase("final_peer_add_clone", lvol=lvol.get_id(), + peer=peer.get_id()): if not peer.rpc_client().bdev_lvol_add_clone(tgt_lvol_composite, tgt_snap_composite): logger.warning( f"add_clone on peer {peer.get_id()[:8]} failed for final lvol (non-fatal)") # Light the target: queued client IO drains here. - enable_target_paths(tgt_node, tgt_node.lvstore, lvol.nqn, lvol.ns_id) + with xfer_timing.phase("enable_target_paths", lvol=lvol.get_id()): + enable_target_paths(tgt_node, tgt_node.lvstore, lvol.nqn, lvol.ns_id) + # The whole client-visible window, fence -> paths live. Compare against the + # client's fast_io_fail_tmo: anything longer surfaces as IO errors, not + # just latency. + xfer_timing.gap("freeze_total", _freeze_started, lvol=lvol.get_id()) return True, None diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index 696e496bcc..c91d9790ce 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -2,7 +2,8 @@ import time import uuid -from simplyblock_core import constants, db_controller, snapshot_retention, utils +from simplyblock_core import (constants, db_controller, snapshot_retention, + utils, xfer_timing) from simplyblock_core.controllers import lvol_controller, snapshot_events, snapshot_controller from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.lvol_model import LVol @@ -213,6 +214,7 @@ def _lvs_transfer_hold(task, snapshot): def process_snap_replicate_start(task, snapshot): # 1 create lvol on remote node logger.info("Starting snapshot replication task") + _t_landing = None # set only if we create the landing volume below hold = _lvs_transfer_hold(task, snapshot) if hold: @@ -389,6 +391,7 @@ def process_snap_replicate_start(task, snapshot): # subsystem cap is a user-admission limit; enforcing it here only stops # replication on a node that is already full, which is precisely when # the transfers that would let retention free those slots are needed. + _t_landing = xfer_timing.now() lv_id, err = lvol_controller.add_lvol_ha( f"REP_{snapshot.snap_name}", snapshot.size, remote_node_uuid.get_id(), snapshot.lvol.ha_type, remote_pool_uuid, internal=True) @@ -420,7 +423,12 @@ def process_snap_replicate_start(task, snapshot): # is not a valid transfer gateway. This mirrors the (working) migration # runner, which has always sent bulk transfers hub+map_id. from simplyblock_core.services.replication_final_step import ensure_hub_attached - _hub_ctrl, hub_bdev, hub_err = ensure_hub_attached(snode.rpc_client(), remote_lv_node) + xfer_timing.gap("landing_volume_create", _t_landing, + snap=snapshot.get_id(), lvol=snapshot.lvol.get_id()) + with xfer_timing.phase("hub_attach", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id(), + tgt=remote_lv_node.get_id()): + _hub_ctrl, hub_bdev, hub_err = ensure_hub_attached(snode.rpc_client(), remote_lv_node) if hub_err: logger.error(f"Transfer hub attach failed: {hub_err}") task.function_result = "transfer hub attach failed, retrying" @@ -491,6 +499,10 @@ def process_snap_replicate_start(task, snapshot): fresh.write_to_db() # 3 start replication + xfer_timing.stamp("transfer_submit", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id(), size=snapshot.size) + task.function_params["xfer_submit_t"] = xfer_timing.now() + task.write_to_db() snode.rpc_client().bdev_lvol_transfer( name=snapshot.snap_bdev, offset=offset, @@ -936,6 +948,8 @@ def process_snap_replicate_finish(task, snapshot): or db.get_storage_node_by_id(snapshot.lvol.node_id)) if remote_snode.transfer_hublvol and remote_snode.transfer_hublvol.bdev_name: if not _other_active_transfers_to_node(task, remote_snode.get_id()): + xfer_timing.stamp("hub_detach", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id()) _src_node.rpc_client().bdev_nvme_detach_controller( remote_snode.transfer_hublvol.bdev_name) replicate_to_source = task.function_params["replicate_to_source"] @@ -965,13 +979,17 @@ def process_snap_replicate_finish(task, snapshot): # chain snaps on primary if target_prev_snap: logger.info(f"Chaining replicated lvol: {remote_lv.top_bdev} to snap: {target_prev_snap['snap_bdev']}") - ret = remote_snode.rpc_client().bdev_lvol_add_clone( remote_lv.top_bdev, target_prev_snap['snap_bdev']) + with xfer_timing.phase("chain_add_clone", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id(), node="primary"): + ret = remote_snode.rpc_client().bdev_lvol_add_clone( remote_lv.top_bdev, target_prev_snap['snap_bdev']) if not ret: logger.error("Failed to chain replicated snapshot on primary node") return False # convert to snapshot on primary - ret = remote_snode.rpc_client().bdev_lvol_convert(remote_lv.top_bdev) + with xfer_timing.phase("chain_convert", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id(), node="primary"): + ret = remote_snode.rpc_client().bdev_lvol_convert(remote_lv.top_bdev) if not ret: logger.error("Failed to convert to snapshot on primary node") return False @@ -981,13 +999,17 @@ def process_snap_replicate_finish(task, snapshot): if sec_node.status == StorageNode.STATUS_ONLINE: if target_prev_snap: logger.info(f"Chaining replicated lvol: {remote_lv.top_bdev} to snap: {target_prev_snap['snap_bdev']}") - ret = sec_node.rpc_client().bdev_lvol_add_clone(remote_lv.top_bdev, target_prev_snap['snap_bdev']) + with xfer_timing.phase("chain_add_clone", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id(), node="secondary"): + ret = sec_node.rpc_client().bdev_lvol_add_clone(remote_lv.top_bdev, target_prev_snap['snap_bdev']) if not ret: logger.error("Failed to chain replicated snapshot on secondary node") return False # convert to snapshot on secondary - ret = sec_node.rpc_client().bdev_lvol_convert(remote_lv.top_bdev) + with xfer_timing.phase("chain_convert", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id(), node="secondary"): + ret = sec_node.rpc_client().bdev_lvol_convert(remote_lv.top_bdev) if not ret: logger.error("Failed to convert to snapshot on secondary node") return False @@ -1154,6 +1176,13 @@ def task_runner(task: JobSchedule): elif task.status == JobSchedule.STATUS_RUNNING: snode = _source_leader_node(snapshot) or db.get_storage_node_by_id(snapshot.lvol.node_id) ret = snode.rpc_client().bdev_lvol_transfer_stat(snapshot.snap_bdev) + if ret: + # offset is the bytes moved so far: the ONLY direct read on actual + # transfer throughput, as distinct from round duration. + xfer_timing.stamp("transfer_running", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id(), + state=str(ret.get("transfer_state")).replace(" ", "_"), + offset=ret.get("offset")) if not ret: logger.error("Failed to get transfer stat") return False @@ -1177,7 +1206,17 @@ def task_runner(task: JobSchedule): task.write_to_db() return False if status == "Done": - new_snapshot_uuid = process_snap_replicate_finish(task, snapshot) + # The transfer proper is submit -> Done. This is the number every + # earlier analysis lacked: round duration includes the chain, + # convert, task-runner latency and DB writes, so it cannot be + # divided into bytes to get a throughput. + xfer_timing.gap("transfer_complete", + task.function_params.get("xfer_submit_t"), + snap=snapshot.get_id(), lvol=snapshot.lvol.get_id(), + bytes=offset) + with xfer_timing.phase("replicate_finish", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id()): + new_snapshot_uuid = process_snap_replicate_finish(task, snapshot) if new_snapshot_uuid: task.function_result = new_snapshot_uuid task.status = JobSchedule.STATUS_DONE diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 904350d3c3..07008ef0bc 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -24,7 +24,7 @@ import uuid as uuid_lib from datetime import datetime -from simplyblock_core import constants, db_controller, utils +from simplyblock_core import constants, db_controller, utils, xfer_timing from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.lvol_model import LVol, LVolReplication from simplyblock_core.models.snapshot import SnapShot @@ -219,6 +219,11 @@ def task_runner(task: JobSchedule): return _finalize(task, False, "source node not found for cutover") if task.status in [JobSchedule.STATUS_NEW, JobSchedule.STATUS_SUSPENDED, JobSchedule.STATUS_RUNNING]: + # One line per runner pass: the spacing between these shows the task + # scheduler's contribution (TASK_EXEC_INTERVAL_SEC per state change), + # which is invisible in any per-phase number. + xfer_timing.stamp("task_pass", lvol=lvol_id, state=task.status, + result=str(task.function_result)[:40].replace(" ", "_")) task.status = JobSchedule.STATUS_RUNNING task.function_params.setdefault("start_time", int(time.time())) # Claim the source LVS for the whole cutover -- the convergence rounds @@ -275,7 +280,8 @@ def task_runner(task: JobSchedule): # ---- CUTOVER PHASE (immediately after the last shrink round) ---- # if "tgt_lvol_composite" not in params: - err = _prepare_cutover(task, lvol, src_node, tgt_node) + with xfer_timing.phase("prepare_cutover", lvol=lvol_id): + err = _prepare_cutover(task, lvol, src_node, tgt_node) if err: return _finalize(task, False, err) params = task.function_params @@ -299,6 +305,8 @@ def task_runner(task: JobSchedule): int(time.time()) + constants.REPL_CUTOVER_PROCEED_TIMEOUT_SEC) task.write_to_db(db.kv_store) if int(time.time()) < params["cutover_proceed_timeout"]: + xfer_timing.stamp("cutover_gate_wait", lvol=lvol_id, + deadline=params["cutover_proceed_timeout"]) task.function_result = "cutover_pending: waiting for preconnect signal" task.status = JobSchedule.STATUS_SUSPENDED task.write_to_db(db.kv_store) @@ -311,10 +319,11 @@ def task_runner(task: JobSchedule): "replication record %s not found; proceeding with cutover", replication_id) try: - ok, err = replication_final_step.run_cutover( - src_node, tgt_node, lvol, - params["tgt_lvol_composite"], params["tgt_map_id"], - params["tgt_snap_composite"], operation=params.get("operation", "replicate")) + with xfer_timing.phase("run_cutover", lvol=lvol_id): + ok, err = replication_final_step.run_cutover( + src_node, tgt_node, lvol, + params["tgt_lvol_composite"], params["tgt_map_id"], + params["tgt_snap_composite"], operation=params.get("operation", "replicate")) except Exception as e: logger.error(f"Cutover raised: {e}", exc_info=True) return _finalize(task, False, str(e)) @@ -328,9 +337,16 @@ def _take_shrink_snapshot(task, lvol): """Snapshot the source and record it as the round in flight.""" from simplyblock_core.controllers import snapshot_controller params = task.function_params - new_snap, err = snapshot_controller.add( - lvol.get_id(), f"repl_commit_{uuid_lib.uuid4()}", - snap_type=SnapShot.TYPE_INTERNAL) + # The gap since the previous round finished: dead time between a round + # completing and the next snapshot existing is pure added delta. + xfer_timing.gap("round_gap_to_next_snapshot", + params.get("shrink_round_done_at"), lvol=lvol.get_id(), + round=params.get("shrink_round", 0)) + with xfer_timing.phase("take_shrink_snapshot", lvol=lvol.get_id(), + round=params.get("shrink_round", 0) + 1): + new_snap, err = snapshot_controller.add( + lvol.get_id(), f"repl_commit_{uuid_lib.uuid4()}", + snap_type=SnapShot.TYPE_INTERNAL) if err: return None, f"shrink round {params.get('shrink_round', 0) + 1} snapshot failed: {err}" params["shrink_round"] = params.get("shrink_round", 0) + 1 @@ -426,6 +442,15 @@ def _shrink_step(task, lvol): params.setdefault("shrink_round_times", []).append(round(elapsed, 2)) logger.info("cutover convergence: lvol=%s round %d transferred in %.2fs", lvol.get_id(), params["shrink_round"], elapsed) + # Structured twin of the line above. round_total is snapshot-taken + # to replicated-and-chained, i.e. it INCLUDES all orchestration -- + # compare it against the transfer phase from snapshot_replication + # to see how much is data movement. + xfer_timing.stamp("round_total", lvol=lvol.get_id(), + snap=params.get("shrink_snap_id"), + round=params["shrink_round"], + ms=elapsed * 1000.0) + params["shrink_round_done_at"] = time.time() # Converged: this round's delta -- the writes made during the previous # round -- moved in low seconds, so the freeze that copies the next diff --git a/simplyblock_core/test/test_replication_final_step.py b/simplyblock_core/test/test_replication_final_step.py index f8b680bf82..b8fa930bbb 100644 --- a/simplyblock_core/test/test_replication_final_step.py +++ b/simplyblock_core/test/test_replication_final_step.py @@ -95,6 +95,11 @@ class _Lvol: # volume's ANA group (group id == namespace id). ns_id = 2 + def get_id(self): + # Real LVol has this; the cutover's phase timing labels every line + # with it. + return self.uuid + def _install_nodes(monkeypatch, nodes_by_id): monkeypatch.setattr(rfs, "db", type("DB", (), { diff --git a/simplyblock_core/xfer_timing.py b/simplyblock_core/xfer_timing.py new file mode 100644 index 0000000000..96e36de827 --- /dev/null +++ b/simplyblock_core/xfer_timing.py @@ -0,0 +1,123 @@ +"""Phase timing for the replication/cutover pipeline. + +Soak analysis 2026-08-27 could not answer a basic question -- of the 588s a +convergence round took, how much was the DATA TRANSFER and how much was +orchestration? Round duration was the only number available, and it spans the +landing-volume create, the hub attach, the transfer, the detach, add_clone and +convert on two nodes, several DB writes, and up to TASK_EXEC_INTERVAL_SEC of +task-runner latency per state change. Every hardware-level explanation we +tested came out an order of magnitude off because the number being explained +was not a throughput. + +So: emit one line per phase, parseable without guessing. + + XFER-TIMING t=1787868791.244 phase=transfer lvol=1c8874f3 snap=a0f48bf5 \ + round=2 ms=1843.2 bytes=33554432 mbps=18.2 ok=1 + +Every line carries its own epoch timestamp because the container clock is +skewed from the host's, so `docker service logs -t` ordering cannot be trusted +across services. Grep for XFER-TIMING and feed it to +scripts/xfer_timing_report.py. + +Instrumentation only -- no behaviour change, and every helper is safe to call +from any thread and cheap enough for the hot path. +""" +import logging +import time +from contextlib import contextmanager + +logger = logging.getLogger() + +_PREFIX = "XFER-TIMING" + + +def _fmt(**fields): + parts = [] + for k, v in fields.items(): + if v is None: + continue + if isinstance(v, float): + parts.append("%s=%.3f" % (k, v)) + else: + parts.append("%s=%s" % (k, v)) + return " ".join(parts) + + +def _short(value): + """Ids are long and the interesting part is the head.""" + if value is None: + return None + text = str(value) + return text[:8] if len(text) > 8 else text + + +def now(): + """Clock for callers that need a start marker to pair with gap(). + + Exposed here so an instrumented module does not have to import time just + to be measured -- a missing `import time` in replication_final_step.py + would have raised NameError mid-cutover. + """ + return time.time() + + +def stamp(phase, lvol=None, snap=None, round=None, **extra): + """A point event: something happened now, with no duration.""" + logger.info("%s %s", _PREFIX, _fmt( + t=time.time(), phase=phase, lvol=_short(lvol), snap=_short(snap), + round=round, **extra)) + + +@contextmanager +def phase(name, lvol=None, snap=None, round=None, **extra): + """Time a block and emit its duration, whether it succeeds or raises. + + Usage: + with xfer_timing.phase("hub_attach", lvol=lvol_id) as ph: + ... + ph["bytes"] = n # optional, folded into the line + """ + started = time.time() + box = {} + ok = 1 + try: + yield box + except BaseException: + ok = 0 + raise + finally: + elapsed_ms = (time.time() - started) * 1000.0 + fields = dict(extra) + fields.update(box) + nbytes = fields.pop("bytes", None) + if nbytes: + try: + fields["bytes"] = int(nbytes) + fields["mbps"] = (int(nbytes) / 1e6) / max(elapsed_ms / 1000.0, 1e-9) + except (TypeError, ValueError): + fields["bytes"] = nbytes + logger.info("%s %s", _PREFIX, _fmt( + t=time.time(), phase=name, lvol=_short(lvol), snap=_short(snap), + round=round, ms=elapsed_ms, ok=ok, **fields)) + + +def gap(name, since, lvol=None, snap=None, round=None, **extra): + """Time from an earlier epoch to now -- for waits nobody is inside of. + + The dead time between one round completing and the next snapshot being + taken is exactly this shape: no call to wrap, just two moments. + """ + if not since: + return + elapsed_ms = (time.time() - float(since)) * 1000.0 + fields = dict(extra) + nbytes = fields.pop("bytes", None) + if nbytes: + try: + fields["bytes"] = int(nbytes) + fields["mbps"] = (int(nbytes) / 1e6) / max(elapsed_ms / 1000.0, 1e-9) + except (TypeError, ValueError): + fields["bytes"] = nbytes + logger.info("%s %s", _PREFIX, _fmt( + t=time.time(), phase=name, lvol=_short(lvol), snap=_short(snap), + round=round, ms=elapsed_ms, ok=1, **fields)) From aa810d74c0d28d5eced667b3a020fecf2cd15e8a Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 28 Aug 2026 12:21:58 +0200 Subject: [PATCH 093/122] Say which retention is in force, and pin the ladder with a test Soak run 20260827_224741: two volumes on a 1-minute cadence for 124 minutes under `--retention-schedule 5m:15m,7m:30m,10m:1h` ended with 2 and 3 internal snapshots, at CONSECUTIVE cadence ticks 63-64s apart. Nothing in the log said which retention path had run, so the investigation could not tell an over-pruning bug from a dead cadence. The ladder itself is provably correct -- the new test feeds it exactly the history that run should have produced (124 x 1-minute snapshots) and it retains 14 points with gaps 300/300/360/420/420/420/180/300/600... So retention is NOT over-pruning. What the test also shows is that an ABSENT schedule keeps exactly the 2 newest -- consecutive cadence ticks -- which is precisely the shape the soak produced. That makes "the schedule never reached _prune_internal_snapshots" the leading explanation, even though the CLI -> add_policy -> policy.retention_schedule -> get_replication_policy_by_id chain all inspect correctly (the lookup even normalises the cluster/uuid form). So _prune_internal_snapshots now logs, on every prune, whether it is applying a schedule (and which) or falling back to FLAT keep-newest, plus the snapshot count and keep floor. The next case-11 run answers the question from its own log instead of needing another two-hour repro. The test's spread assertion allows smaller gaps at TIER BOUNDARIES, where the last bucket of one tier legitimately sits close to the first of the next. --- .../services/snapshot_replication.py | 12 +++ .../test/test_case11_retention_ladder.py | 89 +++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 simplyblock_core/test/test_case11_retention_ladder.py diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index c91d9790ce..07050cb97a 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -750,6 +750,18 @@ def _prune_internal_snapshots(source_lvol): # Either way the newest `keep` are protected, because deleting a snapshot # swap-merges its segments into the successor chained to it. schedule = _retention_schedule_for(source_lvol) + # Say which retention is in force, every time it prunes. Soak run + # 20260827_224741 ended with 2 snapshots at consecutive cadence ticks after + # 124 minutes under `5m:15m,7m:30m,10m:1h` -- which is exactly what the FLAT + # keep-N path produces, and nothing in the log said which path ran. The + # ladder itself is provably correct (test_case11_retention_ladder), so the + # open question is whether the schedule reaches this function at all. + logger.info( + "Retention for lvol %s: %s (replicated internal snapshots: %d, keep=%d)", + source_lvol.get_id(), + ("schedule %s" % snapshot_retention.describe(schedule)) if schedule + else "FLAT keep-newest (no schedule on the policy)", + len(replicated_internal), keep) if schedule: retained_ts = snapshot_retention.select_retained( [s.created_at for s in replicated_internal], schedule, diff --git a/simplyblock_core/test/test_case11_retention_ladder.py b/simplyblock_core/test/test_case11_retention_ladder.py new file mode 100644 index 0000000000..6b07849ed7 --- /dev/null +++ b/simplyblock_core/test/test_case11_retention_ladder.py @@ -0,0 +1,89 @@ +"""Case 11's schedule must retain a ladder, not collapse to the keep-floor. + +Soak run 20260827_224741: two volumes on a 1-minute cadence for 124 minutes +with `--retention-schedule 5m:15m,7m:30m,10m:1h` ended with 2 and 3 internal +snapshots, all at CONSECUTIVE cadence ticks (63-64s apart). A working ladder +cannot produce consecutive minutes -- its finest tier keeps one per 5 minutes +-- so either the cadence never ran or retention collapsed to the flat +keep-newest floor. + +This pins the retention half of that question with no cluster: feed the real +schedule the snapshot history the run should have produced and assert the +ladder survives. +""" +import unittest + +from simplyblock_core.snapshot_retention import ( + horizon_sec, parse_schedule, select_retained, +) + +CASE11_SCHEDULE = "5m:15m,7m:30m,10m:1h" +MINUTE = 60 + + +class TestCase11Ladder(unittest.TestCase): + + def setUp(self): + self.tiers = parse_schedule(CASE11_SCHEDULE) + self.now = 1_787_870_000.0 + # 124 minutes of a 1-minute cadence, as the run actually took them + self.history = [self.now - m * MINUTE for m in range(1, 125)] + + def test_schedule_parses_to_the_expected_ladder(self): + self.assertEqual([(t.every_sec, t.span_sec) for t in self.tiers], + [(300, 900), (420, 1800), (600, 3600)]) + # 15m + 30m + 1h of coverage + self.assertEqual(horizon_sec(self.tiers), 900 + 1800 + 3600) + + def test_the_ladder_retains_a_thinning_history_not_two_snapshots(self): + keep = select_retained(self.history, self.tiers, self.now, + always_keep_newest=2) + # 15m/5m = 3, then 30m/7m ~= 4-5, then 60m/10m = 6 + self.assertGreaterEqual( + len(keep), 12, + "the ladder should retain roughly a dozen points across 105 " + "minutes of horizon; collapsing to the keep-floor is the bug this " + "test exists for (got %d)" % len(keep)) + self.assertLessEqual(len(keep), 16, "and it must not keep everything") + + def test_retained_points_are_spread_not_consecutive(self): + """The observed failure looked like consecutive cadence ticks.""" + keep = sorted(select_retained(self.history, self.tiers, self.now, + always_keep_newest=2), reverse=True) + # Beyond the protected newest pair, retained points must be spread by + # roughly a tier interval. Gaps at a TIER BOUNDARY are legitimately + # smaller (the last bucket of one tier can sit close to the first of + # the next), so assert on the typical gap, not the minimum. + gaps = sorted(keep[i] - keep[i + 1] for i in range(2, len(keep) - 1)) + self.assertTrue(gaps, "expected several retained points") + median = gaps[len(gaps) // 2] + self.assertGreaterEqual( + median, 300, + "retained snapshots a cadence tick apart mean the ladder is not " + "being applied; the flat keep-N path produces exactly that " + "(gaps: %s)" % [int(g) for g in gaps]) + + def test_snapshots_older_than_the_horizon_are_dropped(self): + older = [self.now - 200 * MINUTE, self.now - 300 * MINUTE] + keep = select_retained(self.history + older, self.tiers, self.now, + always_keep_newest=2) + for t in older: + self.assertNotIn(t, keep, + "past the 105-minute horizon nothing is retained") + + def test_an_absent_schedule_falls_back_to_the_keep_floor(self): + """The suspected production path: no tiers -> only the newest N. + + This is what the soak looked like, and it is CORRECT behaviour for an + empty schedule -- which is why an empty schedule reaching retention + silently destroys the history an operator asked for. + """ + keep = select_retained(self.history, [], self.now, always_keep_newest=2) + self.assertEqual(len(keep), 2) + self.assertEqual(sorted(keep, reverse=True), self.history[:2], + "the two newest, i.e. consecutive cadence ticks -- " + "exactly the shape the soak produced") + + +if __name__ == "__main__": + unittest.main() From dd4934be89559b62f8356990f4d0ac5b3f7c4fb3 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 28 Aug 2026 12:45:22 +0200 Subject: [PATCH 094/122] Act on a finished transfer within 100ms, not on a later runner pass The instrumented run finally located the 588s. Run 20260828_115307, case 7 fail-back, 1087 timing events: polls per in-flight transfer : p50 = 1 max = 2 gap between polls : p50 = 81.2s p90 = 105.0s transfer states ever seen : {'Done': 86} -- never once "In progress" duration vs bytes : pearson r = -0.532 zero-byte transfers : p50 34.1s, as costly as data-carrying ones total moved : 0.18 GiB across 4781s of "transfer" time Every poll found the transfer already finished, and transfers that moved NOTHING cost the same tens of seconds as those that moved data. So this was never data movement: transfers complete fast and then sit COMPLETED BUT UNNOTICED until the runner's next pass revisits that task. A convergence round cannot take its next snapshot until the previous one is marked replicated, so ~50s of observation latency per round, over a dozen rounds, IS the 588s. Every hardware theory we tested -- qpair fair share, the single nvmf poll group, 2 MiB granularity, chain depth, COW, network saturation -- was aimed at making data move faster, and none of them could have changed this number. Two sources of latency, both removed: 1. Submit and the Done check happened on DIFFERENT passes. The submitting pass now waits for the transfer, polling every 100ms, and runs the finish (chain, convert -- what sets target_replicated_snap_uuid) in the same pass. The wait is bounded so one slow transfer cannot hold the single-threaded runner: a volume whose cutover owns its lvstore gets a generous budget (nothing else on that lvstore can run anyway), everything else gets 5s and falls back to the old pass-based path. 2. The cutover runner's loop slept 3s after EVERY task that returned False -- the normal result for a queued or mid-round task. With 20 volumes that was 20*3 + 10 = ~70s per pass, matching the measured 75s per-volume spacing. The per-task backoff is gone and the pass interval is now adaptive: 200ms while any cutover is mid-round, the normal interval when the cluster is idle. The owner lookup also takes the prefetched task list -- it re-read get_job_tasks per task, which is O(N^2) DB reads and unaffordable at a 200ms cadence. Also: the snapshot monitor now logs a deferred cadence tick with the snapshot it is waiting on. Case 11 ended with 2 internal snapshots after 124 minutes at a 1-minute cadence and nothing said why. Worst-case path is now 100ms to notice completion plus 200ms for the converging loop to see the marker. 2063 tests pass, including ones that pin both intervals below a second. --- simplyblock_core/constants.py | 24 ++- simplyblock_core/services/snapshot_monitor.py | 8 + .../services/snapshot_replication.py | 120 +++++++++++--- .../tasks_runner_replication_final.py | 54 +++++-- .../test/test_transfer_completion_latency.py | 148 ++++++++++++++++++ 5 files changed, 322 insertions(+), 32 deletions(-) create mode 100644 simplyblock_core/test/test_transfer_completion_latency.py diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index e8ce1eef2d..17c07408f0 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -393,7 +393,11 @@ def get_config_var(name, default=None): # Always worth polling inline for at least this long: a round that finishes # just after the pass is handed back costs a full TASK_EXEC_INTERVAL_SEC of # writes in the next round. -REPL_CUTOVER_MIN_INLINE_SEC = 5 +# The snapshot-replication runner now finishes a transfer in the pass that +# submitted it, so a convergence round completes in about the transfer's own +# duration. Staying inline across that is what makes "next snapshot within a +# second of completion" true; yielding mid-round reintroduces pass latency. +REPL_CUTOVER_MIN_INLINE_SEC = 30 # Whether to block the cutover on the operator's preconnect signal. The wait # sits BETWEEN the cutover clone's base snapshot and the freeze, so every @@ -404,6 +408,24 @@ def get_config_var(name, default=None): # clone's base can be advanced after the signal. REPL_CUTOVER_PROCEED_REQUIRED = False +# --- noticing a finished transfer ---------------------------------------- +# A transfer that has completed must be acted on within a second: the next +# convergence snapshot cannot be taken until the previous one is marked +# replicated, so observation latency lands directly in the IO freeze. +REPL_XFER_POLL_INTERVAL_SEC = 0.1 +# How long the submitting pass may wait inline for the transfer. The runner is +# single-threaded, so this is a starvation budget, not a timeout: exceeding it +# just falls back to being noticed on a later pass. +REPL_XFER_INLINE_WAIT_SEC = 5.0 +# A volume in its final cutover already owns its lvstore and every other +# transfer on it is held, so there is nothing to starve -- wait as long as the +# transfer needs, because this is exactly the window the client freeze pays for. +REPL_XFER_INLINE_WAIT_CUTOVER_SEC = 300.0 +# Pass interval for the cutover runner while any cutover is mid-round. The +# freeze pays for every millisecond between a transfer completing and the next +# snapshot starting, so this must stay well under a second. +REPL_CUTOVER_ACTIVE_POLL_SEC = 0.2 + SPDK_PROXY_MULTI_THREADING_ENABLED=True SPDK_PROXY_TIMEOUT=60*5 LVOL_NVME_CONNECT_RECONNECT_DELAY=2 diff --git a/simplyblock_core/services/snapshot_monitor.py b/simplyblock_core/services/snapshot_monitor.py index 99978cb316..95a101b060 100644 --- a/simplyblock_core/services/snapshot_monitor.py +++ b/simplyblock_core/services/snapshot_monitor.py @@ -614,6 +614,14 @@ def take_due_internal_snapshots(cluster_id, now_ts): for lv in members: outstanding = _outstanding_internal_snapshot(lv, all_snaps) if outstanding is not None: + # Case 11 (run 20260827_224741) ended with 2 internal + # snapshots after 124 minutes at a 1-minute cadence and + # nothing said why. A skipped cadence tick must be + # visible, or the next investigation needs another + # two-hour repro to find out. + logger.info( + "Cadence snapshot for lvol %s deferred: %s has not " + "replicated yet", lv.get_id(), outstanding.get_id()) blocked = (lv, outstanding) break if blocked: diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index 07050cb97a..3d7ac866ce 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -211,6 +211,94 @@ def _lvs_transfer_hold(task, snapshot): return "" +def _finish_completed_transfer(task, snapshot, offset): + """Chain, convert and mark the snapshot replicated. Returns True. + + This is what sets target_replicated_snap_uuid, which is the signal a + cutover's convergence loop waits on -- so it must run as soon as the + transfer is known to be finished, not on some later pass. + """ + # submit -> Done, measured. NOT a throughput: see fix_xfer_latency notes. + xfer_timing.gap("transfer_complete", + task.function_params.get("xfer_submit_t"), + snap=snapshot.get_id(), lvol=snapshot.lvol.get_id(), + bytes=offset) + with xfer_timing.phase("replicate_finish", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id()): + new_snapshot_uuid = process_snap_replicate_finish(task, snapshot) + if new_snapshot_uuid: + task.function_result = new_snapshot_uuid + task.status = JobSchedule.STATUS_DONE + task.function_params["end_time"] = int(time.time()) + task.write_to_db() + else: + task.function_result = "complete repl failed, retrying" + task.status = JobSchedule.STATUS_SUSPENDED + task.retry += 1 + task.write_to_db() + return True + + +def _cutover_owns(lvol_id, cluster_id): + """True while a final cutover is running for this volume. + + Such a volume already holds its lvstore and every other transfer on it is + held, so waiting inline for its transfer starves nothing -- and this is the + window the client's IO freeze is paying for. + """ + if not lvol_id: + return False + try: + tasks = db.get_job_tasks(cluster_id) + except Exception: # noqa: BLE001 + return False + for other in tasks: + if other.function_name != JobSchedule.FN_REPLICATION_FINAL: + continue + if other.status == JobSchedule.STATUS_DONE or other.canceled: + continue + if (other.function_params or {}).get("lvol_id") == lvol_id: + return True + return False + + +def _await_transfer_completion(task, snapshot, snode): + """Poll the just-submitted transfer at 100ms and finish it in this pass. + + Returns True when the transfer completed and was finished here; False to + leave it for the pass-based path (still in flight, or the budget ran out). + + Before this, submit and the Done check happened on DIFFERENT passes of the + runner loop, so a transfer that finished in milliseconds was not acted on + for a median of 81 SECONDS (run 20260828_115307: every poll found state + already 'Done', never once 'In progress'). + """ + budget = (constants.REPL_XFER_INLINE_WAIT_CUTOVER_SEC + if _cutover_owns(snapshot.lvol.get_id(), task.cluster_id) + else constants.REPL_XFER_INLINE_WAIT_SEC) + deadline = time.time() + budget + rpc = snode.rpc_client() + while time.time() < deadline: + try: + ret = rpc.bdev_lvol_transfer_stat(snapshot.snap_bdev) + except Exception as e: # noqa: BLE001 + logger.warning("transfer_stat for %s raised while waiting inline " + "(%s); leaving it to the next pass", + snapshot.get_id(), e) + return False + if not ret: + return False + state = ret.get("transfer_state") + if state == "Done": + return _finish_completed_transfer(task, snapshot, ret.get("offset")) + if state == "Failed": + return False # the pass-based path records the retry + time.sleep(constants.REPL_XFER_POLL_INTERVAL_SEC) + xfer_timing.stamp("inline_wait_expired", snap=snapshot.get_id(), + lvol=snapshot.lvol.get_id(), budget=budget) + return False + + def process_snap_replicate_start(task, snapshot): # 1 create lvol on remote node logger.info("Starting snapshot replication task") @@ -520,6 +608,15 @@ def process_snap_replicate_start(task, snapshot): task.function_params["start_time"] = int(time.time()) task.write_to_db() + # Do not hand the transfer back to the pass loop and forget about it: wait + # for it here, polling every 100ms, and finish it in this same pass. Before + # this, submit and the Done check happened on different passes and a + # transfer that finished in milliseconds went unnoticed for a median of 81 + # SECONDS (run 20260828_115307). Returns False if it is still running when + # the budget expires, in which case the pass-based path picks it up as + # before. + _await_transfer_completion(task, snapshot, snode) + def _receiving_leader_node(remote_lv): """The node that currently leads *remote_lv*'s lvstore, or None. @@ -1218,28 +1315,7 @@ def task_runner(task: JobSchedule): task.write_to_db() return False if status == "Done": - # The transfer proper is submit -> Done. This is the number every - # earlier analysis lacked: round duration includes the chain, - # convert, task-runner latency and DB writes, so it cannot be - # divided into bytes to get a throughput. - xfer_timing.gap("transfer_complete", - task.function_params.get("xfer_submit_t"), - snap=snapshot.get_id(), lvol=snapshot.lvol.get_id(), - bytes=offset) - with xfer_timing.phase("replicate_finish", snap=snapshot.get_id(), - lvol=snapshot.lvol.get_id()): - new_snapshot_uuid = process_snap_replicate_finish(task, snapshot) - if new_snapshot_uuid: - task.function_result = new_snapshot_uuid - task.status = JobSchedule.STATUS_DONE - task.function_params["end_time"] = int(time.time()) - task.write_to_db() - else: - task.function_result = "complete repl failed, retrying" - task.status = JobSchedule.STATUS_SUSPENDED - task.retry += 1 - task.write_to_db() - return True + return _finish_completed_transfer(task, snapshot, offset) def main(): diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 07008ef0bc..cf45e2de4b 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -37,16 +37,19 @@ db = db_controller.DBController() -def _lvs_cutover_owner(task, lvs_name): +def _lvs_cutover_owner(task, lvs_name, tasks=None): """The task that already owns *lvs_name* for a cutover, or None. Deterministic: the earliest-created active claim wins, so two tasks racing the same lvstore agree on who owns it instead of each seeing the other. + + ``tasks`` may be a list already read this pass. Re-reading it per task is + O(N^2) DB reads, which the sub-second poll interval cannot afford. """ if not lvs_name: return None owners = [] - for other in db.get_job_tasks(task.cluster_id): + for other in (tasks if tasks is not None else db.get_job_tasks(task.cluster_id)): if other.function_name != JobSchedule.FN_REPLICATION_FINAL: continue if other.get_id() == task.get_id(): @@ -168,7 +171,7 @@ def _finalize(task, ok, err): return False -def task_runner(task: JobSchedule): +def task_runner(task: JobSchedule, tasks=None): params = task.function_params lvol_id = params.get("lvol_id") if not lvol_id: @@ -234,7 +237,7 @@ def task_runner(task: JobSchedule): # the freeze. lvs_name = getattr(lvol, "lvs_name", "") own_group = _group_id_for_lvol(lvol) - owner = _lvs_cutover_owner(task, lvs_name) + owner = _lvs_cutover_owner(task, lvs_name, tasks) if owner is not None: owner_group = (owner.function_params or {}).get("cutover_group") or "" owner_lvol = str((owner.function_params or {}).get("lvol_id")) @@ -376,6 +379,11 @@ def _inline_window(last_round_secs): the next snapshot must follow within milliseconds -- and yields early when rounds are still long, which is where yielding costs nothing because the freeze is far away regardless. + + The floor is REPL_CUTOVER_MIN_INLINE_SEC rather than zero because yielding + hands the round back to the pass loop, and being picked up again costs far + more than the poll it replaces: the requirement is that NO time is lost + between a transfer completing and the next snapshot starting. """ return min(constants.REPL_CUTOVER_CONVERGE_BUDGET_SEC, max(constants.REPL_CUTOVER_MIN_INLINE_SEC, last_round_secs * 3)) @@ -549,6 +557,24 @@ def _prepare_cutover(task, lvol, src_node, tgt_node): return None +def _any_cutover_in_flight(tasks): + """True while some cutover is between its first snapshot and its freeze. + + Only then is a sub-second pass interval worth paying for: that is the + window where a completed transfer must be picked up immediately. + """ + for t in tasks: + if t.function_name != JobSchedule.FN_REPLICATION_FINAL: + continue + if t.status == JobSchedule.STATUS_DONE or t.canceled: + continue + params = t.function_params or {} + # claimed the lvstore, or already has a round in flight + if params.get("cutover_lvs") or params.get("shrink_snap_id"): + return True + return False + + def main(): logger.info("Starting replication-final tasks runner...") while True: @@ -558,21 +584,31 @@ def main(): logger.error(f"Failed to get clusters: {e}") time.sleep(3) continue + active = False for cl in clusters: - for task in db.get_job_tasks(cl.get_id(), reverse=False): + # Read once per cluster per pass and reuse: the owner lookup used to + # re-read this for every task. + cluster_tasks = db.get_job_tasks(cl.get_id(), reverse=False) + if _any_cutover_in_flight(cluster_tasks): + active = True + for task in cluster_tasks: if task.function_name != JobSchedule.FN_REPLICATION_FINAL: continue if task.status == JobSchedule.STATUS_DONE: continue task = db.get_task_by_id(task.uuid) try: - res = task_runner(task) + res = task_runner(task, cluster_tasks) except Exception as e: logger.error(f"replication-final task {task.uuid} failed: {e}", exc_info=True) res = False - if not res: - time.sleep(3) - time.sleep(constants.TASK_EXEC_INTERVAL_SEC) + # No blanket backoff here. `res is False` is the NORMAL result + # for a task that is queued or mid-round, and sleeping 3s per + # such task cost ~70s per pass with 20 volumes -- which landed + # directly in the client's IO freeze. + # Poll fast only while it matters; an idle cluster keeps the old cadence. + time.sleep(constants.REPL_CUTOVER_ACTIVE_POLL_SEC if active + else constants.TASK_EXEC_INTERVAL_SEC) if __name__ == "__main__": diff --git a/simplyblock_core/test/test_transfer_completion_latency.py b/simplyblock_core/test/test_transfer_completion_latency.py new file mode 100644 index 0000000000..1701b9f8d8 --- /dev/null +++ b/simplyblock_core/test/test_transfer_completion_latency.py @@ -0,0 +1,148 @@ +"""No time may be lost between a transfer completing and the next snapshot. + +Measured before this fix (run 20260828_115307, case 7 fail-back): + + polls per in-flight transfer : p50 = 1 + gap between polls : p50 = 81.2s + states ever seen : {'Done': 86} -- never once "In progress" + +Every poll found the transfer already finished. Submit happened on one pass of +the runner loop and the Done check on a later one, so a transfer that completed +in milliseconds went unnoticed for ~81 seconds. A convergence round cannot take +its next snapshot until the previous one is marked replicated, so that latency +landed directly in the client's IO freeze. + +Requirement: completion must be picked up and the next snapshot initiated in +under one second, in the WORST case, not just typically. +""" +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import constants +from simplyblock_core.services import snapshot_replication as sr +from simplyblock_core.services import tasks_runner_replication_final as final + + +class TestInlineCompletion(unittest.TestCase): + """The submitting pass waits for the transfer and finishes it itself.""" + + def setUp(self): + self.db = patch.object(sr, "db").start() + self.addCleanup(patch.stopall) + self.finish = patch.object(sr, "_finish_completed_transfer", + return_value=True).start() + patch.object(sr, "_cutover_owns", return_value=False).start() + self.slept = [] + patch.object(sr.time, "sleep", side_effect=self.slept.append).start() + + self.snap = MagicMock() + self.snap.get_id.return_value = "S1" + self.snap.snap_bdev = "LVS_1/SNAP_1" + self.snap.lvol.get_id.return_value = "LV1" + + self.task = MagicMock() + self.task.cluster_id = "CL" + self.task.function_params = {} + + self.node = MagicMock() + self.rpc = MagicMock() + self.node.rpc_client.return_value = self.rpc + + def test_a_transfer_already_done_is_finished_without_sleeping(self): + self.rpc.bdev_lvol_transfer_stat.return_value = { + "transfer_state": "Done", "offset": 2 * 1024 * 1024} + self.assertTrue( + sr._await_transfer_completion(self.task, self.snap, self.node)) + self.finish.assert_called_once() + self.assertEqual(self.slept, [], + "a finished transfer must be acted on immediately") + + def test_completion_is_noticed_within_one_poll_interval(self): + states = [{"transfer_state": "In progress", "offset": 1}, + {"transfer_state": "Done", "offset": 2}] + self.rpc.bdev_lvol_transfer_stat.side_effect = states + self.assertTrue( + sr._await_transfer_completion(self.task, self.snap, self.node)) + self.finish.assert_called_once() + self.assertEqual(self.slept, [constants.REPL_XFER_POLL_INTERVAL_SEC]) + self.assertLess(constants.REPL_XFER_POLL_INTERVAL_SEC, 1.0, + "the poll interval IS the worst-case detection delay") + + def test_a_failed_transfer_is_left_to_the_retry_path(self): + self.rpc.bdev_lvol_transfer_stat.return_value = { + "transfer_state": "Failed", "offset": 0} + self.assertFalse( + sr._await_transfer_completion(self.task, self.snap, self.node)) + self.finish.assert_not_called() + + def test_a_volume_in_cutover_gets_the_generous_budget(self): + """Its lvstore is already claimed, so waiting starves nothing.""" + with patch.object(sr, "_cutover_owns", return_value=True): + self.rpc.bdev_lvol_transfer_stat.return_value = { + "transfer_state": "Done", "offset": 1} + self.assertTrue( + sr._await_transfer_completion(self.task, self.snap, self.node)) + self.assertGreater(constants.REPL_XFER_INLINE_WAIT_CUTOVER_SEC, + constants.REPL_XFER_INLINE_WAIT_SEC) + + def test_the_submit_path_calls_the_wait(self): + import inspect + src = inspect.getsource(sr.process_snap_replicate_start) + self.assertIn("_await_transfer_completion", src, + "submitting and forgetting is what cost 81 seconds") + + +class TestPassLatency(unittest.TestCase): + """A round that yields must be picked up again in well under a second.""" + + def test_the_active_poll_interval_is_sub_second(self): + self.assertLess(constants.REPL_CUTOVER_ACTIVE_POLL_SEC, 1.0) + self.assertLess(constants.REPL_CUTOVER_ACTIVE_POLL_SEC, + constants.TASK_EXEC_INTERVAL_SEC) + + def test_a_claimed_or_mid_round_task_counts_as_in_flight(self): + from simplyblock_core.models.job_schedule import JobSchedule + def t(**params): + x = MagicMock() + x.function_name = JobSchedule.FN_REPLICATION_FINAL + x.status = JobSchedule.STATUS_RUNNING + x.canceled = False + x.function_params = params + return x + self.assertTrue(final._any_cutover_in_flight([t(cutover_lvs="LVS_1")])) + self.assertTrue(final._any_cutover_in_flight([t(shrink_snap_id="S1")])) + self.assertFalse(final._any_cutover_in_flight([t()])) + + def test_a_finished_cutover_does_not_hold_the_fast_interval(self): + from simplyblock_core.models.job_schedule import JobSchedule + done = MagicMock() + done.function_name = JobSchedule.FN_REPLICATION_FINAL + done.status = JobSchedule.STATUS_DONE + done.canceled = False + done.function_params = {"cutover_lvs": "LVS_1"} + self.assertFalse(final._any_cutover_in_flight([done]), + "an idle cluster must not be spun on") + + def test_the_per_task_backoff_is_gone(self): + """It punished the COMMON case: 20 queued tasks cost ~70s per pass. + + The DB-failure backoff in the except branch is a different thing and + stays -- this asserts only that a yielding TASK no longer sleeps. + """ + import inspect + src = inspect.getsource(final.main) + self.assertNotIn("if not res:", src, + "a queued or mid-round task must not cost a sleep") + self.assertIn("REPL_CUTOVER_ACTIVE_POLL_SEC", src) + + def test_the_owner_lookup_reuses_the_prefetched_task_list(self): + """Re-reading per task is O(N^2) DB reads, unaffordable at 200ms.""" + import inspect + self.assertIn("tasks=None", + inspect.signature(final._lvs_cutover_owner).__str__() + .replace(" ", "").replace("'", "")) + self.assertIn("cluster_tasks", inspect.getsource(final.main)) + + +if __name__ == "__main__": + unittest.main() From ebe0d51e581e819e63b3c79d797220883fe33022 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 28 Aug 2026 12:58:48 +0100 Subject: [PATCH 095/122] fix: break LVS cutover circular stall by writing claim before arbitrating and including self in owner sort --- .../tasks_runner_replication_final.py | 71 +++++++++++++------ .../test/test_cutover_convergence.py | 32 +++++++++ 2 files changed, 80 insertions(+), 23 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index fc43698f45..19f97a278d 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -39,28 +39,43 @@ def _lvs_cutover_owner(task, lvs_name): - """The task that already owns *lvs_name* for a cutover, or None. - - Deterministic: the earliest-created active claim wins, so two tasks racing - the same lvstore agree on who owns it instead of each seeing the other. + """Return the task that should own *lvs_name*, or None if *task* itself wins. + + Deterministic: the earliest-created claimant wins (sorted by create_dt then + id), so concurrent threads that both wrote a claim agree on the same winner. + + Callers MUST write their own ``cutover_lvs`` claim to DB **before** calling + this function so they appear in the scan and participate in the sort. + + Returns None → the caller is the rightful owner; proceed. + Returns → that other task is the rightful owner; the caller should + remove its own claim and yield. + + Note: the previous implementation excluded the calling task from the scan + (``if other.get_id() == task.get_id(): continue``). When two threads both + raced past the guard before either wrote its claim, both ended up with + ``cutover_lvs`` set, and on every subsequent pass each task's candidate set + was exactly {the other} — so each always deferred to the other, creating a + permanent circular stall (e.g. 36418f5d queued behind 309d3aeb AND + 309d3aeb queued behind 36418f5d). Including self in the sort eliminates + the self-exclusion blindspot that made the cycle possible. """ if not lvs_name: return None - owners = [] + claimants = [] for other in db.get_job_tasks(task.cluster_id): if other.function_name != JobSchedule.FN_REPLICATION_FINAL: continue - if other.get_id() == task.get_id(): - continue if other.status == JobSchedule.STATUS_DONE or other.canceled: continue if (other.function_params or {}).get("cutover_lvs") != lvs_name: continue - owners.append(other) - if not owners: + claimants.append(other) + if not claimants: return None - owners.sort(key=lambda t: (str(getattr(t, "create_dt", "")), t.get_id())) - return owners[0] + claimants.sort(key=lambda t: (str(getattr(t, "create_dt", "")), t.get_id())) + winner = claimants[0] + return None if winner.get_id() == task.get_id() else winner def _group_id_for_lvol(lvol): @@ -331,6 +346,22 @@ def task_runner(task: JobSchedule): # the freeze. lvs_name = getattr(lvol, "lvs_name", "") own_group = _group_id_for_lvol(lvol) + + # Write our claim BEFORE arbitrating. The scheduler starts one thread + # per active task, so two tasks for the same lvstore run concurrently. + # If we check first and write second, both threads can read "no owner" + # before either writes, then both write the claim, and on every + # subsequent pass each sees only the other as the claimant — a circular + # stall (e.g. 36418f5d queued behind 309d3aeb AND 309d3aeb queued + # behind 36418f5d) that never resolves. Writing first means both + # claims are visible to the sort, which then elects a single deterministic + # winner regardless of interleaving. + task.function_params["cutover_lvs"] = lvs_name + # Recording the group is what lets the replication side tell a sibling + # member from an unrelated volume. + task.function_params["cutover_group"] = own_group + task.write_to_db(db.kv_store) + owner = _lvs_cutover_owner(task, lvs_name) if owner is not None: owner_group = (owner.function_params or {}).get("cutover_group") or "" @@ -343,6 +374,11 @@ def task_runner(task: JobSchedule): # lvstore), and their deadline would run down until the task # died of max retries -- which is exactly what happened to 17 # tasks in run 20260827_185009. + # + # Remove our (losing) claim so the winner's _lvs_cutover_owner + # scan does not keep finding two claimants on subsequent passes. + task.function_params.pop("cutover_lvs", None) + task.function_params.pop("cutover_group", None) task.function_params["shrink_deadline"] = ( int(time.time()) + constants.REPL_CUTOVER_SHRINK_TIMEOUT_SEC) task.function_result = ( @@ -350,18 +386,7 @@ def task_runner(task: JobSchedule): task.status = JobSchedule.STATUS_SUSPENDED task.write_to_db(db.kv_store) return False # no retry burned: this is a queue, not a failure - - # Claim the source LVS for the whole cutover -- the convergence rounds - # AND the freeze. Other volumes' snapshot transfers on this LVS queue - # behind it (see snapshot_replication._lvs_transfer_hold): they compete - # for the same lvstore and hub bandwidth, and every second they steal - # from a convergence round is a second of writes that lands in the - # freeze. - task.function_params["cutover_lvs"] = lvs_name - # Recording the group is what lets the replication side tell a sibling - # member from an unrelated volume. - task.function_params["cutover_group"] = own_group - task.write_to_db(db.kv_store) + # We are the rightful owner — claim already written above; proceed. # ---- SHRINK PHASE ----------------------------------------------- # if "shrink_snap_id" in params and params.get("shrink_round", 0) > 0: diff --git a/simplyblock_core/test/test_cutover_convergence.py b/simplyblock_core/test/test_cutover_convergence.py index c02a12aeac..9bc47d7704 100644 --- a/simplyblock_core/test/test_cutover_convergence.py +++ b/simplyblock_core/test/test_cutover_convergence.py @@ -464,6 +464,38 @@ def test_a_claim_on_another_lvstore_is_irrelevant(self): self.db.get_job_tasks.return_value = [me, other] self.assertIsNone(runner._lvs_cutover_owner(me, "LVS_1")) + def test_circular_stall_is_broken_when_both_tasks_hold_the_claim(self): + """Both tasks race and both write cutover_lvs — only one wins. + + Regression for production stall: + 36418f5d queued_for_lvstore_LVS_1_behind_309d3aeb + 309d3aeb queued_for_lvstore_LVS_1_behind_36418f5d + + Root cause: _lvs_cutover_owner excluded the calling task from the scan. + When two threads both raced through the check before either wrote its + claim, both stored cutover_lvs. On every subsequent pass each task's + candidate set was exactly {the other}, so each always deferred to the + other — a permanent cycle. The fix: include self in the sort, return + None iff the caller is the winner. + """ + # Simulate the post-race DB state: both have cutover_lvs set. + early = self._task("T1", lvs="LVS_1", created="2026-01-01") + late = self._task("T2", lvs="LVS_1", created="2026-06-01") + self.db.get_job_tasks.return_value = [early, late] + + # From T1's perspective: T1 is the earliest claimant → it is the owner. + self.assertIsNone( + runner._lvs_cutover_owner(early, "LVS_1"), + "the earliest claimant must see itself as the winner (None), " + "not defer to the only other claimant") + + # From T2's perspective: T1 is the earliest claimant → T2 must yield. + owner_seen_by_late = runner._lvs_cutover_owner(late, "LVS_1") + self.assertIsNotNone(owner_seen_by_late, + "the later claimant must see an owner") + self.assertEqual(owner_seen_by_late.get_id(), "T1", + "the later claimant must defer to the earlier one") + class TestQueuedCutoverDoesNotStarve(unittest.TestCase): """A cutover that cannot have the lvstore waits without cost.""" From 3d1f4855d4a4eb97c0264a57945de7febb60cd42 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 28 Aug 2026 13:43:22 +0100 Subject: [PATCH 096/122] fix: reset shrink deadline and state on cutover retry to prevent instant re-failure --- .../services/tasks_runner_replication_final.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 19f97a278d..44e9da7ee0 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -199,6 +199,15 @@ def _finalize(task, ok, err): # A retry re-claims the LVS on its next pass; holding the claim across the # wait would stall every other volume's replication for nothing. _release_lvs_claim(task) + # Give the next attempt a fresh shrink window. Without this, a + # deadline-timeout failure leaves the expired timestamp in place and every + # subsequent retry fails on the first deadline check, burning all retries + # within seconds without doing any useful work. + task.function_params["shrink_deadline"] = ( + int(time.time()) + constants.REPL_CUTOVER_SHRINK_TIMEOUT_SEC) + for _k in ("shrink_snap_id", "shrink_started_at", "shrink_round", + "shrink_round_times", "shrink_round_done_at"): + task.function_params.pop(_k, None) task.write_to_db(db.kv_store) return False From 1e49b95ec1e6e1ff85b2ce5c872ccafbad32866b Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 28 Aug 2026 15:52:07 +0100 Subject: [PATCH 097/122] chore: revert _lvs_cutover_owner to per-lvstore serialization matching replication-features branch --- .../tasks_runner_replication_final.py | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 7237e996d3..7ccfcbefcf 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -39,33 +39,26 @@ def _lvs_cutover_owner(task, lvs_name, tasks=None): - """Return the task that should own *lvs_name*, or None if *task* itself wins. + """Return the task that should own the active cutover slot, or None if *task* wins. - Deterministic: the earliest-created claimant wins (sorted by create_dt then - id), so concurrent threads that both wrote a claim agree on the same winner. + Serializes all cutovers globally: only one may run at a time across all + volumes and lvstores. Deterministic: the earliest-created claimant wins + (sorted by create_dt then id), so two tasks that both wrote a claim agree on + the same winner regardless of interleaving. Callers MUST write their own ``cutover_lvs`` claim to DB **before** calling this function so they appear in the scan and participate in the sort. Returns None → the caller is the rightful owner; proceed. - Returns → that other task is the rightful owner; the caller should - remove its own claim and yield. - - Note: the previous implementation excluded the calling task from the scan - (``if other.get_id() == task.get_id(): continue``). When two threads both - raced past the guard before either wrote its claim, both ended up with - ``cutover_lvs`` set, and on every subsequent pass each task's candidate set - was exactly {the other} — so each always deferred to the other, creating a - permanent circular stall (e.g. 36418f5d queued behind 309d3aeb AND - 309d3aeb queued behind 36418f5d). Including self in the sort eliminates - the self-exclusion blindspot that made the cycle possible. + Returns → that other task owns the slot; the caller should remove + its own claim and yield. ``tasks`` may be a pre-fetched list from this pass. Re-reading per task is O(N^2) DB reads, which the sub-second poll interval cannot afford. """ if not lvs_name: return None - claimants = [] + owners = [] for other in (tasks if tasks is not None else db.get_job_tasks(task.cluster_id)): if other.function_name != JobSchedule.FN_REPLICATION_FINAL: continue @@ -73,11 +66,11 @@ def _lvs_cutover_owner(task, lvs_name, tasks=None): continue if (other.function_params or {}).get("cutover_lvs") != lvs_name: continue - claimants.append(other) - if not claimants: + owners.append(other) + if not owners: return None - claimants.sort(key=lambda t: (str(getattr(t, "create_dt", "")), t.get_id())) - winner = claimants[0] + owners.sort(key=lambda t: (str(getattr(t, "create_dt", "")), t.get_id())) + winner = owners[0] return None if winner.get_id() == task.get_id() else winner From 95ab5e612ef0de29d26dc0b95c487fd7761fd164 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 28 Aug 2026 18:09:52 +0200 Subject: [PATCH 098/122] Exclusivity is the endgame, not the whole cutover Run 20260828_124859 produced 0/20 fail-back cutovers. The instrumented round times say why, and it was not the transfer: lvol 60ddb170 round 1: 340.9s lvol 277a66bc round 1: 1505.6s lvol 3a4f5a27 round 1: 2073.6s lvol d078e9e7 round 1: 2583.5s lvol 83a18454 round 1: 12.4s <- never queued Each successive volume's round took longer than the last, and the one that did not queue finished in 12 seconds. That is a queue charged to the round: the task claimed its lvstore on its FIRST pass and held it for its entire life, so volumes 2..10 sat holding an ageing round-1 snapshot while volume 1 converged and cut over. Only 5 of 20 volumes got a single round in 64 minutes. The exclusive window is meant to cover the iterative convergence snapshots and the freeze -- the tail, once the delta is already small. The bulk catch-up runs concurrently, like ordinary replication. So the claim now happens at the END of convergence: unexclusive rounds run until one completes within REPL_CUTOVER_EXCLUSIVE_ENTRY_FACTOR x the target (3 x 2s), and only then is the lvstore taken for the final tight rounds and the freeze. A queued volume holds no snapshot, so nothing ages while it waits, and the exclusive window is seconds rather than the whole convergence. Also in this commit, on the earlier latency work: The RPC-based inline wait stays -- the measured effect is exactly what was asked for. round_gap_to_next_snapshot is now 0.00-0.18s across five rounds, against a median 81s before, and take_shrink_snapshot is 0.24s. But the 200ms adaptive pass interval is backed out to 1s. That loop reads the task table and every task per pass, so polling it at 5Hz burns transactions proportional to (clusters x tasks) to learn nothing almost every time -- polling a database to detect an event is the wrong shape regardless of whether FDB survives it. Sub-second reaction lives in the RPC poll, which touches no DB. The half-written HTTP callback module is removed rather than left in the tree: with the gap already at 0.0-0.18s, the handoff is not the bottleneck and a webhook would solve a problem the data says is gone. Tests updated where they encoded the old design (converged-in-the-open now asks for the lvstore instead of freezing; the queue happens at endgame entry, not on the first pass). 2065 pass. --- simplyblock_core/constants.py | 11 +- .../tasks_runner_replication_final.py | 119 ++++++++++++------ .../test/test_cutover_convergence.py | 42 ++++++- .../test_tasks_runner_replication_final.py | 4 + .../test/test_transfer_completion_latency.py | 18 ++- 5 files changed, 147 insertions(+), 47 deletions(-) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 17c07408f0..fb7026422c 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -384,6 +384,12 @@ def get_config_var(name, default=None): # Safety bound: a volume written faster than it replicates never converges, so # stop and freeze rather than looping forever. REPL_CUTOVER_MAX_SHRINK_ROUNDS = 12 +# When to stop converging in the open and take the lvstore for the endgame. +# A round completing within this multiple of the target means the delta is +# nearly converged, so the exclusive window that follows will be short. Claiming +# earlier serialises the bulk catch-up, which is what produced 0/20 cutovers in +# run 20260828_124859 (round 1 growing 340s -> 2584s purely from queueing). +REPL_CUTOVER_EXCLUSIVE_ENTRY_FACTOR = 3.0 # Rounds must follow each other within MILLISECONDS. Returning to the task # scheduler between them costs TASK_EXEC_INTERVAL_SEC (10s) of fresh writes # each time, which puts a floor under the delta no number of rounds can beat. @@ -424,7 +430,10 @@ def get_config_var(name, default=None): # Pass interval for the cutover runner while any cutover is mid-round. The # freeze pays for every millisecond between a transfer completing and the next # snapshot starting, so this must stay well under a second. -REPL_CUTOVER_ACTIVE_POLL_SEC = 0.2 +# Pass interval while a cutover is mid-round. NOT sub-second: this loop reads +# the task table per pass, and polling a database at 5Hz to detect an event is +# the wrong shape. Sub-second reaction lives in the RPC-based inline wait. +REPL_CUTOVER_ACTIVE_POLL_SEC = 1.0 SPDK_PROXY_MULTI_THREADING_ENABLED=True SPDK_PROXY_TIMEOUT=60*5 diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index cf45e2de4b..694e433673 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -171,6 +171,41 @@ def _finalize(task, ok, err): return False +def _acquire_lvs_claim(task, lvol, tasks=None): + """Take the lvstore for this cutover's endgame. False means queued. + + Queueing here is cheap: it happens with a nearly-converged delta and NO + snapshot in hand. The previous design claimed on the task's first pass and + held the lvstore through the entire catch-up, so queued volumes sat on an + ageing round-1 snapshot and their "round" measured the queue rather than + the transfer. + """ + params = task.function_params + lvs_name = getattr(lvol, "lvs_name", "") + own_group = _group_id_for_lvol(lvol) + owner = _lvs_cutover_owner(task, lvs_name, tasks) + if owner is not None: + owner_group = (owner.function_params or {}).get("cutover_group") or "" + owner_lvol = str((owner.function_params or {}).get("lvol_id")) + # A consistency group cuts over AS A GROUP, so a sibling member joins + # the owner rather than queueing behind it. + if not (own_group and own_group == owner_group): + params["shrink_deadline"] = ( + int(time.time()) + constants.REPL_CUTOVER_SHRINK_TIMEOUT_SEC) + task.function_result = ( + f"queued for lvstore {lvs_name} behind {owner_lvol[:8]} " + f"(delta already converged)") + task.status = JobSchedule.STATUS_SUSPENDED + task.write_to_db(db.kv_store) + return False + params["cutover_lvs"] = lvs_name + params["cutover_group"] = own_group + xfer_timing.stamp("lvs_claim_acquired", lvol=lvol.get_id(), lvs=lvs_name, + round=params.get("shrink_round")) + task.write_to_db(db.kv_store) + return True + + def task_runner(task: JobSchedule, tasks=None): params = task.function_params lvol_id = params.get("lvol_id") @@ -235,38 +270,13 @@ def task_runner(task: JobSchedule, tasks=None): # compete for the same lvstore and hub bandwidth, and every second they # steal from a convergence round is a second of writes that lands in # the freeze. - lvs_name = getattr(lvol, "lvs_name", "") - own_group = _group_id_for_lvol(lvol) - owner = _lvs_cutover_owner(task, lvs_name, tasks) - if owner is not None: - owner_group = (owner.function_params or {}).get("cutover_group") or "" - owner_lvol = str((owner.function_params or {}).get("lvol_id")) - # A consistency group cuts over AS A GROUP, so a sibling member - # joins the owner rather than queueing behind it. - if not (own_group and own_group == owner_group): - # WAIT, do not start. Beginning the shrink phase here would - # take snapshots that cannot replicate (the owner holds the - # lvstore), and their deadline would run down until the task - # died of max retries -- which is exactly what happened to 17 - # tasks in run 20260827_185009. - task.function_params["shrink_deadline"] = ( - int(time.time()) + constants.REPL_CUTOVER_SHRINK_TIMEOUT_SEC) - task.function_result = ( - f"queued for lvstore {lvs_name} behind {owner_lvol[:8]}") - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return False # no retry burned: this is a queue, not a failure - - # Claim the source LVS for the whole cutover -- the convergence rounds - # AND the freeze. Other volumes' snapshot transfers on this LVS queue - # behind it (see snapshot_replication._lvs_transfer_hold): they compete - # for the same lvstore and hub bandwidth, and every second they steal - # from a convergence round is a second of writes that lands in the - # freeze. - task.function_params["cutover_lvs"] = lvs_name - # Recording the group is what lets the replication side tell a sibling - # member from an unrelated volume. - task.function_params["cutover_group"] = own_group + # Exclusivity is the ENDGAME, not the whole cutover. The bulk catch-up + # converges in the open, concurrently with every other volume; only + # once a round is already fast is the lvstore taken, for the final + # tight rounds and the freeze (see _acquire_lvs_claim). + if params.get("ready_for_exclusive") and not params.get("cutover_lvs"): + if not _acquire_lvs_claim(task, lvol, tasks): + return False # queued, holding nothing and ageing nothing task.write_to_db(db.kv_store) # ---- SHRINK PHASE ----------------------------------------------- # @@ -282,6 +292,11 @@ def task_runner(task: JobSchedule, tasks=None): return False # ---- CUTOVER PHASE (immediately after the last shrink round) ---- # + # The freeze runs under the claim. A volume whose very first round was + # already fast enough reaches here without having taken it. + if not params.get("cutover_lvs"): + if not _acquire_lvs_claim(task, lvol, tasks): + return False if "tgt_lvol_composite" not in params: with xfer_timing.phase("prepare_cutover", lvol=lvol_id): err = _prepare_cutover(task, lvol, src_node, tgt_node) @@ -460,14 +475,33 @@ def _shrink_step(task, lvol): ms=elapsed * 1000.0) params["shrink_round_done_at"] = time.time() - # Converged: this round's delta -- the writes made during the previous - # round -- moved in low seconds, so the freeze that copies the next - # such window will be about as short. - if elapsed <= constants.REPL_CUTOVER_CONVERGE_TARGET_SEC: + exclusive = bool(params.get("cutover_lvs")) + + # Converged AND holding the lvstore: hand over to the freeze. + if exclusive and elapsed <= constants.REPL_CUTOVER_CONVERGE_TARGET_SEC: task.function_result = (f"converged in {params['shrink_round']} rounds " f"(last {elapsed:.2f}s)") return True, None + # Nearly converged but still converging in the OPEN. Stop and go take + # the lvstore for the endgame. Claiming it any earlier serialises the + # bulk catch-up: in run 20260828_124859 the claim was taken on the + # task's first pass and held for its whole life, so volumes queued + # while holding an ageing round-1 snapshot and their "round" measured + # the queue -- 340s, 1505s, 2073s, 2584s for successive volumes, while + # the one that never queued finished its round in 12.4s. + if not exclusive and elapsed <= ( + constants.REPL_CUTOVER_CONVERGE_TARGET_SEC + * constants.REPL_CUTOVER_EXCLUSIVE_ENTRY_FACTOR): + params["ready_for_exclusive"] = True + task.function_result = (f"delta converged in {params['shrink_round']} " + f"open rounds (last {elapsed:.2f}s); taking " + f"the lvstore for the endgame") + xfer_timing.stamp("ready_for_exclusive", lvol=lvol.get_id(), + round=params["shrink_round"], ms=elapsed * 1000.0) + task.write_to_db(db.kv_store) + return False, None + if params["shrink_round"] >= constants.REPL_CUTOVER_MAX_SHRINK_ROUNDS: # Written faster than it replicates. Freezing now is still the best # move -- the freeze at least stops the writes -- but say so. @@ -478,6 +512,11 @@ def _shrink_step(task, lvol): constants.REPL_CUTOVER_CONVERGE_TARGET_SEC) task.function_result = (f"not converged after {params['shrink_round']} " f"rounds (last {elapsed:.2f}s)") + if not params.get("cutover_lvs"): + # Give up converging, but the freeze still wants the lvstore. + params["ready_for_exclusive"] = True + task.write_to_db(db.kv_store) + return False, None return True, None # IMMEDIATELY take the next snapshot. This is the whole mechanism: the @@ -606,7 +645,13 @@ def main(): # for a task that is queued or mid-round, and sleeping 3s per # such task cost ~70s per pass with 20 volumes -- which landed # directly in the client's IO freeze. - # Poll fast only while it matters; an idle cluster keeps the old cadence. + # Deliberately NOT a sub-second poll: this loop reads the task table + # (and each task) per pass, so polling it at 5Hz burns transactions + # proportional to clusters x tasks to learn nothing almost every time. + # The latency that mattered is gone from the hot path instead -- a + # transfer is now awaited and finished in the pass that submitted it + # (snapshot_replication._await_transfer_completion, an RPC poll), and a + # converging round stays inside its own inline loop. time.sleep(constants.REPL_CUTOVER_ACTIVE_POLL_SEC if active else constants.TASK_EXEC_INTERVAL_SEC) diff --git a/simplyblock_core/test/test_cutover_convergence.py b/simplyblock_core/test/test_cutover_convergence.py index c02a12aeac..d46a5ed61d 100644 --- a/simplyblock_core/test/test_cutover_convergence.py +++ b/simplyblock_core/test/test_cutover_convergence.py @@ -65,11 +65,18 @@ def sleep(self, seconds): class TestConvergence(unittest.TestCase): """_shrink_step loops until a round is fast, without leaving the pass.""" - def _run(self, round_times, max_rounds=None): - """Drive _shrink_step with round i taking round_times[i] seconds.""" + def _run(self, round_times, max_rounds=None, exclusive=True): + """Drive _shrink_step with round i taking round_times[i] seconds. + + exclusive=True means the task already holds its lvstore, which is the + endgame. Unexclusive rounds converge in the open and then ASK for the + lvstore instead of handing straight over to the freeze. + """ clock = _Clock() task = _Task(shrink_snap_id="S0", shrink_round=1, shrink_deadline=10 ** 9, lvol_id="LV1") + if exclusive: + task.function_params["cutover_lvs"] = "LVS_1" task.function_params["shrink_started_at"] = clock.now state = {"i": 0} taken = [] @@ -104,14 +111,27 @@ def _take(task_, lvol_): done, err = runner._shrink_step(task, _lvol()) return done, err, task, taken, clock - def test_a_fast_round_converges_and_hands_over(self): - """Round transferred inside the target -> freeze immediately.""" + def test_a_fast_round_while_holding_the_lvstore_hands_over(self): + """Converged AND exclusive -> freeze immediately.""" done, err, task, taken, _ = self._run([0.5]) self.assertTrue(done) self.assertIsNone(err) self.assertIn("converged", task.function_result) self.assertEqual(taken, [], "a fast first round needs no further rounds") + def test_a_fast_round_in_the_open_asks_for_the_lvstore(self): + """Nearly converged unexclusively -> take the lvstore for the endgame. + + Claiming it earlier serialised the bulk catch-up: run 20260828_124859 + charged the queue wait to round 1 (340s, 1505s, 2073s, 2584s for + successive volumes) while the unqueued one finished in 12.4s. + """ + done, err, task, taken, _ = self._run([0.5], exclusive=False) + self.assertFalse(done, "it must not freeze without holding the lvstore") + self.assertIsNone(err) + self.assertTrue(task.function_params.get("ready_for_exclusive")) + self.assertIn("endgame", task.function_result) + def test_a_slow_round_takes_another_snapshot_without_leaving_the_pass(self): """The whole point: rounds follow each other in milliseconds.""" done, err, task, taken, clock = self._run([3.0, 3.0, 0.4]) @@ -132,6 +152,13 @@ def test_it_gives_up_after_the_round_cap_and_freezes_anyway(self): self.assertIsNone(err) self.assertIn("not converged", task.function_result) + def test_the_cap_takes_the_lvstore_before_freezing(self): + """Giving up converging still requires the lvstore for the freeze.""" + done, err, task, taken, _ = self._run([3.0] * 20, max_rounds=3, + exclusive=False) + self.assertFalse(done) + self.assertTrue(task.function_params.get("ready_for_exclusive")) + def test_a_vanished_snapshot_is_an_error(self): # This one runs on the real clock, so the deadline has to be a real # future epoch -- 10**9 is 2001 and would trip the timeout instead. @@ -491,10 +518,10 @@ def setUp(self): gp = patch.object(runner, "_group_id_for_lvol", return_value="") gp.start() self.addCleanup(gp.stop) - # If the shrink phase ran, the test would see it here. + # A task waiting for the lvstore must not proceed into the endgame. sp = patch.object(runner, "_shrink_step", side_effect=AssertionError( - "a queued cutover must not start its shrink phase")) + "a queued cutover must not run its endgame rounds")) sp.start() self.addCleanup(sp.stop) @@ -512,6 +539,9 @@ def _me(self): "lvol_id": "LV_me", "src_node_id": "N1", "tgt_node_id": "N2", "shrink_round": 1, "shrink_snap_id": "S1", "shrink_deadline": 1, # already expired + # asking for the endgame: the delta has converged in the open, so + # this is the point at which queueing happens + "ready_for_exclusive": True, } return t diff --git a/simplyblock_core/test/test_tasks_runner_replication_final.py b/simplyblock_core/test/test_tasks_runner_replication_final.py index b678f28cee..4466640f56 100644 --- a/simplyblock_core/test/test_tasks_runner_replication_final.py +++ b/simplyblock_core/test/test_tasks_runner_replication_final.py @@ -208,6 +208,9 @@ def test_a_fast_round_converges_instead_of_taking_another(monkeypatch): runner, task = _mk(monkeypatch, {"S1": _ShrinkSnap(replicated=True)}, {"shrink_round": 1, "shrink_snap_id": "S1", "shrink_deadline": 2**60, + # holding the lvstore == the endgame; converging in the + # open asks for it instead of freezing + "cutover_lvs": "LVS_1", "shrink_started_at": __import__("time").time()}) taken = [] @@ -264,6 +267,7 @@ def test_shrink_hands_over_when_it_cannot_converge(monkeypatch): runner, task = _mk(monkeypatch, {"S1": _ShrinkSnap(replicated=True)}, {"shrink_round": 3, "shrink_snap_id": "S1", "shrink_deadline": 2**60, + "cutover_lvs": "LVS_1", "shrink_started_at": _time.time() - 60}) done, err = runner._shrink_step(task, _ShrinkLvol()) assert (done, err) == (True, None), \ diff --git a/simplyblock_core/test/test_transfer_completion_latency.py b/simplyblock_core/test/test_transfer_completion_latency.py index 1701b9f8d8..4516bb9c8e 100644 --- a/simplyblock_core/test/test_transfer_completion_latency.py +++ b/simplyblock_core/test/test_transfer_completion_latency.py @@ -95,10 +95,22 @@ def test_the_submit_path_calls_the_wait(self): class TestPassLatency(unittest.TestCase): """A round that yields must be picked up again in well under a second.""" - def test_the_active_poll_interval_is_sub_second(self): - self.assertLess(constants.REPL_CUTOVER_ACTIVE_POLL_SEC, 1.0) + def test_sub_second_reaction_does_not_come_from_polling_the_database(self): + """The DB pass interval is deliberately NOT sub-second. + + This loop reads the task table (and each task) per pass, so polling it + at 5Hz burns transactions proportional to clusters x tasks to learn + nothing almost every time -- the wrong shape for detecting an event. + Sub-second reaction comes from the RPC-based inline wait instead. + """ + self.assertGreaterEqual(constants.REPL_CUTOVER_ACTIVE_POLL_SEC, 1.0, + "do not poll a database sub-second") self.assertLess(constants.REPL_CUTOVER_ACTIVE_POLL_SEC, - constants.TASK_EXEC_INTERVAL_SEC) + constants.TASK_EXEC_INTERVAL_SEC, + "but a mid-round cutover still deserves a tighter pass") + # This is the interval the guarantee actually rests on, and it polls + # SPDK over RPC, not the DB. + self.assertLess(constants.REPL_XFER_POLL_INTERVAL_SEC, 1.0) def test_a_claimed_or_mid_round_task_counts_as_in_flight(self): from simplyblock_core.models.job_schedule import JobSchedule From 99ca46c0f3392e9d6b06f245fa5618f0d47c8b8b Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 28 Aug 2026 17:50:24 +0100 Subject: [PATCH 099/122] remove shrink reset on retry; deadline expiry now falls through to cutover --- .../tasks_runner_replication_final.py | 44 ++++++++++--------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 7ccfcbefcf..ee82e68282 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -195,15 +195,6 @@ def _finalize(task, ok, err): # A retry re-claims the LVS on its next pass; holding the claim across the # wait would stall every other volume's replication for nothing. _release_lvs_claim(task) - # Give the next attempt a fresh shrink window. Without this, a - # deadline-timeout failure leaves the expired timestamp in place and every - # subsequent retry fails on the first deadline check, burning all retries - # within seconds without doing any useful work. - task.function_params["shrink_deadline"] = ( - int(time.time()) + constants.REPL_CUTOVER_SHRINK_TIMEOUT_SEC) - for _k in ("shrink_snap_id", "shrink_started_at", "shrink_round", - "shrink_round_times", "shrink_round_done_at"): - task.function_params.pop(_k, None) task.write_to_db(db.kv_store) return False @@ -393,16 +384,21 @@ def task_runner(task: JobSchedule, tasks=None): # We are the rightful owner — claim already written above; proceed. # ---- SHRINK PHASE ----------------------------------------------- # - if "shrink_snap_id" in params and params.get("shrink_round", 0) > 0: - done, err = _shrink_step(task, lvol) - if err: - return _finalize(task, False, err) - if not done: - # waiting on replication of the current shrink snapshot; come - # back next pass WITHOUT burning a retry (bounded by deadline) - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return False + # Skip shrink entirely once the cutover clone is prepared: tgt_lvol_composite + # being set means shrink already completed on a prior pass and the clone was + # created from the resulting snapshot. Every retry after a failed run_cutover + # should jump straight to run_cutover without redoing any shrink rounds. + if "tgt_lvol_composite" not in params: + if "shrink_snap_id" in params and params.get("shrink_round", 0) > 0: + done, err = _shrink_step(task, lvol) + if err: + return _finalize(task, False, err) + if not done: + # waiting on replication of the current shrink snapshot; come + # back next pass WITHOUT burning a retry (bounded by deadline) + task.status = JobSchedule.STATUS_SUSPENDED + task.write_to_db(db.kv_store) + return False # ---- CUTOVER PHASE (immediately after the last shrink round) ---- # if "tgt_lvol_composite" not in params: @@ -540,7 +536,15 @@ def _shrink_step(task, lvol): while True: if int(time.time()) > deadline: - return False, "shrink phase timed out waiting for replication" + # Deadline expired: stop adding rounds and fall through to cutover + # rather than failing and burning a retry. The freeze that follows + # is slightly larger than if we had converged, but proceeding is + # always better than another 900-second wait. + logger.warning( + "cutover convergence: lvol=%s shrink deadline expired after %d " + "rounds; proceeding to cutover", lvol.get_id(), + params.get("shrink_round", 0)) + return True, None snap_id = params["shrink_snap_id"] done = _shrink_round_done(snap_id) From 5c4af766dbfc6ed393c83f4363486abfc03f0dd8 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 28 Aug 2026 19:43:33 +0100 Subject: [PATCH 100/122] replication: skip shrink rounds when transfer is zero bytes --- simplyblock_core/constants.py | 6 ++ simplyblock_core/models/snapshot.py | 4 ++ .../services/snapshot_replication.py | 1 + .../tasks_runner_replication_final.py | 69 ++++++++++++++++++- 4 files changed, 77 insertions(+), 3 deletions(-) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 17c07408f0..40000e49cc 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -425,6 +425,12 @@ def get_config_var(name, default=None): # freeze pays for every millisecond between a transfer completing and the next # snapshot starting, so this must stay well under a second. REPL_CUTOVER_ACTIVE_POLL_SEC = 0.2 +# Cooldown between hub-attach retry attempts when the target node is down or +# recovering (covers control-plane lag before the DB reflects the down state). +REPL_CUTOVER_HUB_RETRY_COOLDOWN_SEC = 30 +# Max consecutive hub-attach failures with the node still appearing online +# before we give up and burn a task.retry. 30s × 20 = 10 min of coverage. +REPL_CUTOVER_MAX_HUB_ATTEMPTS = 10 SPDK_PROXY_MULTI_THREADING_ENABLED=True SPDK_PROXY_TIMEOUT=60*5 diff --git a/simplyblock_core/models/snapshot.py b/simplyblock_core/models/snapshot.py index 1476e03ef9..be119bbed8 100644 --- a/simplyblock_core/models/snapshot.py +++ b/simplyblock_core/models/snapshot.py @@ -51,6 +51,10 @@ class SnapShot(BaseModel): # On Snapshot transfer or replicate this field is the same # This value can be used to identify the same snapshot on other nodes data_uuid: str = "" + # Bytes transferred when this snapshot was replicated. -1 = not measured + # (old records or in-flight), 0 = transferred but no data (zero-delta + # round): the cutover convergence loop uses this to stop early. + replication_bytes: int = -1 def write_to_db(self, kv_store=None): super().write_to_db(kv_store) diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index 3d7ac866ce..4ab63fdb54 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -223,6 +223,7 @@ def _finish_completed_transfer(task, snapshot, offset): task.function_params.get("xfer_submit_t"), snap=snapshot.get_id(), lvol=snapshot.lvol.get_id(), bytes=offset) + snapshot.replication_bytes = offset or 0 with xfer_timing.phase("replicate_finish", snap=snapshot.get_id(), lvol=snapshot.lvol.get_id()): new_snapshot_uuid = process_snap_replicate_finish(task, snapshot) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index ee82e68282..5af7f325c0 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -191,10 +191,46 @@ def _finalize(task, ok, err): task.retry + 1, task.max_retry, task.function_params.get("lvol_id"), task.function_result) task.status = JobSchedule.STATUS_SUSPENDED - task.retry += 1 # A retry re-claims the LVS on its next pass; holding the claim across the # wait would stall every other volume's replication for nothing. _release_lvs_claim(task) + + # When the failure happened inside run_cutover (tgt_lvol_composite already + # set), it is likely a connectivity issue: the target node restarted, or the + # control plane hasn't yet reflected the down state in DB. Hammering at the + # 0.2s poll interval burns all retries in seconds — long before the node + # recovers. Instead, re-read the node status and add a cooldown. + if task.function_params.get("tgt_lvol_composite"): + tgt_node_id = task.function_params.get("tgt_node_id") + node_offline = False + if tgt_node_id: + try: + current_tgt = db.get_storage_node_by_id(tgt_node_id) + node_offline = current_tgt.status != StorageNode.STATUS_ONLINE + except KeyError: + node_offline = True + hub_attempts = task.function_params.get("cutover_hub_attempts", 0) + 1 + task.function_params["cutover_hub_attempts"] = hub_attempts + if node_offline or hub_attempts <= constants.REPL_CUTOVER_MAX_HUB_ATTEMPTS: + # Transient: add a cooldown, do NOT burn task.retry. + task.function_params["cutover_retry_after"] = ( + int(time.time()) + constants.REPL_CUTOVER_HUB_RETRY_COOLDOWN_SEC) + logger.warning( + "cutover for lvol %s: connectivity failure (attempt %d, node_offline=%s); " + "waiting %ds before retry", + task.function_params.get("lvol_id"), hub_attempts, node_offline, + constants.REPL_CUTOVER_HUB_RETRY_COOLDOWN_SEC) + task.write_to_db(db.kv_store) + return False + # Exceeded transient cap with node appearing online — real failure. + logger.warning( + "cutover for lvol %s: hub attach failed %d times with node appearing " + "online; treating as real failure and burning a retry", + task.function_params.get("lvol_id"), hub_attempts) + task.function_params.pop("cutover_hub_attempts", None) + task.function_params.pop("cutover_retry_after", None) + + task.retry += 1 task.write_to_db(db.kv_store) return False @@ -316,9 +352,19 @@ def task_runner(task: JobSchedule, tasks=None): params.get("lvol_id"), tgt_node.get_id(), tgt_node.status) task.function_params["last_error"] = ( f"target node {tgt_node.get_id()[:8]} is {tgt_node.status}") - task.function_result = "target node not online, retrying" + task.function_result = "target node not online, waiting" + task.status = JobSchedule.STATUS_SUSPENDED + # Do NOT burn task.retry — node offline is transient; hammering at + # 0.2s would exhaust all retries before the node recovers. + task.write_to_db(db.kv_store) + return False + + # Cooldown set after a hub-attach failure to give the target node time + # to recover and the control plane time to reflect a down state in DB. + retry_after = params.get("cutover_retry_after", 0) + if int(time.time()) < retry_after: + task.function_result = "waiting after connectivity failure" task.status = JobSchedule.STATUS_SUSPENDED - task.retry += 1 task.write_to_db(db.kv_store) return False @@ -561,6 +607,23 @@ def _shrink_step(task, lvol): time.sleep(constants.REPL_CUTOVER_POLL_INTERVAL_SEC) continue + # Zero-delta early exit: if this round transferred 0 bytes the volume + # had no new writes since the previous snapshot. Another round would + # also transfer 0 bytes, so stop immediately rather than burning the + # remaining rounds (and their cross-cluster add_lvol_ha cost) for nothing. + try: + snap_rec = db.get_snapshot_by_id(snap_id) + if snap_rec.replication_bytes == 0: + logger.info( + "cutover convergence: lvol=%s round %d transferred 0 bytes; " + "no new writes — proceeding to cutover immediately", + lvol.get_id(), params.get("shrink_round", 0)) + task.function_result = ( + f"zero-delta after round {params.get('shrink_round', 0)}; converged") + return True, None + except KeyError: + pass + started_at = params.get("shrink_started_at") if started_at is None: # Unmeasurable round (an older task, or one enqueued without the From c8cfedfae8215f0b2ad82108f436f14cc7b17686 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 28 Aug 2026 19:46:17 +0100 Subject: [PATCH 101/122] replication: fix retry burn and node-offline stall during cutover --- simplyblock_core/models/snapshot.py | 4 ---- .../services/snapshot_replication.py | 1 - .../services/tasks_runner_replication_final.py | 17 ----------------- 3 files changed, 22 deletions(-) diff --git a/simplyblock_core/models/snapshot.py b/simplyblock_core/models/snapshot.py index be119bbed8..1476e03ef9 100644 --- a/simplyblock_core/models/snapshot.py +++ b/simplyblock_core/models/snapshot.py @@ -51,10 +51,6 @@ class SnapShot(BaseModel): # On Snapshot transfer or replicate this field is the same # This value can be used to identify the same snapshot on other nodes data_uuid: str = "" - # Bytes transferred when this snapshot was replicated. -1 = not measured - # (old records or in-flight), 0 = transferred but no data (zero-delta - # round): the cutover convergence loop uses this to stop early. - replication_bytes: int = -1 def write_to_db(self, kv_store=None): super().write_to_db(kv_store) diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index 4ab63fdb54..3d7ac866ce 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -223,7 +223,6 @@ def _finish_completed_transfer(task, snapshot, offset): task.function_params.get("xfer_submit_t"), snap=snapshot.get_id(), lvol=snapshot.lvol.get_id(), bytes=offset) - snapshot.replication_bytes = offset or 0 with xfer_timing.phase("replicate_finish", snap=snapshot.get_id(), lvol=snapshot.lvol.get_id()): new_snapshot_uuid = process_snap_replicate_finish(task, snapshot) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 5af7f325c0..8eece94743 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -607,23 +607,6 @@ def _shrink_step(task, lvol): time.sleep(constants.REPL_CUTOVER_POLL_INTERVAL_SEC) continue - # Zero-delta early exit: if this round transferred 0 bytes the volume - # had no new writes since the previous snapshot. Another round would - # also transfer 0 bytes, so stop immediately rather than burning the - # remaining rounds (and their cross-cluster add_lvol_ha cost) for nothing. - try: - snap_rec = db.get_snapshot_by_id(snap_id) - if snap_rec.replication_bytes == 0: - logger.info( - "cutover convergence: lvol=%s round %d transferred 0 bytes; " - "no new writes — proceeding to cutover immediately", - lvol.get_id(), params.get("shrink_round", 0)) - task.function_result = ( - f"zero-delta after round {params.get('shrink_round', 0)}; converged") - return True, None - except KeyError: - pass - started_at = params.get("shrink_started_at") if started_at is None: # Unmeasurable round (an older task, or one enqueued without the From cafc9042b58e20f9c9af1a3afbeb1244c285239c Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 28 Aug 2026 14:56:19 +0200 Subject: [PATCH 102/122] Land replication transfers in a COW clone so the delta path can run The dirty bitmap has been in the SPDK fork for a long time and has never executed. bdev_lvol_transfer takes an optional allow_partial; the control plane never sent it, and could not: a partial transfer ships only the ranges written since the previous snapshot, so the destination must already hold everything else -- and this pipeline created a FRESH EMPTY landing volume, transferred the whole snapshot into it, and only chained it afterwards. A delta into an empty volume silently drops every cluster it does not cover. Chain the landing volume onto the destination's copy of the previous snapshot BEFORE the transfer instead of after it. That is the same bdev_lvol_add_clone the finish step already issued, moved earlier, and it is what makes the volume a valid delta target: a cluster the new snapshot does not own stays unallocated and reads through to the parent, and the first write into a cluster the delta does touch makes the blobstore copy-on-write the whole cluster from that same parent before applying the incoming range. Both halves of the destination image come from the same predecessor the source computed its delta against. allow_partial is then requested only when that chaining actually succeeded on the leader AND on an online secondary -- the transfer lands on the leader and the lvstore mirrors it to the secondary, so an unchained secondary would read zeros wherever the delta did not write. Everything short of that logs the reason and sends a full transfer, which is always correct. The fork also gates the bitmap path on the snapshot's dirty generation being complete and falls back to a full transfer on its own, so the two checks are independent. Nodes chained up front are recorded on the task and skipped by the finish step, because adding the same clone entry twice is not idempotent. convert is untouched. The RPC sends the allow_partial key only when opting in, so every existing caller keeps its current wire form. Retention already keeps the COW parent: _KEEP_REPLICATED_INTERNAL and ReplicationPolicy.MIN_KEEP_REPLICATED are both 2, the schedule path passes always_keep_newest=keep, and _prune_internal_snapshots additionally defers a prune whose successor is not yet chained onto it. Pinned with tests rather than changed. Co-Authored-By: Claude Opus 5 (1M context) --- simplyblock_core/rpc_client.py | 27 +- .../services/snapshot_replication.py | 182 ++++++++- .../test/test_replication_partial_transfer.py | 379 ++++++++++++++++++ 3 files changed, 582 insertions(+), 6 deletions(-) create mode 100644 simplyblock_core/test/test_replication_partial_transfer.py diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index 257593b87a..21f0250916 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -1918,22 +1918,43 @@ def bdev_lvol_set_migration_flag(self, name): """Mark *name* (composite lvol bdev) as a migration-target lvol.""" return self._request("bdev_lvol_set_migration_flag", {"lvol_name": name}) - def bdev_lvol_transfer(self, name, offset, batch_size, bdev_name, operation="migrate", lvol_id=0): + def bdev_lvol_transfer(self, name, offset, batch_size, bdev_name, operation="migrate", lvol_id=0, + allow_partial=False): """ Start an async blob transfer from *name* (source composite bdev) to the NVMe-oF bdev *bdev_name* attached on the caller's node. Returns the RPC result (truthy on success) or None on error. Poll progress with :meth:`bdev_lvol_transfer_stat`. + + *allow_partial* opts this transfer into the dirty-bitmap delta path: the + SPDK side then ships only the ranges written since the previous snapshot + instead of every allocated cluster. It is a REQUEST, not a guarantee -- + the fork gates the bitmap path on the snapshot carrying a COMPLETE + dirty generation and silently sends a full transfer whenever it does + not, so passing it can never produce less data than the destination + needs *from this transfer*. + + What it does NOT excuse is the destination's starting content: a partial + transfer only ships the delta, so everything outside the delta must + already be on the destination. Pass it only when the landing volume is a + clone of the destination's copy of the PREVIOUS snapshot. Passing it for + a transfer into a fresh empty volume would silently drop every cluster + the delta does not cover. """ - return self._request("bdev_lvol_transfer", { + params = { "lvol_name": name, "lvol_id": lvol_id, "offset": offset, "cluster_batch": batch_size, "gateway": bdev_name, "operation": operation, - }) + } + # Send the key only when opting in: the RPC parameter is optional on the + # SPDK side, so every existing caller keeps its exact current wire form. + if allow_partial: + params["allow_partial"] = True + return self._request("bdev_lvol_transfer", params) def bdev_lvol_transfer_stat(self, name): """ diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index 3d7ac866ce..9c37091664 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -555,6 +555,15 @@ def process_snap_replicate_start(task, snapshot): task.write_to_db() return + allow_partial = _partial_transfer_decision( + task, snapshot, replicate_to_source, remote_lv, remote_lv_node) + logger.info("Transfer of %s into %s: %s", snapshot.get_id(), + remote_lv.top_bdev, + "PARTIAL allowed (landing volume is a clone of the previous " + "replicated snapshot on every online member)" if allow_partial + else "FULL (no delta basis on every online member; see above " + "for which check declined)") + offset = 0 if "offset" in task.function_params and task.function_params["offset"]: offset = task.function_params["offset"] @@ -602,7 +611,11 @@ def process_snap_replicate_start(task, snapshot): batch_size=16, bdev_name=hub_bdev, operation="replicate", - lvol_id=remote_map_id + lvol_id=remote_map_id, + # Safe only because the landing volume was chained onto the previous + # replicated snapshot above; the fork independently refuses the delta + # path unless the snapshot's dirty generation is complete. + allow_partial=allow_partial, ) task.status = JobSchedule.STATUS_RUNNING task.function_params["start_time"] = int(time.time()) @@ -1038,6 +1051,154 @@ def _resolve_chain_target(snapshot, replicate_to_source, remote_snode): return {"snap_bdev": _snap_obj.snap_bdev}, _snap_obj, True +def _prechain_landing_volume(task, snapshot, replicate_to_source, remote_lv, + remote_lv_node): + """Chain the landing volume onto the destination's copy of the PREVIOUS + snapshot, before the transfer runs, so only the delta has to be sent. + + Returns True when the landing volume (and every online member of its HA + pair) is chained, which is the precondition for asking for a PARTIAL + transfer. Returns False to mean "send a full transfer", which is always + correct and is what this pipeline did unconditionally until now. + + Why chaining first is what makes a partial transfer correct + ----------------------------------------------------------- + ``bdev_lvol_transfer`` sends a blob's OWN cluster map and nothing else, so a + partial transfer ships only the ranges written since the previous snapshot. + Everything OUTSIDE that delta therefore has to be on the destination + already, and a fresh empty landing volume does not have it -- that is + exactly why the delta path has never been switched on. Chaining the landing + volume onto the predecessor's remote copy supplies it: a cluster the new + snapshot does not own stays unallocated and reads through to the parent, + and the first write into a cluster the delta DOES touch makes the blobstore + copy-on-write the whole cluster from that same parent before applying the + incoming range. Both halves of the destination image thus come from the + same predecessor the source computed its delta against. + + This is the very ``bdev_lvol_add_clone`` the finish step used to issue AFTER + the transfer; issuing it BEFORE is what turns the landing volume into a + valid delta target. Nodes chained here are recorded on the task so the + finish step does not add the same clone entry twice. + + The bar for returning True is deliberately high: + * a predecessor copy must resolve cleanly on this node (``ok`` and a + non-empty ``target_prev_snap`` from :func:`_resolve_chain_target`); + * the LVS leader must accept the chain; + * the secondary must be ONLINE and accept it too. The transfer lands on + the leader and the lvstore mirrors it to the secondary; an unchained + secondary would read zeros wherever the delta did not write, so a + degraded pair gets a full transfer rather than a delta. + Anything short of that logs why and falls back to a full transfer. + """ + target_prev_snap, _prev_snap_for_db, ok = _resolve_chain_target( + snapshot, replicate_to_source, remote_lv_node) + if not ok: + logger.info( + "Landing volume %s stays a full-transfer target: the predecessor's " + "copy on the destination could not be resolved", remote_lv.top_bdev) + return False + if not target_prev_snap: + logger.info( + "Landing volume %s stays a full-transfer target: %s starts the " + "chain, so there is no predecessor to clone from", + remote_lv.top_bdev, snapshot.get_id()) + return False + + sec_node = None + if remote_lv_node.secondary_node_id: + try: + sec_node = db.get_storage_node_by_id(remote_lv_node.secondary_node_id) + except KeyError: + sec_node = None + if sec_node is None or sec_node.status != StorageNode.STATUS_ONLINE: + logger.info( + "Landing volume %s stays a full-transfer target: the secondary of " + "%s is not online, and a delta would leave holes on it", + remote_lv.top_bdev, remote_lv_node.get_id()) + return False + + prechained = list(task.function_params.get("prechained_node_ids") or []) + + def _chain_on(node, role): + if node.get_id() in prechained: + return True + logger.info("Pre-chaining landing volume %s onto %s on %s (%s) so the " + "transfer can ship only the delta", + remote_lv.top_bdev, target_prev_snap['snap_bdev'], + node.get_id(), role) + try: + ret = node.rpc_client().bdev_lvol_add_clone( + remote_lv.top_bdev, target_prev_snap['snap_bdev']) + except Exception as e: + logger.warning("Pre-chain of %s on %s (%s) raised %s; falling back " + "to a full transfer", remote_lv.top_bdev, + node.get_id(), role, e) + return False + if not ret: + logger.warning("Pre-chain of %s onto %s failed on %s (%s); falling " + "back to a full transfer", remote_lv.top_bdev, + target_prev_snap['snap_bdev'], node.get_id(), role) + return False + prechained.append(node.get_id()) + return True + + primary_ok = _chain_on(remote_lv_node, "primary") + # Record whatever actually landed even on the failure path: the entry exists + # on that node now, and the finish step must not add it a second time. + secondary_ok = _chain_on(sec_node, "secondary") if primary_ok else False + if prechained != (task.function_params.get("prechained_node_ids") or []): + task.function_params["prechained_node_ids"] = prechained + task.write_to_db() + + if not (primary_ok and secondary_ok): + # A full transfer into a partially chained volume is still correct: it + # writes every cluster the snapshot owns, and the chained node reads the + # remainder from the parent exactly as it should. + return False + return True + + +def _partial_transfer_decision(task, snapshot, replicate_to_source, remote_lv, + remote_lv_node): + """Whether this transfer may ship only the delta, chaining the landing + volume first if that has not happened yet. + + The verdict is cached on the task so a resumed transfer (offset > 0) keeps + the mode its first attempt used -- but it is KEYED to the landing volume it + was made about. An interrupted attempt can delete a half-created landing + volume and build a fresh, unchained one (see the adoption block in + process_snap_replicate_start); carrying "partial is fine" onto that volume + would ship a delta into something that holds nothing, which is precisely + the silent data loss this design exists to avoid. A key mismatch throws + away the old verdict AND the old chain record and re-derives both. + """ + landing_key = remote_lv.get_id() + if task.function_params.get("allow_partial_landing") == landing_key: + return bool(task.function_params.get("allow_partial")) + + # Chain state recorded against a previous landing volume says nothing about + # this one. + task.function_params["prechained_node_ids"] = [] + allow_partial = _prechain_landing_volume( + task, snapshot, replicate_to_source, remote_lv, remote_lv_node) + task.function_params["allow_partial"] = allow_partial + task.function_params["allow_partial_landing"] = landing_key + task.write_to_db() + return allow_partial + + +def _prechained_nodes_for(task, remote_lv): + """Nodes already carrying the landing volume's clone entry. + + Empty unless the record was made about THIS landing volume: adding the same + clone entry twice is not idempotent on the SPDK side, and trusting a stale + record would skip a chain the volume actually needs. + """ + if task.function_params.get("allow_partial_landing") != remote_lv.get_id(): + return set() + return set(task.function_params.get("prechained_node_ids") or []) + + def process_snap_replicate_finish(task, snapshot): # Close the transfer session — but ONLY when this was the last active @@ -1085,8 +1246,13 @@ def process_snap_replicate_finish(task, snapshot): if not _require_lvs_leader(remote_snode, remote_lv.lvs_name, "add_clone/convert"): return False + # Nodes whose landing volume was already chained BEFORE the transfer, so it + # could receive a delta (see _prechain_landing_volume). Those must be + # skipped here: adding the same clone entry twice is not idempotent. + _prechained = _prechained_nodes_for(task, remote_lv) + # chain snaps on primary - if target_prev_snap: + if target_prev_snap and remote_snode.get_id() not in _prechained: logger.info(f"Chaining replicated lvol: {remote_lv.top_bdev} to snap: {target_prev_snap['snap_bdev']}") with xfer_timing.phase("chain_add_clone", snap=snapshot.get_id(), lvol=snapshot.lvol.get_id(), node="primary"): @@ -1094,6 +1260,11 @@ def process_snap_replicate_finish(task, snapshot): if not ret: logger.error("Failed to chain replicated snapshot on primary node") return False + elif target_prev_snap: + logger.info("Landing volume %s was already chained to %s on %s before " + "the transfer; skipping the redundant add_clone", + remote_lv.top_bdev, target_prev_snap['snap_bdev'], + remote_snode.get_id()) # convert to snapshot on primary with xfer_timing.phase("chain_convert", snap=snapshot.get_id(), @@ -1106,7 +1277,7 @@ def process_snap_replicate_finish(task, snapshot): # chain snaps on secondary sec_node = db.get_storage_node_by_id(remote_snode.secondary_node_id) if sec_node.status == StorageNode.STATUS_ONLINE: - if target_prev_snap: + if target_prev_snap and sec_node.get_id() not in _prechained: logger.info(f"Chaining replicated lvol: {remote_lv.top_bdev} to snap: {target_prev_snap['snap_bdev']}") with xfer_timing.phase("chain_add_clone", snap=snapshot.get_id(), lvol=snapshot.lvol.get_id(), node="secondary"): @@ -1114,6 +1285,11 @@ def process_snap_replicate_finish(task, snapshot): if not ret: logger.error("Failed to chain replicated snapshot on secondary node") return False + elif target_prev_snap: + logger.info("Landing volume %s was already chained to %s on %s " + "before the transfer; skipping the redundant add_clone", + remote_lv.top_bdev, target_prev_snap['snap_bdev'], + sec_node.get_id()) # convert to snapshot on secondary with xfer_timing.phase("chain_convert", snap=snapshot.get_id(), diff --git a/simplyblock_core/test/test_replication_partial_transfer.py b/simplyblock_core/test/test_replication_partial_transfer.py new file mode 100644 index 0000000000..dfe53b0f19 --- /dev/null +++ b/simplyblock_core/test/test_replication_partial_transfer.py @@ -0,0 +1,379 @@ +"""Partial (dirty-bitmap delta) replication transfers. + +The in-memory dirty bitmap has been in the SPDK fork for a long time but never +executed: ``bdev_lvol_transfer`` takes an optional ``allow_partial`` and the +control plane never sent it. It could not be sent, either -- a partial transfer +ships only the ranges written since the previous snapshot, so the destination +has to hold everything else already, and this pipeline landed every transfer in +a FRESH EMPTY volume and only chained it afterwards. A delta into an empty +volume silently loses every cluster it does not cover. + +These tests pin the change that makes it safe: chain the landing volume onto the +destination's copy of the previous snapshot BEFORE the transfer, and ask for a +delta only when that actually succeeded on every online member of the target's +HA pair. +""" +from simplyblock_core.models.snapshot import SnapShot +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import snapshot_replication as sr + + +# -------------------------------------------------------------------------- +# fakes +# -------------------------------------------------------------------------- + +def _mk_snap(uuid, created_at, lvol_uuid, node_id, target="", source="", + status=SnapShot.STATUS_ONLINE, snap_type=None): + lv = LVol() + lv.uuid = lvol_uuid + lv.node_id = node_id + s = SnapShot() + s.uuid = uuid + s.created_at = created_at + s.status = status + s.snap_ref_id = "" + s.target_replicated_snap_uuid = target + s.source_replicated_snap_uuid = source + s.snap_bdev = f"LVS/{uuid}" + s.lvol = lv + if snap_type is not None: + s.snap_type = snap_type + return s + + +class _FakeRPC: + def __init__(self, owner, add_clone_ok=True): + self._owner = owner + self._add_clone_ok = add_clone_ok + + def bdev_lvol_add_clone(self, clone_bdev, snap_bdev): + self._owner.add_clone_calls.append((clone_bdev, snap_bdev)) + return self._add_clone_ok + + +class _Node: + def __init__(self, uuid, secondary_node_id=None, add_clone_ok=True, + status=StorageNode.STATUS_ONLINE): + self._uuid = uuid + self.secondary_node_id = secondary_node_id + self.status = status + self.add_clone_calls = [] + self._add_clone_ok = add_clone_ok + + def get_id(self): + return self._uuid + + def rpc_client(self): + return _FakeRPC(self, self._add_clone_ok) + + +class _FakeDB: + def __init__(self, snaps, nodes): + self._snaps = list(snaps) + self._nodes = dict(nodes) + + def get_snapshots_by_node_id(self, node_id): + return [s for s in self._snaps if s.lvol.node_id == node_id] + + def get_snapshot_by_id(self, uuid): + for s in self._snaps: + if s.uuid == uuid: + return s + raise KeyError(uuid) + + def get_storage_node_by_id(self, uuid): + if uuid not in self._nodes: + raise KeyError(uuid) + return self._nodes[uuid] + + +class _Task: + def __init__(self, **params): + self.function_params = dict(params) + self.writes = 0 + + def write_to_db(self): + self.writes += 1 + return True + + +class _LandingLV: + top_bdev = "LVS/REP_SNAP_2" + + def __init__(self, uuid="REP_LV_1"): + self.uuid = uuid + + def get_id(self): + return self.uuid + + +# -------------------------------------------------------------------------- +# the landing volume becomes a clone of the predecessor when eligible +# -------------------------------------------------------------------------- + +def _eligible_setup(monkeypatch, primary_ok=True, secondary_ok=True, + secondary_online=True): + """A snapshot with a replicated predecessor whose remote copy sits on the + receiving leader -- the case a delta is legitimate in.""" + cur = _mk_snap("SNAP_2", 200, "LV1", "N_SRC") + prev = _mk_snap("SNAP_1", 100, "LV1", "N_SRC", target="T_SNAP_1") + # the predecessor's copy on the destination, on the receiving leader + remote_copy = _mk_snap("T_SNAP_1", 150, "REP_LV", "N_TGT") + + sec = _Node("N_TGT_SEC", add_clone_ok=secondary_ok, + status=(StorageNode.STATUS_ONLINE if secondary_online + else StorageNode.STATUS_OFFLINE)) + leader = _Node("N_TGT", secondary_node_id="N_TGT_SEC", + add_clone_ok=primary_ok) + monkeypatch.setattr(sr, "db", _FakeDB([cur, prev, remote_copy], + {"N_TGT": leader, "N_TGT_SEC": sec})) + return cur, leader, sec + + +def test_landing_volume_is_chained_to_predecessor_when_eligible(monkeypatch): + cur, leader, sec = _eligible_setup(monkeypatch) + task = _Task() + + assert sr._prechain_landing_volume( + task, cur, False, _LandingLV(), leader) is True + + # chained on BOTH members, onto the predecessor's remote copy + assert leader.add_clone_calls == [("LVS/REP_SNAP_2", "LVS/T_SNAP_1")] + assert sec.add_clone_calls == [("LVS/REP_SNAP_2", "LVS/T_SNAP_1")] + # and recorded so the finish step does not add the entry a second time + assert task.function_params["prechained_node_ids"] == ["N_TGT", "N_TGT_SEC"] + + +def test_no_predecessor_falls_back_to_a_fresh_full_transfer(monkeypatch): + """First snapshot of a volume: nothing on the destination to build on.""" + cur = _mk_snap("SNAP_1", 100, "LV1", "N_SRC") + leader = _Node("N_TGT", secondary_node_id="N_TGT_SEC") + sec = _Node("N_TGT_SEC") + monkeypatch.setattr(sr, "db", + _FakeDB([cur], {"N_TGT": leader, "N_TGT_SEC": sec})) + task = _Task() + + assert sr._prechain_landing_volume( + task, cur, False, _LandingLV(), leader) is False + + assert leader.add_clone_calls == [] + assert sec.add_clone_calls == [] + assert "prechained_node_ids" not in task.function_params + + +def test_unreplicated_predecessor_falls_back(monkeypatch): + """A predecessor exists but was never replicated -- no remote copy to + clone from, so this snapshot starts the chain.""" + cur = _mk_snap("SNAP_2", 200, "LV1", "N_SRC") + prev = _mk_snap("SNAP_1", 100, "LV1", "N_SRC") # no target copy + leader = _Node("N_TGT", secondary_node_id="N_TGT_SEC") + sec = _Node("N_TGT_SEC") + monkeypatch.setattr(sr, "db", _FakeDB([cur, prev], + {"N_TGT": leader, "N_TGT_SEC": sec})) + task = _Task() + + assert sr._prechain_landing_volume( + task, cur, False, _LandingLV(), leader) is False + assert leader.add_clone_calls == [] + + +def test_offline_secondary_forces_a_full_transfer(monkeypatch): + """The transfer lands on the leader and the lvstore mirrors it to the + secondary. A secondary that is not there to be chained would read zeros + wherever the delta did not write, so a degraded pair gets a full copy.""" + cur, leader, sec = _eligible_setup(monkeypatch, secondary_online=False) + task = _Task() + + assert sr._prechain_landing_volume( + task, cur, False, _LandingLV(), leader) is False + # nothing was chained at all -- we bail before touching either node + assert leader.add_clone_calls == [] + assert sec.add_clone_calls == [] + + +def test_failed_primary_chain_forces_a_full_transfer(monkeypatch): + cur, leader, sec = _eligible_setup(monkeypatch, primary_ok=False) + task = _Task() + + assert sr._prechain_landing_volume( + task, cur, False, _LandingLV(), leader) is False + # the secondary is not touched once the primary refused + assert sec.add_clone_calls == [] + + +def test_failed_secondary_chain_forces_full_but_records_the_primary(monkeypatch): + """A full transfer into a half-chained volume is still correct, but the + entry that DID land must be remembered or the finish step adds it twice.""" + cur, leader, sec = _eligible_setup(monkeypatch, secondary_ok=False) + task = _Task() + + assert sr._prechain_landing_volume( + task, cur, False, _LandingLV(), leader) is False + assert task.function_params["prechained_node_ids"] == ["N_TGT"] + + +def test_prechain_is_not_repeated_for_a_node_already_chained(monkeypatch): + """A retried attempt must not add the same clone entry a second time.""" + cur, leader, sec = _eligible_setup(monkeypatch) + task = _Task(prechained_node_ids=["N_TGT"]) + + assert sr._prechain_landing_volume( + task, cur, False, _LandingLV(), leader) is True + assert leader.add_clone_calls == [] # already done + assert sec.add_clone_calls == [("LVS/REP_SNAP_2", "LVS/T_SNAP_1")] + + +def test_decision_is_cached_for_the_same_landing_volume(monkeypatch): + """A resumed transfer keeps the mode its first attempt used, without + re-issuing add_clone.""" + cur, leader, sec = _eligible_setup(monkeypatch) + lv = _LandingLV("REP_LV_1") + task = _Task() + + assert sr._partial_transfer_decision(task, cur, False, lv, leader) is True + assert len(leader.add_clone_calls) == 1 + + # second pass over the same landing volume: cached, nothing re-chained + leader.add_clone_calls.clear() + sec.add_clone_calls.clear() + assert sr._partial_transfer_decision(task, cur, False, lv, leader) is True + assert leader.add_clone_calls == [] + assert sec.add_clone_calls == [] + + +def test_a_replaced_landing_volume_re_derives_and_never_inherits_partial(monkeypatch): + """An interrupted attempt can delete a half-created landing volume and + build a fresh, UNCHAINED one. Inheriting the old "partial is fine" verdict + would ship a delta into something holding nothing. + """ + cur = _mk_snap("SNAP_2", 200, "LV1", "N_SRC") + prev = _mk_snap("SNAP_1", 100, "LV1", "N_SRC") # predecessor NOT replicated + leader = _Node("N_TGT", secondary_node_id="N_TGT_SEC") + sec = _Node("N_TGT_SEC") + monkeypatch.setattr(sr, "db", _FakeDB([cur, prev], + {"N_TGT": leader, "N_TGT_SEC": sec})) + # a stale verdict from a landing volume that no longer exists + task = _Task(allow_partial=True, allow_partial_landing="REP_LV_OLD", + prechained_node_ids=["N_TGT", "N_TGT_SEC"]) + + got = sr._partial_transfer_decision( + task, cur, False, _LandingLV("REP_LV_NEW"), leader) + + assert got is False, "a replaced landing volume must not inherit partial" + assert task.function_params["allow_partial"] is False + assert task.function_params["allow_partial_landing"] == "REP_LV_NEW" + # the stale chain record is discarded too + assert task.function_params["prechained_node_ids"] == [] + + +def test_prechained_nodes_are_ignored_for_a_different_landing_volume(): + """Skipping add_clone on the strength of a stale record would leave the + volume unchained.""" + task = _Task(allow_partial=True, allow_partial_landing="REP_LV_1", + prechained_node_ids=["N_TGT", "N_TGT_SEC"]) + + assert sr._prechained_nodes_for(task, _LandingLV("REP_LV_1")) == { + "N_TGT", "N_TGT_SEC"} + assert sr._prechained_nodes_for(task, _LandingLV("REP_LV_2")) == set() + + +# -------------------------------------------------------------------------- +# allow_partial reaches the RPC only when it was asked for +# -------------------------------------------------------------------------- + +def _transfer_params(**kwargs): + """Run the REAL bdev_lvol_transfer body and return the params it would + put on the wire.""" + from simplyblock_core.rpc_client import RPCClient + + class _C(RPCClient): + def __init__(self): + self.sent = None + + def _request(self, method, params): + self.sent = (method, params) + return True + + c = _C() + c.bdev_lvol_transfer(name="LVS/SNAP_2", offset=0, batch_size=16, + bdev_name="hub0", operation="replicate", lvol_id=7, + **kwargs) + method, params = c.sent + assert method == "bdev_lvol_transfer" + return params + + +def test_allow_partial_is_sent_only_when_requested(): + # opted in + assert _transfer_params(allow_partial=True)["allow_partial"] is True + # opted out -- the key is absent, so every pre-existing caller keeps its + # exact current wire form and the fork's default (full) applies + assert "allow_partial" not in _transfer_params(allow_partial=False) + # and the default is opted out + assert "allow_partial" not in _transfer_params() + + +def test_transfer_still_carries_the_routing_fields(): + """allow_partial must not disturb the map-id routing the hub demux needs.""" + p = _transfer_params(allow_partial=True) + assert p["lvol_name"] == "LVS/SNAP_2" + assert p["lvol_id"] == 7 + assert p["gateway"] == "hub0" + assert p["operation"] == "replicate" + assert p["cluster_batch"] == 16 + + +# -------------------------------------------------------------------------- +# retention keeps the COW parent +# -------------------------------------------------------------------------- + +def test_retention_floor_keeps_the_cow_parent_alive(): + """The delta's correctness depends on the PREVIOUS replicated snapshot + still existing on the destination when the next one lands -- it is the COW + parent the landing volume is chained onto. Retention must therefore never + prune down to a single replicated internal snapshot. + """ + from simplyblock_core.models.replication import ReplicationPolicy + + # the flat default and the policy floor both keep a PAIR + assert sr._KEEP_REPLICATED_INTERNAL >= 2 + assert ReplicationPolicy.MIN_KEEP_REPLICATED >= 2 + # a policy cannot be configured below the floor + assert ReplicationPolicy.keep_replicated >= ReplicationPolicy.MIN_KEEP_REPLICATED + + +def test_retention_keep_count_never_drops_below_the_floor(monkeypatch): + """_keep_replicated_for clamps a policy that asks for fewer than a pair.""" + from simplyblock_core.models.replication import ReplicationPolicy + + class _Policy: + keep_replicated = 1 # below the floor + retention_schedule = None + + def get_id(self): + return "P1" + + class _DB: + def get_replication_policy_for_lvol(self, lvol): + return _Policy() + + monkeypatch.setattr(sr, "db", _DB()) + lv = LVol() + lv.uuid = "LV1" + assert sr._keep_replicated_for(lv) == ReplicationPolicy.MIN_KEEP_REPLICATED + + +def test_scheduled_retention_still_keeps_the_newest_pair(): + """The ladder thins history but always_keep_newest protects the COW parent + regardless of how coarse the schedule's finest tier is.""" + from simplyblock_core.snapshot_retention import parse_schedule, select_retained + + tiers = parse_schedule("1h:24h") + now = 1_787_900_000.0 + # a fast cadence: two snapshots a minute apart, far finer than the tier + history = [now - 60, now - 120] + keep = select_retained(history, tiers, now, always_keep_newest=2) + # BOTH survive: the newest is the next delta's base, the one before it is + # the COW parent the current landing volume is chained onto + assert set(keep) == set(history) From ae3a6d0c501affaae9cae3f2aab22e2f57431f98 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 28 Aug 2026 23:04:33 +0200 Subject: [PATCH 103/122] test: never lose the timing bundle to a control-plane hiccup Twice now a case-7 run has ended with no usable evidence. The fail-back poll calls sbctl on the mgmt node; when that returned rc=1 (a transient CP blip -- `sbctl pool list` worked again minutes later) the exception propagated out of the poll loop, skipped collect_xfer_timing, and left the PREVIOUS run's xfer_timing_case7_failback.log in place. Downloading it produced a file byte-identical to the earlier run -- same five volume ids, same 340.93/1505.63/ 2073.55/2583.53/12.38s durations -- which reads exactly like fresh data until you notice the new phases are missing. Two changes: the poll retries a failed CP call instead of ending the case (the deadline still bounds it), and collection moved into a `finally` so the bundle survives any failure in the loop. The bundle is the point of an instrumented run; losing it costs the whole run. --- scripts/test_async_replication.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/scripts/test_async_replication.py b/scripts/test_async_replication.py index 49299b1649..5a189b477f 100644 --- a/scripts/test_async_replication.py +++ b/scripts/test_async_replication.py @@ -2030,14 +2030,27 @@ def test_case_7(meta): start = time.time() done = 0 - while time.time() - start < CUTOVER_WAIT_TIMEOUT * 2: - states = replication_states(mgmt_ip, key_path, tgt_lvols) - done = sum(1 for s in states.values() if s in ("cutover_done", "failed_over")) - print(f" fail-back cutovers done: {done}/{len(tgt_lvols)}") - if done == len(tgt_lvols): - break - time.sleep(15) - collect_xfer_timing(mgmt_ip, key_path, "case7_failback") + # The timing bundle is the whole point of the run, so collect it even when + # the poll itself blows up. Twice now a transient control-plane error inside + # this loop (`sbctl` rc=1) propagated out and skipped the collection, leaving + # the previous run's file in place -- which then looked like fresh data. + try: + while time.time() - start < CUTOVER_WAIT_TIMEOUT * 2: + try: + states = replication_states(mgmt_ip, key_path, tgt_lvols) + except Exception as e: # noqa: BLE001 + # A CP hiccup must not end the case: log it, keep polling, and + # let the deadline decide. + print(f" fail-back poll failed ({str(e)[:120]}); retrying") + time.sleep(15) + continue + done = sum(1 for s in states.values() if s in ("cutover_done", "failed_over")) + print(f" fail-back cutovers done: {done}/{len(tgt_lvols)}") + if done == len(tgt_lvols): + break + time.sleep(15) + finally: + collect_xfer_timing(mgmt_ip, key_path, "case7_failback") if done != len(tgt_lvols): # The breakdown matters MOST here: a stalled fail-back is the case we # have failed to explain seven times. From 636fb3186380e148f5aff73bf380872edcff352e Mon Sep 17 00:00:00 2001 From: michael Date: Sat, 29 Aug 2026 00:47:30 +0200 Subject: [PATCH 104/122] cutover: converge on the ordinary cadence, then run an exclusive endgame The iterative snapshots ARE the endgame. They were being taken from the moment commit ran, so a volume waiting its turn behind 19 others held an ageing round-1 snapshot, and the "round" that followed measured the wait rather than the transfer (run 20260828_124859: round 1 growing 340s -> 1505s -> 2073s -> 2584s across successive volumes, while the one volume that never queued finished in 12.4s). A cutover now costs the cluster nothing until it can finish quickly: * commit only enqueues the task -- no snapshot, no round in flight * the runner waits until ordinary replication has the volume within REPL_CUTOVER_ENDGAME_LAG_SEC (50s) of the source, suspending without burning a retry and pushing its deadline out, because catching up is progress rather than a stall * only then does it claim the lvstore, and every round runs under that claim, down to the freeze So the exclusive window is the tail -- the tight rounds and the freeze -- and the bulk catch-up rides the cadence that was already running. This removes the "open rounds" of the previous attempt along with ready_for_exclusive and REPL_CUTOVER_EXCLUSIVE_ENTRY_FACTOR: with the claim taken before round 1, a round without the lvstore cannot happen. Co-Authored-By: Claude Opus 5 (1M context) --- simplyblock_core/constants.py | 5 +- .../controllers/lvol_controller.py | 30 +++-- .../tasks_runner_replication_final.py | 107 +++++++++++------- .../test/test_cutover_convergence.py | 84 +++++++++----- .../test/test_replication_commit.py | 26 +++-- .../test_tasks_runner_replication_final.py | 6 + 6 files changed, 160 insertions(+), 98 deletions(-) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index fb7026422c..eeac0b2b01 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -389,7 +389,10 @@ def get_config_var(name, default=None): # nearly converged, so the exclusive window that follows will be short. Claiming # earlier serialises the bulk catch-up, which is what produced 0/20 cutovers in # run 20260828_124859 (round 1 growing 340s -> 2584s purely from queueing). -REPL_CUTOVER_EXCLUSIVE_ENTRY_FACTOR = 3.0 +# The endgame starts once ordinary replication has the target within this many +# seconds. Before that the cutover waits and takes NO snapshots of its own -- +# the iterative snapshots ARE the endgame. +REPL_CUTOVER_ENDGAME_LAG_SEC = 50 # Rounds must follow each other within MILLISECONDS. Returning to the task # scheduler between them costs TASK_EXEC_INTERVAL_SEC (10s) of fresh writes # each time, which puts a floor under the delta no number of rounds can beat. diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index a61c3b2d3b..82895193ab 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -4402,14 +4402,14 @@ def replication_commit(lvol_id, delete_source=False): source_node = db_controller.get_storage_node_by_id(lvol.node_id) - # Shrink round 1: freeze the current top delta and let the normal - # replication pipeline carry it (snapshot_controller.add auto-enqueues the - # replication task for do_replicate volumes). - snap_uuid, snap_err = snapshot_controller.add( - lvol_id, f"repl_commit_{uuid.uuid4()}", snap_type=SnapShot.TYPE_INTERNAL) - if snap_err: - logger.error(f"Shrink snapshot failed: {snap_err}") - return False, f"Shrink snapshot failed: {snap_err}" + # NO snapshot here. The iterative snapshots ARE the endgame, and the + # endgame does not start until ordinary replication has the target within + # REPL_CUTOVER_ENDGAME_LAG_SEC. Taking one at commit time meant every + # volume held an ageing snapshot while it waited its turn, and the "round" + # that followed measured that wait rather than the transfer (run + # 20260828_124859: round 1 growing 341s -> 2584s across five volumes, while + # the one that never waited finished in 12.4s). The runner takes the first + # one when it enters the endgame, so its delta covers only the residual. task = tasks_controller.add_replication_final_task( source_node.cluster_id, source_node.get_id(), @@ -4419,14 +4419,12 @@ def replication_commit(lvol_id, delete_source=False): "tgt_node_id": target_node.get_id(), "operation": "replicate", "final_state": LVolReplication.STATE_CUTOVER_DONE, - "shrink_round": 1, - "shrink_snap_id": snap_uuid, - # When this round started transferring. The convergence loop - # measures each round against it to decide whether the delta is - # small enough to freeze; without it round 1 measures as 0.00s and - # "converges" instantly, which is how the freeze stayed at 9-55s - # with the loop deployed (run 20260827_172734). - "shrink_started_at": time.time(), + # The runner takes the first iterative snapshot when it enters the + # endgame and stamps shrink_started_at then; a round measured + # without that stamp reads as 0.00s and "converges" instantly, + # which is how the freeze stayed at 9-55s with the convergence loop + # deployed (run 20260827_172734). + "shrink_round": 0, "shrink_deadline": int(time.time()) + constants.REPL_CUTOVER_SHRINK_TIMEOUT_SEC, # Migration semantics on request: retire the source volume once # the cutover state is durable (see _finalize in the final runner). diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 694e433673..cd281dcfd3 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -264,19 +264,42 @@ def task_runner(task: JobSchedule, tasks=None): result=str(task.function_result)[:40].replace(" ", "_")) task.status = JobSchedule.STATUS_RUNNING task.function_params.setdefault("start_time", int(time.time())) - # Claim the source LVS for the whole cutover -- the convergence rounds - # AND the freeze. Other volumes' snapshot transfers on this LVS queue - # behind it (see snapshot_replication._lvs_locked_by_cutover): they - # compete for the same lvstore and hub bandwidth, and every second they - # steal from a convergence round is a second of writes that lands in - # the freeze. - # Exclusivity is the ENDGAME, not the whole cutover. The bulk catch-up - # converges in the open, concurrently with every other volume; only - # once a round is already fast is the lvstore taken, for the final - # tight rounds and the freeze (see _acquire_lvs_claim). - if params.get("ready_for_exclusive") and not params.get("cutover_lvs"): + + # ---- WAIT FOR THE VOLUME TO CATCH UP ---------------------------- # + # The iterative snapshots ARE the endgame. Until then the volume just + # replicates on its ordinary cadence: the cutover takes no snapshots of + # its own and holds nothing, so it adds no load to a cluster that is + # still catching up -- exactly when it can least afford it. + if not params.get("cutover_lvs"): + lag = _replication_lag_sec(lvol) + if lag is None or lag > constants.REPL_CUTOVER_ENDGAME_LAG_SEC: + task.function_result = ( + "waiting for replication to catch up before the endgame " + "(lag %s > %ds)" + % ("unknown" if lag is None else "%.0fs" % lag, + constants.REPL_CUTOVER_ENDGAME_LAG_SEC)) + xfer_timing.stamp("await_catchup", lvol=lvol_id, + ms=(lag or 0) * 1000.0) + # Not a failure: no retry burned, and the deadline is pushed out + # because catching up is legitimate progress, not a stall. + params["shrink_deadline"] = ( + int(time.time()) + constants.REPL_CUTOVER_SHRINK_TIMEOUT_SEC) + task.status = JobSchedule.STATUS_SUSPENDED + task.write_to_db(db.kv_store) + return False + + # Caught up: take the lvstore for the endgame. Queueing here is + # cheap -- nothing is held and no snapshot is ageing. if not _acquire_lvs_claim(task, lvol, tasks): - return False # queued, holding nothing and ageing nothing + return False + xfer_timing.stamp("endgame_entered", lvol=lvol_id, ms=lag * 1000.0) + + # The first iterative snapshot belongs to the endgame, not to task + # creation: taken here, its delta covers only the catch-up residual. + if not params.get("shrink_snap_id"): + _, snap_err = _take_shrink_snapshot(task, lvol) + if snap_err: + return _finalize(task, False, snap_err) task.write_to_db(db.kv_store) # ---- SHRINK PHASE ----------------------------------------------- # @@ -351,6 +374,35 @@ def task_runner(task: JobSchedule, tasks=None): +def _replication_lag_sec(lvol): + """How far the target is behind: age of the newest REPLICATED snapshot. + + None when the volume has no replicated snapshot yet, which counts as "not + caught up" -- there is nothing for the endgame's first delta to chain onto. + + This is the entry gate for the endgame. Measuring it from ordinary cadence + replication costs nothing: the alternative (taking rounds to find out how + fast a round is) adds snapshot and transfer load to a cluster that is still + catching up, which is precisely when it can least afford it. + """ + newest = None + try: + snaps = db.get_snapshots_by_node_id(lvol.node_id) + except Exception as e: # noqa: BLE001 + logger.warning("Cannot read snapshots of %s to measure replication lag: %s", + lvol.get_id(), e) + return None + for s in snaps: + if (s.lvol.get_id() == lvol.get_id() + and s.snap_type == SnapShot.TYPE_INTERNAL + and getattr(s, "target_replicated_snap_uuid", "") + and (newest is None or s.created_at > newest.created_at)): + newest = s + if newest is None: + return None + return max(0.0, time.time() - float(newest.created_at)) + + def _take_shrink_snapshot(task, lvol): """Snapshot the source and record it as the round in flight.""" from simplyblock_core.controllers import snapshot_controller @@ -475,33 +527,13 @@ def _shrink_step(task, lvol): ms=elapsed * 1000.0) params["shrink_round_done_at"] = time.time() - exclusive = bool(params.get("cutover_lvs")) - - # Converged AND holding the lvstore: hand over to the freeze. - if exclusive and elapsed <= constants.REPL_CUTOVER_CONVERGE_TARGET_SEC: + # Every round is an endgame round now -- the lvstore is held before the + # first iterative snapshot is taken. + if elapsed <= constants.REPL_CUTOVER_CONVERGE_TARGET_SEC: task.function_result = (f"converged in {params['shrink_round']} rounds " f"(last {elapsed:.2f}s)") return True, None - # Nearly converged but still converging in the OPEN. Stop and go take - # the lvstore for the endgame. Claiming it any earlier serialises the - # bulk catch-up: in run 20260828_124859 the claim was taken on the - # task's first pass and held for its whole life, so volumes queued - # while holding an ageing round-1 snapshot and their "round" measured - # the queue -- 340s, 1505s, 2073s, 2584s for successive volumes, while - # the one that never queued finished its round in 12.4s. - if not exclusive and elapsed <= ( - constants.REPL_CUTOVER_CONVERGE_TARGET_SEC - * constants.REPL_CUTOVER_EXCLUSIVE_ENTRY_FACTOR): - params["ready_for_exclusive"] = True - task.function_result = (f"delta converged in {params['shrink_round']} " - f"open rounds (last {elapsed:.2f}s); taking " - f"the lvstore for the endgame") - xfer_timing.stamp("ready_for_exclusive", lvol=lvol.get_id(), - round=params["shrink_round"], ms=elapsed * 1000.0) - task.write_to_db(db.kv_store) - return False, None - if params["shrink_round"] >= constants.REPL_CUTOVER_MAX_SHRINK_ROUNDS: # Written faster than it replicates. Freezing now is still the best # move -- the freeze at least stops the writes -- but say so. @@ -512,11 +544,6 @@ def _shrink_step(task, lvol): constants.REPL_CUTOVER_CONVERGE_TARGET_SEC) task.function_result = (f"not converged after {params['shrink_round']} " f"rounds (last {elapsed:.2f}s)") - if not params.get("cutover_lvs"): - # Give up converging, but the freeze still wants the lvstore. - params["ready_for_exclusive"] = True - task.write_to_db(db.kv_store) - return False, None return True, None # IMMEDIATELY take the next snapshot. This is the whole mechanism: the diff --git a/simplyblock_core/test/test_cutover_convergence.py b/simplyblock_core/test/test_cutover_convergence.py index d46a5ed61d..a2b8c43832 100644 --- a/simplyblock_core/test/test_cutover_convergence.py +++ b/simplyblock_core/test/test_cutover_convergence.py @@ -119,18 +119,23 @@ def test_a_fast_round_while_holding_the_lvstore_hands_over(self): self.assertIn("converged", task.function_result) self.assertEqual(taken, [], "a fast first round needs no further rounds") - def test_a_fast_round_in_the_open_asks_for_the_lvstore(self): - """Nearly converged unexclusively -> take the lvstore for the endgame. - - Claiming it earlier serialised the bulk catch-up: run 20260828_124859 - charged the queue wait to round 1 (340s, 1505s, 2073s, 2584s for - successive volumes) while the unqueued one finished in 12.4s. + def test_no_round_ever_runs_outside_the_lvstore_claim(self): + """Rounds are the endgame, and the endgame is exclusive by definition. + + There used to be "open" rounds converging before the claim, so that the + bulk catch-up was not serialised. That was the wrong instrument: the + volume is caught up by its ORDINARY replication cadence, which costs the + cutover nothing, and only then does it ask for the lvstore. The claim is + therefore taken before round 1, and the loop below need not consider a + round without it. """ - done, err, task, taken, _ = self._run([0.5], exclusive=False) - self.assertFalse(done, "it must not freeze without holding the lvstore") - self.assertIsNone(err) - self.assertTrue(task.function_params.get("ready_for_exclusive")) - self.assertIn("endgame", task.function_result) + import inspect + src = inspect.getsource(runner.task_runner) + entry = src.index("_acquire_lvs_claim") + self.assertLess(entry, src.index("_take_shrink_snapshot"), + "the lvstore is taken before the first round, not after") + self.assertNotIn("ready_for_exclusive", inspect.getsource(runner), + "open rounds are gone") def test_a_slow_round_takes_another_snapshot_without_leaving_the_pass(self): """The whole point: rounds follow each other in milliseconds.""" @@ -152,12 +157,16 @@ def test_it_gives_up_after_the_round_cap_and_freezes_anyway(self): self.assertIsNone(err) self.assertIn("not converged", task.function_result) - def test_the_cap_takes_the_lvstore_before_freezing(self): - """Giving up converging still requires the lvstore for the freeze.""" - done, err, task, taken, _ = self._run([3.0] * 20, max_rounds=3, - exclusive=False) - self.assertFalse(done) - self.assertTrue(task.function_params.get("ready_for_exclusive")) + def test_the_cap_freezes_under_the_claim_it_already_holds(self): + """Giving up converging goes straight to the freeze. + + The claim was taken on entry to the endgame, so reaching the round cap + needs no further acquisition -- it just stops converging and freezes. + """ + done, err, task, taken, _ = self._run([3.0] * 20, max_rounds=3) + self.assertTrue(done) + self.assertIsNone(err) + self.assertIn("not converged", task.function_result) def test_a_vanished_snapshot_is_an_error(self): # This one runs on the real clock, so the deadline has to be a real @@ -375,22 +384,28 @@ def test_a_group_transferring_on_another_lvstore_does_not_hold_us(self): class TestRoundOneIsMeasured(unittest.TestCase): """The regression that let the freeze survive the convergence loop. - replicate/commit creates the cutover task itself, and its params are the - ONLY ones the loop ever sees for round 1. Every earlier test supplied - shrink_started_at by hand, so none of them noticed the controller did not: - the round then measured as 0.00s, counted as converged, and the freeze - copied the whole delta (run 20260827_172734, 9-55s server-side). + Round 1 used to be created by replicate/commit without the stamp the loop + measures against, so it measured as 0.00s, counted as converged, and the + freeze copied the whole delta (run 20260827_172734, 9-55s server-side). + + Every round is now born in one place -- _take_shrink_snapshot, in the + endgame -- so there is a single stamp to get right. """ - def test_the_controller_stamps_the_start_of_round_one(self): + def test_every_round_is_stamped_where_it_is_taken(self): + import inspect + src = inspect.getsource(runner._take_shrink_snapshot) + self.assertIn('params["shrink_started_at"] = time.time()', src) + self.assertIn('params["shrink_round"] = params.get("shrink_round", 0) + 1', + src, "the round number and its stamp must move together") + + def test_the_controller_enqueues_no_round_of_its_own(self): + """Commit takes no snapshot, so it must not claim a round in flight.""" import inspect from simplyblock_core.controllers import lvol_controller as lc src = inspect.getsource(lc.replication_commit) - self.assertIn('"shrink_started_at"', src, - "round 1 must carry the stamp the loop measures against") - self.assertLess(src.index('"shrink_round": 1'), - src.index('"shrink_deadline"'), - "sanity: this is the cutover task's param block") + self.assertIn('"shrink_round": 0', src) + self.assertNotIn('"shrink_snap_id"', src) def test_an_unmeasured_round_is_not_treated_as_converged(self): """Belt and braces for tasks enqueued without the stamp.""" @@ -524,6 +539,16 @@ def setUp(self): "a queued cutover must not run its endgame rounds")) sp.start() self.addCleanup(sp.stop) + # Caught up: the endgame is asked for at this point, and the answer is + # either the lvstore or a queue slot. + lp = patch.object(runner, "_replication_lag_sec", return_value=1.0) + lp.start() + self.addCleanup(lp.stop) + tp = patch.object(runner, "_take_shrink_snapshot", + side_effect=AssertionError( + "a queued cutover must not take a snapshot")) + tp.start() + self.addCleanup(tp.stop) def _me(self): t = MagicMock() @@ -539,9 +564,6 @@ def _me(self): "lvol_id": "LV_me", "src_node_id": "N1", "tgt_node_id": "N2", "shrink_round": 1, "shrink_snap_id": "S1", "shrink_deadline": 1, # already expired - # asking for the endgame: the delta has converged in the open, so - # this is the point at which queueing happens - "ready_for_exclusive": True, } return t diff --git a/simplyblock_core/test/test_replication_commit.py b/simplyblock_core/test/test_replication_commit.py index 59ab9054b7..1c32bd21f2 100644 --- a/simplyblock_core/test/test_replication_commit.py +++ b/simplyblock_core/test/test_replication_commit.py @@ -133,15 +133,16 @@ def _add_task(cluster_id, node_id, params): def test_commit_enqueues_final_task(patched): - """Commit takes shrink snapshot #1 and enqueues the task; the clone/base - selection moved into the runner (after the shrink rounds), so the enqueued - params carry the shrink state instead of tgt_* composites.""" + """Commit only enqueues the task. + + It takes NO snapshot: the iterative snapshots are the endgame, which does + not start until ordinary replication has caught up. Taking one here left + every volume holding an ageing snapshot while it waited its turn, and the + round that followed measured the wait rather than the transfer.""" result = lvol_controller.replication_commit("LV1") assert result["cutover_task_queued"] is True - - # Shrink snapshot #1 taken at commit time. - assert patched["snap_add"] and patched["snap_add"][0][1] == SnapShot.TYPE_INTERNAL + assert patched["snap_add"] == [], "the endgame takes the first snapshot" # The clone is NOT built at commit time any more: the base must be the # LAST replicated shrink snapshot, which only exists after the runner's # shrink rounds complete. @@ -154,15 +155,20 @@ def test_commit_enqueues_final_task(patched): assert p["tgt_node_id"] == "N_tgt" assert p["operation"] == "replicate" assert p["final_state"] == LVolReplication.STATE_CUTOVER_DONE - assert p["shrink_round"] == 1 - assert p["shrink_snap_id"] == "snap" + assert p["shrink_round"] == 0, "no round is in flight yet" + assert "shrink_snap_id" not in p, "the endgame takes the first snapshot" assert p["shrink_deadline"] > 0 assert "tgt_lvol_composite" not in p, "clone base chosen before shrink completed" assert p["_cluster"] == "CL_src" and p["_node"] == "N_src" -def test_commit_fails_when_shrink_snapshot_fails(patched, monkeypatch): +def test_commit_does_not_depend_on_taking_a_snapshot(patched, monkeypatch): + """Commit is now pure bookkeeping. + + It used to take shrink snapshot #1 and fail if that failed. The snapshot + moved into the endgame, so a full lvstore no longer blocks the enqueue -- + the runner reports it when it actually gets there.""" monkeypatch.setattr(lvol_controller.snapshot_controller, "add", lambda lid, name, snap_type="user": (None, "no space")) result = lvol_controller.replication_commit("LV1") - assert result[0] is False + assert result["cutover_task_queued"] is True diff --git a/simplyblock_core/test/test_tasks_runner_replication_final.py b/simplyblock_core/test/test_tasks_runner_replication_final.py index 4466640f56..85db1130b4 100644 --- a/simplyblock_core/test/test_tasks_runner_replication_final.py +++ b/simplyblock_core/test/test_tasks_runner_replication_final.py @@ -25,6 +25,12 @@ def _task(**params): "operation": "replicate", "replication_id": "REP1", "final_state": LVolReplication.STATE_CUTOVER_DONE, + # These tests cover the FREEZE, so the task starts past the endgame + # entry: the lvstore is already claimed and the convergence rounds are + # behind it (round 0 = no round in flight). + "cutover_lvs": "lvs_src", + "shrink_snap_id": "S_endgame", + "shrink_round": 0, } t.function_params.update(params) return t From 2720fd84a6568f2a343c82644e985c372d2884e7 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 31 Aug 2026 10:35:53 +0100 Subject: [PATCH 105/122] =?UTF-8?q?fix:=20treat=20lag=3DNone=20as=20procee?= =?UTF-8?q?d=20in=20await=5Fcatchup=20gate=20=E2=80=94=20unblocks=20failba?= =?UTF-8?q?ck=20cutover=20on=20volumes=20with=20no=20reverse-direction=20r?= =?UTF-8?q?eplication=20history?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tasks_runner_replication_final.py | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index ebaa865287..45053589ca 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -422,14 +422,19 @@ def task_runner(task: JobSchedule, tasks=None): # still catching up -- exactly when it can least afford it. if not params.get("cutover_lvs"): lag = _replication_lag_sec(lvol) - if lag is None or lag > constants.REPL_CUTOVER_ENDGAME_LAG_SEC: + # lag is None when the volume has no internal snapshot replicated in + # the current source→target direction yet (e.g. a freshly set-up + # failback pair that has not completed even one reverse-direction + # cycle). Treat it as "no measurement" and proceed: the first + # shrink round captures all outstanding delta, and _prepare_cutover + # will surface a proper error if there is truly no base to chain + # onto. Only block when lag is a real number that exceeds the gate. + if lag is not None and lag > constants.REPL_CUTOVER_ENDGAME_LAG_SEC: task.function_result = ( "waiting for replication to catch up before the endgame " - "(lag %s > %ds)" - % ("unknown" if lag is None else "%.0fs" % lag, - constants.REPL_CUTOVER_ENDGAME_LAG_SEC)) + "(lag %.0fs > %ds)" % (lag, constants.REPL_CUTOVER_ENDGAME_LAG_SEC)) xfer_timing.stamp("await_catchup", lvol=lvol_id, - ms=(lag or 0) * 1000.0) + ms=lag * 1000.0) # Not a failure: no retry burned, and the deadline is pushed out # because catching up is legitimate progress, not a stall. params["shrink_deadline"] = ( @@ -438,11 +443,11 @@ def task_runner(task: JobSchedule, tasks=None): task.write_to_db(db.kv_store) return False - # Caught up: take the lvstore for the endgame. Queueing here is - # cheap -- nothing is held and no snapshot is ageing. + # Caught up (or no measurement): take the lvstore for the endgame. + # Queueing here is cheap -- nothing is held and no snapshot is ageing. if not _acquire_lvs_claim(task, lvol, tasks): return False - xfer_timing.stamp("endgame_entered", lvol=lvol_id, ms=lag * 1000.0) + xfer_timing.stamp("endgame_entered", lvol=lvol_id, ms=(lag or 0) * 1000.0) # The first iterative snapshot belongs to the endgame, not to task # creation: taken here, its delta covers only the catch-up residual. @@ -532,13 +537,16 @@ def task_runner(task: JobSchedule, tasks=None): def _replication_lag_sec(lvol): """How far the target is behind: age of the newest REPLICATED snapshot. - None when the volume has no replicated snapshot yet, which counts as "not - caught up" -- there is nothing for the endgame's first delta to chain onto. + Returns None when no internal snapshot with target_replicated_snap_uuid + exists for this volume in the current source→target direction. The caller + treats None as "no measurement available" and proceeds rather than blocking: + this happens legitimately on failback pairs that have not yet completed + their first reverse-direction replication cycle. - This is the entry gate for the endgame. Measuring it from ordinary cadence - replication costs nothing: the alternative (taking rounds to find out how - fast a round is) adds snapshot and transfer load to a cluster that is still - catching up, which is precisely when it can least afford it. + Measuring lag from ordinary cadence replication costs nothing: the + alternative (taking rounds to find out how fast a round is) adds snapshot + and transfer load to a cluster that is still catching up, which is + precisely when it can least afford it. """ newest = None try: From 1c4d97f5d13998fd5d37b9cccdd7946868cbeba5 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 31 Aug 2026 11:19:53 +0100 Subject: [PATCH 106/122] =?UTF-8?q?fix:=20unblock=20failback=20cutover=20?= =?UTF-8?q?=E2=80=94=20bypass=20await=5Fcatchup=20when=20lag=3DNone,=20exi?= =?UTF-8?q?t=20inline=20wait=20immediately=20on=20No=20process=20transfer?= =?UTF-8?q?=20state?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- simplyblock_core/services/snapshot_replication.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index 9c37091664..c340b016a2 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -293,6 +293,8 @@ def _await_transfer_completion(task, snapshot, snode): return _finish_completed_transfer(task, snapshot, ret.get("offset")) if state == "Failed": return False # the pass-based path records the retry + if state == "No process": + return False # transfer never started; pass-based path retries time.sleep(constants.REPL_XFER_POLL_INTERVAL_SEC) xfer_timing.stamp("inline_wait_expired", snap=snapshot.get_id(), lvol=snapshot.lvol.get_id(), budget=budget) From 52e75afb2cdc0d45fe3780e437bb58bcbb8906f0 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 31 Aug 2026 13:17:18 +0100 Subject: [PATCH 107/122] commented out allow_partial --- simplyblock_core/rpc_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index 21f0250916..fc1908664c 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -1952,8 +1952,8 @@ def bdev_lvol_transfer(self, name, offset, batch_size, bdev_name, operation="mig } # Send the key only when opting in: the RPC parameter is optional on the # SPDK side, so every existing caller keeps its exact current wire form. - if allow_partial: - params["allow_partial"] = True + # if allow_partial: + # params["allow_partial"] = True return self._request("bdev_lvol_transfer", params) def bdev_lvol_transfer_stat(self, name): From ce8bc2a544db1629af0b894576f481e881d57c76 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 31 Aug 2026 15:45:20 +0100 Subject: [PATCH 108/122] fix: treat transfer_state=Failed as failure in final-step cutover --- .../services/replication_final_step.py | 17 +++++++-- .../services/tasks_runner_lvol_migration.py | 13 ++++++- .../test/test_replication_final_step.py | 36 ++++++++++++++++--- 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/simplyblock_core/services/replication_final_step.py b/simplyblock_core/services/replication_final_step.py index 2239a2410d..320c1e9fb4 100644 --- a/simplyblock_core/services/replication_final_step.py +++ b/simplyblock_core/services/replication_final_step.py @@ -267,15 +267,28 @@ def run_cutover(src_node, tgt_node, lvol, tgt_lvol_composite, tgt_map_id, ret = src_rpc.bdev_lvol_transfer_final_step( src_lvol_composite, tgt_map_id, tgt_snap_composite, _FINAL_STEP_BATCH, hub_bdev, operation) - if ret is None: + # The RPC can return normally (no exception) while still reporting the + # transfer failed -- transfer_state is one of "No process" | "In progress" | + # "Failed" | "Done". Checking only for None lets a "Failed" response slip + # through to the ANA flip as if the data had moved (seen 2026-08-31, + # lvol=1fc7911f: {"transfer_state": "Failed"} logged, then [IO-RESUME] + # fired unconditionally, leaving the target with missing delta data). + transfer_state = ret.get("transfer_state") if isinstance(ret, dict) else None + if transfer_state != "Done": # The freeze failed with the source fenced: restore the source paths so # the client resumes there (nothing moved; source is still authoritative) # rather than leaving the volume dark until a retry succeeds. + logger.error( + f"bdev_lvol_transfer_final_step: transfer_state={transfer_state!r} " + f"(expected 'Done'): {ret!r} lvol={lvol.uuid}") with xfer_timing.phase("restore_source_paths", lvol=lvol.get_id()): restore_source_paths(src_node, src_node.lvstore, lvol.nqn, lvol.ns_id) xfer_timing.gap("freeze_total", _freeze_started, lvol=lvol.get_id(), aborted=1) - return False, "bdev_lvol_transfer_final_step failed" + return False, ( + f"bdev_lvol_transfer_final_step: transfer_state={transfer_state!r}" + f" (expected 'Done'): {ret!r}" + ) logger.info(f"[IO-RESUME] final step Done: lvol={lvol.uuid} io now live on target") # Link the final lvol to its predecessor snapshot on the target peers. diff --git a/simplyblock_core/services/tasks_runner_lvol_migration.py b/simplyblock_core/services/tasks_runner_lvol_migration.py index 5b8cf53024..16d04c20a1 100644 --- a/simplyblock_core/services/tasks_runner_lvol_migration.py +++ b/simplyblock_core/services/tasks_runner_lvol_migration.py @@ -2022,7 +2022,12 @@ def _revert_src_replicas(reason): stat = src_rpc.bdev_lvol_transfer_stat(src_lvol_composite) state = (stat or {}).get('transfer_state') if stat is not None else None else: - state = None + # The RPC can return normally (no exception) while still reporting + # transfer failure — transfer_state is one of "No process" | + # "In progress" | "Failed" | "Done". Checking only for falsy + # lets a "Failed" dict slip through to the ANA flip as if the + # data had moved (same class of bug as batch migration 2026-08-22). + state = ret.get("transfer_state") if isinstance(ret, dict) else None except Exception: # SRC secondary/tertiary were just flipped inaccessible above; if # either RPC above raises (e.g. source unreachable — RPCException @@ -2047,6 +2052,12 @@ def _revert_src_replicas(reason): logger.info( f"[IO-RESUME] {_now_ms()} final migration complete (recovered from RPC error, " f"transfer_state={state}): lvol={migration.lvol_id} io now live on target") + elif state != "Done": + logger.error( + f"bdev_lvol_final_migration: transfer_state={state!r} " + f"(expected 'Done'): {ret!r} lvol={migration.lvol_id}") + _revert_src_replicas(f"final migration failed: transfer_state={state!r}") + return False, True, f"bdev_lvol_final_migration failed: transfer_state={state!r}" else: logger.info( f"[IO-RESUME] {_now_ms()} final migration Done: " diff --git a/simplyblock_core/test/test_replication_final_step.py b/simplyblock_core/test/test_replication_final_step.py index b8fa930bbb..6f1bec9b90 100644 --- a/simplyblock_core/test/test_replication_final_step.py +++ b/simplyblock_core/test/test_replication_final_step.py @@ -19,10 +19,11 @@ def get_remote_bdev_name(self): class _RPC: - def __init__(self, node_id, events, final_step_ret=True): + def __init__(self, node_id, events, final_step_ret=True, final_step_state="Done"): self.node_id = node_id self.events = events self._final_step_ret = final_step_ret + self._final_step_state = final_step_state self.final_step_gateways = [] self.ana_groups = [] @@ -42,7 +43,9 @@ def bdev_lvol_transfer_final_step(self, lvol_name, lvol_id, snapshot_name, # which is how the controller-name-instead-of-bdev bug went unnoticed. self.events.append(("final_step", operation, lvol_name, snapshot_name, batch)) self.final_step_gateways.append(gateway) - return ["ok"] if self._final_step_ret else None + if not self._final_step_ret: + return None + return {"transfer_state": self._final_step_state, "offset": 0} def bdev_lvol_add_clone(self, lvol_name, parent): self.events.append(("add_clone", self.node_id, lvol_name, parent)) @@ -60,7 +63,7 @@ def nvmf_subsystem_listener_set_ana_state(self, nqn, ip, port, trtype="TCP", ana class _Node: def __init__(self, uuid, events, ip, lvstore, status=StorageNode.STATUS_ONLINE, - secondary="", tertiary="", final_step_ret=True): + secondary="", tertiary="", final_step_ret=True, final_step_state="Done"): self.uuid = uuid self._events = events self._ip = ip @@ -72,7 +75,7 @@ def __init__(self, uuid, events, ip, lvstore, status=StorageNode.STATUS_ONLINE, self.data_nics = [_Nic(ip)] self.mgmt_ip = ip self.transfer_hublvol = _Hub() - self._rpc = _RPC(uuid, events, final_step_ret) + self._rpc = _RPC(uuid, events, final_step_ret, final_step_state) def get_id(self): return self.uuid @@ -224,6 +227,31 @@ def test_run_cutover_final_step_failure_no_ana(monkeypatch): assert not [e for e in events if e[0] == "add_clone"] +def test_run_cutover_final_step_failed_state_treated_as_failure(monkeypatch): + """Regression: bdev_lvol_transfer_final_step returning {"transfer_state": "Failed"} + (a truthy non-None dict) must be treated as a failure, not silently as success. + Observed 2026-08-31, lvol=1fc7911f: SPDK returned 'Failed' but [IO-RESUME] + fired unconditionally because the old code only checked `if ret is None`.""" + events: list = [] + tgt = _Node("T1", events, "t1", "lvs_tgt") + src = _Node("S1", events, "s1", "lvs_src", + final_step_ret=True, final_step_state="Failed") + _install_nodes(monkeypatch, {"T1": tgt, "S1": src}) + + ok, err = rfs.run_cutover( + src, tgt, _Lvol(), "lvs_tgt/LVOL_1", 42, "lvs_tgt/SNAP1") + + assert ok is False + assert "Failed" in err + # target must never be enabled when the transfer didn't complete + assert not [e for e in events if e[0] == "ana" and e[1].startswith("T") + and e[2] in ("optimized", "non_optimized")] + assert not [e for e in events if e[0] == "add_clone"] + # source must be restored so the client can resume IO on the unfailed side + ana = [(e[1], e[2]) for e in events if e[0] == "ana"] + assert ana[-1] == ("S1", "optimized"), "source must be unfenced after failed freeze" + + def test_final_step_failure_unfences_the_source(monkeypatch): """If the freeze fails after the source was fenced, the source must be restored (it is still the authoritative copy) — never left dark.""" From 937a3abad5f24159f8ef3f6308407a9a9f88af93 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 1 Sep 2026 17:09:46 +0100 Subject: [PATCH 109/122] fix(failback): gate pre-cutover delete of superseded original behind REPL_FAILBACK_RETIRE_ORIGINAL_BEFORE_CUTOVER (off) to stop blob-id reuse breaking cutover delta writes --- simplyblock_core/constants.py | 11 +++++++++++ simplyblock_core/controllers/lvol_controller.py | 17 +++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 62f33b56cc..86778c578e 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -440,6 +440,17 @@ def get_config_var(name, default=None): # the task table per pass, and polling a database at 5Hz to detect an event is # the wrong shape. Sub-second reaction lives in the RPC-based inline wait. REPL_CUTOVER_ACTIVE_POLL_SEC = 1.0 +# Delete the superseded original volume BEFORE building the fail-back clone +# (_retire_superseded_original). Disabled 2026-09-01: that delete frees the +# original's blob id while its parent snapshot's clone registry is already +# inconsistent ("Clone entry not found for blob ... under snapshot ..."), the +# clone created seconds later reuses the freed id, and every final-step delta +# write to it fails rc -1 (-EPERM) -> transfer_state Failed on all fail-back +# cutovers. The SPDK-side namespace slot is still freed by +# _evict_stale_namespace, and the original's DB record is removed after a +# successful cutover by _swap_failback_lvol_uuid. Re-enable once the fork's +# clone-entry/blob-id-reuse defect is fixed. +REPL_FAILBACK_RETIRE_ORIGINAL_BEFORE_CUTOVER = False SPDK_PROXY_MULTI_THREADING_ENABLED=True SPDK_PROXY_TIMEOUT=60*5 diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 53a58da147..db9ffb7f7b 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -3773,10 +3773,19 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps # A fail-back returns into the subsystem the original still occupies, so # the original has to go first (its snapshots stay: they are the delta # base). A first fail-over has no original here and this does nothing. - ok, err = _retire_superseded_original(db_controller, lvol, - target_node.cluster_id) - if not ok: - return None, err + # + # Gated off while the fork's blob-id-reuse defect is open: this delete + # frees the original's blob id, the clone reuses it seconds later, and the + # cutover's delta writes then fail rc -1 (see the constant's comment). + # The namespace slot the delete used to free is still reclaimed by + # _evict_stale_namespace below; the DB-level slot claim may report the + # subsystem full on groups at max_namespaces until the original is + # removed post-cutover by _swap_failback_lvol_uuid. + if constants.REPL_FAILBACK_RETIRE_ORIGINAL_BEFORE_CUTOVER: + ok, err = _retire_superseded_original(db_controller, lvol, + target_node.cluster_id) + if not ok: + return None, err # Last line of defence for the one-subsystem-one-primary invariant. The # target node was chosen when the policy was attached, long before this From 43f7d58413cee5faabc131a22954823462ce34f7 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 1 Sep 2026 18:14:50 +0100 Subject: [PATCH 110/122] fix(failback): give cutover clone a fresh bdev name when the original is kept, so the create probe cannot silently adopt the original's bdev/blobid --- simplyblock_core/controllers/lvol_controller.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index db9ffb7f7b..de4c9c74a7 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -3815,6 +3815,17 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps new_lvol = copy.deepcopy(lvol) new_lvol.uuid = str(uuid.uuid4()) + if not constants.REPL_FAILBACK_RETIRE_ORIGINAL_BEFORE_CUTOVER: + # The superseded original is left alive (see the constant) and — on a + # fail-back — still owns this deep-copied bdev name on the target + # lvstore. Creating the clone under the same name would not fail: the + # idempotency probe in _create_bdev_stack finds the original's bdev, + # skips the create, and add_lvol_on_node returns the ORIGINAL's + # uuid/blobid as the clone's — aiming the cutover delta at the + # original's stale data. Give the clone its own name; client identity + # is carried by NQN/nsid/namespace-UUID, never the bdev name. + new_lvol.vuid = utils.get_random_vuid() + new_lvol.lvol_bdev = f"LVOL_{new_lvol.vuid}" new_lvol.create_dt = str(datetime.now()) new_lvol.node_id = target_node.get_id() new_lvol.nodes = [target_node.get_id()] From 3a00915bfe49c7733dad9c817a46d2cdafd11085 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 2 Sep 2026 11:47:39 +0100 Subject: [PATCH 111/122] fix(failback): report the relationship source id in connect entries so CSI device lookup finds the failed-back volume --- .../controllers/lvol_controller.py | 117 ++++++++++++------ .../test/test_connect_path_resolution.py | 54 ++++++++ 2 files changed, 133 insertions(+), 38 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index de4c9c74a7..8d5595eb37 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -2735,6 +2735,27 @@ def connect_lvol(uuid, ctrl_loss_tmo=constants.LVOL_NVME_CONNECT_CTRL_LOSS_TMO, for entry in entries: entry.target_lvol_id = clone_id out.extend(entries) + + # Device-lookup identity for the FAIL-BACK shape. The clone's wire NSUUID + # is inherited from the migration SOURCE (the kernel merges multipath + # paths only on matching NSUUIDs), so the client's /dev/disk/by-id links + # carry the source volume's id — for clients that rode through the + # cutover AND for fresh connects, since the NSUUID is baked into the + # namespace. After a forward migration the requested volume is the + # relationship's source end, the loop above redirects to the target and + # target_lvol_id points at the right device. After a fail-back the UUID + # swap makes the requested volume the TARGET end under its original id, + # clone_id == uuid for every entry, and nothing reports the source id the + # device actually advertises — the CSI then cannot find the block device. + # Emit the other end here; only its ID is needed, so a source record that + # was since deleted still resolves. + if not any(e.target_lvol_id for e in out): + rep = _replication_for_lvol(db_controller, uuid) + if (rep is not None and rep.state == LVolReplication.STATE_CUTOVER_DONE + and rep.target_lvol and rep.target_lvol.get_id() == uuid + and rep.source_lvol and rep.source_lvol.get_id() != uuid): + for entry in out: + entry.target_lvol_id = rep.source_lvol.get_id() return out, None @@ -2761,16 +2782,7 @@ def _connect_path_volumes(db_controller, lvol): destination primary and may differ -- so every path returned here aggregates into one multipath device on the client. """ - from simplyblock_core.models.lvol_model import LVolReplication - - lvol_id = lvol.get_id() - rep = None - for candidate in reversed(db_controller.get_lvol_replication_objects()): - source_id = candidate.source_lvol.get_id() if candidate.source_lvol else "" - target_id = candidate.target_lvol.get_id() if candidate.target_lvol else "" - if lvol_id in (source_id, target_id): - rep = candidate - break + rep = _replication_for_lvol(db_controller, lvol.get_id()) if rep is None or rep.state == LVolReplication.STATE_REPLICATING: return [lvol] @@ -2793,6 +2805,20 @@ def _live(candidate): return [target or source or lvol] +def _replication_for_lvol(db_controller, lvol_id): + """The most recent replication relationship *lvol_id* is an end of, or None. + + Newest-first so a volume that went through several cycles (fail-over then + fail-back) resolves to the relationship describing its CURRENT location. + """ + for candidate in reversed(db_controller.get_lvol_replication_objects()): + source_id = candidate.source_lvol.get_id() if candidate.source_lvol else "" + target_id = candidate.target_lvol.get_id() if candidate.target_lvol else "" + if lvol_id in (source_id, target_id): + return candidate + return None + + def _connect_entries_for_volume(db_controller, lvol, ctrl_loss_tmo, host_entry, host_nqn): out = [] nodes_ids = [] @@ -3684,6 +3710,34 @@ def _claim_target_nsid(db_controller, new_lvol, target_node): return 0 +def _superseded_original(db_controller, lvol, dest_cluster_id): + """The still-live volume a fail-back of *lvol* supersedes on + *dest_cluster_id* — the original this copy descends from — or None when + nothing is in the way (first fail-over, original already deleted, or the + original lives on some other cluster).""" + original = None + for rep in db_controller.get_lvol_replication_objects(): + target = getattr(rep, "target_lvol", None) + if target and target.get_id() == lvol.get_id() and rep.source_lvol: + original = rep.source_lvol # keep the LAST match: the + # most recent fail-over wins + if not original: + return None + try: + current = db_controller.get_lvol_by_id(original.get_id()) + except KeyError: + return None # already gone + if current.status in (LVol.STATUS_DELETED, LVol.STATUS_IN_DELETION): + return None + try: + node = db_controller.get_storage_node_by_id(current.node_id) + except KeyError: + return None + if node.cluster_id != dest_cluster_id: + return None # not in our way + return current + + def _retire_superseded_original(db_controller, lvol, dest_cluster_id): """Delete the volume a fail-back replaces. Returns (ok, error). @@ -3705,26 +3759,9 @@ def _retire_superseded_original(db_controller, lvol, dest_cluster_id): A no-op unless *lvol* is itself a fail-over copy whose original still lives on the destination cluster, so a first fail-over deletes nothing. """ - original = None - for rep in db_controller.get_lvol_replication_objects(): - target = getattr(rep, "target_lvol", None) - if target and target.get_id() == lvol.get_id() and rep.source_lvol: - original = rep.source_lvol # keep the LAST match: the - # most recent fail-over wins - if not original: - return True, "" - try: - current = db_controller.get_lvol_by_id(original.get_id()) - except KeyError: - return True, "" # already gone - if current.status in (LVol.STATUS_DELETED, LVol.STATUS_IN_DELETION): + current = _superseded_original(db_controller, lvol, dest_cluster_id) + if current is None: return True, "" - try: - node = db_controller.get_storage_node_by_id(current.node_id) - except KeyError: - return True, "" - if node.cluster_id != dest_cluster_id: - return True, "" # not in our way logger.info( "Fail-back: deleting the superseded original %s (nsid %s of subsystem " @@ -3781,11 +3818,15 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps # _evict_stale_namespace below; the DB-level slot claim may report the # subsystem full on groups at max_namespaces until the original is # removed post-cutover by _swap_failback_lvol_uuid. + superseded = None if constants.REPL_FAILBACK_RETIRE_ORIGINAL_BEFORE_CUTOVER: ok, err = _retire_superseded_original(db_controller, lvol, target_node.cluster_id) if not ok: return None, err + else: + superseded = _superseded_original(db_controller, lvol, + target_node.cluster_id) # Last line of defence for the one-subsystem-one-primary invariant. The # target node was chosen when the policy was attached, long before this @@ -3815,15 +3856,15 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps new_lvol = copy.deepcopy(lvol) new_lvol.uuid = str(uuid.uuid4()) - if not constants.REPL_FAILBACK_RETIRE_ORIGINAL_BEFORE_CUTOVER: - # The superseded original is left alive (see the constant) and — on a - # fail-back — still owns this deep-copied bdev name on the target - # lvstore. Creating the clone under the same name would not fail: the - # idempotency probe in _create_bdev_stack finds the original's bdev, - # skips the create, and add_lvol_on_node returns the ORIGINAL's - # uuid/blobid as the clone's — aiming the cutover delta at the - # original's stale data. Give the clone its own name; client identity - # is carried by NQN/nsid/namespace-UUID, never the bdev name. + if superseded is not None: + # The superseded original is left alive (see the constant) and still + # owns this deep-copied bdev name on the target lvstore. Creating the + # clone under the same name would not fail: the idempotency probe in + # _create_bdev_stack finds the original's bdev, skips the create, and + # add_lvol_on_node returns the ORIGINAL's uuid/blobid as the clone's — + # aiming the cutover delta at the original's stale data. Give the + # clone its own name; client identity is carried by + # NQN/nsid/namespace-UUID, never the bdev name. new_lvol.vuid = utils.get_random_vuid() new_lvol.lvol_bdev = f"LVOL_{new_lvol.vuid}" new_lvol.create_dt = str(datetime.now()) diff --git a/simplyblock_core/test/test_connect_path_resolution.py b/simplyblock_core/test/test_connect_path_resolution.py index fff74c5dd7..1c7fec345a 100644 --- a/simplyblock_core/test/test_connect_path_resolution.py +++ b/simplyblock_core/test/test_connect_path_resolution.py @@ -120,3 +120,57 @@ def test_resolution_never_consults_cluster_status(): db = _FakeDB([src, tgt], [_rep(src, tgt, LVolReplication.STATE_FAILED_OVER)]) assert not hasattr(db, "get_cluster_by_id") assert _ids(lvol_controller._connect_path_volumes(db, src)) == ["LV_TGT"] + + +# ---- device-lookup identity (entry.target_lvol_id) ------------------------ # +# +# The clone's wire NSUUID is inherited from the migration SOURCE (the kernel +# merges multipath paths only on matching NSUUIDs), so the client's +# /dev/disk/by-id links carry the source volume's id. After a fail-back the +# UUID swap makes the requested volume the relationship's TARGET end under its +# original id — every path volume equals the requested one, and without the +# explicit emission below the CSI has no id that matches the block device. + + +class _FakeEntry: + target_lvol_id = None + + +def _connect(monkeypatch, db, requested_id): + """Run connect_lvol against the fake DB with the entry builder stubbed.""" + monkeypatch.setattr(lvol_controller, "DBController", lambda: db) + monkeypatch.setattr(lvol_controller.HostConnectAuth, "resolve", + classmethod(lambda cls, lvol, host_nqn, db_controller: None)) + monkeypatch.setattr(lvol_controller, "_connect_entries_for_volume", + lambda *a, **kw: [_FakeEntry()]) + entries, err = lvol_controller.connect_lvol(requested_id) + assert err is None + return entries + + +def test_failback_connect_reports_the_source_id_for_device_lookup(monkeypatch): + """cutover_done with the requested volume on the TARGET end (the fail-back + shape): the device advertises the SOURCE volume's NSUUID, so connect must + hand that id out.""" + dr, back = _lvol("LV_DR"), _lvol("LV_BACK") + db = _FakeDB([dr, back], [_rep(dr, back, LVolReplication.STATE_CUTOVER_DONE)]) + entries = _connect(monkeypatch, db, "LV_BACK") + assert [e.target_lvol_id for e in entries] == ["LV_DR"] + + +def test_forward_migration_keeps_the_target_id_for_device_lookup(monkeypatch): + """Requested volume on the SOURCE end: the redirect loop already reports the + target id; the fail-back emission must not overwrite it.""" + src, tgt = _lvol("LV_SRC"), _lvol("LV_TGT") + db = _FakeDB([src, tgt], [_rep(src, tgt, LVolReplication.STATE_CUTOVER_DONE)]) + entries = _connect(monkeypatch, db, "LV_SRC") + assert [e.target_lvol_id for e in entries] == ["LV_TGT"] + + +def test_failed_over_target_end_reports_no_device_lookup_id(monkeypatch): + """A fail-over clone's NSUUID is its own uuid — connecting it by its own id + needs no redirect, so nothing must be emitted.""" + src, tgt = _lvol("LV_SRC"), _lvol("LV_TGT") + db = _FakeDB([src, tgt], [_rep(src, tgt, LVolReplication.STATE_FAILED_OVER)]) + entries = _connect(monkeypatch, db, "LV_TGT") + assert [e.target_lvol_id for e in entries] == [None] From 7031835a7f89747c935f6c1d772817b5aa162ebc Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 2 Sep 2026 13:44:59 +0100 Subject: [PATCH 112/122] fix(cutover): enable the cutover-proceed preconnect gate now that the operator signals for migration and failback --- simplyblock_core/constants.py | 9 ++++++++- simplyblock_core/test/test_cutover_convergence.py | 15 +++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 0a6506bc72..d4d28bebfb 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -415,7 +415,14 @@ def get_config_var(name, default=None): # fed the 25-72s freezes. Deployments whose operator posts # .../replication/cutover-proceed set this True and accept that cost until the # clone's base can be advanced after the signal. -REPL_CUTOVER_PROCEED_REQUIRED = False +# +# Enabled 2026-09-02: the operator's reconcileCutoverPending runs the +# preconnect Job and posts cutover-proceed for both migration and failback +# (annotFailbackTarget routes the call to the target cluster on failback). +# Without the gate, the flip races the client: the 2026-09-02 failback run +# flipped ANA on listeners no client had connected to and deleted the DR-side +# subsystem 150ms later, orphaning every connected client for ctrl_loss_tmo. +REPL_CUTOVER_PROCEED_REQUIRED = True # --- noticing a finished transfer ---------------------------------------- # A transfer that has completed must be acted on within a second: the next diff --git a/simplyblock_core/test/test_cutover_convergence.py b/simplyblock_core/test/test_cutover_convergence.py index 87aa3ec2f6..71b2900264 100644 --- a/simplyblock_core/test/test_cutover_convergence.py +++ b/simplyblock_core/test/test_cutover_convergence.py @@ -205,11 +205,18 @@ def test_it_yields_the_pass_when_the_budget_runs_out(self): class TestProceedGate(unittest.TestCase): """The preconnect wait must be opt-in: it costs freeze time.""" - def test_disabled_by_default(self): - self.assertFalse( + def test_enabled_now_that_the_operator_signals(self): + """Flipped 2026-09-02: the operator's reconcileCutoverPending posts + cutover-proceed for migration AND failback (annotFailbackTarget routes + the call to the target cluster). Without the gate the ANA flip races + the client's preconnect: the 2026-09-02 failback run flipped listeners + no client had connected to and deleted the DR-side subsystem 150ms + later, orphaning every connected client for ctrl_loss_tmo.""" + self.assertTrue( constants.REPL_CUTOVER_PROCEED_REQUIRED, - "waiting for a signal nobody sends put 120s of writes into the " - "frozen final step") + "cutover must wait for the operator's preconnect signal; " + "flipping ANA on listeners no client is connected to and then " + "deleting the source subsystem strands every live client") def test_the_wait_is_guarded_by_the_flag(self): import inspect From f5b0b77b84037385175d6d85f507297ab3ea8386 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 2 Sep 2026 15:34:42 +0100 Subject: [PATCH 113/122] fix(failback): evict the superseded original's namespace by its identity so shared-subsystem cutovers can claim their preserved nsid --- .../controllers/lvol_controller.py | 27 ++++- .../test/test_evict_stale_namespace.py | 98 +++++++++++++++++++ .../test_replication_chain_completeness.py | 2 +- 3 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 simplyblock_core/test/test_evict_stale_namespace.py diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 76f5385784..47c87115b0 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -4015,7 +4015,7 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps new_lvol.write_to_db(db_controller.kv_store) - _evict_stale_namespace(new_lvol, target_node) + _evict_stale_namespace(new_lvol, target_node, superseded=superseded) # The target clone shares the source NQN. During preconnect the host has # live paths to the source (cntlids 1, 1000, 2000) AND tries to add paths @@ -4076,7 +4076,7 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps # succeed while the peer's failed with the same -32602, and the # peer failure rolled the whole cutover back (run 20260824_113711: # primary add_ns result:1, peer -32602, 0/5 cutovers). - _evict_stale_namespace(new_lvol, peer_node) + _evict_stale_namespace(new_lvol, peer_node, superseded=superseded) _peer_cntlid = next(_tgt_cntlid_iter, None) lvol_bdev, error = add_lvol_on_node(new_lvol, peer_node, is_primary=False, min_cntlid=_peer_cntlid, ns_uuid=_src_ns_uuid) @@ -4173,9 +4173,18 @@ def _last_replicated_target_snapshot(db_controller, lvol_id, cluster_id, generat return None -def _evict_stale_namespace(new_lvol, target_node): +def _evict_stale_namespace(new_lvol, target_node, superseded=None): """Make room for the preserved identity on a RECOVERED fail-back target. + ``superseded`` is the still-live original volume this fail-back replaces + (from :func:`_superseded_original`, present since the pre-cutover retire + was disabled). Its namespace carries the ORIGINAL record's uuid — not the + clone's — so on a SHARED subsystem the own-uuid match below cannot see it + and the clone's add_ns fails -32602 forever (2026-09-02: shared subsystem + holding nsids 1-5, clone wanted nsid 1, occupant was the original). + Matching on the superseded original's identity is safe on shared + subsystems: it names exactly one namespace, never a sibling's. + The cutover clone keeps the ORIGINAL volume's NQN and nsid so the client reconnects to the same identity. Failing back to a recovered source means that subsystem usually still exists there WITH the original volume's @@ -4225,8 +4234,20 @@ def _evict_stale_namespace(new_lvol, target_node): single_namespace_subsystem = len(namespaces) <= 1 own_uuid = getattr(new_lvol, "uuid", None) own_nsid = getattr(new_lvol, "ns_id", None) + # The superseded original's namespace is identified by ITS record + # uuid (the ns uuid it was added with) or its bdev — SPDK reports + # lvol namespaces' bdev_name as the raw lvol_uuid. + superseded_ids = set() + if superseded is not None: + superseded_ids = { + v for v in (getattr(superseded, "uuid", None), + getattr(superseded, "lvol_uuid", None), + getattr(superseded, "top_bdev", None)) + if v} stale = [ns for ns in namespaces if ((own_uuid is not None and ns.get("uuid") == own_uuid) + or ns.get("uuid") in superseded_ids + or ns.get("bdev_name") in superseded_ids or (single_namespace_subsystem and own_nsid is not None and ns.get("nsid") == own_nsid)) diff --git a/simplyblock_core/test/test_evict_stale_namespace.py b/simplyblock_core/test/test_evict_stale_namespace.py new file mode 100644 index 0000000000..0538576a31 --- /dev/null +++ b/simplyblock_core/test/test_evict_stale_namespace.py @@ -0,0 +1,98 @@ +"""Which namespace the fail-back eviction removes. + +The cutover clone needs its preserved nsid; the still-live superseded original +occupies it. On a SHARED (namespaced) subsystem the original's namespace +carries the ORIGINAL record's uuid, so the own-uuid match cannot see it and the +nsid fallback is forbidden (it could evict a live sibling). The eviction must +therefore match the superseded original's identity explicitly — and never touch +siblings (2026-09-02: shared subsystem holding nsids 1-5, clone wanted nsid 1, +add_ns -32602 on every retry). +""" +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from simplyblock_core.controllers import lvol_controller + + +class _FakeRpc: + """Subsystem with mutable namespaces; remove_ns takes effect immediately.""" + + def __init__(self, namespaces): + self.namespaces = list(namespaces) + self.removed = [] + + def subsystem_get(self, nqn): + return {"nqn": nqn, "namespaces": list(self.namespaces)} + + def nvmf_subsystem_remove_ns(self, nqn, nsid): + self.removed.append(nsid) + self.namespaces = [ns for ns in self.namespaces if ns["nsid"] != nsid] + return True + + +def _node(rpc): + node = MagicMock() + node.secondary_node_id = "" + node.tertiary_node_id = "" + node.rpc_client.return_value = rpc + node.get_id.return_value = "N_tgt" + return node + + +def _clone(nsid=1): + return SimpleNamespace(uuid="CLONE_UUID", ns_id=nsid, + top_bdev="LVS/CLONE_BDEV", nqn="nqn.shared") + + +_ORIGINAL = SimpleNamespace(uuid="ORIG_UUID", lvol_uuid="ORIG_BDEV", + top_bdev="LVS/ORIG") + +_SHARED = [ + {"nsid": 1, "uuid": "ORIG_UUID", "bdev_name": "ORIG_BDEV"}, + {"nsid": 2, "uuid": "SIB_A", "bdev_name": "SIB_A_BDEV"}, + {"nsid": 3, "uuid": "SIB_B", "bdev_name": "SIB_B_BDEV"}, +] + + +def _evict(rpc, superseded): + with patch.object(lvol_controller, "DBController", MagicMock()): + lvol_controller._evict_stale_namespace(_clone(), _node(rpc), + superseded=superseded) + + +def test_superseded_original_is_evicted_on_a_shared_subsystem(): + rpc = _FakeRpc(_SHARED) + _evict(rpc, _ORIGINAL) + assert rpc.removed == [1] + + +def test_siblings_survive_even_when_the_clone_wants_their_nsid(): + """nsid never identifies a namespace on a shared subsystem.""" + rpc = _FakeRpc([ns for ns in _SHARED if ns["nsid"] != 1]) + _evict(rpc, None) + assert rpc.removed == [] + + +def test_without_superseded_the_shared_original_stays(): + """The pre-fix behavior: own-uuid match alone cannot see the original. + Guards that passing superseded is what makes the eviction possible.""" + rpc = _FakeRpc(_SHARED) + _evict(rpc, None) + assert rpc.removed == [] + + +def test_matches_by_bdev_name_when_ns_uuid_diverged(): + """SPDK reports lvol namespaces' bdev_name as the raw lvol_uuid; a + namespace registered with a different ns uuid is still the original's.""" + rpc = _FakeRpc([{"nsid": 1, "uuid": "SOMETHING_ELSE", + "bdev_name": "ORIG_BDEV"}] + _SHARED[1:]) + _evict(rpc, _ORIGINAL) + assert rpc.removed == [1] + + +def test_dedicated_subsystem_still_evicts_by_nsid(): + """Single-namespace subsystems keep the nsid fallback: there is nobody + else the match could hit.""" + rpc = _FakeRpc([{"nsid": 1, "uuid": "OLD", "bdev_name": "OLD_BDEV"}]) + _evict(rpc, None) + assert rpc.removed == [1] diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index 5d2ea848dc..9bf671f93c 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -384,7 +384,7 @@ def test_failback_evicts_on_every_ha_node_not_just_the_primary(monkeypatch): evicted, added = [], [] monkeypatch.setattr(lc, "_evict_stale_namespace", - lambda lvol, node: evicted.append(node.get_id())) + lambda lvol, node, **kw: evicted.append(node.get_id())) def _fake_add_lvol_on_node(lvol, node, is_primary=True, **kw): # The real signature also takes min_cntlid / ns_uuid / primary_nsid; From 08c076104f5ac5d0af229ac492b6052e121f6139 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 2 Sep 2026 17:34:09 +0100 Subject: [PATCH 114/122] fix(failback): register the clone's namespace with the superseded original's uuid and nguid so connected clients keep their multipath heads across cutover --- .../controllers/lvol_controller.py | 89 +++++++++++++------ simplyblock_core/models/lvol_model.py | 6 ++ .../test/test_connect_path_resolution.py | 44 ++++++--- .../test_replication_chain_completeness.py | 86 ++++++++++++++++++ 4 files changed, 189 insertions(+), 36 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 47c87115b0..3829a4c6d4 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -2799,26 +2799,37 @@ def connect_lvol(uuid, ctrl_loss_tmo=constants.LVOL_NVME_CONNECT_CTRL_LOSS_TMO, entry.target_lvol_id = clone_id out.extend(entries) - # Device-lookup identity for the FAIL-BACK shape. The clone's wire NSUUID - # is inherited from the migration SOURCE (the kernel merges multipath - # paths only on matching NSUUIDs), so the client's /dev/disk/by-id links - # carry the source volume's id — for clients that rode through the - # cutover AND for fresh connects, since the NSUUID is baked into the - # namespace. After a forward migration the requested volume is the - # relationship's source end, the loop above redirects to the target and - # target_lvol_id points at the right device. After a fail-back the UUID - # swap makes the requested volume the TARGET end under its original id, - # clone_id == uuid for every entry, and nothing reports the source id the - # device actually advertises — the CSI then cannot find the block device. - # Emit the other end here; only its ID is needed, so a source record that - # was since deleted still resolves. + # Device-lookup identity for the FAIL-BACK shape. A clone's wire NSUUID + # may be borrowed from another volume (the kernel merges multipath paths + # only on matching NSUUIDs), so the client's /dev/disk/by-id links can + # carry an id that is not the record's own. After a forward migration the + # requested volume is the relationship's source end, the loop above + # redirects to the target and target_lvol_id points at the right device. + # After a fail-back the UUID swap makes the requested volume the TARGET + # end under its original id, clone_id == uuid for every entry, and + # nothing would report the id the device actually advertises — the CSI + # then cannot find the block device. The record's ns_uuid carries the + # wire identity: for a fail-back over a still-live original it equals the + # restored original uuid (no override needed — the CSI's own-id lookup + # matches), while a fail-back to a fresh cluster keeps the DR source's + # id and must be reported. if not any(e.target_lvol_id for e in out): - rep = _replication_for_lvol(db_controller, uuid) - if (rep is not None and rep.state == LVolReplication.STATE_CUTOVER_DONE - and rep.target_lvol and rep.target_lvol.get_id() == uuid - and rep.source_lvol and rep.source_lvol.get_id() != uuid): - for entry in out: - entry.target_lvol_id = rep.source_lvol.get_id() + wire_id = getattr(lvol, "ns_uuid", "") + if wire_id: + if wire_id != uuid: + for entry in out: + entry.target_lvol_id = wire_id + else: + # Legacy records created before ns_uuid was persisted: the only + # fail-back shape that existed then inherited the DR source's + # NSUUID, recoverable from the relationship's other end (only its + # ID is needed, so a source record since deleted still resolves). + rep = _replication_for_lvol(db_controller, uuid) + if (rep is not None and rep.state == LVolReplication.STATE_CUTOVER_DONE + and rep.target_lvol and rep.target_lvol.get_id() == uuid + and rep.source_lvol and rep.source_lvol.get_id() != uuid): + for entry in out: + entry.target_lvol_id = rep.source_lvol.get_id() return out, None @@ -3930,6 +3941,17 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps # NQN/nsid/namespace-UUID, never the bdev name. new_lvol.vuid = utils.get_random_vuid() new_lvol.lvol_bdev = f"LVOL_{new_lvol.vuid}" + # And precisely BECAUSE identity is NQN/nsid/UUID/NGUID: the clone must + # present the ORIGINAL's NGUID, not the deep-copied source guid. The + # client's multipath head for this nsid was built from the original's + # ids; a namespace re-added at the same nsid with different ids is + # rejected forever ("IDs don't match for shared namespace N") and on a + # shared subsystem that leaves the head pathless — the sibling pods + # keep the controllers alive, so no reconnect ever rebuilds it + # (run 2026-09-02 16:00: nsid 3 evicted at prepare_cutover, clone + # re-added with the DR ids, XFS shutdown on the client). + if superseded.guid: + new_lvol.guid = superseded.guid new_lvol.create_dt = str(datetime.now()) new_lvol.node_id = target_node.get_id() new_lvol.nodes = [target_node.get_id()] @@ -4013,6 +4035,28 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps if not for_migration: new_lvol.ns_id = _claim_target_nsid(db_controller, new_lvol, target_node) + # Which identity the clone's namespace advertises on the wire: + # + # Migration: preserve the source UUID as the NVMe namespace UUID so the kernel + # can merge source and target paths into the same multipath namespace during + # the preconnect phase (ANA flip requires matching NSUUID on both paths). + # Failover: the source is gone — use the clone's own UUID so it appears as + # nvme-uuid. in /dev/disk/by-id, consistent with standalone volumes. + # Fail-back over a still-live original: the namespace identity must be the + # ORIGINAL's uuid (the very identity _swap_failback_lvol_uuid restores on + # the record after cutover). The client's multipath head at this nsid was + # built from the original's ids; re-adding the slot under the DR lvol's + # uuid makes the kernel reject the path ("IDs don't match for shared + # namespace N") and on a shared subsystem the head stays pathless until + # the pod is restaged (run 2026-09-02 16:00, nsids 2 and 3). + if superseded is not None: + _src_ns_uuid = superseded.uuid + else: + _src_ns_uuid = lvol.uuid if for_migration else new_lvol.uuid + # Persist the wire identity when it is borrowed, so connect_lvol can tell + # the CSI which /dev/disk/by-id/nvme-uuid. the device really carries. + new_lvol.ns_uuid = _src_ns_uuid if _src_ns_uuid != new_lvol.uuid else "" + new_lvol.write_to_db(db_controller.kv_store) _evict_stale_namespace(new_lvol, target_node, superseded=superseded) @@ -4028,13 +4072,6 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps random.randint(6001, 6500), # tertiary ] - # Migration: preserve the source UUID as the NVMe namespace UUID so the kernel - # can merge source and target paths into the same multipath namespace during - # the preconnect phase (ANA flip requires matching NSUUID on both paths). - # Failover: the source is gone — use the clone's own UUID so it appears as - # nvme-uuid. in /dev/disk/by-id, consistent with standalone volumes. - _src_ns_uuid = lvol.uuid if for_migration else new_lvol.uuid - # For migration/failover, preserve the source nsid so the kernel can # match target paths to source paths under the same NQN. new_lvol is a # deepcopy of the source lvol, so new_lvol.ns_id is already the source diff --git a/simplyblock_core/models/lvol_model.py b/simplyblock_core/models/lvol_model.py index 78c77f91f5..5335ac6985 100644 --- a/simplyblock_core/models/lvol_model.py +++ b/simplyblock_core/models/lvol_model.py @@ -50,6 +50,12 @@ class LVol(BaseModel): nodes: List[str] = default_factory(list) nqn: str = "" ns_id: int = 1 + # The UUID the NVMe namespace advertises on the wire when it differs from + # the record's uuid (migration/fail-back clones inherit another volume's + # identity so the client's multipath head keeps its paths). Empty means + # the namespace carries the record's own uuid. connect_lvol reports it as + # target_lvol_id so the CSI globs /dev/disk/by-id/nvme-uuid.. + ns_uuid: str = "" max_namespace_per_subsys: int = 1 subsys_port: int = 9090 # Node ids whose sync delete already completed inline in the API delete diff --git a/simplyblock_core/test/test_connect_path_resolution.py b/simplyblock_core/test/test_connect_path_resolution.py index 1c7fec345a..d82115c7f9 100644 --- a/simplyblock_core/test/test_connect_path_resolution.py +++ b/simplyblock_core/test/test_connect_path_resolution.py @@ -124,12 +124,13 @@ def test_resolution_never_consults_cluster_status(): # ---- device-lookup identity (entry.target_lvol_id) ------------------------ # # -# The clone's wire NSUUID is inherited from the migration SOURCE (the kernel -# merges multipath paths only on matching NSUUIDs), so the client's -# /dev/disk/by-id links carry the source volume's id. After a fail-back the -# UUID swap makes the requested volume the relationship's TARGET end under its -# original id — every path volume equals the requested one, and without the -# explicit emission below the CSI has no id that matches the block device. +# A clone's wire NSUUID may be borrowed from another volume (the kernel merges +# multipath paths only on matching NSUUIDs), so the client's /dev/disk/by-id +# links can carry an id that is not the record's own. The record's ns_uuid +# persists that wire identity; connect reports it whenever it differs from the +# requested id. Records predating ns_uuid fall back to the relationship's +# other end (the only fail-back shape that existed then inherited the DR +# source's NSUUID). class _FakeEntry: @@ -148,16 +149,39 @@ def _connect(monkeypatch, db, requested_id): return entries -def test_failback_connect_reports_the_source_id_for_device_lookup(monkeypatch): - """cutover_done with the requested volume on the TARGET end (the fail-back - shape): the device advertises the SOURCE volume's NSUUID, so connect must - hand that id out.""" +def test_legacy_failback_record_falls_back_to_the_relationship(monkeypatch): + """cutover_done with the requested volume on the TARGET end and no ns_uuid + persisted (records from before the field existed): those clones inherited + the DR SOURCE's NSUUID, so connect must hand that id out.""" dr, back = _lvol("LV_DR"), _lvol("LV_BACK") db = _FakeDB([dr, back], [_rep(dr, back, LVolReplication.STATE_CUTOVER_DONE)]) entries = _connect(monkeypatch, db, "LV_BACK") assert [e.target_lvol_id for e in entries] == ["LV_DR"] +def test_failback_with_restored_identity_needs_no_override(monkeypatch): + """A fail-back over a still-live original registers the clone's namespace + under the ORIGINAL's uuid — after the UUID swap the wire identity equals + the volume's own id, so the CSI's own-id lookup matches and emitting the + DR source id would point it at a NSUUID that does not exist.""" + dr, back = _lvol("LV_DR"), _lvol("LV_BACK") + back.ns_uuid = "LV_BACK" # what _swap_failback_lvol_uuid leaves behind + db = _FakeDB([dr, back], [_rep(dr, back, LVolReplication.STATE_CUTOVER_DONE)]) + entries = _connect(monkeypatch, db, "LV_BACK") + assert [e.target_lvol_id for e in entries] == [None] + + +def test_fresh_cluster_failback_reports_the_wire_identity(monkeypatch): + """A fail-back to a fresh cluster has no still-live original: the clone + keeps the DR source's NSUUID, persisted in ns_uuid, and connect reports + exactly that — no relationship walk needed.""" + back = _lvol("LV_BACK") + back.ns_uuid = "LV_DR" + db = _FakeDB([back]) + entries = _connect(monkeypatch, db, "LV_BACK") + assert [e.target_lvol_id for e in entries] == ["LV_DR"] + + def test_forward_migration_keeps_the_target_id_for_device_lookup(monkeypatch): """Requested volume on the SOURCE end: the redirect loop already reports the target id; the fail-back emission must not overwrite it.""" diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index 9bf671f93c..14a3fcd824 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -461,6 +461,92 @@ def get_id(self): assert ("S", False) in added +def test_failback_clone_keeps_the_superseded_originals_wire_identity(monkeypatch): + """Run 2026-09-02 16:00: the fail-back clone was re-added at the preserved + nsid with the DR lvol's uuid and nguid (deep-copied), but the client's + multipath head for that nsid was built from the ORIGINAL's ids — the + kernel rejected every new path ("IDs don't match for shared namespace N") + and, on the shared subsystem, the eviction at prepare_cutover had already + removed the head's only live path: XFS shut down on running pods. The + clone must advertise the superseded original's uuid AND nguid, the exact + identity _swap_failback_lvol_uuid restores on the record after cutover.""" + from simplyblock_core.controllers import lvol_controller as lc + + added = [] + monkeypatch.setattr(lc, "_evict_stale_namespace", lambda lvol, node, **kw: None) + monkeypatch.setattr(lc.utils, "get_random_vuid", lambda *a, **kw: 999) + + def _fake_add_lvol_on_node(lvol, node, is_primary=True, ns_uuid=None, **kw): + added.append((node.get_id(), ns_uuid, lvol.guid)) + return {"uuid": "U", "driver_specific": {"lvol": {"blobid": 9}}}, None + + monkeypatch.setattr(lc, "add_lvol_on_node", _fake_add_lvol_on_node) + + class _Original: + uuid = "ORIG_ID" + guid = "ORIG_NGUID" + + monkeypatch.setattr(lc, "_superseded_original", lambda *a, **kw: _Original()) + + class _N: + def __init__(self, nid, secondary=""): + self._id, self.secondary_node_id, self.tertiary_node_id = nid, secondary, "" + self.lvstore = "LVS_1" + self.status = lc.StorageNode.STATUS_ONLINE + self.cluster_id = "CL_tgt" + + def get_lvol_subsys_port(self, lvstore): + return 4420 + + def get_id(self): + return self._id + + primary = _N("P", secondary="S") + peer = _N("S") + + class _DB: + kv_store = None + + def get_storage_node_by_id(self, nid): + return {"P": primary, "S": peer}[nid] + + def release_lvol_ns_slot(self, lvol): + pass + + def get_lvols(self): + return [] + + class _Lvol: + uuid = "DR_ID"; nqn = "nqn.test:lvol:SHARED"; ns_id = 3 + guid = "DR_NGUID" + namespace = "" + max_namespace_per_subsys = 10 + lvol_bdev = "LVOL_28"; crypto_bdev = "" + + def __deepcopy__(self, memo): + c = _Lvol(); c.__dict__.update(self.__dict__); return c + + def write_to_db(self, kv=None): + pass + + class _Snap: + cluster_id = "C1"; snap_bdev = "LVS_1/SNAP_1" + + def get_id(self): + return "SNAP1" + + new_lvol, error = lc._create_target_lvol_clone( + _DB(), _Lvol(), primary, "POOL", _Snap(), for_migration=True) + assert error is None + # Every node's add_ns must carry the original's identity, never the DR's. + assert [(nid, ns) for nid, ns, _ in added] == [("P", "ORIG_ID"), ("S", "ORIG_ID")] + assert all(guid == "ORIG_NGUID" for _, _, guid in added) + # The wire identity is persisted so connect_lvol can report it. + assert new_lvol.ns_uuid == "ORIG_ID" + # And the clone still got its own bdev name (adoption guard). + assert new_lvol.lvol_bdev == "LVOL_999" + + def test_interrupted_landing_volume_is_adopted_or_cleared(): """Case 6, run 20260824_144226: a node outage mid-create left a REP_* landing volume whose id was never stored on the task; every retry then From 5916276875af5c5d7bc53f80f7e37775b4e81356 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 2 Sep 2026 19:45:24 +0100 Subject: [PATCH 115/122] fix(failback): advertise the DR source's wire identity on the clone namespace so restaged clients' multipath heads survive the cutover --- .../controllers/lvol_controller.py | 44 +++++++++---------- .../test/test_connect_path_resolution.py | 20 ++++----- .../test_replication_chain_completeness.py | 44 +++++++++++++------ 3 files changed, 61 insertions(+), 47 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 3829a4c6d4..ac61cd26cc 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -3941,17 +3941,6 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps # NQN/nsid/namespace-UUID, never the bdev name. new_lvol.vuid = utils.get_random_vuid() new_lvol.lvol_bdev = f"LVOL_{new_lvol.vuid}" - # And precisely BECAUSE identity is NQN/nsid/UUID/NGUID: the clone must - # present the ORIGINAL's NGUID, not the deep-copied source guid. The - # client's multipath head for this nsid was built from the original's - # ids; a namespace re-added at the same nsid with different ids is - # rejected forever ("IDs don't match for shared namespace N") and on a - # shared subsystem that leaves the head pathless — the sibling pods - # keep the controllers alive, so no reconnect ever rebuilds it - # (run 2026-09-02 16:00: nsid 3 evicted at prepare_cutover, clone - # re-added with the DR ids, XFS shutdown on the client). - if superseded.guid: - new_lvol.guid = superseded.guid new_lvol.create_dt = str(datetime.now()) new_lvol.node_id = target_node.get_id() new_lvol.nodes = [target_node.get_id()] @@ -4037,22 +4026,29 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps # Which identity the clone's namespace advertises on the wire: # - # Migration: preserve the source UUID as the NVMe namespace UUID so the kernel - # can merge source and target paths into the same multipath namespace during - # the preconnect phase (ANA flip requires matching NSUUID on both paths). + # Migration and fail-back: preserve the SOURCE's wire identity — what the + # connected client's multipath head currently holds — so the kernel can + # merge source and target paths during the preconnect phase (the ANA flip + # requires matching NSUUIDs on both paths). The source's own wire identity + # may itself be borrowed (its ns_uuid, e.g. a fail-back clone from an + # earlier cycle), so propagate the chain, not the record uuid. # Failover: the source is gone — use the clone's own UUID so it appears as # nvme-uuid. in /dev/disk/by-id, consistent with standalone volumes. - # Fail-back over a still-live original: the namespace identity must be the - # ORIGINAL's uuid (the very identity _swap_failback_lvol_uuid restores on - # the record after cutover). The client's multipath head at this nsid was - # built from the original's ids; re-adding the slot under the DR lvol's - # uuid makes the kernel reject the path ("IDs don't match for shared - # namespace N") and on a shared subsystem the head stays pathless until - # the pod is restaged (run 2026-09-02 16:00, nsids 2 and 3). - if superseded is not None: - _src_ns_uuid = superseded.uuid + # + # Do NOT use the superseded original's uuid here. That was tried (run + # 2026-09-02 17:00) to serve clients whose heads still held the ORIGINAL + # identity — but such clients only exist because failover leaves the + # original unfenced and their writes land on superseded data that fail-back + # discards anyway. For the canonical shape — pods restaged onto the DR side + # after failover, heads built from the DR identity — restoring the original + # uuid made the kernel reject every preconnected target path ("IDs don't + # match for shared namespace N"), and deleteSource then removed the head's + # only live paths: no available path, XFS shutdown (run 2026-09-02 ~19:00, + # subsystem 20d8a917 nsid 1). + if for_migration: + _src_ns_uuid = getattr(lvol, "ns_uuid", "") or lvol.uuid else: - _src_ns_uuid = lvol.uuid if for_migration else new_lvol.uuid + _src_ns_uuid = new_lvol.uuid # Persist the wire identity when it is borrowed, so connect_lvol can tell # the CSI which /dev/disk/by-id/nvme-uuid. the device really carries. new_lvol.ns_uuid = _src_ns_uuid if _src_ns_uuid != new_lvol.uuid else "" diff --git a/simplyblock_core/test/test_connect_path_resolution.py b/simplyblock_core/test/test_connect_path_resolution.py index d82115c7f9..f16912ff58 100644 --- a/simplyblock_core/test/test_connect_path_resolution.py +++ b/simplyblock_core/test/test_connect_path_resolution.py @@ -159,22 +159,22 @@ def test_legacy_failback_record_falls_back_to_the_relationship(monkeypatch): assert [e.target_lvol_id for e in entries] == ["LV_DR"] -def test_failback_with_restored_identity_needs_no_override(monkeypatch): - """A fail-back over a still-live original registers the clone's namespace - under the ORIGINAL's uuid — after the UUID swap the wire identity equals - the volume's own id, so the CSI's own-id lookup matches and emitting the - DR source id would point it at a NSUUID that does not exist.""" +def test_wire_identity_equal_to_own_id_needs_no_override(monkeypatch): + """A volume whose persisted wire identity equals its own id needs no + redirect: the CSI's own-id lookup already matches the device, and any + emission would point it elsewhere.""" dr, back = _lvol("LV_DR"), _lvol("LV_BACK") - back.ns_uuid = "LV_BACK" # what _swap_failback_lvol_uuid leaves behind + back.ns_uuid = "LV_BACK" db = _FakeDB([dr, back], [_rep(dr, back, LVolReplication.STATE_CUTOVER_DONE)]) entries = _connect(monkeypatch, db, "LV_BACK") assert [e.target_lvol_id for e in entries] == [None] -def test_fresh_cluster_failback_reports_the_wire_identity(monkeypatch): - """A fail-back to a fresh cluster has no still-live original: the clone - keeps the DR source's NSUUID, persisted in ns_uuid, and connect reports - exactly that — no relationship walk needed.""" +def test_failback_reports_the_persisted_wire_identity(monkeypatch): + """After a fail-back the UUID swap gives the record its original id while + the namespace keeps advertising the DR source's wire identity (that is + what kept the client's multipath head alive through the cutover). Connect + must report the persisted ns_uuid — no relationship walk needed.""" back = _lvol("LV_BACK") back.ns_uuid = "LV_DR" db = _FakeDB([back]) diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index 14a3fcd824..f236a360ea 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -461,15 +461,16 @@ def get_id(self): assert ("S", False) in added -def test_failback_clone_keeps_the_superseded_originals_wire_identity(monkeypatch): - """Run 2026-09-02 16:00: the fail-back clone was re-added at the preserved - nsid with the DR lvol's uuid and nguid (deep-copied), but the client's - multipath head for that nsid was built from the ORIGINAL's ids — the - kernel rejected every new path ("IDs don't match for shared namespace N") - and, on the shared subsystem, the eviction at prepare_cutover had already - removed the head's only live path: XFS shut down on running pods. The - clone must advertise the superseded original's uuid AND nguid, the exact - identity _swap_failback_lvol_uuid restores on the record after cutover.""" +def test_failback_clone_keeps_the_client_visible_wire_identity(monkeypatch): + """The fail-back clone must advertise the SOURCE's wire identity — what + the connected client's multipath head currently holds — or the kernel + rejects every preconnected target path ("IDs don't match for shared + namespace N") and deleteSource then removes the head's only live paths + (run 2026-09-02 ~19:00, subsystem 20d8a917: no available path, XFS + shutdown on a restaged pod). Explicitly NOT the superseded original's + uuid: that variant was tried (run 2026-09-02 17:00) and only served + clients still riding the unfenced superseded original, whose writes + fail-back discards anyway.""" from simplyblock_core.controllers import lvol_controller as lc added = [] @@ -519,6 +520,7 @@ def get_lvols(self): class _Lvol: uuid = "DR_ID"; nqn = "nqn.test:lvol:SHARED"; ns_id = 3 guid = "DR_NGUID" + ns_uuid = "" namespace = "" max_namespace_per_subsys = 10 lvol_bdev = "LVOL_28"; crypto_bdev = "" @@ -538,14 +540,30 @@ def get_id(self): new_lvol, error = lc._create_target_lvol_clone( _DB(), _Lvol(), primary, "POOL", _Snap(), for_migration=True) assert error is None - # Every node's add_ns must carry the original's identity, never the DR's. - assert [(nid, ns) for nid, ns, _ in added] == [("P", "ORIG_ID"), ("S", "ORIG_ID")] - assert all(guid == "ORIG_NGUID" for _, _, guid in added) + # Every node's add_ns must carry the DR source's wire identity — what the + # client's head holds — never the superseded original's. + assert [(nid, ns) for nid, ns, _ in added] == [("P", "DR_ID"), ("S", "DR_ID")] + assert all(guid == "DR_NGUID" for _, _, guid in added) # The wire identity is persisted so connect_lvol can report it. - assert new_lvol.ns_uuid == "ORIG_ID" + assert new_lvol.ns_uuid == "DR_ID" # And the clone still got its own bdev name (adoption guard). assert new_lvol.lvol_bdev == "LVOL_999" + # A second fail-back cycle: the DR source's own wire identity is already + # borrowed (its ns_uuid points at an earlier generation). The chain must + # propagate — the client's head knows only the ORIGINAL wire id. + added.clear() + + class _Gen2Lvol(_Lvol): + ns_uuid = "GEN0_WIRE_ID" + + new_lvol, error = lc._create_target_lvol_clone( + _DB(), _Gen2Lvol(), primary, "POOL", _Snap(), for_migration=True) + assert error is None + assert [(nid, ns) for nid, ns, _ in added] == [ + ("P", "GEN0_WIRE_ID"), ("S", "GEN0_WIRE_ID")] + assert new_lvol.ns_uuid == "GEN0_WIRE_ID" + def test_interrupted_landing_volume_is_adopted_or_cleared(): """Case 6, run 20260824_144226: a node outage mid-create left a REP_* From b5ea6d8bef29f79c71927ec5d7d27b50063f60a6 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 3 Sep 2026 00:01:07 +0100 Subject: [PATCH 116/122] fix(failback): gate delete-source on client coverage of the target subsystem so a late preconnect stalls IO instead of severing a mounted namespace --- simplyblock_core/constants.py | 12 ++ .../controllers/health_controller.py | 5 +- .../controllers/lvol_controller.py | 16 ++- .../controllers/migration_controller.py | 5 +- simplyblock_core/models/lvol_model.py | 13 ++ simplyblock_core/services/lvol_monitor.py | 10 +- .../tasks_runner_replication_final.py | 96 ++++++++++++- simplyblock_core/storage_node_ops.py | 16 ++- .../test/test_delete_source_coverage.py | 131 ++++++++++++++++++ .../test/test_wire_ns_identity.py | 105 ++++++++++++++ 10 files changed, 393 insertions(+), 16 deletions(-) create mode 100644 simplyblock_core/test/test_delete_source_coverage.py create mode 100644 simplyblock_core/test/test_wire_ns_identity.py diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index d4d28bebfb..18d6dc6f74 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -373,6 +373,18 @@ def get_config_var(name, default=None): # waiting for POST .../replication/cutover-proceed; this is the fallback deadline # if the operator is unavailable. Cutover proceeds regardless after this many seconds. REPL_CUTOVER_PROCEED_TIMEOUT_SEC = 120 +# delete-source coverage gate: how long the finalize step waits for every client +# connected to the retiring SOURCE subsystem to show a controller on the TARGET +# subsystem before deleting the source. On a SHARED subsystem the delete is an +# nvmf_subsystem_remove_ns over a live connection — the client kernel drops the +# path immediately, with none of the ~60s reconnect grace that masks the same +# race on dedicated subsystems — so deleting before the client holds a target +# path shuts its filesystem down (run 2026-09-02 ~19:40: path removed at t+0, +# replacement attached t+2s, XFS dead in between). On timeout the delete is +# SKIPPED, loudly: the source stays fenced (ANA inaccessible since cutover), +# which is safe indefinitely, and can be deleted once the client is covered. +REPL_DELETE_SOURCE_COVERAGE_TIMEOUT_SEC = 60 +REPL_DELETE_SOURCE_COVERAGE_POLL_SEC = 2 # --- cutover delta convergence ------------------------------------------- # The IO freeze copies everything written since the last replicated snapshot, diff --git a/simplyblock_core/controllers/health_controller.py b/simplyblock_core/controllers/health_controller.py index 6cbb5bb26b..01ed604f56 100644 --- a/simplyblock_core/controllers/health_controller.py +++ b/simplyblock_core/controllers/health_controller.py @@ -1113,7 +1113,10 @@ def check_lvol_on_node(lvol_id, node_id, node_bdev_names=None, node_lvols_nqns=N bdev_check = check_bdev(lvol.top_bdev, rpc_client=rpc_client) passed &= bdev_check - passed &= check_subsystem(lvol.nqn, rpc_client=rpc_client, ns_uuid=lvol.uuid) + # The namespace advertises the WIRE identity, which differs from the + # record uuid after a fail-back — checking the record uuid flags every + # failed-back volume unhealthy and triggers the monitor's self-heal. + passed &= check_subsystem(lvol.nqn, rpc_client=rpc_client, ns_uuid=lvol.get_ns_uuid()) except Exception as e: logger.error(e) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index ac61cd26cc..bab2dfbeb6 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -1493,7 +1493,8 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid f"maps across the shared subsystem's paths)", is_primary=is_primary) requested_nsid = lvol.ns_id ret, err = rpc_client.nvmf_subsystem_add_ns2( - lvol.nqn, lvol.top_bdev, ns_uuid or lvol.uuid, lvol.guid, nsid=requested_nsid) + lvol.nqn, lvol.top_bdev, ns_uuid or lvol.get_ns_uuid(), lvol.guid, + nsid=requested_nsid) if err: def _ns_add_detail(): """What the node actually holds for this subsystem. @@ -1703,7 +1704,7 @@ def recreate_lvol_on_node(lvol, snode, ha_inode_self=None, ana_state=None): # correct nsid as well. Only a record with ns_id unset falls back to # auto-assignment. ret = rpc_client.nvmf_subsystem_add_ns( - lvol.nqn, lvol.top_bdev, lvol.uuid, lvol.guid, + lvol.nqn, lvol.top_bdev, lvol.get_ns_uuid(), lvol.guid, nsid=lvol.ns_id if lvol.ns_id else None) if not ret: # FATAL, deliberately. This used to log and fall through to the @@ -1919,7 +1920,10 @@ def _remove_lvol_subsys_from_node(lvol, rpc_client): return True for ns in subsystem["namespaces"]: - if ns["uuid"] == lvol.uuid: + # Match by the WIRE identity: a failed-back volume's namespace + # advertises ns_uuid, not the record uuid — matching the record uuid + # here leaves the namespace behind on a shared subsystem forever. + if ns["uuid"] == lvol.get_ns_uuid(): logger.info("Removing namespace %s from subsystem %s", ns["uuid"], lvol.nqn) ret = bool(rpc_client.nvmf_subsystem_remove_ns(lvol.nqn, ns['nsid'])) if not ret: @@ -4272,8 +4276,12 @@ def _evict_stale_namespace(new_lvol, target_node, superseded=None): # lvol namespaces' bdev_name as the raw lvol_uuid. superseded_ids = set() if superseded is not None: + # ns_uuid first: a multi-cycle superseded original is itself an + # earlier fail-back clone whose namespace advertises a borrowed + # wire identity, not its record uuid. superseded_ids = { - v for v in (getattr(superseded, "uuid", None), + v for v in (getattr(superseded, "ns_uuid", None), + getattr(superseded, "uuid", None), getattr(superseded, "lvol_uuid", None), getattr(superseded, "top_bdev", None)) if v} diff --git a/simplyblock_core/controllers/migration_controller.py b/simplyblock_core/controllers/migration_controller.py index 49a173f874..8195e04253 100644 --- a/simplyblock_core/controllers/migration_controller.py +++ b/simplyblock_core/controllers/migration_controller.py @@ -1280,8 +1280,11 @@ def create_migration(lvol_id, target_node_id, # namespace occupying a low nsid on the target (or add_ns calls # racing/reordering across nodes) would silently diverge the # source and target nsid maps for this lvol. + # WIRE identity, not the record uuid: a failed-back volume's + # namespace advertises its ns_uuid, and the client's multipath + # head only merges the migration target's path when it matches. _ns = _rpc.nvmf_subsystem_add_ns( - nqn, _ns_bdev, lvol.uuid, lvol.guid, + nqn, _ns_bdev, lvol.get_ns_uuid(), lvol.guid, nsid=lvol.ns_id if lvol.ns_id else None) if _ns: logger.info( diff --git a/simplyblock_core/models/lvol_model.py b/simplyblock_core/models/lvol_model.py index 5335ac6985..f2c48a6239 100644 --- a/simplyblock_core/models/lvol_model.py +++ b/simplyblock_core/models/lvol_model.py @@ -57,6 +57,19 @@ class LVol(BaseModel): # target_lvol_id so the CSI globs /dev/disk/by-id/nvme-uuid.. ns_uuid: str = "" max_namespace_per_subsys: int = 1 + + def get_ns_uuid(self) -> str: + """The UUID this volume's NVMe namespace advertises on the wire. + + Every site that registers or verifies the namespace must use this, + never the record ``uuid``: after a fail-back the two differ, and + probing or re-adding by the record uuid flags a healthy volume + unhealthy — or re-registers the namespace under an identity the + client's multipath head rejects ("IDs don't match for shared + namespace N"), severing its paths (run 2026-09-02 ~19:40: the lvol + monitor's self-heal fought the fail-back identity every cycle). + """ + return self.ns_uuid or self.uuid subsys_port: int = 9090 # Node ids whose sync delete already completed inline in the API delete # call (lvol_controller._delete_lvol_from_all_nodes). lvol_monitor skips diff --git a/simplyblock_core/services/lvol_monitor.py b/simplyblock_core/services/lvol_monitor.py index d7cfd87ab6..ca90799d5e 100644 --- a/simplyblock_core/services/lvol_monitor.py +++ b/simplyblock_core/services/lvol_monitor.py @@ -614,7 +614,12 @@ def check_node(cluster, snode, all_lvols, subsys_check=False): passed = True try: - passed &= health_controller.check_subsystem(lvol.nqn, rpc_client=snode.rpc_client(), ns_uuid=lvol.uuid) + # Verify against the WIRE identity: after a fail-back the record + # uuid differs from the namespace's advertised uuid, and checking + # the record uuid here marked healthy volumes unhealthy and drove + # the self-heal below into re-registering the wrong identity. + passed &= health_controller.check_subsystem( + lvol.nqn, rpc_client=snode.rpc_client(), ns_uuid=lvol.get_ns_uuid()) except Exception as e: logger.error(f"Failed to check lvol:{lvol.get_id()} on node: {lvol.node_id}") logger.error(e) @@ -628,7 +633,8 @@ def check_node(cluster, snode, all_lvols, subsys_check=False): if sec_node and sec_node.status == StorageNode.STATUS_ONLINE: try: ret = health_controller.check_subsystem( - lvol.nqn, rpc_client=sec_node.rpc_client(), ns_uuid=lvol.uuid) + lvol.nqn, rpc_client=sec_node.rpc_client(), + ns_uuid=lvol.get_ns_uuid()) if not ret: passed = False # Explicit, greppable degraded-path signal. Without diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 45053589ca..31f90f0c7e 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -100,6 +100,86 @@ def _release_lvs_claim(task): task.write_to_db(db.kv_store) +def _subsystem_client_hosts(lvol): + """Host NQNs holding a controller on *lvol*'s subsystem, across its HA set. + + Best-effort: an offline or unresponsive node contributes nothing. Every + controller on an lvol subsystem belongs to a client — internal fabric + connections (hublvol, transferhub, JM, devices) use their own NQNs. + """ + hosts = set() + node_ids = getattr(lvol, "nodes", None) or [getattr(lvol, "node_id", "")] + for node_id in node_ids: + if not node_id: + continue + try: + snode = db.get_storage_node_by_id(node_id) + except KeyError: + continue + if snode.status != StorageNode.STATUS_ONLINE: + continue + try: + controllers = snode.rpc_client(timeout=3, retry=1) \ + .nvmf_subsystem_get_controllers(lvol.nqn) or [] + except Exception as e: # noqa: BLE001 + logger.warning("Could not list controllers of %s on %s: %s", + lvol.nqn, node_id, e) + continue + hosts.update(c.get("hostnqn") for c in controllers if c.get("hostnqn")) + return hosts + + +def _await_delete_source_coverage(src_lvol, rep): + """Whether every client of the retiring SOURCE also reached the TARGET. + + deleteSource on a SHARED subsystem is an nvmf_subsystem_remove_ns over a + live connection: the client kernel removes the path immediately and + administratively, so if the client has not yet attached the target path + (late or missed preconnect), its filesystem dies on the spot. A dedicated + subsystem only survives the same race by accident — its subsystem delete + masks the removal behind a connection loss and the ~60s reconnect grace. + + The delete is therefore gated on evidence, not timing: every host NQN + connected to the source subsystem must hold a controller on the target + subsystem. Polls until covered or the deadline; the caller SKIPS the + delete on False — the source stays fenced (ANA inaccessible since the + cutover), which is safe indefinitely. + """ + if rep is None or rep.target_lvol is None: + # Nothing to verify against; keep the pre-gate behavior rather than + # blocking a delete we cannot reason about. + logger.warning("delete-source coverage: no replication target to " + "verify against for %s; deleting without the gate", + src_lvol.get_id()) + return True + + deadline = time.time() + constants.REPL_DELETE_SOURCE_COVERAGE_TIMEOUT_SEC + while True: + src_hosts = _subsystem_client_hosts(src_lvol) + if not src_hosts: + # No client is connected to the source (or none of its nodes + # answered) — there is no path to yank from under anyone. + return True + try: + tgt_lvol = db.get_lvol_by_id(rep.target_lvol.get_id()) + except KeyError: + logger.warning("delete-source coverage: target volume %s not " + "found; deleting without the gate", + rep.target_lvol.get_id()) + return True + uncovered = src_hosts - _subsystem_client_hosts(tgt_lvol) + if not uncovered: + return True + if time.time() >= deadline: + logger.error( + "delete-source coverage: client(s) %s still hold no controller " + "on the target subsystem %s after %ss", + sorted(uncovered), tgt_lvol.nqn, + constants.REPL_DELETE_SOURCE_COVERAGE_TIMEOUT_SEC) + return False + time.sleep(constants.REPL_DELETE_SOURCE_COVERAGE_POLL_SEC) + + def _finalize(task, ok, err): if ok: replication_id = task.function_params.get("replication_id") @@ -171,9 +251,19 @@ def _finalize(task, ok, err): try: from simplyblock_core.controllers import lvol_controller src_lvol = db.get_lvol_by_id(src_lvol_id) - logger.info(f"Cutover committed with --delete-source: deleting " - f"source volume {src_lvol_id}") - lvol_controller.delete_lvol(src_lvol) + if _await_delete_source_coverage(src_lvol, rep): + logger.info(f"Cutover committed with --delete-source: deleting " + f"source volume {src_lvol_id}") + lvol_controller.delete_lvol(src_lvol) + else: + # Deleting now would remove a connected client's only path + # (namespace removal has no reconnect grace on shared + # subsystems). The source is already fenced by the cutover, + # so leaving it is safe; delete it manually once the client + # holds target paths. + logger.error( + f"delete-source SKIPPED for {src_lvol_id}: client not " + f"yet covered on the target; source stays fenced") except Exception as e: # The cutover itself succeeded; a failed source delete is # reported loudly but does not un-succeed the task. diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index f68eeb2990..65a54464e5 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -11061,14 +11061,18 @@ def add_lvol_thread(lvol, snode: StorageNode, lvol_ana_state="optimized"): return False, msg # Add NS to subsystem (idempotent: skip if already bound with matching NSID). + # Probe and register by the WIRE identity (get_ns_uuid), never the record + # uuid: after a fail-back the two differ, and re-adding under the record + # uuid presents an identity the client's multipath head rejects ("IDs + # don't match for shared namespace N"), severing its paths. if _rpc_subsystem_has_ns(rpc_client, lvol.nqn, nsid=lvol.ns_id, - bdev_name=lvol.top_bdev, uuid=lvol.uuid): + bdev_name=lvol.top_bdev, uuid=lvol.get_ns_uuid()): logger.info("Namespace nsid=%s already on subsystem %s, skipping add_ns", lvol.ns_id, lvol.nqn) else: logger.info("Add BDev to subsystem " + f"{lvol.vuid:016X}") if not rpc_client.nvmf_subsystem_add_ns( - lvol.nqn, lvol.top_bdev, lvol.uuid, lvol.guid, nsid=lvol.ns_id): + lvol.nqn, lvol.top_bdev, lvol.get_ns_uuid(), lvol.guid, nsid=lvol.ns_id): # An add_ns error is not by itself a reason to abandon the whole # registration. What matters for the client is whether the # namespace is on the subsystem now — it may already have been, @@ -11084,7 +11088,8 @@ def add_lvol_thread(lvol, snode: StorageNode, lvol_ana_state="optimized"): # Re-read the subsystem and only give up if the namespace is # genuinely absent. if _rpc_wait_subsystem_has_ns(rpc_client, lvol.nqn, nsid=lvol.ns_id, - bdev_name=lvol.top_bdev, uuid=lvol.uuid): + bdev_name=lvol.top_bdev, + uuid=lvol.get_ns_uuid()): logger.warning( "add_ns for nsid=%s (%s) on %s reported failure but the " "namespace is present; continuing to listener setup", @@ -11114,9 +11119,10 @@ def add_lvol_thread(lvol, snode: StorageNode, lvol_ana_state="optimized"): # path loss the control plane never flagged, re-refused by the lvol-monitor # repair loop on every cycle. if not _rpc_wait_subsystem_has_ns(rpc_client, lvol.nqn, nsid=lvol.ns_id, - bdev_name=lvol.top_bdev, uuid=lvol.uuid): + bdev_name=lvol.top_bdev, + uuid=lvol.get_ns_uuid()): msg = (f"Subsystem {lvol.nqn} on {snode.get_id()} has no namespace " - f"nsid={lvol.ns_id} ({lvol.top_bdev}, uuid={lvol.uuid}) after " + f"nsid={lvol.ns_id} ({lvol.top_bdev}, uuid={lvol.get_ns_uuid()}) after " f"registration; refusing to add a listener for an empty subsystem") logger.error(msg) return False, msg diff --git a/simplyblock_core/test/test_delete_source_coverage.py b/simplyblock_core/test/test_delete_source_coverage.py new file mode 100644 index 0000000000..f6f195736e --- /dev/null +++ b/simplyblock_core/test/test_delete_source_coverage.py @@ -0,0 +1,131 @@ +"""The delete-source coverage gate: never remove a client's source path until +that client demonstrably holds a controller on the target subsystem. + +deleteSource on a SHARED subsystem is an nvmf_subsystem_remove_ns over a live +connection — the client kernel drops the path immediately, with none of the +~60s reconnect grace that masks the same race on dedicated subsystems. Run +2026-09-02 ~19:40: the source namespace was removed at t+0, the target path +attached at t+2s, and XFS shut down in between. The gate replaces the blind +time-based delete with an evidence-based one. +""" +import inspect +from unittest.mock import MagicMock + +import pytest + +from simplyblock_core.models.lvol_model import LVol, LVolReplication +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import tasks_runner_replication_final as runner + + +CLIENT = "nqn.2014-08.io.simplyblock:uuid:client-1" + + +def _lvol(uuid, nqn, node_id): + lvol = LVol() + lvol.uuid = uuid + lvol.nqn = nqn + lvol.node_id = node_id + lvol.nodes = [node_id] + return lvol + + +def _node(controllers_by_nqn): + node = MagicMock() + node.status = StorageNode.STATUS_ONLINE + rpc = MagicMock() + rpc.nvmf_subsystem_get_controllers.side_effect = \ + lambda nqn: controllers_by_nqn.get(nqn, []) + node.rpc_client.return_value = rpc + return node + + +@pytest.fixture() +def _fast(monkeypatch): + """Collapse the poll loop so uncovered cases fail fast.""" + monkeypatch.setattr(runner.constants, + "REPL_DELETE_SOURCE_COVERAGE_TIMEOUT_SEC", 0) + monkeypatch.setattr(runner.time, "sleep", lambda s: None) + + +def _wire(monkeypatch, src_node, tgt_node, tgt_lvol): + db = MagicMock() + db.get_storage_node_by_id.side_effect = \ + lambda nid: {"SRC_N": src_node, "TGT_N": tgt_node}[nid] + db.get_lvol_by_id.return_value = tgt_lvol + monkeypatch.setattr(runner, "db", db) + + +def _rep(tgt_lvol): + rep = LVolReplication() + rep.target_lvol = tgt_lvol + return rep + + +def test_covered_client_lets_the_delete_proceed(monkeypatch, _fast): + src = _lvol("SRC", "nqn.test:lvol:S", "SRC_N") + tgt = _lvol("TGT", "nqn.test:lvol:S", "TGT_N") + src_node = _node({"nqn.test:lvol:S": [{"hostnqn": CLIENT}]}) + tgt_node = _node({"nqn.test:lvol:S": [{"hostnqn": CLIENT}]}) + _wire(monkeypatch, src_node, tgt_node, tgt) + assert runner._await_delete_source_coverage(src, _rep(tgt)) is True + + +def test_uncovered_client_blocks_the_delete(monkeypatch, _fast): + """The exact 2026-09-02 shape: the client still rides only the source.""" + src = _lvol("SRC", "nqn.test:lvol:S", "SRC_N") + tgt = _lvol("TGT", "nqn.test:lvol:S", "TGT_N") + src_node = _node({"nqn.test:lvol:S": [{"hostnqn": CLIENT}]}) + tgt_node = _node({"nqn.test:lvol:S": []}) + _wire(monkeypatch, src_node, tgt_node, tgt) + assert runner._await_delete_source_coverage(src, _rep(tgt)) is False + + +def test_no_connected_client_means_nothing_to_protect(monkeypatch, _fast): + src = _lvol("SRC", "nqn.test:lvol:S", "SRC_N") + tgt = _lvol("TGT", "nqn.test:lvol:S", "TGT_N") + src_node = _node({"nqn.test:lvol:S": []}) + tgt_node = _node({"nqn.test:lvol:S": []}) + _wire(monkeypatch, src_node, tgt_node, tgt) + assert runner._await_delete_source_coverage(src, _rep(tgt)) is True + + +def test_coverage_arriving_during_the_poll_unblocks(monkeypatch): + """A late preconnect attaching mid-wait releases the gate — the normal + recovery path when the operator's preconnect lands after the flip.""" + monkeypatch.setattr(runner.constants, + "REPL_DELETE_SOURCE_COVERAGE_TIMEOUT_SEC", 60) + monkeypatch.setattr(runner.constants, + "REPL_DELETE_SOURCE_COVERAGE_POLL_SEC", 0) + monkeypatch.setattr(runner.time, "sleep", lambda s: None) + + src = _lvol("SRC", "nqn.test:lvol:S", "SRC_N") + tgt = _lvol("TGT", "nqn.test:lvol:S", "TGT_N") + src_node = _node({"nqn.test:lvol:S": [{"hostnqn": CLIENT}]}) + tgt_answers = iter([[], [], [{"hostnqn": CLIENT}]]) + tgt_node = MagicMock() + tgt_node.status = StorageNode.STATUS_ONLINE + tgt_rpc = MagicMock() + tgt_rpc.nvmf_subsystem_get_controllers.side_effect = \ + lambda nqn: next(tgt_answers) + tgt_node.rpc_client.return_value = tgt_rpc + _wire(monkeypatch, src_node, tgt_node, tgt) + assert runner._await_delete_source_coverage(src, _rep(tgt)) is True + + +def test_without_a_target_reference_the_gate_stands_aside(monkeypatch, _fast): + """No relationship to verify against: keep the pre-gate delete behavior + rather than blocking a delete the gate cannot reason about.""" + src = _lvol("SRC", "nqn.test:lvol:S", "SRC_N") + monkeypatch.setattr(runner, "db", MagicMock()) + assert runner._await_delete_source_coverage(src, None) is True + + +def test_finalize_delete_is_gated(): + """Guard: the delete-source block must consult the coverage gate and skip + (not fail the task) when it reports False.""" + src = inspect.getsource(runner._finalize) + gate = src.index("_await_delete_source_coverage") + delete = src.index("lvol_controller.delete_lvol") + assert gate < delete, "coverage must be checked before deleting the source" + assert "delete-source SKIPPED" in src diff --git a/simplyblock_core/test/test_wire_ns_identity.py b/simplyblock_core/test/test_wire_ns_identity.py new file mode 100644 index 0000000000..da5579599b --- /dev/null +++ b/simplyblock_core/test/test_wire_ns_identity.py @@ -0,0 +1,105 @@ +"""The wire namespace identity (LVol.get_ns_uuid) is what every verify, +register, and remove site must use — never the record uuid. + +After a fail-back the two differ by design: the record carries the restored +original uuid while the namespace keeps advertising the DR-generation wire +identity the client's multipath head holds. Run 2026-09-02 ~19:40: the health +check compared the record uuid, flagged every failed-back volume unhealthy +(Health: False), and the lvol monitor's self-heal then re-registered the +namespace under the record uuid — an identity the client kernel rejects +("IDs don't match for shared namespace N") — severing live paths. +""" +import inspect +from unittest.mock import MagicMock, patch + +from simplyblock_core.models.lvol_model import LVol + + +def _failed_back_lvol(): + lvol = LVol() + lvol.uuid = "REC" + lvol.ns_uuid = "WIRE" + lvol.nqn = "nqn.test:lvol:SHARED" + lvol.ns_id = 1 + lvol.top_bdev = "LVS_1/LVOL_C" + lvol.status = LVol.STATUS_ONLINE + return lvol + + +def test_get_ns_uuid_prefers_the_borrowed_wire_identity(): + lvol = _failed_back_lvol() + assert lvol.get_ns_uuid() == "WIRE" + lvol.ns_uuid = "" + assert lvol.get_ns_uuid() == "REC" + + +def test_health_check_verifies_the_wire_identity(monkeypatch): + """check_lvol_on_node must hand check_subsystem the WIRE identity, or a + healthy failed-back volume reports unhealthy forever.""" + from simplyblock_core.controllers import health_controller as hc + + lvol = _failed_back_lvol() + lvol.bdev_stack = [] + + db = MagicMock() + db.get_lvol_by_id.return_value = lvol + db.get_storage_node_by_id.return_value = MagicMock() + + captured = {} + + def _fake_check_subsystem(nqn, *, rpc_client=None, nqns=None, ns_uuid=None): + captured["ns_uuid"] = ns_uuid + return True + + with patch.object(hc, "DBController", lambda: db), \ + patch.object(hc, "check_subsystem", _fake_check_subsystem): + assert hc.check_lvol_on_node("REC", "N1") is True + assert captured["ns_uuid"] == "WIRE" + + +def test_delete_removes_the_namespace_by_its_wire_identity(monkeypatch): + """_remove_lvol_subsys_from_node matched ns['uuid'] against the record + uuid; for a failed-back volume that never matches, and the namespace is + left behind on the shared subsystem forever.""" + from simplyblock_core.controllers import lvol_controller as lc + + lvol = _failed_back_lvol() + + removed = [] + + class _Rpc: + def __init__(self): + self.namespaces = [ + {"nsid": 1, "uuid": "WIRE", "bdev_name": "LVS_1/LVOL_C"}, + {"nsid": 2, "uuid": "SIBLING", "bdev_name": "LVS_1/LVOL_S"}, + ] + + def subsystem_get(self, nqn): + return {"nqn": nqn, "namespaces": list(self.namespaces)} + + def nvmf_subsystem_remove_ns(self, nqn, nsid): + removed.append(nsid) + self.namespaces = [n for n in self.namespaces if n["nsid"] != nsid] + return True + + monkeypatch.setattr(lc.time, "sleep", lambda s: None) + assert lc._remove_lvol_subsys_from_node(lvol, _Rpc()) is True + assert removed == [1] + + +def test_registration_sites_use_the_wire_identity(): + """Every path that (re)registers an lvol namespace must pass + get_ns_uuid(): the monitor's self-heal, node-restart recreation, and + intra-cluster migration all re-run against records whose namespace may + carry a borrowed identity.""" + from simplyblock_core import storage_node_ops + from simplyblock_core.controllers import lvol_controller, migration_controller + + for func in (storage_node_ops.add_lvol_thread, + lvol_controller.recreate_lvol_on_node, + migration_controller.create_migration): + src = inspect.getsource(func) + assert "get_ns_uuid()" in src, \ + f"{func.__name__} must register/verify by the wire identity" + assert "lvol.uuid, lvol.guid" not in src, \ + f"{func.__name__} still registers by the record uuid" From 00206ddebc8661a802b99b24abd8a11eca6de198 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 3 Sep 2026 11:17:38 +0100 Subject: [PATCH 117/122] fix(failover): fence and unpublish the source volume's data path so a still-alive source cannot keep serving superseded data --- .../controllers/lvol_controller.py | 54 +++++++++ simplyblock_core/services/lvol_monitor.py | 8 ++ simplyblock_core/storage_node_ops.py | 6 + .../test/test_failover_retires_source.py | 108 ++++++++++++++++++ 4 files changed, 176 insertions(+) create mode 100644 simplyblock_core/test/test_failover_retires_source.py diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index bab2dfbeb6..9bb2f14323 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -4417,6 +4417,54 @@ def resolve_replication_destination(db_controller, lvol, target_node, source_nod return target_cluster, "" +def _retire_source_data_path(db_controller, lvol): + """Fence and unpublish the SOURCE volume's data path after a fail-over. + + The source is only ASSUMED dead. When it is in fact alive (administrative + or test fail-over), leaving its path published breaks everything + downstream: the client keeps writing to the superseded original (all of + it discarded at fail-back), and a namespaced volume's restage re-attaches + to the ORIGINAL device — the shared subsystem's controllers survive the + unstage, the stale device matches the same model/nsid glob, and the DR + clone's namespace is rejected against the original-identity head ("IDs + don't match for shared namespace N") — so pods never actually reach the + DR cluster (run 2026-09-03 ~09:00: all three namespaced pods remounted + on the originals after fail-over; fail-back's eviction then shut down + all three filesystems mid-IO). + + Best-effort: an unreachable source (a genuine DR event) changes nothing + here, and a source that recovers later comes back without the namespace + registered — fenced by absence. The DB record stays (it is the fail-back + delta base); its from_source=False marks it retired so the lvol monitor + and the non-leader repair do not register the namespace back. + """ + try: + if not suspend_lvol(lvol.get_id()): + logger.warning("Fail-over: could not fence the source listeners " + "of %s", lvol.get_id()) + except Exception as e: # noqa: BLE001 + logger.warning("Fail-over: fencing the source of %s failed: %s", + lvol.get_id(), e) + + for node_id in (lvol.nodes or [lvol.node_id]): + if not node_id: + continue + try: + snode = db_controller.get_storage_node_by_id(node_id) + except KeyError: + continue + if snode.status != StorageNode.STATUS_ONLINE: + continue + try: + if not _remove_lvol_subsys_from_node( + lvol, snode.rpc_client(timeout=5, retry=1)): + logger.warning("Fail-over: source namespace of %s not removed " + "on %s", lvol.get_id(), node_id) + except Exception as e: # noqa: BLE001 + logger.warning("Fail-over: removing the source namespace of %s " + "on %s failed: %s", lvol.get_id(), node_id, e) + + def replicate_lvol_on_target_cluster(lvol_id, generation=0): db_controller = DBController() try: @@ -4522,6 +4570,12 @@ def replicate_lvol_on_target_cluster(lvol_id, generation=0): lvol_events.lvol_replicated(lvol, new_lvol) + # The relationship is durable and the DR copy is online: retire the + # source's data path NOW (fence + namespace removal, best-effort) so a + # still-alive source cannot keep serving superseded data. See + # _retire_source_data_path for the failure chain this closes. + _retire_source_data_path(db_controller, lvol) + # Provide the new connection paths (primary/secondary/tertiary) — identical # NQN, different IP/port — so the client can fail over to the target cluster. connection_strings = [] diff --git a/simplyblock_core/services/lvol_monitor.py b/simplyblock_core/services/lvol_monitor.py index ca90799d5e..c11c9e98e0 100644 --- a/simplyblock_core/services/lvol_monitor.py +++ b/simplyblock_core/services/lvol_monitor.py @@ -609,6 +609,14 @@ def check_node(cluster, snode, all_lvols, subsys_check=False): if lvol.status not in (LVol.STATUS_ONLINE, LVol.STATUS_OFFLINE): continue + if not getattr(lvol, "from_source", True): + # The retired SOURCE of a fail-over: its namespace was removed + # deliberately (_retire_source_data_path) and the record only + # survives as the fail-back delta base. Checking it always fails, + # and the self-heal below would register the namespace back — + # resurrecting the stale device the retirement removed. + continue + if snode.lvstore_status != "ready": continue diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 65a54464e5..3719efb5b9 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -11235,6 +11235,12 @@ def repair_lvol_registration_on_non_leader(lvol, sec_node: StorageNode, secondar if lvol.status not in (LVol.STATUS_ONLINE, LVol.STATUS_OFFLINE): return False, (f"LVol {lvol.get_id()} status is {lvol.status}, " f"not repairing registration") + if not getattr(lvol, "from_source", True): + # Retired fail-over source: its namespace was removed deliberately + # (_retire_source_data_path). Registering it back republishes the + # superseded data path the fail-over just fenced. + return False, (f"LVol {lvol.get_id()} is the retired source of a " + f"fail-over, not re-registering its namespace") rpc_client = sec_node.rpc_client(timeout=10, retry=2) if rpc_client.subsystem_get(lvol.nqn) is None: diff --git a/simplyblock_core/test/test_failover_retires_source.py b/simplyblock_core/test/test_failover_retires_source.py new file mode 100644 index 0000000000..87389489c6 --- /dev/null +++ b/simplyblock_core/test/test_failover_retires_source.py @@ -0,0 +1,108 @@ +"""Fail-over must retire the still-alive source's data path. + +The source is only assumed dead. Left published, the client keeps writing to +the superseded original (discarded at fail-back), and a namespaced volume's +restage re-attaches to the stale ORIGINAL device — same model/nsid glob, DR +clone rejected against the original-identity head — so pods never reach the +DR cluster (run 2026-09-03 ~09:00: all three namespaced pods remounted on the +originals after fail-over; fail-back's eviction then shut down all three +filesystems mid-IO). +""" +import inspect +from unittest.mock import MagicMock + +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.controllers import lvol_controller as lc + + +def _lvol(): + lvol = LVol() + lvol.uuid = "SRC" + lvol.nqn = "nqn.test:lvol:S" + lvol.node_id = "N1" + lvol.nodes = ["N1", "N2"] + return lvol + + +def _node(status=StorageNode.STATUS_ONLINE): + node = MagicMock() + node.status = status + return node + + +def test_retire_fences_then_removes_on_every_online_node(monkeypatch): + fenced, removed = [], [] + monkeypatch.setattr(lc, "suspend_lvol", + lambda lvol_id: fenced.append(lvol_id) or True) + monkeypatch.setattr(lc, "_remove_lvol_subsys_from_node", + lambda lvol, rpc: removed.append(lvol.get_id()) or True) + db = MagicMock() + db.get_storage_node_by_id.side_effect = \ + lambda nid: {"N1": _node(), "N2": _node()}[nid] + lc._retire_source_data_path(db, _lvol()) + assert fenced == ["SRC"] + assert removed == ["SRC", "SRC"], "namespace removed on primary AND peer" + + +def test_retire_skips_offline_nodes(monkeypatch): + removed = [] + monkeypatch.setattr(lc, "suspend_lvol", lambda lvol_id: True) + monkeypatch.setattr(lc, "_remove_lvol_subsys_from_node", + lambda lvol, rpc: removed.append(rpc) or True) + db = MagicMock() + db.get_storage_node_by_id.side_effect = lambda nid: { + "N1": _node(), "N2": _node(status=StorageNode.STATUS_OFFLINE)}[nid] + lc._retire_source_data_path(db, _lvol()) + assert len(removed) == 1 + + +def test_retire_tolerates_an_unreachable_source(monkeypatch): + """A genuine DR event: fencing raises, nodes unknown — the fail-over must + not be aborted by its own best-effort cleanup.""" + def _boom(lvol_id): + raise RuntimeError("source cluster unreachable") + + monkeypatch.setattr(lc, "suspend_lvol", _boom) + db = MagicMock() + db.get_storage_node_by_id.side_effect = KeyError("gone") + lc._retire_source_data_path(db, _lvol()) # must not raise + + +def test_failover_retires_the_source_after_the_relationship_is_durable(): + """Ordering guard: by the time the source path disappears, connect_lvol + must already resolve the volume to the DR copy — so the relationship + write comes first, the retire after.""" + src = inspect.getsource(lc.replicate_lvol_on_target_cluster) + rel = src.index("lvol_replication.write_to_db") + retire = src.index("_retire_source_data_path") + assert rel < retire + + +def test_monitor_skips_retired_sources(): + """The monitor's health check would fail on the deliberately-removed + namespace and its self-heal would register it back — resurrecting the + stale device the retirement removed.""" + from simplyblock_core.services import lvol_monitor + src = inspect.getsource(lvol_monitor) + guard = src.index('from_source') + check = src.index("check_subsystem") + assert guard < check, "the from_source guard must run before any check" + + +def test_repair_refuses_retired_sources(monkeypatch): + from simplyblock_core import storage_node_ops as ops + + lvol = _lvol() + lvol.status = LVol.STATUS_ONLINE + lvol.from_source = False + lvol.lvs_name = "LVS_1" + + monkeypatch.setattr(ops, "get_restart_phase", lambda *a: "") + db = MagicMock() + db.get_lvol_by_id.return_value = lvol + monkeypatch.setattr(ops, "DBController", lambda: db) + + ok, err = ops.repair_lvol_registration_on_non_leader(lvol, _node(), 0) + assert ok is False + assert "retired source" in err From 167a4c56d1d64320b4264319219e349b5885d946 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 3 Sep 2026 12:03:03 +0100 Subject: [PATCH 118/122] =?UTF-8?q?revert(replication):=20drop=20delete-so?= =?UTF-8?q?urce=20coverage=20gate=20=E2=80=94=20failover=20source=20retire?= =?UTF-8?q?ment=20makes=20it=20redundant?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- simplyblock_core/constants.py | 12 -- .../tasks_runner_replication_final.py | 96 +------------ .../test/test_delete_source_coverage.py | 131 ------------------ 3 files changed, 3 insertions(+), 236 deletions(-) delete mode 100644 simplyblock_core/test/test_delete_source_coverage.py diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 18d6dc6f74..d4d28bebfb 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -373,18 +373,6 @@ def get_config_var(name, default=None): # waiting for POST .../replication/cutover-proceed; this is the fallback deadline # if the operator is unavailable. Cutover proceeds regardless after this many seconds. REPL_CUTOVER_PROCEED_TIMEOUT_SEC = 120 -# delete-source coverage gate: how long the finalize step waits for every client -# connected to the retiring SOURCE subsystem to show a controller on the TARGET -# subsystem before deleting the source. On a SHARED subsystem the delete is an -# nvmf_subsystem_remove_ns over a live connection — the client kernel drops the -# path immediately, with none of the ~60s reconnect grace that masks the same -# race on dedicated subsystems — so deleting before the client holds a target -# path shuts its filesystem down (run 2026-09-02 ~19:40: path removed at t+0, -# replacement attached t+2s, XFS dead in between). On timeout the delete is -# SKIPPED, loudly: the source stays fenced (ANA inaccessible since cutover), -# which is safe indefinitely, and can be deleted once the client is covered. -REPL_DELETE_SOURCE_COVERAGE_TIMEOUT_SEC = 60 -REPL_DELETE_SOURCE_COVERAGE_POLL_SEC = 2 # --- cutover delta convergence ------------------------------------------- # The IO freeze copies everything written since the last replicated snapshot, diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 31f90f0c7e..45053589ca 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -100,86 +100,6 @@ def _release_lvs_claim(task): task.write_to_db(db.kv_store) -def _subsystem_client_hosts(lvol): - """Host NQNs holding a controller on *lvol*'s subsystem, across its HA set. - - Best-effort: an offline or unresponsive node contributes nothing. Every - controller on an lvol subsystem belongs to a client — internal fabric - connections (hublvol, transferhub, JM, devices) use their own NQNs. - """ - hosts = set() - node_ids = getattr(lvol, "nodes", None) or [getattr(lvol, "node_id", "")] - for node_id in node_ids: - if not node_id: - continue - try: - snode = db.get_storage_node_by_id(node_id) - except KeyError: - continue - if snode.status != StorageNode.STATUS_ONLINE: - continue - try: - controllers = snode.rpc_client(timeout=3, retry=1) \ - .nvmf_subsystem_get_controllers(lvol.nqn) or [] - except Exception as e: # noqa: BLE001 - logger.warning("Could not list controllers of %s on %s: %s", - lvol.nqn, node_id, e) - continue - hosts.update(c.get("hostnqn") for c in controllers if c.get("hostnqn")) - return hosts - - -def _await_delete_source_coverage(src_lvol, rep): - """Whether every client of the retiring SOURCE also reached the TARGET. - - deleteSource on a SHARED subsystem is an nvmf_subsystem_remove_ns over a - live connection: the client kernel removes the path immediately and - administratively, so if the client has not yet attached the target path - (late or missed preconnect), its filesystem dies on the spot. A dedicated - subsystem only survives the same race by accident — its subsystem delete - masks the removal behind a connection loss and the ~60s reconnect grace. - - The delete is therefore gated on evidence, not timing: every host NQN - connected to the source subsystem must hold a controller on the target - subsystem. Polls until covered or the deadline; the caller SKIPS the - delete on False — the source stays fenced (ANA inaccessible since the - cutover), which is safe indefinitely. - """ - if rep is None or rep.target_lvol is None: - # Nothing to verify against; keep the pre-gate behavior rather than - # blocking a delete we cannot reason about. - logger.warning("delete-source coverage: no replication target to " - "verify against for %s; deleting without the gate", - src_lvol.get_id()) - return True - - deadline = time.time() + constants.REPL_DELETE_SOURCE_COVERAGE_TIMEOUT_SEC - while True: - src_hosts = _subsystem_client_hosts(src_lvol) - if not src_hosts: - # No client is connected to the source (or none of its nodes - # answered) — there is no path to yank from under anyone. - return True - try: - tgt_lvol = db.get_lvol_by_id(rep.target_lvol.get_id()) - except KeyError: - logger.warning("delete-source coverage: target volume %s not " - "found; deleting without the gate", - rep.target_lvol.get_id()) - return True - uncovered = src_hosts - _subsystem_client_hosts(tgt_lvol) - if not uncovered: - return True - if time.time() >= deadline: - logger.error( - "delete-source coverage: client(s) %s still hold no controller " - "on the target subsystem %s after %ss", - sorted(uncovered), tgt_lvol.nqn, - constants.REPL_DELETE_SOURCE_COVERAGE_TIMEOUT_SEC) - return False - time.sleep(constants.REPL_DELETE_SOURCE_COVERAGE_POLL_SEC) - - def _finalize(task, ok, err): if ok: replication_id = task.function_params.get("replication_id") @@ -251,19 +171,9 @@ def _finalize(task, ok, err): try: from simplyblock_core.controllers import lvol_controller src_lvol = db.get_lvol_by_id(src_lvol_id) - if _await_delete_source_coverage(src_lvol, rep): - logger.info(f"Cutover committed with --delete-source: deleting " - f"source volume {src_lvol_id}") - lvol_controller.delete_lvol(src_lvol) - else: - # Deleting now would remove a connected client's only path - # (namespace removal has no reconnect grace on shared - # subsystems). The source is already fenced by the cutover, - # so leaving it is safe; delete it manually once the client - # holds target paths. - logger.error( - f"delete-source SKIPPED for {src_lvol_id}: client not " - f"yet covered on the target; source stays fenced") + logger.info(f"Cutover committed with --delete-source: deleting " + f"source volume {src_lvol_id}") + lvol_controller.delete_lvol(src_lvol) except Exception as e: # The cutover itself succeeded; a failed source delete is # reported loudly but does not un-succeed the task. diff --git a/simplyblock_core/test/test_delete_source_coverage.py b/simplyblock_core/test/test_delete_source_coverage.py deleted file mode 100644 index f6f195736e..0000000000 --- a/simplyblock_core/test/test_delete_source_coverage.py +++ /dev/null @@ -1,131 +0,0 @@ -"""The delete-source coverage gate: never remove a client's source path until -that client demonstrably holds a controller on the target subsystem. - -deleteSource on a SHARED subsystem is an nvmf_subsystem_remove_ns over a live -connection — the client kernel drops the path immediately, with none of the -~60s reconnect grace that masks the same race on dedicated subsystems. Run -2026-09-02 ~19:40: the source namespace was removed at t+0, the target path -attached at t+2s, and XFS shut down in between. The gate replaces the blind -time-based delete with an evidence-based one. -""" -import inspect -from unittest.mock import MagicMock - -import pytest - -from simplyblock_core.models.lvol_model import LVol, LVolReplication -from simplyblock_core.models.storage_node import StorageNode -from simplyblock_core.services import tasks_runner_replication_final as runner - - -CLIENT = "nqn.2014-08.io.simplyblock:uuid:client-1" - - -def _lvol(uuid, nqn, node_id): - lvol = LVol() - lvol.uuid = uuid - lvol.nqn = nqn - lvol.node_id = node_id - lvol.nodes = [node_id] - return lvol - - -def _node(controllers_by_nqn): - node = MagicMock() - node.status = StorageNode.STATUS_ONLINE - rpc = MagicMock() - rpc.nvmf_subsystem_get_controllers.side_effect = \ - lambda nqn: controllers_by_nqn.get(nqn, []) - node.rpc_client.return_value = rpc - return node - - -@pytest.fixture() -def _fast(monkeypatch): - """Collapse the poll loop so uncovered cases fail fast.""" - monkeypatch.setattr(runner.constants, - "REPL_DELETE_SOURCE_COVERAGE_TIMEOUT_SEC", 0) - monkeypatch.setattr(runner.time, "sleep", lambda s: None) - - -def _wire(monkeypatch, src_node, tgt_node, tgt_lvol): - db = MagicMock() - db.get_storage_node_by_id.side_effect = \ - lambda nid: {"SRC_N": src_node, "TGT_N": tgt_node}[nid] - db.get_lvol_by_id.return_value = tgt_lvol - monkeypatch.setattr(runner, "db", db) - - -def _rep(tgt_lvol): - rep = LVolReplication() - rep.target_lvol = tgt_lvol - return rep - - -def test_covered_client_lets_the_delete_proceed(monkeypatch, _fast): - src = _lvol("SRC", "nqn.test:lvol:S", "SRC_N") - tgt = _lvol("TGT", "nqn.test:lvol:S", "TGT_N") - src_node = _node({"nqn.test:lvol:S": [{"hostnqn": CLIENT}]}) - tgt_node = _node({"nqn.test:lvol:S": [{"hostnqn": CLIENT}]}) - _wire(monkeypatch, src_node, tgt_node, tgt) - assert runner._await_delete_source_coverage(src, _rep(tgt)) is True - - -def test_uncovered_client_blocks_the_delete(monkeypatch, _fast): - """The exact 2026-09-02 shape: the client still rides only the source.""" - src = _lvol("SRC", "nqn.test:lvol:S", "SRC_N") - tgt = _lvol("TGT", "nqn.test:lvol:S", "TGT_N") - src_node = _node({"nqn.test:lvol:S": [{"hostnqn": CLIENT}]}) - tgt_node = _node({"nqn.test:lvol:S": []}) - _wire(monkeypatch, src_node, tgt_node, tgt) - assert runner._await_delete_source_coverage(src, _rep(tgt)) is False - - -def test_no_connected_client_means_nothing_to_protect(monkeypatch, _fast): - src = _lvol("SRC", "nqn.test:lvol:S", "SRC_N") - tgt = _lvol("TGT", "nqn.test:lvol:S", "TGT_N") - src_node = _node({"nqn.test:lvol:S": []}) - tgt_node = _node({"nqn.test:lvol:S": []}) - _wire(monkeypatch, src_node, tgt_node, tgt) - assert runner._await_delete_source_coverage(src, _rep(tgt)) is True - - -def test_coverage_arriving_during_the_poll_unblocks(monkeypatch): - """A late preconnect attaching mid-wait releases the gate — the normal - recovery path when the operator's preconnect lands after the flip.""" - monkeypatch.setattr(runner.constants, - "REPL_DELETE_SOURCE_COVERAGE_TIMEOUT_SEC", 60) - monkeypatch.setattr(runner.constants, - "REPL_DELETE_SOURCE_COVERAGE_POLL_SEC", 0) - monkeypatch.setattr(runner.time, "sleep", lambda s: None) - - src = _lvol("SRC", "nqn.test:lvol:S", "SRC_N") - tgt = _lvol("TGT", "nqn.test:lvol:S", "TGT_N") - src_node = _node({"nqn.test:lvol:S": [{"hostnqn": CLIENT}]}) - tgt_answers = iter([[], [], [{"hostnqn": CLIENT}]]) - tgt_node = MagicMock() - tgt_node.status = StorageNode.STATUS_ONLINE - tgt_rpc = MagicMock() - tgt_rpc.nvmf_subsystem_get_controllers.side_effect = \ - lambda nqn: next(tgt_answers) - tgt_node.rpc_client.return_value = tgt_rpc - _wire(monkeypatch, src_node, tgt_node, tgt) - assert runner._await_delete_source_coverage(src, _rep(tgt)) is True - - -def test_without_a_target_reference_the_gate_stands_aside(monkeypatch, _fast): - """No relationship to verify against: keep the pre-gate delete behavior - rather than blocking a delete the gate cannot reason about.""" - src = _lvol("SRC", "nqn.test:lvol:S", "SRC_N") - monkeypatch.setattr(runner, "db", MagicMock()) - assert runner._await_delete_source_coverage(src, None) is True - - -def test_finalize_delete_is_gated(): - """Guard: the delete-source block must consult the coverage gate and skip - (not fail the task) when it reports False.""" - src = inspect.getsource(runner._finalize) - gate = src.index("_await_delete_source_coverage") - delete = src.index("lvol_controller.delete_lvol") - assert gate < delete, "coverage must be checked before deleting the source" - assert "delete-source SKIPPED" in src From 9832bd0144324acaf8f60b17842c1ec448304eb3 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 3 Sep 2026 16:02:51 +0100 Subject: [PATCH 119/122] =?UTF-8?q?fix:=20green=20the=20unit=20suite=20?= =?UTF-8?q?=E2=80=94=203=20production=20bugs,=20stale=20cutover=20tests,?= =?UTF-8?q?=20lint=20and=20types?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/repl_soak.py | 0 scripts/setup_perf_test1.py | 2 +- scripts/xfer_timing_report.py | 4 +- .../consistency_group_controller.py | 11 +++-- .../controllers/lvol_controller.py | 15 +++++-- .../models/lvol_migration_group.py | 11 ----- simplyblock_core/models/replication.py | 4 +- simplyblock_core/services/snapshot_monitor.py | 13 ++++-- .../services/snapshot_replication.py | 2 +- .../tasks_runner_replication_final.py | 13 +++--- .../test/test_cutover_convergence.py | 18 +++++--- .../test/test_failover_retires_source.py | 18 +++++--- .../test_replication_chain_completeness.py | 6 +-- .../test/test_replication_partial_transfer.py | 19 +++++---- .../test_tasks_runner_replication_final.py | 41 +++++++++++++++++-- .../test/test_transfer_completion_latency.py | 2 +- simplyblock_core/xfer_timing.py | 2 +- tests/unit/tasks/test_retry_ceiling.py | 19 +++++++-- tests/unit/test_jm_rep_wait.py | 20 +++++++-- 19 files changed, 150 insertions(+), 70 deletions(-) mode change 100644 => 100755 scripts/repl_soak.py mode change 100644 => 100755 scripts/xfer_timing_report.py diff --git a/scripts/repl_soak.py b/scripts/repl_soak.py old mode 100644 new mode 100755 diff --git a/scripts/setup_perf_test1.py b/scripts/setup_perf_test1.py index 6f69f9da9d..3aea191e65 100644 --- a/scripts/setup_perf_test1.py +++ b/scripts/setup_perf_test1.py @@ -687,7 +687,7 @@ def add_one_node(priv_ip): if flag in sys.argv: env += f"{var}={sys.argv[sys.argv.index(flag) + 1]} " start_soak(mgmt_ip, env_prefix=env) - print("") + print() print("Soak running on mgmt. Follow it with:") print(f" ssh -i {KEY_PATH} ec2-user@{mgmt_ip} " "\"tail -f ~/soak_base_$(cat ~/soak_ts).out\"") diff --git a/scripts/xfer_timing_report.py b/scripts/xfer_timing_report.py old mode 100644 new mode 100755 index 5b0395add4..389b89d9e8 --- a/scripts/xfer_timing_report.py +++ b/scripts/xfer_timing_report.py @@ -21,7 +21,7 @@ import argparse import re import sys -from collections import defaultdict, OrderedDict +from collections import defaultdict LINE = re.compile(r"XFER-TIMING\s+(.*)$") KV = re.compile(r"(\w+)=(\S+)") @@ -143,7 +143,7 @@ def report(events, csv_path=None): if envelope > 0: unaccounted = envelope - measured pct = 100.0 * unaccounted / envelope - print("") + print() print("round envelopes %8.1fs" % (envelope / 1000.0)) print("measured phases %8.1fs" % (measured / 1000.0)) print("UNACCOUNTED %8.1fs (%.1f%% of the envelope)" diff --git a/simplyblock_core/controllers/consistency_group_controller.py b/simplyblock_core/controllers/consistency_group_controller.py index abe79a52a8..fb2df7ccaf 100644 --- a/simplyblock_core/controllers/consistency_group_controller.py +++ b/simplyblock_core/controllers/consistency_group_controller.py @@ -33,8 +33,7 @@ from simplyblock_core import utils from simplyblock_core.controllers import snapshot_events, tasks_controller from simplyblock_core.controllers.snapshot_controller import ( - _find_lvs_leader, _rollback_snapshot_bdev, lvstore_op_lock, - object_mutation_lock) + _find_lvs_leader, _rollback_snapshot_bdev, lvstore_op_lock) from simplyblock_core.models.lvol_model import LVol from simplyblock_core.models.replication import ConsistencyGroup from simplyblock_core.models.snapshot import SnapShot @@ -305,15 +304,19 @@ def create_group_snapshot(policy_id, snap_type=SnapShot.TYPE_INTERNAL, lock=True return None, (f"Group snapshot RPC failed on {primary_node.get_id()}; " f"SPDK rolled the partial group back") + # Bound outside the closure: mypy does not carry the ``group is None`` + # guard's narrowing into nested functions. + group_lvs_name = group.lvs_name + def _rollback_all(): for p in plan: - _rollback_snapshot_bdev(pool.cluster_id, group.lvs_name, + _rollback_snapshot_bdev(pool.cluster_id, group_lvs_name, primary_node, p["snap_bdev_name"], all_nodes, lock=lock) # Everything below mirrors snapshot_controller.add's tail per member: # read back uuid/blobid, register on the HA peers, then the record. - created_ids = [] + created_ids: list = [] for p in plan: lvol = p["lvol"] snap_bdev = rpc_client.get_bdevs(f"{group.lvs_name}/{p['snap_bdev_name']}") diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 9bb2f14323..e17844324c 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -4242,10 +4242,17 @@ def _evict_stale_namespace(new_lvol, target_node, superseded=None): # that still holds the old namespace at this nsid also fails with -32602. peer_ids = [target_node.secondary_node_id, target_node.tertiary_node_id] db = DBController() - nodes_to_evict = [target_node] + [ - db.get_storage_node_by_id(pid) - for pid in peer_ids if pid - ] + nodes_to_evict = [target_node] + for pid in peer_ids: + if not pid: + continue + try: + nodes_to_evict.append(db.get_storage_node_by_id(pid)) + except KeyError: + # A peer id with no record holds no stale namespace to evict, and + # this eviction is best-effort — it must not abort the fail-over. + logger.warning("Stale-namespace eviction: peer node %s of %s not " + "found; skipping it", pid, target_node.get_id()) for node in nodes_to_evict: try: rpc = node.rpc_client() diff --git a/simplyblock_core/models/lvol_migration_group.py b/simplyblock_core/models/lvol_migration_group.py index 36c5fc43b0..f5edee7e47 100644 --- a/simplyblock_core/models/lvol_migration_group.py +++ b/simplyblock_core/models/lvol_migration_group.py @@ -102,17 +102,6 @@ class LVolMigrationGroup(BaseModel): # triggers another synchronized round for every member. intermediate_more_needed: List[str] = default_factory(list) - # Which intermediate round is currently in flight (0-indexed; round 0 is - # always taken unconditionally). Incremented when the orchestrator starts - # another synchronized round. - intermediate_round: int = 0 - - # migration_ids that reported their dirty delta still exceeded the - # threshold after finishing intermediate_round. Cleared when a new round - # starts. Non-empty at the end of a round (and under the round cap) - # triggers another synchronized round for every member. - intermediate_more_needed: List[str] = [] - # migration_ids that have completed CLEANUP_SOURCE. cleanup_source_done: List[str] = default_factory(list) diff --git a/simplyblock_core/models/replication.py b/simplyblock_core/models/replication.py index 0364d35236..497f80a621 100644 --- a/simplyblock_core/models/replication.py +++ b/simplyblock_core/models/replication.py @@ -10,7 +10,7 @@ from typing import ClassVar import datetime -from simplyblock_core.models.base_model import BaseModel +from simplyblock_core.models.base_model import BaseModel, default_factory class ReplicationTarget(BaseModel): @@ -117,7 +117,7 @@ class ConsistencyGroup(BaseModel): #: monotonically increasing generation counter; group snapshot N stamps #: every member snapshot it takes with group_seq = N. last_group_seq: int = 0 - members: dict = {} + members: dict = default_factory(dict) status: str = "active" def get_id(self): diff --git a/simplyblock_core/services/snapshot_monitor.py b/simplyblock_core/services/snapshot_monitor.py index 95a101b060..b0d2c3c81d 100644 --- a/simplyblock_core/services/snapshot_monitor.py +++ b/simplyblock_core/services/snapshot_monitor.py @@ -184,15 +184,20 @@ def set_snap_offline(snap): def _warn_leaderless(lvs_name): now = time.monotonic() - last, suppressed = _leaderless_warn_memo.get(lvs_name, (0.0, 0)) - if now - last >= _LEADERLESS_WARN_INTERVAL_SEC: + entry = _leaderless_warn_memo.get(lvs_name) + # An unseen lvs must warn NOW. Seeding "last" with 0.0 instead suppressed + # the FIRST warning whenever monotonic() itself was still under the + # interval — i.e. for the first 60s after boot, which is exactly when a + # leadership flap is most likely to be happening. + if entry is None or now - entry[0] >= _LEADERLESS_WARN_INTERVAL_SEC: + suppressed = entry[1] if entry else 0 logger.warning( f"No confirmed leader for {lvs_name} — snapshot deletes needing " f"phase-1 are deferred ({suppressed} similar messages suppressed " f"in the last {_LEADERLESS_WARN_INTERVAL_SEC}s)") _leaderless_warn_memo[lvs_name] = (now, 0) else: - _leaderless_warn_memo[lvs_name] = (last, suppressed + 1) + _leaderless_warn_memo[lvs_name] = (entry[0], entry[1] + 1) def _poll_delete_status(node, bdev_name): @@ -594,7 +599,7 @@ def take_due_internal_snapshots(cluster_id, now_ts): if getattr(p, "consistency_group", False)} if cg_policies: from simplyblock_core.controllers import consistency_group_controller - grouped_ids = set() + grouped_ids: set = set() for policy_id, policy in cg_policies.items(): members = [lv for lv in repl_lvols if getattr(lv, "replication_policy_id", "") == policy_id] diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index c340b016a2..6833356599 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -164,7 +164,7 @@ def _lvs_transfer_hold(task, snapshot): """ own_lvol = getattr(snapshot, "lvol", None) lvs_name = getattr(own_lvol, "lvs_name", "") if own_lvol else "" - if not lvs_name: + if own_lvol is None or not lvs_name: return "" own_id = own_lvol.get_id() own_group = _group_id_for_lvol(own_lvol) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 45053589ca..aaabc48521 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -21,7 +21,6 @@ the cutover phase has prepared them) """ import time -import threading import uuid as uuid_lib from datetime import datetime @@ -851,14 +850,14 @@ def main(): continue task = db.get_task_by_id(task.uuid) try: - res = task_runner(task, cluster_tasks) + task_runner(task, cluster_tasks) except Exception as e: logger.error(f"replication-final task {task.uuid} failed: {e}", exc_info=True) - res = False - # No blanket backoff here. `res is False` is the NORMAL result - # for a task that is queued or mid-round, and sleeping 3s per - # such task cost ~70s per pass with 20 volumes -- which landed - # directly in the client's IO freeze. + # No blanket backoff here, and the return value is deliberately + # ignored: False is the NORMAL result for a task that is queued + # or mid-round, and sleeping 3s per such task cost ~70s per + # pass with 20 volumes -- which landed directly in the + # client's IO freeze. # Deliberately NOT a sub-second poll: this loop reads the task table # (and each task) per pass, so polling it at 5Hz burns transactions # proportional to clusters x tasks to learn nothing almost every time. diff --git a/simplyblock_core/test/test_cutover_convergence.py b/simplyblock_core/test/test_cutover_convergence.py index 71b2900264..6f0caa55e7 100644 --- a/simplyblock_core/test/test_cutover_convergence.py +++ b/simplyblock_core/test/test_cutover_convergence.py @@ -30,6 +30,8 @@ def __init__(self, **params): self.function_result = "" self.status = "" self.retry = 0 + self.max_retry = 0 + self.canceled = False self.cluster_id = "CL" def write_to_db(self, *a, **kw): @@ -95,7 +97,7 @@ def _take(task_, lvol_): task_.function_params["shrink_started_at"] = clock.now return "S%d" % state["i"], None - patches = [ + patches: list = [ patch.object(runner.time, "time", clock), patch.object(runner.time, "sleep", clock.sleep), patch.object(runner, "_shrink_round_done", side_effect=_done), @@ -179,12 +181,15 @@ def test_a_vanished_snapshot_is_an_error(self): self.assertIn("disappeared", err) def test_the_deadline_still_bounds_the_phase(self): + """The deadline bounds the phase by handing over to the freeze, not by + failing: proceeding with a slightly larger residual always beats + burning a retry on another 900-second shrink window.""" task = _Task(shrink_snap_id="S0", shrink_round=1, shrink_deadline=0, lvol_id="LV1") with patch.object(runner, "_shrink_round_done", return_value=False): done, err = runner._shrink_step(task, _lvol()) - self.assertFalse(done) - self.assertIn("timed out", err) + self.assertTrue(done) + self.assertIsNone(err) def test_it_yields_the_pass_when_the_budget_runs_out(self): """A very slow transfer must not hog the runner forever.""" @@ -222,9 +227,12 @@ def test_the_wait_is_guarded_by_the_flag(self): import inspect src = inspect.getsource(runner.task_runner) self.assertIn("constants.REPL_CUTOVER_PROCEED_REQUIRED", src) + # Compare against the actual freeze CALL SITE, not the first "run_cutover" + # occurrence — the string also appears earlier in a comment about the + # retry path, which is not the freeze. self.assertLess( src.index("REPL_CUTOVER_PROCEED_REQUIRED"), - src.index("run_cutover"), + src.index("replication_final_step.run_cutover"), "the gate must be evaluated before the freeze") @@ -247,7 +255,7 @@ def setUp(self): patcher = patch.object(sr, "db") self.db = patcher.start() self.addCleanup(patcher.stop) - self.groups = {} # lvol id -> group id + self.groups: dict = {} # lvol id -> group id gp = patch.object(sr, "_group_id_for_lvol", side_effect=lambda lv: self.groups.get(lv.get_id(), "")) gp.start() diff --git a/simplyblock_core/test/test_failover_retires_source.py b/simplyblock_core/test/test_failover_retires_source.py index 87389489c6..4d6b060b1c 100644 --- a/simplyblock_core/test/test_failover_retires_source.py +++ b/simplyblock_core/test/test_failover_retires_source.py @@ -31,12 +31,20 @@ def _node(status=StorageNode.STATUS_ONLINE): return node +def _record(bucket, value): + """Append and report success — a lambda-safe stand-in for the patched + call sites, which must return truthy.""" + bucket.append(value) + return True + + def test_retire_fences_then_removes_on_every_online_node(monkeypatch): - fenced, removed = [], [] + fenced: list = [] + removed: list = [] monkeypatch.setattr(lc, "suspend_lvol", - lambda lvol_id: fenced.append(lvol_id) or True) + lambda lvol_id: _record(fenced, lvol_id)) monkeypatch.setattr(lc, "_remove_lvol_subsys_from_node", - lambda lvol, rpc: removed.append(lvol.get_id()) or True) + lambda lvol, rpc: _record(removed, lvol.get_id())) db = MagicMock() db.get_storage_node_by_id.side_effect = \ lambda nid: {"N1": _node(), "N2": _node()}[nid] @@ -46,10 +54,10 @@ def test_retire_fences_then_removes_on_every_online_node(monkeypatch): def test_retire_skips_offline_nodes(monkeypatch): - removed = [] + removed: list = [] monkeypatch.setattr(lc, "suspend_lvol", lambda lvol_id: True) monkeypatch.setattr(lc, "_remove_lvol_subsys_from_node", - lambda lvol, rpc: removed.append(rpc) or True) + lambda lvol, rpc: _record(removed, rpc)) db = MagicMock() db.get_storage_node_by_id.side_effect = lambda nid: { "N1": _node(), "N2": _node(status=StorageNode.STATUS_OFFLINE)}[nid] diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index f236a360ea..dbf87c0c79 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -17,7 +17,7 @@ already existent on the target by every other descendant. """ -from typing import Optional +from typing import ClassVar, Optional from simplyblock_core.models.snapshot import SnapShot from simplyblock_core.services import snapshot_replication as sr @@ -784,7 +784,6 @@ def test_dedicated_subsystem_needs_no_probe(): def test_probe_failure_falls_back_to_the_record(): """A probe that raises must not decide: assume the record is right (attach), which is the pre-existing behaviour.""" - from simplyblock_core.controllers import lvol_controller as lc class _Boom: def subsystem_get(self, nqn): @@ -829,7 +828,8 @@ def test_replica_rollback_clears_its_namespace_and_syncs_the_delete(): class _Lvol: nqn = "nqn.test:lvol:SHARED" top_bdev = "LVS_1/LVOL_NEW" - bdev_stack = [{"type": "bdev_lvol_clone", "name": "LVS_1/LVOL_NEW"}] + bdev_stack: ClassVar[list] = [ + {"type": "bdev_lvol_clone", "name": "LVS_1/LVOL_NEW"}] status = "" def get_id(self): return "LV_NEW" diff --git a/simplyblock_core/test/test_replication_partial_transfer.py b/simplyblock_core/test/test_replication_partial_transfer.py index dfe53b0f19..33619a6afc 100644 --- a/simplyblock_core/test/test_replication_partial_transfer.py +++ b/simplyblock_core/test/test_replication_partial_transfer.py @@ -289,9 +289,9 @@ def _transfer_params(**kwargs): class _C(RPCClient): def __init__(self): - self.sent = None + self.sent: tuple = () - def _request(self, method, params): + def _request(self, method, params=None, request_timeout=None): self.sent = (method, params) return True @@ -304,13 +304,16 @@ def _request(self, method, params): return params -def test_allow_partial_is_sent_only_when_requested(): - # opted in - assert _transfer_params(allow_partial=True)["allow_partial"] is True - # opted out -- the key is absent, so every pre-existing caller keeps its - # exact current wire form and the fork's default (full) applies +def test_allow_partial_is_never_sent_while_the_fork_workaround_holds(): + """The delta path is DISABLED: bdev_lvol_transfer never emits allow_partial, + so every transfer is a full one regardless of what the caller requests + (commit 52e75afb2, 2026-08-31 — the SPDK fork's fragment write path + corrupts partial transfers). When the fork is fixed and the emission in + rpc_client.bdev_lvol_transfer is re-enabled, restore the opt-in + assertions: allow_partial=True must put the key on the wire, and + False/default must keep it absent.""" + assert "allow_partial" not in _transfer_params(allow_partial=True) assert "allow_partial" not in _transfer_params(allow_partial=False) - # and the default is opted out assert "allow_partial" not in _transfer_params() diff --git a/simplyblock_core/test/test_tasks_runner_replication_final.py b/simplyblock_core/test/test_tasks_runner_replication_final.py index 85db1130b4..674a20c09d 100644 --- a/simplyblock_core/test/test_tasks_runner_replication_final.py +++ b/simplyblock_core/test/test_tasks_runner_replication_final.py @@ -97,7 +97,13 @@ def test_happy_path_marks_done_and_updates_state(monkeypatch): assert rep.state == LVolReplication.STATE_CUTOVER_DONE -def test_failure_suspends_and_retries(monkeypatch): +def test_failure_enters_hub_cooldown_without_burning_a_retry(monkeypatch): + """With the clone already prepared (tgt_lvol_composite set), a run_cutover + failure is treated as likely connectivity trouble: the task suspends + behind a cooldown and task.retry stays intact for the first + REPL_CUTOVER_MAX_HUB_ATTEMPTS attempts — burning retries at the poll + interval would exhaust the ceiling long before a restarting node + recovers.""" rep = LVolReplication() rep.cutover_proceed = True nodes = {"S1": _node("S1"), "T1": _node("T1")} @@ -108,8 +114,30 @@ def test_failure_suspends_and_retries(monkeypatch): assert res is False assert task.status == JobSchedule.STATUS_SUSPENDED - assert task.retry == 1 assert task.function_result == "boom" + assert task.retry == 0, "a transient hub attempt must not burn task.retry" + assert task.function_params["cutover_hub_attempts"] == 1 + assert task.function_params["cutover_retry_after"] > 0 + + +def test_failure_burns_a_retry_once_hub_attempts_are_exhausted(monkeypatch): + """Past the hub-attempt cap with the target node online, the failure is + real: the cooldown state resets and one retry is burned, so the ceiling + in task_runner can eventually end a cutover that keeps failing.""" + from simplyblock_core import constants + rep = LVolReplication() + rep.cutover_proceed = True + nodes = {"S1": _node("S1"), "T1": _node("T1")} + _install(monkeypatch, nodes, rep, (False, "boom")) + + task = _task(cutover_hub_attempts=constants.REPL_CUTOVER_MAX_HUB_ATTEMPTS) + res = runner.task_runner(task) + + assert res is False + assert task.status == JobSchedule.STATUS_SUSPENDED + assert task.retry == 1 + assert "cutover_hub_attempts" not in task.function_params + assert "cutover_retry_after" not in task.function_params def test_max_retry_marks_done_without_cutover(monkeypatch): @@ -281,9 +309,14 @@ def test_shrink_hands_over_when_it_cannot_converge(monkeypatch): assert "not converged" in task.function_result -def test_shrink_deadline_aborts(monkeypatch): +def test_shrink_deadline_proceeds_to_cutover(monkeypatch): + """An expired deadline stops adding rounds and hands straight over to the + freeze: the residual delta is slightly larger than a converged one, but + proceeding always beats failing the task and waiting out another + 900-second shrink window.""" runner, task = _mk(monkeypatch, {"S1": _ShrinkSnap(replicated=False)}, {"shrink_round": 1, "shrink_snap_id": "S1", "shrink_deadline": 1}) done, err = runner._shrink_step(task, _ShrinkLvol()) - assert done is False and err and "timed out" in err + assert (done, err) == (True, None), \ + "the deadline must hand over to the freeze, not fail the cutover" diff --git a/simplyblock_core/test/test_transfer_completion_latency.py b/simplyblock_core/test/test_transfer_completion_latency.py index 4516bb9c8e..c6a1f5b714 100644 --- a/simplyblock_core/test/test_transfer_completion_latency.py +++ b/simplyblock_core/test/test_transfer_completion_latency.py @@ -32,7 +32,7 @@ def setUp(self): self.finish = patch.object(sr, "_finish_completed_transfer", return_value=True).start() patch.object(sr, "_cutover_owns", return_value=False).start() - self.slept = [] + self.slept: list = [] patch.object(sr.time, "sleep", side_effect=self.slept.append).start() self.snap = MagicMock() diff --git a/simplyblock_core/xfer_timing.py b/simplyblock_core/xfer_timing.py index 96e36de827..a6f1196e20 100644 --- a/simplyblock_core/xfer_timing.py +++ b/simplyblock_core/xfer_timing.py @@ -78,7 +78,7 @@ def phase(name, lvol=None, snap=None, round=None, **extra): ph["bytes"] = n # optional, folded into the line """ started = time.time() - box = {} + box: dict = {} ok = 1 try: yield box diff --git a/tests/unit/tasks/test_retry_ceiling.py b/tests/unit/tasks/test_retry_ceiling.py index 0b4a429b70..83648dd8f2 100644 --- a/tests/unit/tasks/test_retry_ceiling.py +++ b/tests/unit/tasks/test_retry_ceiling.py @@ -325,13 +325,24 @@ def _spec_node_add(runner, monkeypatch): def _spec_replication_final(runner, monkeypatch): + # The endgame params are pre-set so each pass skips the shrink machinery + # and goes straight to run_cutover, which is mocked to fail every cycle. + # The target node stays ONLINE on purpose: an offline target deliberately + # does NOT burn task.retry (waiting out an outage is transient), so the + # ceiling must be driven by the work itself failing. Each failure walks + # the hub-attempt ladder (REPL_CUTOVER_MAX_HUB_ATTEMPTS cooldown attempts + # per burned retry); the fake clock's giant steps make every cooldown + # already elapsed by the next poll. task = _make_task( JobSchedule.FN_REPLICATION_FINAL, - lvol_id="lv-1", tgt_node_id="tgt-1", src_node_id="src-1") - db, _cluster, node = _wire_base(runner, monkeypatch, task) - # Target node never comes online -> cutover cannot proceed, retry each poll. - node.status = StorageNode.STATUS_OFFLINE + lvol_id="lv-1", tgt_node_id="tgt-1", src_node_id="src-1", + tgt_lvol_composite="lvs_tgt/LVOL_1", tgt_map_id=1, + tgt_snap_composite="lvs_tgt/SNAP_1", + shrink_snap_id="S_endgame", shrink_round=0) + db, _cluster, _node = _wire_base(runner, monkeypatch, task) db.get_lvol_by_id.return_value = MagicMock() + monkeypatch.setattr(runner.replication_final_step, "run_cutover", + lambda *a, **k: (False, "boom")) return task diff --git a/tests/unit/test_jm_rep_wait.py b/tests/unit/test_jm_rep_wait.py index 35b013a6f3..098652000c 100644 --- a/tests/unit/test_jm_rep_wait.py +++ b/tests/unit/test_jm_rep_wait.py @@ -49,6 +49,8 @@ def test_dead_peer_abandons_the_wait_immediately(self): rpc.bdev_lvol_get_lvstores.side_effect = RuntimeError("connection refused") node = _node(rpc=rpc) with self._patch_db(StorageNode.STATUS_OFFLINE), \ + patch("simplyblock_core.models.storage_node.time.time", + return_value=1000.0), \ patch("simplyblock_core.models.storage_node.time.sleep") as sleep: self.assertFalse(node.wait_for_jm_rep_tasks_to_finish(10)) sleep.assert_not_called() @@ -64,6 +66,8 @@ def test_unreachable_rpc_on_a_live_peer_is_bounded(self): rpc.bdev_lvol_get_lvstores.side_effect = RuntimeError("timeout") node = _node(rpc=rpc) with self._patch_db(StorageNode.STATUS_ONLINE), \ + patch("simplyblock_core.models.storage_node.time.time", + return_value=1000.0), \ patch("simplyblock_core.models.storage_node.time.sleep") as sleep: self.assertFalse( node.wait_for_jm_rep_tasks_to_finish(10, retry=4, delay=5)) @@ -77,6 +81,8 @@ def test_pre_check_failure_no_longer_escapes(self): rpc.bdev_lvol_get_lvstores.side_effect = RuntimeError("connection refused") node = _node(rpc=rpc) with self._patch_db(StorageNode.STATUS_ONLINE), \ + patch("simplyblock_core.models.storage_node.time.time", + return_value=1000.0), \ patch("simplyblock_core.models.storage_node.time.sleep"): result = node.wait_for_jm_rep_tasks_to_finish(10, retry=2, delay=1) self.assertFalse(result) # returned, did not raise @@ -85,7 +91,9 @@ def test_no_lvstore_returns_immediately(self): rpc = MagicMock() rpc.bdev_lvol_get_lvstores.return_value = [] node = _node(rpc=rpc) - with patch("simplyblock_core.models.storage_node.time.sleep") as sleep: + with patch("simplyblock_core.models.storage_node.time.time", + return_value=1000.0), \ + patch("simplyblock_core.models.storage_node.time.sleep") as sleep: self.assertTrue(node.wait_for_jm_rep_tasks_to_finish(10)) sleep.assert_not_called() rpc.jc_get_jm_status.assert_not_called() @@ -98,7 +106,9 @@ def test_busy_then_free_returns_true(self): {"jm_a": True, "jm_b": True}, # free ] node = _node(rpc=rpc) - with patch("simplyblock_core.models.storage_node.time.sleep") as sleep: + with patch("simplyblock_core.models.storage_node.time.time", + return_value=1000.0), \ + patch("simplyblock_core.models.storage_node.time.sleep") as sleep: self.assertTrue(node.wait_for_jm_rep_tasks_to_finish(10, delay=7)) self.assertEqual(sleep.call_count, 1) sleep.assert_called_once_with(7) @@ -108,7 +118,9 @@ def test_persistently_busy_peer_exhausts_the_budget(self): rpc.bdev_lvol_get_lvstores.return_value = [{"name": "LVS_10"}] rpc.jc_get_jm_status.return_value = {"jm_a": False} node = _node(rpc=rpc) - with patch("simplyblock_core.models.storage_node.time.sleep") as sleep: + with patch("simplyblock_core.models.storage_node.time.time", + return_value=1000.0), \ + patch("simplyblock_core.models.storage_node.time.sleep") as sleep: self.assertFalse( node.wait_for_jm_rep_tasks_to_finish(10, retry=3, delay=2)) self.assertEqual(rpc.jc_get_jm_status.call_count, 3) @@ -122,6 +134,8 @@ def test_db_lookup_failure_keeps_the_bounded_retries(self): node = _node(rpc=rpc) with patch("simplyblock_core.db_controller.DBController", side_effect=RuntimeError("fdb down")), \ + patch("simplyblock_core.models.storage_node.time.time", + return_value=1000.0), \ patch("simplyblock_core.models.storage_node.time.sleep") as sleep: self.assertFalse( node.wait_for_jm_rep_tasks_to_finish(10, retry=3, delay=1)) From a216dc53596cdd5b9e6f7fbde958494ecf189105 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 3 Sep 2026 16:46:10 +0100 Subject: [PATCH 120/122] =?UTF-8?q?fix(cli):=20declare=20retention-schedul?= =?UTF-8?q?e,=20consistency-group,=20and=20replication-policy-snapshot=20i?= =?UTF-8?q?n=20cli-reference.yaml=20=E2=80=94=20cli.py=20was=20hand-edited?= =?UTF-8?q?=20and=20regeneration=20also=20restores=20the=20missing=20snaps?= =?UTF-8?q?hot=20dispatch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- simplyblock_cli/cli-reference.yaml | 16 ++++++++++++++++ simplyblock_cli/cli.py | 10 ++++++---- tests/integration/test_clone_namespace_race.py | 6 ++++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index f237c17a10..e9767f8c8e 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -1695,6 +1695,15 @@ commands: help: "Replicated internal snapshots to retain on each side. Minimum (and default): `2`." dest: keep_replicated type: int + - name: "--retention-schedule" + help: "Tiered retention, e.g. `15m:2h,1h:11h,1d:7d` - one snapshot every 15 minutes for the last 2 hours, then hourly for 11 hours, then daily for 7 days. Snapshots older than the total span are pruned. Empty (default) keeps the flat --keep behaviour." + dest: retention_schedule + type: str + - name: "--consistency-group" + help: "All volumes attached to this policy form ONE consistency group: they must share an LVS (creation pins them to it), cadence snapshots are taken as one frozen group, and fail-over generations resolve group-wide." + dest: consistency_group + type: bool + action: store_true - name: replication-policy-list help: Lists the replication policies of a cluster arguments: @@ -1727,6 +1736,13 @@ commands: dest: json type: bool action: store_true + - name: replication-policy-snapshot + help: "Takes ONE crash-consistent snapshot of every volume in the policy's consistency group, as a new group generation" + arguments: + - name: "policy_id" + help: "Replication policy id (must be a consistency-group policy)" + dest: policy_id + type: str - name: "volume" help: "Logical Volume Commands" aliases: diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index c6b7a2fcd5..e449d2c5bf 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -675,15 +675,15 @@ def init_cluster__replication_policy_remove(self, subparser): subcommand = self.add_sub_command(subparser, 'replication-policy-remove', 'Removes a replication policy. Refused while a volume still follows it.') subcommand.add_argument('policy_id', help='Replication policy id', type=str) - def init_cluster__replication_policy_snapshot(self, subparser): - subcommand = self.add_sub_command(subparser, 'replication-policy-snapshot', "Takes ONE crash-consistent snapshot of every volume in the policy's consistency group, as a new group generation") - subcommand.add_argument('policy_id', help='Replication policy id (must be a consistency-group policy)', type=str) - def init_cluster__replication_policy_failover(self, subparser): subcommand = self.add_sub_command(subparser, 'replication-policy-failover', 'Fails over EVERY volume following this policy') subcommand.add_argument('policy_id', help='Replication policy id', type=str) subcommand.add_argument('--json', help='Print outputs in json format.', dest='json', action='store_true') + def init_cluster__replication_policy_snapshot(self, subparser): + subcommand = self.add_sub_command(subparser, 'replication-policy-snapshot', 'Takes ONE crash-consistent snapshot of every volume in the policy\'s consistency group, as a new group generation') + subcommand.add_argument('policy_id', help='Replication policy id (must be a consistency-group policy)', type=str) + def init_volume(self): subparser = self.add_command('volume', 'Logical Volume Commands', aliases=['lvol',]) @@ -1510,6 +1510,8 @@ def run(self): ret = self.cluster__replication_policy_remove(sub_command, args) elif sub_command in ['replication-policy-failover']: ret = self.cluster__replication_policy_failover(sub_command, args) + elif sub_command in ['replication-policy-snapshot']: + ret = self.cluster__replication_policy_snapshot(sub_command, args) else: self.parser.print_help() diff --git a/tests/integration/test_clone_namespace_race.py b/tests/integration/test_clone_namespace_race.py index a9bd313d05..f8e8e9475e 100644 --- a/tests/integration/test_clone_namespace_race.py +++ b/tests/integration/test_clone_namespace_race.py @@ -65,6 +65,12 @@ def _lvol_for_add(uuid, namespace="", nqn=None): lv.ha_type = "single" lv.nqn = nqn or ("nqn.test:cluster-1:lvol:" + uuid) lv.namespace = namespace + # Create-flow invariant (see add_lvol): ns_id is 0 = "not assigned yet" + # until the PRIMARY namespace add persists SPDK's assignment. The model + # default is 1 — a legitimate nsid — and a non-zero ns_id reads as + # "dictated by a migration/fail-over caller", which forbids the -32602 + # re-claim these tests exercise. + lv.ns_id = 0 lv.allowed_hosts = [] lv.fabric = "tcp" lv.max_namespace_per_subsys = 32 From 272a3a0aec83ac5d1e92a858ab5861fb2bf9615e Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 4 Sep 2026 11:41:01 +0100 Subject: [PATCH 121/122] =?UTF-8?q?fix(proxy):=20start=20the=20server=20an?= =?UTF-8?q?d=20stats=20thread=20only=20under=20=5F=5Fmain=5F=5F=20?= =?UTF-8?q?=E2=80=94=20the=20import=20side=20effect=20crashed=20green=20CI?= =?UTF-8?q?=20runs=20at=20interpreter=20shutdown?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/spdk_http_proxy_server.py | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/simplyblock_core/services/spdk_http_proxy_server.py b/simplyblock_core/services/spdk_http_proxy_server.py index c83e8902ef..aecd2146a5 100644 --- a/simplyblock_core/services/spdk_http_proxy_server.py +++ b/simplyblock_core/services/spdk_http_proxy_server.py @@ -15,10 +15,14 @@ from simplyblock_core.settings import Settings -logger_handler = logging.StreamHandler(stream=sys.stdout) -logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s')) logger = logging.getLogger() -logger.addHandler(logger_handler) +if __name__ == "__main__": + # Attach the stdout handler only when running as the proxy script. + # Attaching on IMPORT handed every importer (e.g. the test suite) a + # duplicate root handler, doubling every log line in the process. + logger_handler = logging.StreamHandler(stream=sys.stdout) + logger_handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(message)s')) + logger.addHandler(logger_handler) logger.setLevel(logging.INFO) read_line_time_diff: dict = {} @@ -319,4 +323,13 @@ def run_server(host, port, user, password, is_threading_enabled=False): logger.info(f"SPDK concurrency limit: {MAX_CONCURRENT_SPDK}") is_threading_enabled = bool(is_threading_enabled) -run_server(server_ip, rpc_port, rpc_username, rpc_password, is_threading_enabled=is_threading_enabled) + +if __name__ == "__main__": + # Start ONLY when executed as the proxy script (deploy_spdk.yaml and the + # docker snode path both run this file directly). Starting on import made + # every importer spawn the print_stats daemon thread too: after the proxy + # e2e tests ran, that thread logged every 3s for the rest of the pytest + # session and could hold the stderr buffer lock at interpreter shutdown — + # "Fatal Python error: _enter_buffered_busy", SIGABRT, a red CI run with + # 1227/1227 tests passed (2026-09-04). + run_server(server_ip, rpc_port, rpc_username, rpc_password, is_threading_enabled=is_threading_enabled) From 0a1fc38b30e7781122e278d18684d9651389a9b6 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 4 Sep 2026 12:14:49 +0100 Subject: [PATCH 122/122] =?UTF-8?q?test(migration):=20mock=20bdev=5Flvol?= =?UTF-8?q?=5Ftransfer=5Ffinal=5Fstep=20synchronously=20=E2=80=94=20the=20?= =?UTF-8?q?runner=20requires=20transfer=5Fstate=3DDone=20on=20the=20return?= =?UTF-8?q?=20since=20the=202026-08-22=20hardening?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../integration/migration/mock_rpc_server.py | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/tests/integration/migration/mock_rpc_server.py b/tests/integration/migration/mock_rpc_server.py index 5bd9df500f..fe86338529 100644 --- a/tests/integration/migration/mock_rpc_server.py +++ b/tests/integration/migration/mock_rpc_server.py @@ -657,21 +657,34 @@ def _bdev_lvol_transfer_stat(s: NodeState, p: dict): def _bdev_lvol_transfer_final_step(s: NodeState, p: dict): - """Start async final-migration (source lvol → target blobstore). - - Wire name for ``RPCClient.bdev_lvol_final_migration`` (a deprecated alias - for ``bdev_lvol_transfer_final_step``). + """Final migration (source lvol → target blobstore) — SYNCHRONOUS. + + The real RPC blocks through the IO drain and delta copy and returns a + stat dict; the runner requires ``transfer_state == 'Done'`` on the return + itself (tasks_runner_lvol_migration, strict check added 2026-08 after the + batch-migration "Failed slipped through" bug). This mock used to model it + as async (register op, return True) — a stale contract that made every + completed migration read as ``transfer_state=None`` and failed 26 tests + while the code worked on real clusters. + + Failure injection still applies: the dispatch layer times out or errors + BEFORE this handler runs, which exercises the runner's not-ret / + exception paths, including the crash-recovery stat poll — served by the + 'Done' op recorded here. """ lvol_name = _req(p, 'lvol_name') composite = lvol_name if lvol_name in s.lvols else s.composite(lvol_name) if composite not in s.lvols: raise _RpcError(-2, f"source lvol {composite} not found") + # Block briefly like the real drain does, but cap the exponential tail — + # a synchronous multi-second stall would serialize the whole suite. + time.sleep(min(_async_delay(s.rng), 0.5)) s.transfer_ops[composite] = { - 'complete_at': time.time() + _async_delay(s.rng), - 'state': 'In progress', + 'complete_at': time.time(), + 'state': 'Done', } - logger.debug("mock bdev_lvol_transfer_final_step started for %s", composite) - return True + logger.debug("mock bdev_lvol_transfer_final_step completed for %s", composite) + return {'transfer_state': 'Done', 'offset': 0} # ---- NVMe-oF subsystems ----