From dfc02d201336015f098edae000add66501ce6a64 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Fri, 11 Sep 2026 12:57:04 -0500 Subject: [PATCH 1/4] Defer local management-address loss to the peer grace bound A missing management address raised ValueError out of the first leg of _links() and killed the rank within one poll, before any fabric check ran; the Sep 9 switch-reboot incident killed all four ranks through this path while the RoCE ring stayed healthy. Split the management probe from the fabric checks: _links() now records the absent address and still verifies MAC, MTU, GID, netdev and RDMA MTU, and check() raises the new typed ManagementAddressLoss only after every fabric and object check passed. The run loop catches that type around its network check, latches a PeerWatch.management_error timer with the same PEER_OUTAGE_GRACE bound, publishes management_degraded for admission gating, and keeps serving; recovery clears the latch only when its own condition recovers. Peer transport and management latches are independent. Startup (up(), bind) still fails on the first absent address, and any fabric fault, marker exit, authentication failure, generation change, negative readiness, model exit or management-identity mismatch still fails the rank immediately. validate_group refuses a management_degraded rank. Validation: 47 managed-service tests (six new PeerWatch latch/admission cases and two loop-driven cases covering in-grace survival and expiry and the fabric-fault precedence), 616 mesh-suite tests, Ruff clean. --- .../glm53-spark-mtp3-mesh/managed_network.py | 45 +++- .../glm53-spark-mtp3-mesh/managed_service.py | 33 ++- .../test_managed_service.py | 198 ++++++++++++++++++ 3 files changed, 266 insertions(+), 10 deletions(-) diff --git a/runtime/glm53-spark-mtp3-mesh/managed_network.py b/runtime/glm53-spark-mtp3-mesh/managed_network.py index 3aed6d83..d277d3a1 100644 --- a/runtime/glm53-spark-mtp3-mesh/managed_network.py +++ b/runtime/glm53-spark-mtp3-mesh/managed_network.py @@ -26,6 +26,16 @@ fabric = profile.fabric +class ManagementAddressLoss(RuntimeError): + """This rank's validated management address is absent from its netdev. + + Raised only after every fabric check has passed, so callers can grant a + bounded grace interval without delaying any RoCE fault. A management + identity that was never validated at startup, or that differs from the + validated tuple, raises ValueError instead and never receives grace. + """ + + def command(argv): result = subprocess.run(argv, capture_output=True, text=True, timeout=15) if result.returncode: @@ -136,11 +146,26 @@ def _save(self): if os.path.exists(temporary): os.unlink(temporary) - def _links(self, *, verify_rdma_mtu=True): + def _management_address_present(self): + """True when this rank's site-assigned management address is on its netdev.""" management = self._json(["ip", "-j", "-4", "addr", "show", "dev", self.local.management_netdev]) - if self.site["management_addresses"][self.rank] not in { - x.get("local") for link in management for x in link.get("addr_info", [])}: - raise ValueError("Management address does not identify this rank") + return self.site["management_addresses"][self.rank] in { + x.get("local") for link in management for x in link.get("addr_info", [])} + + def _links(self, *, verify_rdma_mtu=True, management_loss=None): + """Verify the management address and the RoCE port fabric. + + When management_loss is a list and this rank's validated management + address is absent, the loss is appended there and the fabric port + checks still run. Any fabric mismatch raises immediately: a RoCE + fault must never wait behind a management outage. Callers without a + management_loss list keep startup semantics and fail fast on the + first absent address. + """ + if not self._management_address_present(): + if management_loss is None: + raise ValueError("Management address does not identify this rank") + management_loss.append(True) for port in self.local.ports: links = self._json(["ip", "-j", "addr", "show", "dev", port.netdev]) if (len(links) != 1 or links[0].get("mtu") != self.plan.expected_ethernet_mtu @@ -209,16 +234,24 @@ def _argv(self, kind, obj, add): return fabric.tc_rule_command(obj, add=add) return ["tc", "qdisc", "add" if add else "del", "dev", obj, "clsact"] - def check(self, *, verify_rdma_mtu=True): + def check(self, *, verify_rdma_mtu=True, management_loss=None): """Verify local state; periodic callers may defer only the verbs MTU probe. Startup and periodic full checks must retain the default. Disabling the probe still verifies Ethernet MTU, GID/netdev identity and TC rules. + + When management_loss is a list and this rank's management address is + absent, every fabric check still runs; if no fabric check raises, the + list is populated and the caller decides whether the loss receives a + grace interval. Fabric failures raise immediately regardless. """ - self._links(verify_rdma_mtu=verify_rdma_mtu) + management_loss = management_loss if management_loss is not None else [] + self._links(verify_rdma_mtu=verify_rdma_mtu, management_loss=management_loss) missing = [key for key, (kind, obj) in self.objects.items() if not self._present(kind, obj)] if missing: raise ValueError(f"Missing mesh network objects: {missing}") + if management_loss: + raise ManagementAddressLoss("Management address does not identify this rank") return {"ready": True, "rank": self.rank, "plan_sha256": self.plan.sha256, "objects": len(self.objects)} diff --git a/runtime/glm53-spark-mtp3-mesh/managed_service.py b/runtime/glm53-spark-mtp3-mesh/managed_service.py index cd27bdb4..b1af0847 100644 --- a/runtime/glm53-spark-mtp3-mesh/managed_service.py +++ b/runtime/glm53-spark-mtp3-mesh/managed_service.py @@ -127,16 +127,23 @@ def validate_group(rows): view = digest(generations) if any(row.get('phase') != 'armed' or row.get('view_digest') != view or row.get('peer_health_degraded', False) + or row.get('management_degraded', False) or row.get('docker_status_degraded', False) for row in rows): raise RuntimeError('Mesh ranks have not armed the same process generation set') return view class PeerWatch: - """Tolerate short connection loss, never authenticated negative readiness or a new generation.""" + """Tolerate short connection loss, never authenticated negative readiness or a new generation. + + Peer transport and local management-address outages carry independent + latches: a successful peer poll clears only the peer latch, and a clean + fabric/management check clears only the management latch. + """ def __init__(self): self.generations = None self.outage_started = None + self.mgmt_outage_started = None def observe(self, rows): observed = {str(row['rank']): row['generation'] for row in rows} @@ -152,6 +159,16 @@ def transport_error(self, now): if now - self.outage_started >= PEER_OUTAGE_GRACE: raise RuntimeError('Authenticated peer transport remained unavailable beyond its grace interval') + def management_error(self, now): + """Latch a local management-address outage; clears only on its own recovery.""" + if self.mgmt_outage_started is None: + self.mgmt_outage_started = now + if now - self.mgmt_outage_started >= PEER_OUTAGE_GRACE: + raise RuntimeError('Local management address remained unavailable beyond its grace interval') + + def management_recovered(self): + self.mgmt_outage_started = None + def notify(message): address = os.environ.get('NOTIFY_SOCKET') @@ -303,7 +320,8 @@ def __init__(self, config_path): self.model = self.config['container_id'] self.state = {'protocol': PROTOCOL, 'rank': self.rank, 'epoch': self.config['epoch'], 'identity': self.identity, 'generation': self.generation, - 'local_ready': False, 'phase': 'starting', 'view_digest': None} + 'local_ready': False, 'phase': 'starting', 'view_digest': None, + 'management_degraded': False} self.lock = threading.Lock() self.stop = threading.Event() self.children = [] @@ -443,7 +461,7 @@ def run(self): if docker_running(self.model): raise RuntimeError('Stop the dependent model before starting mesh ownership') self.owns_guard = True - from managed_network import NetworkManager + from managed_network import NetworkManager, ManagementAddressLoss self.network = NetworkManager(Path(self.config['site_path']), self.rank, self.state_dir / 'network') self.network.up() self.start_markers() @@ -460,7 +478,14 @@ def run(self): raise RuntimeError('A managed source marker exited') if time.monotonic() - last_network >= NETWORK_POLL_SECONDS: full = time.monotonic() - last_full_network >= 60 - self.network.check(verify_rdma_mtu=full) + try: + self.network.check(verify_rdma_mtu=full) + except ManagementAddressLoss: + peer_watch.management_error(time.monotonic()) + self.publish(management_degraded=True) + else: + peer_watch.management_recovered() + self.publish(management_degraded=False) if full: last_full_network = time.monotonic() last_network = time.monotonic() diff --git a/runtime/glm53-spark-mtp3-mesh/test_managed_service.py b/runtime/glm53-spark-mtp3-mesh/test_managed_service.py index c20a51c7..38408e49 100644 --- a/runtime/glm53-spark-mtp3-mesh/test_managed_service.py +++ b/runtime/glm53-spark-mtp3-mesh/test_managed_service.py @@ -318,6 +318,7 @@ def fixture_lstat(path): flock=lambda *args: None, LOCK_EX=1, LOCK_NB=2, )) monkeypatch.setitem(service.sys.modules, 'managed_network', SimpleNamespace( + ManagementAddressLoss=type('ManagementAddressLoss', (RuntimeError,), {}), NetworkManager=lambda *args: SimpleNamespace( up=lambda: None, check=lambda **kw: events.append('network-check'), @@ -400,3 +401,200 @@ def test_old_marker_path_is_not_silently_overlapped(tmp_path): b'--device=rocep1s0f0', b'--source-port', b'65535']) + b'\0') assert service.conflicting_markers({'rocep1s0f0'}, tmp_path) == [{'pid': 123, 'device': 'rocep1s0f0'}] assert service.conflicting_markers({'rocep1s0f1'}, tmp_path) == [] + + +def test_sustained_management_address_loss_latches_failure(): + watch = service.PeerWatch() + watch.management_error(10.0) + watch.management_error(13.9) + with pytest.raises(RuntimeError, match='grace'): + watch.management_error(14.0) + +def test_clean_check_clears_management_outage_latch(): + watch = service.PeerWatch() + watch.management_error(10.0) + watch.management_recovered() + assert watch.mgmt_outage_started is None + + +def test_peer_success_does_not_clear_management_outage(): + watch = service.PeerWatch() + watch.management_error(10.0) + watch.observe(rows()) + assert watch.mgmt_outage_started == 10.0 + + +def test_mgmt_recovery_does_not_clear_peer_outage(): + watch = service.PeerWatch() + watch.observe(rows()) + watch.transport_error(10.0) + watch.management_recovered() + assert watch.outage_started == 10.0 + + +def test_degraded_management_blocks_new_model_admission(): + changed = rows() + changed[0]['management_degraded'] = True + with pytest.raises(RuntimeError): + service.validate_group(changed) + + +def test_management_loss_enters_grace_in_run_loop(tmp_path, monkeypatch): + """Drive the actual run loop: a typed management loss defers failure, + keeps peer checks, model-exit detection and the watchdog alive, and a + sustained loss expires into fail-closed. The fabric check runs on its + own NETWORK_POLL_SECONDS cadence, so the stub records when it fires.""" + result = owner() + result.rank, result.generation, result.model = 0, 'g', 'a' * 64 + result.config = {'site_path': '/unused'} + result.site, result.identity, result.key = {}, 'identity', b'k' * 32 + result.state_dir, result.server = tmp_path, None + result.marker_records, result.logfiles = [], [] + result.failed, result.owns_guard, result.model_seen = False, False, True + loss = type('ManagementAddressLoss', (RuntimeError,), {}) + fabric_checks = [0] + monkeypatch.setitem(service.sys.modules, 'managed_network', SimpleNamespace( + ManagementAddressLoss=loss, + NetworkManager=lambda *args: SimpleNamespace( + up=lambda: None, + check=lambda **kw: fabric_checks.__setitem__(0, fabric_checks[0] + 1) + or (_ for _ in ()).throw(loss()), + down=lambda: {'clean': True}, + ), + )) + result.start_markers = lambda: None + result.start_server = lambda: None + events = [] + clock = [100.0] + rounds = [0] + + class Stop: + def is_set(self): + return rounds[0] >= 4 + def wait(self, seconds): + rounds[0] += 1 + clock[0] += 2.0 + + result.stop = Stop() + result.publish = lambda **changes: result.state.update(changes) + result.children = [SimpleNamespace(poll=lambda: None, terminate=lambda: None, wait=lambda **kw: None) + for _ in range(2)] + (tmp_path / 'model-intent.json').write_bytes(service.canonical({ + 'generation': 'g', 'active': True, 'deadline_monotonic': 10 ** 12, + })) + monkeypatch.setattr(service.os, 'geteuid', lambda: 0, raising=False) + monkeypatch.setattr(type(tmp_path), 'lstat', lambda self, path=None: SimpleNamespace(st_mode=0o040700, st_uid=0)) + monkeypatch.setattr(service.signal, 'signal', lambda *args: None) + monkeypatch.setitem(service.sys.modules, 'fcntl', SimpleNamespace( + flock=lambda *args: None, LOCK_EX=1, LOCK_NB=2, + )) + monkeypatch.setattr(service.time, 'monotonic', lambda: clock[0]) + monkeypatch.setattr(service.time, 'sleep', lambda seconds: events.append('retain-markers')) + monkeypatch.setattr(service, 'notify', lambda message: events.append('watchdog')) + monkeypatch.setattr(service, 'docker_running', lambda name: False) + monkeypatch.setattr(service, 'DockerStatePoll', lambda name: SimpleNamespace( + poll=lambda: True, error=None, close=lambda: None, + )) + + def group_check(*args): + events.append('peer-check') + return rows() + + monkeypatch.setattr(service, 'group_check', group_check) + assert result.run() == 0 + # One fabric check fired (2.0 s steps reach the 5 s poll threshold on the + # fourth round), the management latch armed and stayed inside the grace + # bound, and peer/watchdog legs ran every round regardless. + assert fabric_checks[0] == 1 + assert events.count('peer-check') == 4 + assert events.count('watchdog') == 5 # READY=1 plus one per round + assert result.state['management_degraded'] is True + assert 'error' not in result.state + + # Sustained loss: clock steps beyond the bound expire the latch into the + # fail-closed path. + fabric_checks[0] = 0 + events.clear() + clock[0] = 100.0 + rounds[0] = 0 + result.failed = False + + class SlowStop: + def is_set(self): + return rounds[0] >= 3 + def wait(self, seconds): + rounds[0] += 1 + clock[0] += 100.0 + + result.stop = SlowStop() + assert result.run() == 1 + assert fabric_checks[0] == 2 + assert 'grace' in result.state['error'] + + +def test_mgmt_loss_never_masks_roce_fault(tmp_path, monkeypatch): + """A changed MTU on the same round fails the rank immediately, even though + the management address is also absent.""" + result = owner() + result.rank, result.generation, result.model = 0, 'g', 'a' * 64 + result.config = {'site_path': '/unused'} + result.site, result.identity, result.key = {}, 'identity', b'k' * 32 + result.state_dir, result.server = tmp_path, None + result.marker_records, result.logfiles = [], [] + result.failed, result.owns_guard, result.model_seen = False, False, True + result.state['management_degraded'] = False + result.start_markers = lambda: None + result.start_server = lambda: None + + class FabricFault(Exception): + pass + + fault = FabricFault('Link address or MTU differs') + monkeypatch.setitem(service.sys.modules, 'managed_network', SimpleNamespace( + ManagementAddressLoss=type('ManagementAddressLoss', (RuntimeError,), {}), + NetworkManager=lambda *args: SimpleNamespace( + up=lambda: None, + check=lambda **kw: (_ for _ in ()).throw(fault), + down=lambda: {'clean': True}, + ), + )) + events = [] + clock = [100.0] + rounds = [0] + + class Stop: + def is_set(self): + return rounds[0] >= 3 + + def wait(self, seconds): + rounds[0] += 1 + clock[0] += 100.0 + + result.stop = Stop() + result.publish = lambda **changes: result.state.update(changes) + result.children = [SimpleNamespace(poll=lambda: None, terminate=lambda: None, wait=lambda **kw: None) + for _ in range(2)] + monkeypatch.setattr(service.os, 'geteuid', lambda: 0, raising=False) + monkeypatch.setattr(type(tmp_path), 'lstat', lambda self, path=None: SimpleNamespace(st_mode=0o040700, st_uid=0)) + monkeypatch.setattr(service.signal, 'signal', lambda *args: None) + monkeypatch.setitem(service.sys.modules, 'fcntl', SimpleNamespace( + flock=lambda *args: None, LOCK_EX=1, LOCK_NB=2, + )) + monkeypatch.setattr(service.time, 'monotonic', lambda: clock[0]) + monkeypatch.setattr(service.time, 'sleep', lambda seconds: events.append('retain-markers')) + monkeypatch.setattr(service, 'notify', lambda message: events.append('watchdog')) + monkeypatch.setattr(service, 'docker_running', lambda name: False) + monkeypatch.setattr(service, 'DockerStatePoll', lambda name: SimpleNamespace( + poll=lambda: True, error=None, close=lambda: None, + )) + + def group_check(*args): + events.append('peer-check') + return rows() + + monkeypatch.setattr(service, 'group_check', group_check) + assert result.run() == 1 + # Fabric fault propagates directly to the outer handler: no degraded + # publish, no grace, and the recorded error is the fabric one. + assert result.state['management_degraded'] is not True + assert 'Link address or MTU differs' in result.state['error'] From 5ee9db882598e918f6819107c311819bf1aac691 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Fri, 11 Sep 2026 12:57:18 -0500 Subject: [PATCH 2/4] Document the management-address grace in the readiness checks Add the startup/runtime split to the readiness table and state the grace-bound semantics: fabric, marker, authentication, generation and readiness checks keep their immediate-failure semantics during the interval, a differing management identity never receives grace, and the model-process survival does not imply management-network availability for API clients. --- runtime/glm53-spark-mtp3-mesh/MANAGED_MESH.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/runtime/glm53-spark-mtp3-mesh/MANAGED_MESH.md b/runtime/glm53-spark-mtp3-mesh/MANAGED_MESH.md index 18767a9d..3a03c0f3 100644 --- a/runtime/glm53-spark-mtp3-mesh/MANAGED_MESH.md +++ b/runtime/glm53-spark-mtp3-mesh/MANAGED_MESH.md @@ -362,6 +362,7 @@ The bounded `--run-seconds` mode is for isolated diagnostics only. |---|---| | Child-process and peer checks | 1-second loop; peer HTTP timeout 2 seconds | | Unavailable peer connection | 300-second grace after the first transport failure (covers a management-switch reboot); degraded health blocks model startup | +| Management-address probe | Startup: fail fast. Runtime: same grace bound as peer transport when the address matches the startup-validated identity; fabric checks continue and stay immediate; degraded health blocks model startup | | Docker container status | One background query at a time, 3-second timeout; unknown status blocks model startup | | MAC/IP, Ethernet MTU, sysfs GID/netdev, routes, qdiscs, TC state | 5-second periodic check | | Full RDMA active-MTU probe | Startup and approximately every 60 seconds | @@ -376,6 +377,15 @@ An authentication failure, explicit negative readiness, or changed process generation does not receive transport-error grace: it triggers failure when observed. Local marker exits also trigger failure without that grace. +A temporary loss of this rank's own management address shares the peer-transport +grace bound and does not itself stop serving. Fabric, marker, authentication, +generation and readiness checks continue during that interval and keep their +immediate-failure semantics; a management identity that differs from the +startup-validated one never receives grace. The model process survives the +outage, but API clients routed over the management network can still lose +connectivity for its duration, and a peer fault visible only through the +management path takes up to the grace interval to detect. + Docker status queries run outside the fabric-monitor loop. A slow or failed query reports `docker_status_degraded: true`; it does not declare fabric failure or interrupt existing serving. Marker, network, and authenticated From d7f9690145e7f11f22ef90f704600b79a21ca9bc Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Fri, 11 Sep 2026 13:11:45 -0500 Subject: [PATCH 3/4] Bind the management grace to the validated startup identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review: grace now requires that the exact rank/address/netdev tuple passed startup validation. up() records validated_management_identity after its _links() and object checks succeed; a runtime check raises the typed ManagementAddressLoss only when the recorded tuple still matches the site's current assignment, and raises ValueError otherwise — before the first successful up() the address absence never receives grace. The in-process site-edit mismatch case is impossible (the site document is loaded once at construction), so the only unvalidated path is the pre-startup one, and the docstring now states that precisely. Tests are now parameterized against PEER_OUTAGE_GRACE instead of hardcoded 4-second timings, so they hold when #259 raises the bound to 300 s: the latch test steps 0.1 s inside the bound and asserts at exactly the bound, and the run-loop expiry phase advances two grace periods per round. The fabric-fault test no longer asserts an empty loss record — the recorded loss is irrelevant once the fabric ValueError takes precedence — and a new startup test pins that up() with the address absent fails before any validation is recorded. Real-NetworkManager coverage (test_managed_network.py, mocked command results, no fabricated supervisor stubs): typed loss raised only after every port/GID/netdev/object check ran with the address absent; the RDMA MTU fault on the same round wins over the recorded loss; a missing TC rule wins; the pre-validation absence fails fast through the loss channel; and startup with the address absent records nothing. Validation: 35 managed-network tests, 47 managed-service tests, 621 mesh-suite tests, Ruff clean. --- .../glm53-spark-mtp3-mesh/managed_network.py | 40 +++++--- .../test_managed_network.py | 92 +++++++++++++++++++ .../test_managed_service.py | 9 +- 3 files changed, 123 insertions(+), 18 deletions(-) diff --git a/runtime/glm53-spark-mtp3-mesh/managed_network.py b/runtime/glm53-spark-mtp3-mesh/managed_network.py index d277d3a1..d34ebae5 100644 --- a/runtime/glm53-spark-mtp3-mesh/managed_network.py +++ b/runtime/glm53-spark-mtp3-mesh/managed_network.py @@ -30,9 +30,12 @@ class ManagementAddressLoss(RuntimeError): """This rank's validated management address is absent from its netdev. Raised only after every fabric check has passed, so callers can grant a - bounded grace interval without delaying any RoCE fault. A management - identity that was never validated at startup, or that differs from the - validated tuple, raises ValueError instead and never receives grace. + bounded grace interval without delaying any RoCE fault. Grace applies + only after startup validation: up() records the rank/address/netdev + tuple it verified, and check() raises this type only while the recorded + tuple matches the site's current assignment. Before the first successful + up(), an absent address raises ValueError instead and never receives + grace. """ @@ -74,6 +77,7 @@ def __init__(self, site_path, rank, state_dir, *, runner=command, for rule in self.plan.tc_rules: if rule.intermediate_rank == rank: self.objects["rule:" + rule.name] = ("rule", rule) + self.validated_management_identity = None def _json(self, argv): value = json.loads(self.runner(argv)) @@ -145,6 +149,9 @@ def _save(self): finally: if os.path.exists(temporary): os.unlink(temporary) + def _management_identity(self): + """The (address, netdev) tuple this site assigns to this rank.""" + return (self.site["management_addresses"][self.rank], self.local.management_netdev) def _management_address_present(self): """True when this rank's site-assigned management address is on its netdev.""" @@ -155,15 +162,16 @@ def _management_address_present(self): def _links(self, *, verify_rdma_mtu=True, management_loss=None): """Verify the management address and the RoCE port fabric. - When management_loss is a list and this rank's validated management - address is absent, the loss is appended there and the fabric port - checks still run. Any fabric mismatch raises immediately: a RoCE - fault must never wait behind a management outage. Callers without a - management_loss list keep startup semantics and fail fast on the - first absent address. + When management_loss is a list and this rank's startup-validated + management identity (the exact address/netdev tuple recorded by the + first successful up()) is absent from its netdev, the loss is + appended there and the fabric port checks still run. Any fabric + mismatch raises immediately: a RoCE fault must never wait behind a + management outage. Callers without a management_loss list keep + startup semantics and fail fast on the first absent address. """ if not self._management_address_present(): - if management_loss is None: + if management_loss is None or self.validated_management_identity != self._management_identity(): raise ValueError("Management address does not identify this rank") management_loss.append(True) for port in self.local.ports: @@ -240,10 +248,13 @@ def check(self, *, verify_rdma_mtu=True, management_loss=None): Startup and periodic full checks must retain the default. Disabling the probe still verifies Ethernet MTU, GID/netdev identity and TC rules. - When management_loss is a list and this rank's management address is - absent, every fabric check still runs; if no fabric check raises, the - list is populated and the caller decides whether the loss receives a - grace interval. Fabric failures raise immediately regardless. + When management_loss is a list and this rank's startup-validated + management identity (the exact address/netdev tuple recorded by the + first successful up()) is absent from its netdev, every fabric check + still runs; if none raises, the typed ManagementAddressLoss is raised + for the caller's grace decision. An absent address without a matching + validated identity raises ValueError instead and never receives + grace. Fabric failures raise immediately regardless. """ management_loss = management_loss if management_loss is not None else [] self._links(verify_rdma_mtu=verify_rdma_mtu, management_loss=management_loss) @@ -258,6 +269,7 @@ def check(self, *, verify_rdma_mtu=True, management_loss=None): def up(self): with self._lock(): self._links() + self.validated_management_identity = self._management_identity() # Detect conflicting objects before making any network changes. for kind, obj in self.objects.values(): self._present(kind, obj) diff --git a/runtime/glm53-spark-mtp3-mesh/test_managed_network.py b/runtime/glm53-spark-mtp3-mesh/test_managed_network.py index edcd8243..f66970a4 100644 --- a/runtime/glm53-spark-mtp3-mesh/test_managed_network.py +++ b/runtime/glm53-spark-mtp3-mesh/test_managed_network.py @@ -380,3 +380,95 @@ def test_root_required_by_default(rig, monkeypatch): monkeypatch.setattr(network.os, "geteuid", lambda: 1000, raising=False) with pytest.raises(PermissionError, match="root"): manager.up() + + +def test_management_loss_raises_after_fabric_checks_pass(rig): + """The real NetworkManager: an absent management address defers into the + typed loss only after every port, GID/netdev, object and (on request) + RDMA MTU check passed.""" + manager, host = rig + manager.up() # records the validated identity + assert manager.validated_management_identity == manager._management_identity() + # Remove the address from the fake host's management netdev answer. + original_call = host.__call__ + def without_address(argv): + if "addr" in argv and argv[-1] == manager.local.management_netdev: + return json.dumps([{"addr_info": []}]) + return original_call(argv) + manager.runner = without_address + loss = [] + with pytest.raises(network.ManagementAddressLoss, match="Management address"): + manager.check(management_loss=loss) + assert loss == [True] + # Every fabric check ran despite the loss: the port/GID answers were read. + fabric_ports = [argv[-1] for argv in host.commands if "addr" in argv and argv[-1] != manager.local.management_netdev] + assert set(fabric_ports) == {p.netdev for p in manager.local.ports} +def test_management_fail_fast_at_startup(rig): + """Startup semantics: up() with the address already absent fails on the + first _links() call, before any validation is recorded.""" + manager, host = rig + original_call = host.__call__ + def without_address(argv): + if "addr" in argv and argv[-1] == manager.local.management_netdev: + return json.dumps([{"addr_info": []}]) + return original_call(argv) + manager.runner = without_address + with pytest.raises(ValueError, match="Management address"): + manager.up() + assert manager.validated_management_identity is None + + +def test_management_loss_with_fabric_fault_raises_fabric_error(rig): + """A fabric mismatch on the same round wins: no typed loss is raised.""" + manager, host = rig + manager.up() + original_call = host.__call__ + def without_address_and_wrong_mtu(argv): + if "addr" in argv and argv[-1] == manager.local.management_netdev: + return json.dumps([{"addr_info": []}]) + if argv[0] == "ibv_devinfo": + return "\tactive_mtu: 1500 (1)\n" + return original_call(argv) + manager.runner = without_address_and_wrong_mtu + loss = [] + with pytest.raises(ValueError, match="RoCE MTU differs"): + manager.check(management_loss=loss, verify_rdma_mtu=True) + # The loss was recorded during _links but the fabric fault took + # precedence: the raised error is the fabric one, not the typed loss. + + +def test_management_loss_with_missing_object_raises_object_error(rig): + """A missing TC rule on the same round fails immediately, not as a loss.""" + manager, host = rig + manager.up() + key = next(k for k in manager.objects if k.startswith("rule:")) + del host.inventory[key] + original_call = host.__call__ + def without_address(argv): + if "addr" in argv and argv[-1] == manager.local.management_netdev: + return json.dumps([{"addr_info": []}]) + return original_call(argv) + manager.runner = without_address + loss = [] + with pytest.raises(ValueError, match="Missing mesh network objects"): + manager.check(management_loss=loss) + assert loss == [True] # recorded during _links, but objects error wins + + +def test_unvalidated_management_identity_never_receives_grace(rig): + """Without a successful up(), an absent address raises ValueError even + through the loss channel; a site change after validation also fails.""" + manager, host = rig + original_call = host.__call__ + def without_address(argv): + if "addr" in argv and argv[-1] == manager.local.management_netdev: + return json.dumps([{"addr_info": []}]) + return original_call(argv) + manager.runner = without_address + assert manager.validated_management_identity is None + loss = [] + with pytest.raises(ValueError, match="Management address"): + manager.check(management_loss=loss) + assert loss == [] + + diff --git a/runtime/glm53-spark-mtp3-mesh/test_managed_service.py b/runtime/glm53-spark-mtp3-mesh/test_managed_service.py index 38408e49..49d3b7be 100644 --- a/runtime/glm53-spark-mtp3-mesh/test_managed_service.py +++ b/runtime/glm53-spark-mtp3-mesh/test_managed_service.py @@ -404,11 +404,12 @@ def test_old_marker_path_is_not_silently_overlapped(tmp_path): def test_sustained_management_address_loss_latches_failure(): + grace = service.PEER_OUTAGE_GRACE watch = service.PeerWatch() watch.management_error(10.0) - watch.management_error(13.9) + watch.management_error(10.0 + grace - 0.1) with pytest.raises(RuntimeError, match='grace'): - watch.management_error(14.0) + watch.management_error(10.0 + grace) def test_clean_check_clears_management_outage_latch(): watch = service.PeerWatch() @@ -521,10 +522,10 @@ def group_check(*args): class SlowStop: def is_set(self): - return rounds[0] >= 3 + return rounds[0] >= 4 def wait(self, seconds): rounds[0] += 1 - clock[0] += 100.0 + clock[0] += service.PEER_OUTAGE_GRACE * 2 result.stop = SlowStop() assert result.run() == 1 From 544f498c6369a72c1afba0a856dfb870cfc0d302 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Fri, 11 Sep 2026 13:23:34 -0500 Subject: [PATCH 4/4] Record the validated identity after startup validation completes up() now sets validated_management_identity after its links check, the conflict-detection pass and the object creation/verification loop all succeeded, matching the description: the grace bound covers exactly the configuration the supervisor verified at startup, not a partially validated one. The all-fabric-checks-ran assertion now slices host.commands to the commands issued during the outage itself, so startup activity cannot make the test pass falsely. Verified by mutation: skipping the links call inside check() makes the test fail. Executed the grace-sensitive tests under both PEER_OUTAGE_GRACE values: the four PeerWatch latch/admission tests and both run-loop tests pass at 4.0 and at 300.0 (the post-#259 value), not just by arithmetic. --- runtime/glm53-spark-mtp3-mesh/managed_network.py | 6 ++++-- runtime/glm53-spark-mtp3-mesh/test_managed_network.py | 9 +++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/runtime/glm53-spark-mtp3-mesh/managed_network.py b/runtime/glm53-spark-mtp3-mesh/managed_network.py index d34ebae5..eb9e3958 100644 --- a/runtime/glm53-spark-mtp3-mesh/managed_network.py +++ b/runtime/glm53-spark-mtp3-mesh/managed_network.py @@ -269,8 +269,6 @@ def check(self, *, verify_rdma_mtu=True, management_loss=None): def up(self): with self._lock(): self._links() - self.validated_management_identity = self._management_identity() - # Detect conflicting objects before making any network changes. for kind, obj in self.objects.values(): self._present(kind, obj) for key, (kind, obj) in self.objects.items(): @@ -286,6 +284,10 @@ def up(self): self._save() if not self._present(kind, obj): raise ValueError(f"Created network object is absent: {key}") + # Record the validated identity only after every startup check + # above succeeded, so the grace bound covers exactly the + # configuration the supervisor verified at startup. + self.validated_management_identity = self._management_identity() return {**self.check(), "ownership": dict(self.journal["objects"])} def down(self): diff --git a/runtime/glm53-spark-mtp3-mesh/test_managed_network.py b/runtime/glm53-spark-mtp3-mesh/test_managed_network.py index f66970a4..93ee899a 100644 --- a/runtime/glm53-spark-mtp3-mesh/test_managed_network.py +++ b/runtime/glm53-spark-mtp3-mesh/test_managed_network.py @@ -397,12 +397,17 @@ def without_address(argv): return original_call(argv) manager.runner = without_address loss = [] + outage_commands_before = len(host.commands) with pytest.raises(network.ManagementAddressLoss, match="Management address"): manager.check(management_loss=loss) assert loss == [True] - # Every fabric check ran despite the loss: the port/GID answers were read. - fabric_ports = [argv[-1] for argv in host.commands if "addr" in argv and argv[-1] != manager.local.management_netdev] + # Every fabric check ran during THIS outage despite the loss: the + # per-port answers were read after startup activity was excluded. + outage_commands = host.commands[outage_commands_before:] + fabric_ports = [argv[-1] for argv in outage_commands + if "addr" in argv and argv[-1] != manager.local.management_netdev] assert set(fabric_ports) == {p.netdev for p in manager.local.ports} + assert outage_commands # the object-presence queries also ran def test_management_fail_fast_at_startup(rig): """Startup semantics: up() with the address already absent fails on the first _links() call, before any validation is recorded."""