Feature/unconfigured fabric

docs/generate_plugin_doc_bundle.py

@@ -212,9 +212,12 @@ def all_subclasses_union(bases: Iterable[type]) -> set[type]:
 
 
 def all_subclasses_single(cls: type) -> set[type]:
-    seen, out, work = set(), set(), [cls]
+    seen: set[type] = set()
+    out: set[type] = set()
+    work: list[type] = [cls]
     while work:
         parent = work.pop()
+        sub: type
         for sub in parent.__subclasses__():
             if sub not in seen:
                 seen.add(sub)

nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py

@@ -391,7 +391,11 @@ def check_amdsmi_metric_pcie(
                             category=EventCategory.IO,
                             description=f"GPU: {gpu} PCIe link speed is not Gen5 (32 GT/s)",
                             priority=EventPriority.ERROR,
-                            data={"gpu": gpu, "pcie_speed": speed_val, "expected": 32.0},
+                            data={
+                                "gpu": gpu,
+                                "pcie_speed": speed_val,
+                                "expected": 32.0,
+                            },
                             console_log=True,
                         )
                 except (ValueError, TypeError):
@@ -414,7 +418,10 @@ def check_amdsmi_metric_pcie(
                     category=EventCategory.IO,
                     description=f"GPU: {gpu} has PCIe replay rollover count: {pcie_data.replay_roll_over_count}",
                     priority=EventPriority.WARNING,
-                    data={"gpu": gpu, "replay_roll_over_count": pcie_data.replay_roll_over_count},
+                    data={
+                        "gpu": gpu,
+                        "replay_roll_over_count": pcie_data.replay_roll_over_count,
+                    },
                     console_log=True,
                 )
 
@@ -458,7 +465,10 @@ def check_amdsmi_metric_pcie(
                     category=EventCategory.IO,
                     description=f"GPU: {gpu} has received {pcie_data.nak_received_count} PCIe NAKs",
                     priority=EventPriority.WARNING,
-                    data={"gpu": gpu, "nak_received_count": pcie_data.nak_received_count},
+                    data={
+                        "gpu": gpu,
+                        "nak_received_count": pcie_data.nak_received_count,
+                    },
                     console_log=True,
                 )
 
@@ -795,7 +805,12 @@ def check_static_data(
                     and device_id[1].upper() not in dev_actual.upper()
                 ):
                     mismatches.append(
-                        (gpu_data.gpu, "device_id", f"{device_id[0]}|{device_id[1]}", dev_actual)
+                        (
+                            gpu_data.gpu,
+                            "device_id",
+                            f"{device_id[0]}|{device_id[1]}",
+                            dev_actual,
+                        )
                     )
 
             if subsystem_id[0] is not None and subsystem_id[1] is not None:
@@ -1125,16 +1140,23 @@ def check_fabric(
 
         def _add(gpu: int, field: str, expected: object, actual: object) -> None:
             issues.append(
-                {"gpu": gpu, "field": field, "expected": str(expected), "actual": str(actual)}
+                {
+                    "gpu": gpu,
+                    "field": field,
+                    "expected": str(expected),
+                    "actual": str(actual),
+                }
             )
 
         for entry in fabric_data:
             info = entry.fabric_info
             gpu = entry.gpu
-
             accel_state = info.accel_state if info else None
             if accel_state is None or accel_state.strip().upper() != expected_state:
                 _add(gpu, "accel_state", expected_state, accel_state or "N/A")
+                if accel_state == "UNCONFIGURED":
+                    # Skip all other fabric checks for this GPU if it is unconfigured
+                    continue
 
             fabric_type = info.fabric_type if info else None
             if fabric_type is None or fabric_type.strip().upper() != expected_type:
@@ -1155,20 +1177,15 @@ def _add(gpu: int, field: str, expected: object, actual: object) -> None:
                     _add(gpu, field, "non-zero", value if value is not None else "N/A")
 
         if issues:
-            details = "; ".join(
-                f"GPU {i['gpu']} {i['field']}: expected {i['expected']}, actual {i['actual']}"
-                for i in issues
-            )
             gpus_affected = len({i["gpu"] for i in issues})
             self._log_event(
                 category=EventCategory.NETWORK,
-                description=f"Fabric data mismatch on {gpus_affected} GPU(s): {details}",
+                description=f"Fabric data mismatch on {gpus_affected} GPU(s)",
                 priority=EventPriority.ERROR,
                 data={
                     "expected_accel_state": expected_state,
                     "expected_fabric_type": expected_type,
                     "mismatches": issues,
-                    "details": details,
                 },
                 console_log=True,
             )

nodescraper/plugins/inband/fabrics/fabrics_collector.py

@@ -24,7 +24,7 @@
 #
 ###############################################################################
 import re
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple, Union
 
 from nodescraper.base import InBandDataCollector
 from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus
@@ -425,7 +425,10 @@ def collect_data(
             ibstat_ports = []
             for device in ibstat_devices:
                 for port_num, attrs in device.ports.items():
-                    port = {"device": device.ca_name, "port": port_num}
+                    port: Dict[str, Union[int, str, None]] = {
+                        "device": device.ca_name,
+                        "port": port_num,
+                    }
                     if "Link layer" in attrs:
                         port["link_layer"] = attrs["Link layer"]
                     if "Rate" in attrs:

test/unit/plugin/test_amdsmi_analyzer.py

@@ -28,7 +28,7 @@
 
 import pytest
 
-from nodescraper.enums import EventPriority
+from nodescraper.enums import EventPriority, ExecutionStatus
 from nodescraper.plugins.inband.amdsmi.amdsmi_analyzer import (
     AmdSmiAnalyzer,
     _gpu_mismatch_description,
@@ -1565,3 +1565,76 @@ def test_check_fabric_zero_ppod_id(mock_analyzer):
     mismatches = analyzer.result.events[0].data["mismatches"]
     assert [m["field"] for m in mismatches] == ["ppod_id"]
     assert mismatches[0]["expected"] == "non-zero"
+
+
+def create_unconfigured_fabric(gpu: int) -> Fabric:
+    """Helper building amd-smi fabric output for a GPU whose fabric is not configured."""
+    return Fabric.model_validate(
+        {
+            "gpu": gpu,
+            "bdf": f"0000:{gpu:02x}:00.0",
+            "fabric_info": {
+                "bdf": f"0000:{gpu:02x}:00.1",
+                "version": 4294967295,
+                "accelerator_id": gpu,
+                "fabric_type": "UNKNOWN",
+                "bandwidth": {"value": 0, "unit": "Mb/s"},
+                "latency": {"value": 0, "unit": "ns"},
+                "ppod_id": "00000000-0000-0000-0000-000000000000",
+                "ppod_size": 0,
+                "vpod_id": 0,
+                "vpod_size": 0,
+                "local_accelerators": "0, 0, 0, 0, 0, 0, 0, 0",
+                "local_active_accelerators": ["0, 0, 0, 0, 0, 0, 0, 0"],
+                "addr_mode": "UNKNOWN",
+                "accel_state": "UNCONFIGURED",
+            },
+        }
+    )
+
+
+def test_check_fabric_unconfigured_many_gpus(mock_analyzer):
+    """8 GPUs failing all 7 fabric fields log one bounded ERROR event with full detail in data."""
+    analyzer = mock_analyzer
+
+    analyzer.check_fabric([create_unconfigured_fabric(gpu) for gpu in range(8)])
+
+    assert len(analyzer.result.events) == 1
+    event = analyzer.result.events[0]
+    assert event.priority == EventPriority.ERROR
+    assert event.category == "NETWORK"
+    assert len(event.data["mismatches"]) == 8
+    assert {m["gpu"] for m in event.data["mismatches"]} == set(range(8))
+    assert {m["field"] for m in event.data["mismatches"]} == {"accel_state"}
+    assert len(event.description.encode("utf-8")) < 2048
+    assert event.description.startswith("Fabric data mismatch on 8 GPU(s)")
+
+
+def test_analyze_data_unconfigured_fabric_completes(mock_analyzer):
+    """An oversized fabric mismatch does not abort analyze_data or skip later checks."""
+    analyzer = mock_analyzer
+
+    data = AmdSmiDataModel(
+        version=None,
+        static=None,
+        process=None,
+        firmware=None,
+        partition=None,
+        gpu_list=None,
+        fabric=[create_unconfigured_fabric(gpu) for gpu in range(8)],
+        xgmi_link=[XgmiLinks(gpu=0, bdf="0000:00:00.0", link_status=[LinkStatusTable.UP])],
+    )
+
+    result = analyzer.analyze_data(data, AmdSmiAnalyzerArgs(check_xgmi_or_peer_links_status=True))
+
+    assert result.status != ExecutionStatus.EXECUTION_FAILURE
+    assert not any(e.priority == EventPriority.CRITICAL for e in result.events)
+    assert not any(e.description == "Validation error during analysis" for e in result.events)
+    error_events = [e for e in result.events if e.priority == EventPriority.ERROR]
+    assert len(error_events) == 1
+    assert error_events[0].description.startswith("Fabric data mismatch")
+    assert any(
+        e.priority == EventPriority.INFO
+        and e.description == "All XGMI/peer GPU links are working fine"
+        for e in result.events
+    )