diff --git a/docs/generate_plugin_doc_bundle.py b/docs/generate_plugin_doc_bundle.py index 4e148d96..3f2ff819 100644 --- a/docs/generate_plugin_doc_bundle.py +++ b/docs/generate_plugin_doc_bundle.py @@ -212,9 +212,12 @@ def all_subclasses_union(bases: Iterable[type]) -> set[type]: def all_subclasses_single(cls: type) -> set[type]: - seen, out, work = set(), set(), [cls] + seen: set[type] = set() + out: set[type] = set() + work: list[type] = [cls] while work: parent = work.pop() + sub: type for sub in parent.__subclasses__(): if sub not in seen: seen.add(sub) diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py index 0a74c710..cf859125 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py @@ -391,7 +391,11 @@ def check_amdsmi_metric_pcie( category=EventCategory.IO, description=f"GPU: {gpu} PCIe link speed is not Gen5 (32 GT/s)", priority=EventPriority.ERROR, - data={"gpu": gpu, "pcie_speed": speed_val, "expected": 32.0}, + data={ + "gpu": gpu, + "pcie_speed": speed_val, + "expected": 32.0, + }, console_log=True, ) except (ValueError, TypeError): @@ -414,7 +418,10 @@ def check_amdsmi_metric_pcie( category=EventCategory.IO, description=f"GPU: {gpu} has PCIe replay rollover count: {pcie_data.replay_roll_over_count}", priority=EventPriority.WARNING, - data={"gpu": gpu, "replay_roll_over_count": pcie_data.replay_roll_over_count}, + data={ + "gpu": gpu, + "replay_roll_over_count": pcie_data.replay_roll_over_count, + }, console_log=True, ) @@ -458,7 +465,10 @@ def check_amdsmi_metric_pcie( category=EventCategory.IO, description=f"GPU: {gpu} has received {pcie_data.nak_received_count} PCIe NAKs", priority=EventPriority.WARNING, - data={"gpu": gpu, "nak_received_count": pcie_data.nak_received_count}, + data={ + "gpu": gpu, + "nak_received_count": pcie_data.nak_received_count, + }, console_log=True, ) @@ -795,7 +805,12 @@ def check_static_data( and device_id[1].upper() not in dev_actual.upper() ): mismatches.append( - (gpu_data.gpu, "device_id", f"{device_id[0]}|{device_id[1]}", dev_actual) + ( + gpu_data.gpu, + "device_id", + f"{device_id[0]}|{device_id[1]}", + dev_actual, + ) ) if subsystem_id[0] is not None and subsystem_id[1] is not None: @@ -1125,16 +1140,23 @@ def check_fabric( def _add(gpu: int, field: str, expected: object, actual: object) -> None: issues.append( - {"gpu": gpu, "field": field, "expected": str(expected), "actual": str(actual)} + { + "gpu": gpu, + "field": field, + "expected": str(expected), + "actual": str(actual), + } ) for entry in fabric_data: info = entry.fabric_info gpu = entry.gpu - accel_state = info.accel_state if info else None if accel_state is None or accel_state.strip().upper() != expected_state: _add(gpu, "accel_state", expected_state, accel_state or "N/A") + if accel_state == "UNCONFIGURED": + # Skip all other fabric checks for this GPU if it is unconfigured + continue fabric_type = info.fabric_type if info else None if fabric_type is None or fabric_type.strip().upper() != expected_type: @@ -1155,20 +1177,15 @@ def _add(gpu: int, field: str, expected: object, actual: object) -> None: _add(gpu, field, "non-zero", value if value is not None else "N/A") if issues: - details = "; ".join( - f"GPU {i['gpu']} {i['field']}: expected {i['expected']}, actual {i['actual']}" - for i in issues - ) gpus_affected = len({i["gpu"] for i in issues}) self._log_event( category=EventCategory.NETWORK, - description=f"Fabric data mismatch on {gpus_affected} GPU(s): {details}", + description=f"Fabric data mismatch on {gpus_affected} GPU(s)", priority=EventPriority.ERROR, data={ "expected_accel_state": expected_state, "expected_fabric_type": expected_type, "mismatches": issues, - "details": details, }, console_log=True, ) diff --git a/nodescraper/plugins/inband/fabrics/fabrics_collector.py b/nodescraper/plugins/inband/fabrics/fabrics_collector.py index 4c7501a2..e1a9afbd 100644 --- a/nodescraper/plugins/inband/fabrics/fabrics_collector.py +++ b/nodescraper/plugins/inband/fabrics/fabrics_collector.py @@ -24,7 +24,7 @@ # ############################################################################### import re -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple, Union from nodescraper.base import InBandDataCollector from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus @@ -425,7 +425,10 @@ def collect_data( ibstat_ports = [] for device in ibstat_devices: for port_num, attrs in device.ports.items(): - port = {"device": device.ca_name, "port": port_num} + port: Dict[str, Union[int, str, None]] = { + "device": device.ca_name, + "port": port_num, + } if "Link layer" in attrs: port["link_layer"] = attrs["Link layer"] if "Rate" in attrs: diff --git a/test/unit/plugin/test_amdsmi_analyzer.py b/test/unit/plugin/test_amdsmi_analyzer.py index 1b2329e8..3674ea40 100644 --- a/test/unit/plugin/test_amdsmi_analyzer.py +++ b/test/unit/plugin/test_amdsmi_analyzer.py @@ -28,7 +28,7 @@ import pytest -from nodescraper.enums import EventPriority +from nodescraper.enums import EventPriority, ExecutionStatus from nodescraper.plugins.inband.amdsmi.amdsmi_analyzer import ( AmdSmiAnalyzer, _gpu_mismatch_description, @@ -1565,3 +1565,76 @@ def test_check_fabric_zero_ppod_id(mock_analyzer): mismatches = analyzer.result.events[0].data["mismatches"] assert [m["field"] for m in mismatches] == ["ppod_id"] assert mismatches[0]["expected"] == "non-zero" + + +def create_unconfigured_fabric(gpu: int) -> Fabric: + """Helper building amd-smi fabric output for a GPU whose fabric is not configured.""" + return Fabric.model_validate( + { + "gpu": gpu, + "bdf": f"0000:{gpu:02x}:00.0", + "fabric_info": { + "bdf": f"0000:{gpu:02x}:00.1", + "version": 4294967295, + "accelerator_id": gpu, + "fabric_type": "UNKNOWN", + "bandwidth": {"value": 0, "unit": "Mb/s"}, + "latency": {"value": 0, "unit": "ns"}, + "ppod_id": "00000000-0000-0000-0000-000000000000", + "ppod_size": 0, + "vpod_id": 0, + "vpod_size": 0, + "local_accelerators": "0, 0, 0, 0, 0, 0, 0, 0", + "local_active_accelerators": ["0, 0, 0, 0, 0, 0, 0, 0"], + "addr_mode": "UNKNOWN", + "accel_state": "UNCONFIGURED", + }, + } + ) + + +def test_check_fabric_unconfigured_many_gpus(mock_analyzer): + """8 GPUs failing all 7 fabric fields log one bounded ERROR event with full detail in data.""" + analyzer = mock_analyzer + + analyzer.check_fabric([create_unconfigured_fabric(gpu) for gpu in range(8)]) + + assert len(analyzer.result.events) == 1 + event = analyzer.result.events[0] + assert event.priority == EventPriority.ERROR + assert event.category == "NETWORK" + assert len(event.data["mismatches"]) == 8 + assert {m["gpu"] for m in event.data["mismatches"]} == set(range(8)) + assert {m["field"] for m in event.data["mismatches"]} == {"accel_state"} + assert len(event.description.encode("utf-8")) < 2048 + assert event.description.startswith("Fabric data mismatch on 8 GPU(s)") + + +def test_analyze_data_unconfigured_fabric_completes(mock_analyzer): + """An oversized fabric mismatch does not abort analyze_data or skip later checks.""" + analyzer = mock_analyzer + + data = AmdSmiDataModel( + version=None, + static=None, + process=None, + firmware=None, + partition=None, + gpu_list=None, + fabric=[create_unconfigured_fabric(gpu) for gpu in range(8)], + xgmi_link=[XgmiLinks(gpu=0, bdf="0000:00:00.0", link_status=[LinkStatusTable.UP])], + ) + + result = analyzer.analyze_data(data, AmdSmiAnalyzerArgs(check_xgmi_or_peer_links_status=True)) + + assert result.status != ExecutionStatus.EXECUTION_FAILURE + assert not any(e.priority == EventPriority.CRITICAL for e in result.events) + assert not any(e.description == "Validation error during analysis" for e in result.events) + error_events = [e for e in result.events if e.priority == EventPriority.ERROR] + assert len(error_events) == 1 + assert error_events[0].description.startswith("Fabric data mismatch") + assert any( + e.priority == EventPriority.INFO + and e.description == "All XGMI/peer GPU links are working fine" + for e in result.events + )