From 9666e7544093f8c6d54efda71dbc8687f0619654 Mon Sep 17 00:00:00 2001 From: graepaul_amdeng Date: Thu, 15 Oct 2026 16:51:54 -0700 Subject: [PATCH 1/3] AmdSmiPlugin analyzer crash fix --- docs/generate_plugin_doc_bundle.py | 5 +- .../plugins/inband/amdsmi/amdsmi_analyzer.py | 6 +- .../inband/fabrics/fabrics_collector.py | 7 +- test/unit/plugin/test_amdsmi_analyzer.py | 89 ++++++++++++++++++- 4 files changed, 102 insertions(+), 5 deletions(-) diff --git a/docs/generate_plugin_doc_bundle.py b/docs/generate_plugin_doc_bundle.py index 4e148d96..3f2ff819 100644 --- a/docs/generate_plugin_doc_bundle.py +++ b/docs/generate_plugin_doc_bundle.py @@ -212,9 +212,12 @@ def all_subclasses_union(bases: Iterable[type]) -> set[type]: def all_subclasses_single(cls: type) -> set[type]: - seen, out, work = set(), set(), [cls] + seen: set[type] = set() + out: set[type] = set() + work: list[type] = [cls] while work: parent = work.pop() + sub: type for sub in parent.__subclasses__(): if sub not in seen: seen.add(sub) diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py index 0a74c710..27ddb670 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py @@ -1160,9 +1160,13 @@ def _add(gpu: int, field: str, expected: object, actual: object) -> None: for i in issues ) gpus_affected = len({i["gpu"] for i in issues}) + fields = sorted({i["field"] for i in issues}) self._log_event( category=EventCategory.NETWORK, - description=f"Fabric data mismatch on {gpus_affected} GPU(s): {details}", + description=( + f"Fabric data mismatch on {gpus_affected} GPU(s): {', '.join(fields)} " + "(per-GPU detail in data.mismatches)" + ), priority=EventPriority.ERROR, data={ "expected_accel_state": expected_state, diff --git a/nodescraper/plugins/inband/fabrics/fabrics_collector.py b/nodescraper/plugins/inband/fabrics/fabrics_collector.py index 4c7501a2..e1a9afbd 100644 --- a/nodescraper/plugins/inband/fabrics/fabrics_collector.py +++ b/nodescraper/plugins/inband/fabrics/fabrics_collector.py @@ -24,7 +24,7 @@ # ############################################################################### import re -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple, Union from nodescraper.base import InBandDataCollector from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus @@ -425,7 +425,10 @@ def collect_data( ibstat_ports = [] for device in ibstat_devices: for port_num, attrs in device.ports.items(): - port = {"device": device.ca_name, "port": port_num} + port: Dict[str, Union[int, str, None]] = { + "device": device.ca_name, + "port": port_num, + } if "Link layer" in attrs: port["link_layer"] = attrs["Link layer"] if "Rate" in attrs: diff --git a/test/unit/plugin/test_amdsmi_analyzer.py b/test/unit/plugin/test_amdsmi_analyzer.py index 1b2329e8..28ef2c8a 100644 --- a/test/unit/plugin/test_amdsmi_analyzer.py +++ b/test/unit/plugin/test_amdsmi_analyzer.py @@ -28,7 +28,7 @@ import pytest -from nodescraper.enums import EventPriority +from nodescraper.enums import EventPriority, ExecutionStatus from nodescraper.plugins.inband.amdsmi.amdsmi_analyzer import ( AmdSmiAnalyzer, _gpu_mismatch_description, @@ -1565,3 +1565,90 @@ def test_check_fabric_zero_ppod_id(mock_analyzer): mismatches = analyzer.result.events[0].data["mismatches"] assert [m["field"] for m in mismatches] == ["ppod_id"] assert mismatches[0]["expected"] == "non-zero" + + +_ALL_FABRIC_FIELDS = { + "accel_state", + "fabric_type", + "ppod_id", + "local_accelerators", + "local_active_accelerators", + "ppod_size", + "vpod_size", +} + + +def create_unconfigured_fabric(gpu: int) -> Fabric: + """Helper building amd-smi fabric output for a GPU whose fabric is not configured.""" + return Fabric.model_validate( + { + "gpu": gpu, + "bdf": f"0000:{gpu:02x}:00.0", + "fabric_info": { + "bdf": f"0000:{gpu:02x}:00.1", + "version": 4294967295, + "accelerator_id": gpu, + "fabric_type": "UNKNOWN", + "bandwidth": {"value": 0, "unit": "Mb/s"}, + "latency": {"value": 0, "unit": "ns"}, + "ppod_id": "00000000-0000-0000-0000-000000000000", + "ppod_size": 0, + "vpod_id": 0, + "vpod_size": 0, + "local_accelerators": "0, 0, 0, 0, 0, 0, 0, 0", + "local_active_accelerators": ["0, 0, 0, 0, 0, 0, 0, 0"], + "addr_mode": "UNKNOWN", + "accel_state": "UNCONFIGURED", + }, + } + ) + + +def test_check_fabric_unconfigured_many_gpus(mock_analyzer): + """8 GPUs failing all 7 fabric fields log one bounded ERROR event with full detail in data.""" + analyzer = mock_analyzer + + analyzer.check_fabric([create_unconfigured_fabric(gpu) for gpu in range(8)]) + + assert len(analyzer.result.events) == 1 + event = analyzer.result.events[0] + assert event.priority == EventPriority.ERROR + assert event.category == "NETWORK" + assert len(event.data["mismatches"]) == 56 + assert {m["gpu"] for m in event.data["mismatches"]} == set(range(8)) + assert {m["field"] for m in event.data["mismatches"]} == _ALL_FABRIC_FIELDS + assert len(event.description.encode("utf-8")) < 2048 + assert event.description.startswith("Fabric data mismatch on 8 GPU(s)") + for field in _ALL_FABRIC_FIELDS: + assert field in event.description + assert "GPU 7 accel_state: expected ACTIVE, actual UNCONFIGURED" in event.data["details"] + + +def test_analyze_data_unconfigured_fabric_completes(mock_analyzer): + """An oversized fabric mismatch does not abort analyze_data or skip later checks.""" + analyzer = mock_analyzer + + data = AmdSmiDataModel( + version=None, + static=None, + process=None, + firmware=None, + partition=None, + gpu_list=None, + fabric=[create_unconfigured_fabric(gpu) for gpu in range(8)], + xgmi_link=[XgmiLinks(gpu=0, bdf="0000:00:00.0", link_status=[LinkStatusTable.UP])], + ) + + result = analyzer.analyze_data(data, AmdSmiAnalyzerArgs(check_xgmi_or_peer_links_status=True)) + + assert result.status != ExecutionStatus.EXECUTION_FAILURE + assert not any(e.priority == EventPriority.CRITICAL for e in result.events) + assert not any(e.description == "Validation error during analysis" for e in result.events) + error_events = [e for e in result.events if e.priority == EventPriority.ERROR] + assert len(error_events) == 1 + assert error_events[0].description.startswith("Fabric data mismatch") + assert any( + e.priority == EventPriority.INFO + and e.description == "All XGMI/peer GPU links are working fine" + for e in result.events + ) From 763d00a3d2c8d7a083648d1293417dd1e35e407d Mon Sep 17 00:00:00 2001 From: graepaul_amdeng Date: Fri, 16 Oct 2026 16:14:28 -0700 Subject: [PATCH 2/3] Reduce errors on UNCONFIGURED fabric --- .../plugins/inband/amdsmi/amdsmi_analyzer.py | 116 +++++++++++++----- test/unit/plugin/test_amdsmi_analyzer.py | 18 +-- 2 files changed, 85 insertions(+), 49 deletions(-) diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py index 27ddb670..770f5b5a 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py @@ -59,10 +59,13 @@ def _gpu_mismatch_description( """Return (event description, console details) for per-GPU expected vs actual.""" gpu_ids = sorted(actual_by_gpu.keys()) exp = f"{expected}{unit}" if unit else str(expected) - per_gpu_short = ", ".join(f"GPU {gpu}={actual_by_gpu[gpu]}{unit}" for gpu in gpu_ids) + per_gpu_short = ", ".join( + f"GPU {gpu}={actual_by_gpu[gpu]}{unit}" for gpu in gpu_ids + ) description = f"{check_name} mismatch (expected {exp}): {per_gpu_short}" details = "; ".join( - f"GPU {gpu}: expected {exp}, actual {actual_by_gpu[gpu]}{unit}" for gpu in gpu_ids + f"GPU {gpu}: expected {exp}, actual {actual_by_gpu[gpu]}{unit}" + for gpu in gpu_ids ) return description, details @@ -391,7 +394,11 @@ def check_amdsmi_metric_pcie( category=EventCategory.IO, description=f"GPU: {gpu} PCIe link speed is not Gen5 (32 GT/s)", priority=EventPriority.ERROR, - data={"gpu": gpu, "pcie_speed": speed_val, "expected": 32.0}, + data={ + "gpu": gpu, + "pcie_speed": speed_val, + "expected": 32.0, + }, console_log=True, ) except (ValueError, TypeError): @@ -414,12 +421,18 @@ def check_amdsmi_metric_pcie( category=EventCategory.IO, description=f"GPU: {gpu} has PCIe replay rollover count: {pcie_data.replay_roll_over_count}", priority=EventPriority.WARNING, - data={"gpu": gpu, "replay_roll_over_count": pcie_data.replay_roll_over_count}, + data={ + "gpu": gpu, + "replay_roll_over_count": pcie_data.replay_roll_over_count, + }, console_log=True, ) if pcie_data.l0_to_recovery_count is not None: - if pcie_data.l0_to_recovery_count > l0_to_recovery_count_error_threshold: + if ( + pcie_data.l0_to_recovery_count + > l0_to_recovery_count_error_threshold + ): self._log_event( category=EventCategory.IO, description=f"GPU: {gpu} has {pcie_data.l0_to_recovery_count} L0 recoveries", @@ -431,7 +444,10 @@ def check_amdsmi_metric_pcie( }, console_log=True, ) - elif pcie_data.l0_to_recovery_count > l0_to_recovery_count_warning_threshold: + elif ( + pcie_data.l0_to_recovery_count + > l0_to_recovery_count_warning_threshold + ): self._log_event( category=EventCategory.IO, description=f"GPU: {gpu} has {pcie_data.l0_to_recovery_count} L0 recoveries", @@ -453,12 +469,18 @@ def check_amdsmi_metric_pcie( console_log=True, ) - if pcie_data.nak_received_count is not None and pcie_data.nak_received_count > 0: + if ( + pcie_data.nak_received_count is not None + and pcie_data.nak_received_count > 0 + ): self._log_event( category=EventCategory.IO, description=f"GPU: {gpu} has received {pcie_data.nak_received_count} PCIe NAKs", priority=EventPriority.WARNING, - data={"gpu": gpu, "nak_received_count": pcie_data.nak_received_count}, + data={ + "gpu": gpu, + "nak_received_count": pcie_data.nak_received_count, + }, console_log=True, ) @@ -625,7 +647,10 @@ def check_amdsmi_metric_ecc(self, amdsmi_metric_data: list[AmdSmiMetric]): if not isinstance(ecc_data, EccData): continue - if ecc_data.correctable_count is not None and ecc_data.correctable_count > 0: + if ( + ecc_data.correctable_count is not None + and ecc_data.correctable_count > 0 + ): self._log_event( category=EventCategory.RAS, description=f"GPU: {gpu} has correctable ECC errors in block {block_name}", @@ -638,7 +663,10 @@ def check_amdsmi_metric_ecc(self, amdsmi_metric_data: list[AmdSmiMetric]): console_log=True, ) - if ecc_data.uncorrectable_count is not None and ecc_data.uncorrectable_count > 0: + if ( + ecc_data.uncorrectable_count is not None + and ecc_data.uncorrectable_count > 0 + ): self._log_event( category=EventCategory.RAS, description=f"GPU: {gpu} has uncorrectable ECC errors in block {block_name}", @@ -726,7 +754,9 @@ def static_consistancy_check(self, amdsmi_static_data: list[AmdSmiStatic]): "subsystem_id": {gpu.asic.subsystem_id for gpu in amdsmi_static_data}, "device_id": {gpu.asic.device_id for gpu in amdsmi_static_data}, "rev_id": {gpu.asic.rev_id for gpu in amdsmi_static_data}, - "num_compute_units": {str(gpu.asic.num_compute_units) for gpu in amdsmi_static_data}, + "num_compute_units": { + str(gpu.asic.num_compute_units) for gpu in amdsmi_static_data + }, "target_graphics_version": { gpu.asic.target_graphics_version for gpu in amdsmi_static_data }, @@ -795,7 +825,12 @@ def check_static_data( and device_id[1].upper() not in dev_actual.upper() ): mismatches.append( - (gpu_data.gpu, "device_id", f"{device_id[0]}|{device_id[1]}", dev_actual) + ( + gpu_data.gpu, + "device_id", + f"{device_id[0]}|{device_id[1]}", + dev_actual, + ) ) if subsystem_id[0] is not None and subsystem_id[1] is not None: @@ -842,7 +877,9 @@ def _format_static_mismatch_payload( for gpu, field, expected, actual in mismatches: field_set.add(field) - per_gpu[gpu].append({"field": field, "expected": expected, "actual": actual}) + per_gpu[gpu].append( + {"field": field, "expected": expected, "actual": actual} + ) per_gpu_list: list[dict[str, Any]] = [ {"gpu": gpu, "mismatches": entries} @@ -1026,7 +1063,9 @@ def check_expected_xgmi_link_speed( if expected_xgmi_speed is None or len(expected_xgmi_speed) == 0: self._log_event( category=EventCategory.IO, - description=("Expected XGMI link speed not set; skipping XGMI link speed analysis"), + description=( + "Expected XGMI link speed not set; skipping XGMI link speed analysis" + ), priority=EventPriority.INFO, console_log=True, ) @@ -1048,7 +1087,9 @@ def check_expected_xgmi_link_speed( "gpu": xgmi_data.gpu, "expected_xgmi_speed": expected_xgmi_speed, "xgmi_bit_rate": ( - link_metric.bit_rate.unit if link_metric.bit_rate else "N/A" + link_metric.bit_rate.unit + if link_metric.bit_rate + else "N/A" ), "details": details, }, @@ -1070,7 +1111,9 @@ def check_expected_xgmi_link_speed( "gpu": xgmi_data.gpu, "expected_xgmi_speed": expected_xgmi_speed, "xgmi_bit_rate": ( - link_metric.bit_rate.value if link_metric.bit_rate else "N/A" + link_metric.bit_rate.value + if link_metric.bit_rate + else "N/A" ), "details": details, }, @@ -1125,16 +1168,23 @@ def check_fabric( def _add(gpu: int, field: str, expected: object, actual: object) -> None: issues.append( - {"gpu": gpu, "field": field, "expected": str(expected), "actual": str(actual)} + { + "gpu": gpu, + "field": field, + "expected": str(expected), + "actual": str(actual), + } ) for entry in fabric_data: info = entry.fabric_info gpu = entry.gpu - accel_state = info.accel_state if info else None if accel_state is None or accel_state.strip().upper() != expected_state: _add(gpu, "accel_state", expected_state, accel_state or "N/A") + if accel_state == "UNCONFIGURED": + # Skip all other fabric checks for this GPU if it is unconfigured + continue fabric_type = info.fabric_type if info else None if fabric_type is None or fabric_type.strip().upper() != expected_type: @@ -1155,29 +1205,22 @@ def _add(gpu: int, field: str, expected: object, actual: object) -> None: _add(gpu, field, "non-zero", value if value is not None else "N/A") if issues: - details = "; ".join( - f"GPU {i['gpu']} {i['field']}: expected {i['expected']}, actual {i['actual']}" - for i in issues - ) gpus_affected = len({i["gpu"] for i in issues}) - fields = sorted({i["field"] for i in issues}) self._log_event( category=EventCategory.NETWORK, - description=( - f"Fabric data mismatch on {gpus_affected} GPU(s): {', '.join(fields)} " - "(per-GPU detail in data.mismatches)" - ), + description=f"Fabric data mismatch on {gpus_affected} GPU(s)", priority=EventPriority.ERROR, data={ "expected_accel_state": expected_state, "expected_fabric_type": expected_type, "mismatches": issues, - "details": details, }, console_log=True, ) - def check_xgmi_or_peer_links_status(self, xgmi_links: Optional[list[XgmiLinks]]) -> None: + def check_xgmi_or_peer_links_status( + self, xgmi_links: Optional[list[XgmiLinks]] + ) -> None: """Check XGMI or peer-link status: U passes, SELF is ignored, D/X warn.""" if not xgmi_links: self._log_event( @@ -1220,7 +1263,9 @@ def check_xgmi_or_peer_links_status(self, xgmi_links: Optional[list[XgmiLinks]]) if down_links: self._log_event( category=EventCategory.IO, - description=(f"XGMI/peer links contain {len(down_links)} down/error links"), + description=( + f"XGMI/peer links contain {len(down_links)} down/error links" + ), priority=EventPriority.WARNING, data={ "down_links": down_links, @@ -1233,7 +1278,8 @@ def check_xgmi_or_peer_links_status(self, xgmi_links: Optional[list[XgmiLinks]]) self._log_event( category=EventCategory.IO, description=( - f"XGMI/peer links contain {len(degraded_links)} " "disabled/degraded links" + f"XGMI/peer links contain {len(degraded_links)} " + "disabled/degraded links" ), priority=EventPriority.WARNING, data={ @@ -1261,7 +1307,9 @@ def analyze_data( if data.metric is not None and len(data.metric) > 0: if args.expected_power_management: - self.check_expected_power_management(data.metric, args.expected_power_management) + self.check_expected_power_management( + data.metric, args.expected_power_management + ) if args.l0_to_recovery_count_error_threshold is not None: self.check_amdsmi_metric_pcie( data.metric, @@ -1293,7 +1341,9 @@ def analyze_data( if args.power: self.check_power_cap_consistency(data.static, args.power) if args.expected_driver_version: - self.check_expected_driver_version(data.static, args.expected_driver_version) + self.check_expected_driver_version( + data.static, args.expected_driver_version + ) self.static_consistancy_check(data.static) if ( diff --git a/test/unit/plugin/test_amdsmi_analyzer.py b/test/unit/plugin/test_amdsmi_analyzer.py index 28ef2c8a..d6899a42 100644 --- a/test/unit/plugin/test_amdsmi_analyzer.py +++ b/test/unit/plugin/test_amdsmi_analyzer.py @@ -1567,17 +1567,6 @@ def test_check_fabric_zero_ppod_id(mock_analyzer): assert mismatches[0]["expected"] == "non-zero" -_ALL_FABRIC_FIELDS = { - "accel_state", - "fabric_type", - "ppod_id", - "local_accelerators", - "local_active_accelerators", - "ppod_size", - "vpod_size", -} - - def create_unconfigured_fabric(gpu: int) -> Fabric: """Helper building amd-smi fabric output for a GPU whose fabric is not configured.""" return Fabric.model_validate( @@ -1614,14 +1603,11 @@ def test_check_fabric_unconfigured_many_gpus(mock_analyzer): event = analyzer.result.events[0] assert event.priority == EventPriority.ERROR assert event.category == "NETWORK" - assert len(event.data["mismatches"]) == 56 + assert len(event.data["mismatches"]) == 8 assert {m["gpu"] for m in event.data["mismatches"]} == set(range(8)) - assert {m["field"] for m in event.data["mismatches"]} == _ALL_FABRIC_FIELDS + assert {m["field"] for m in event.data["mismatches"]} == {'accel_state'} assert len(event.description.encode("utf-8")) < 2048 assert event.description.startswith("Fabric data mismatch on 8 GPU(s)") - for field in _ALL_FABRIC_FIELDS: - assert field in event.description - assert "GPU 7 accel_state: expected ACTIVE, actual UNCONFIGURED" in event.data["details"] def test_analyze_data_unconfigured_fabric_completes(mock_analyzer): From dc6ef8a578c288677be5d0abeaa32d13a6a982d1 Mon Sep 17 00:00:00 2001 From: graepaul_amdeng Date: Fri, 16 Oct 2026 16:15:58 -0700 Subject: [PATCH 3/3] Format --- .../plugins/inband/amdsmi/amdsmi_analyzer.py | 71 +++++-------------- test/unit/plugin/test_amdsmi_analyzer.py | 2 +- 2 files changed, 18 insertions(+), 55 deletions(-) diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py index 770f5b5a..cf859125 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py @@ -59,13 +59,10 @@ def _gpu_mismatch_description( """Return (event description, console details) for per-GPU expected vs actual.""" gpu_ids = sorted(actual_by_gpu.keys()) exp = f"{expected}{unit}" if unit else str(expected) - per_gpu_short = ", ".join( - f"GPU {gpu}={actual_by_gpu[gpu]}{unit}" for gpu in gpu_ids - ) + per_gpu_short = ", ".join(f"GPU {gpu}={actual_by_gpu[gpu]}{unit}" for gpu in gpu_ids) description = f"{check_name} mismatch (expected {exp}): {per_gpu_short}" details = "; ".join( - f"GPU {gpu}: expected {exp}, actual {actual_by_gpu[gpu]}{unit}" - for gpu in gpu_ids + f"GPU {gpu}: expected {exp}, actual {actual_by_gpu[gpu]}{unit}" for gpu in gpu_ids ) return description, details @@ -429,10 +426,7 @@ def check_amdsmi_metric_pcie( ) if pcie_data.l0_to_recovery_count is not None: - if ( - pcie_data.l0_to_recovery_count - > l0_to_recovery_count_error_threshold - ): + if pcie_data.l0_to_recovery_count > l0_to_recovery_count_error_threshold: self._log_event( category=EventCategory.IO, description=f"GPU: {gpu} has {pcie_data.l0_to_recovery_count} L0 recoveries", @@ -444,10 +438,7 @@ def check_amdsmi_metric_pcie( }, console_log=True, ) - elif ( - pcie_data.l0_to_recovery_count - > l0_to_recovery_count_warning_threshold - ): + elif pcie_data.l0_to_recovery_count > l0_to_recovery_count_warning_threshold: self._log_event( category=EventCategory.IO, description=f"GPU: {gpu} has {pcie_data.l0_to_recovery_count} L0 recoveries", @@ -469,10 +460,7 @@ def check_amdsmi_metric_pcie( console_log=True, ) - if ( - pcie_data.nak_received_count is not None - and pcie_data.nak_received_count > 0 - ): + if pcie_data.nak_received_count is not None and pcie_data.nak_received_count > 0: self._log_event( category=EventCategory.IO, description=f"GPU: {gpu} has received {pcie_data.nak_received_count} PCIe NAKs", @@ -647,10 +635,7 @@ def check_amdsmi_metric_ecc(self, amdsmi_metric_data: list[AmdSmiMetric]): if not isinstance(ecc_data, EccData): continue - if ( - ecc_data.correctable_count is not None - and ecc_data.correctable_count > 0 - ): + if ecc_data.correctable_count is not None and ecc_data.correctable_count > 0: self._log_event( category=EventCategory.RAS, description=f"GPU: {gpu} has correctable ECC errors in block {block_name}", @@ -663,10 +648,7 @@ def check_amdsmi_metric_ecc(self, amdsmi_metric_data: list[AmdSmiMetric]): console_log=True, ) - if ( - ecc_data.uncorrectable_count is not None - and ecc_data.uncorrectable_count > 0 - ): + if ecc_data.uncorrectable_count is not None and ecc_data.uncorrectable_count > 0: self._log_event( category=EventCategory.RAS, description=f"GPU: {gpu} has uncorrectable ECC errors in block {block_name}", @@ -754,9 +736,7 @@ def static_consistancy_check(self, amdsmi_static_data: list[AmdSmiStatic]): "subsystem_id": {gpu.asic.subsystem_id for gpu in amdsmi_static_data}, "device_id": {gpu.asic.device_id for gpu in amdsmi_static_data}, "rev_id": {gpu.asic.rev_id for gpu in amdsmi_static_data}, - "num_compute_units": { - str(gpu.asic.num_compute_units) for gpu in amdsmi_static_data - }, + "num_compute_units": {str(gpu.asic.num_compute_units) for gpu in amdsmi_static_data}, "target_graphics_version": { gpu.asic.target_graphics_version for gpu in amdsmi_static_data }, @@ -877,9 +857,7 @@ def _format_static_mismatch_payload( for gpu, field, expected, actual in mismatches: field_set.add(field) - per_gpu[gpu].append( - {"field": field, "expected": expected, "actual": actual} - ) + per_gpu[gpu].append({"field": field, "expected": expected, "actual": actual}) per_gpu_list: list[dict[str, Any]] = [ {"gpu": gpu, "mismatches": entries} @@ -1063,9 +1041,7 @@ def check_expected_xgmi_link_speed( if expected_xgmi_speed is None or len(expected_xgmi_speed) == 0: self._log_event( category=EventCategory.IO, - description=( - "Expected XGMI link speed not set; skipping XGMI link speed analysis" - ), + description=("Expected XGMI link speed not set; skipping XGMI link speed analysis"), priority=EventPriority.INFO, console_log=True, ) @@ -1087,9 +1063,7 @@ def check_expected_xgmi_link_speed( "gpu": xgmi_data.gpu, "expected_xgmi_speed": expected_xgmi_speed, "xgmi_bit_rate": ( - link_metric.bit_rate.unit - if link_metric.bit_rate - else "N/A" + link_metric.bit_rate.unit if link_metric.bit_rate else "N/A" ), "details": details, }, @@ -1111,9 +1085,7 @@ def check_expected_xgmi_link_speed( "gpu": xgmi_data.gpu, "expected_xgmi_speed": expected_xgmi_speed, "xgmi_bit_rate": ( - link_metric.bit_rate.value - if link_metric.bit_rate - else "N/A" + link_metric.bit_rate.value if link_metric.bit_rate else "N/A" ), "details": details, }, @@ -1218,9 +1190,7 @@ def _add(gpu: int, field: str, expected: object, actual: object) -> None: console_log=True, ) - def check_xgmi_or_peer_links_status( - self, xgmi_links: Optional[list[XgmiLinks]] - ) -> None: + def check_xgmi_or_peer_links_status(self, xgmi_links: Optional[list[XgmiLinks]]) -> None: """Check XGMI or peer-link status: U passes, SELF is ignored, D/X warn.""" if not xgmi_links: self._log_event( @@ -1263,9 +1233,7 @@ def check_xgmi_or_peer_links_status( if down_links: self._log_event( category=EventCategory.IO, - description=( - f"XGMI/peer links contain {len(down_links)} down/error links" - ), + description=(f"XGMI/peer links contain {len(down_links)} down/error links"), priority=EventPriority.WARNING, data={ "down_links": down_links, @@ -1278,8 +1246,7 @@ def check_xgmi_or_peer_links_status( self._log_event( category=EventCategory.IO, description=( - f"XGMI/peer links contain {len(degraded_links)} " - "disabled/degraded links" + f"XGMI/peer links contain {len(degraded_links)} " "disabled/degraded links" ), priority=EventPriority.WARNING, data={ @@ -1307,9 +1274,7 @@ def analyze_data( if data.metric is not None and len(data.metric) > 0: if args.expected_power_management: - self.check_expected_power_management( - data.metric, args.expected_power_management - ) + self.check_expected_power_management(data.metric, args.expected_power_management) if args.l0_to_recovery_count_error_threshold is not None: self.check_amdsmi_metric_pcie( data.metric, @@ -1341,9 +1306,7 @@ def analyze_data( if args.power: self.check_power_cap_consistency(data.static, args.power) if args.expected_driver_version: - self.check_expected_driver_version( - data.static, args.expected_driver_version - ) + self.check_expected_driver_version(data.static, args.expected_driver_version) self.static_consistancy_check(data.static) if ( diff --git a/test/unit/plugin/test_amdsmi_analyzer.py b/test/unit/plugin/test_amdsmi_analyzer.py index d6899a42..3674ea40 100644 --- a/test/unit/plugin/test_amdsmi_analyzer.py +++ b/test/unit/plugin/test_amdsmi_analyzer.py @@ -1605,7 +1605,7 @@ def test_check_fabric_unconfigured_many_gpus(mock_analyzer): assert event.category == "NETWORK" assert len(event.data["mismatches"]) == 8 assert {m["gpu"] for m in event.data["mismatches"]} == set(range(8)) - assert {m["field"] for m in event.data["mismatches"]} == {'accel_state'} + assert {m["field"] for m in event.data["mismatches"]} == {"accel_state"} assert len(event.description.encode("utf-8")) < 2048 assert event.description.startswith("Fabric data mismatch on 8 GPU(s)")