From 136d54d84317b9a9d8f4d48abfe7ac58b7277a11 Mon Sep 17 00:00:00 2001 From: sunnyhe2 Date: Mon, 6 Jul 2026 10:36:00 -0700 Subject: [PATCH 1/9] sync with errorscraper --- .../plugins/inband/network/analyzer_args.py | 4 + .../plugins/inband/network/collector_args.py | 4 + .../plugins/inband/network/ethtool_vendor.py | 96 +++---- .../inband/network/network_analyzer.py | 79 +++--- .../inband/network/network_collector.py | 34 ++- .../plugins/inband/rdma/analyzer_args.py | 39 +++ .../plugins/inband/rdma/rdma_analyzer.py | 70 +++-- .../plugins/inband/rdma/rdma_collector.py | 10 +- .../plugins/inband/rdma/rdma_plugin.py | 4 +- nodescraper/plugins/inband/rdma/rdmadata.py | 7 +- test/unit/plugin/test_network_analyzer.py | 252 +++++++++++++----- test/unit/plugin/test_rdma_analyzer.py | 81 +++++- test/unit/plugin/test_rdma_collector.py | 2 + 13 files changed, 497 insertions(+), 185 deletions(-) create mode 100644 nodescraper/plugins/inband/rdma/analyzer_args.py diff --git a/nodescraper/plugins/inband/network/analyzer_args.py b/nodescraper/plugins/inband/network/analyzer_args.py index f2e63047..d2eda7fa 100644 --- a/nodescraper/plugins/inband/network/analyzer_args.py +++ b/nodescraper/plugins/inband/network/analyzer_args.py @@ -38,3 +38,7 @@ class NetworkAnalyzerArgs(AnalyzerArgs): default=None, description="Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern.", ) + exclusion_regex: Optional[list[str]] = Field( + default=None, + description="Regex patterns matched (search) against netdev/interface names; any match is skipped (not analyzed).", + ) diff --git a/nodescraper/plugins/inband/network/collector_args.py b/nodescraper/plugins/inband/network/collector_args.py index 61ca7f23..1c0cfc89 100644 --- a/nodescraper/plugins/inband/network/collector_args.py +++ b/nodescraper/plugins/inband/network/collector_args.py @@ -40,3 +40,7 @@ class NetworkCollectorArgs(CollectorArgs): default=None, description="Tool to use for network connectivity probe: ping, wget, or curl.", ) + exclusion_regex: Optional[list[str]] = Field( + default=None, + description="Regex patterns matched (search) against netdev/interface names; any match is skipped (ethtool not run against it).", + ) diff --git a/nodescraper/plugins/inband/network/ethtool_vendor.py b/nodescraper/plugins/inband/network/ethtool_vendor.py index 47e91be9..d43791b4 100644 --- a/nodescraper/plugins/inband/network/ethtool_vendor.py +++ b/nodescraper/plugins/inband/network/ethtool_vendor.py @@ -111,9 +111,15 @@ class PollaraEthtoolStatistics(BaseModel): "frames_rx_dropped", "frames_rx_less_than_64b", "frames_tx_bad", + "frames_tx_less_than_64b", + "frames_tx_truncated", + ] + + warning_fields: ClassVar[list[str]] = [ + "frames_rx_pause", + "frames_rx_pripause", "frames_tx_pause", "frames_tx_pripause", - "frames_tx_less_than_64b", "frames_tx_pri_0", "frames_tx_pri_1", "frames_tx_pri_2", @@ -130,12 +136,6 @@ class PollaraEthtoolStatistics(BaseModel): "tx_pripause_5_1us_count", "tx_pripause_6_1us_count", "tx_pripause_7_1us_count", - "frames_tx_truncated", - ] - - warning_fields: ClassVar[list[str]] = [ - "frames_rx_pause", - "frames_rx_pripause", "frames_rx_pri_0", "frames_rx_pri_1", "frames_rx_pri_2", @@ -274,28 +274,14 @@ class Thor2EthtoolStatistics(BaseModel): "rx_runt_frames", "rx_stat_discard", "rx_stat_err", - "tx_pause_frames", - "tx_pfc_frames", "tx_jabber_frames", "tx_fcs_err_frames", "tx_err", "tx_fifo_underruns", - "tx_pfc_ena_frames_pri0", - "tx_pfc_ena_frames_pri1", - "tx_pfc_ena_frames_pri2", - "tx_pfc_ena_frames_pri3", - "tx_pfc_ena_frames_pri4", - "tx_pfc_ena_frames_pri5", - "tx_pfc_ena_frames_pri6", - "tx_pfc_ena_frames_pri7", "tx_total_collisions", "tx_stat_discard", "tx_stat_error", "link_down_events", - "continuous_pause_events", - "resume_pause_events", - "continuous_roce_pause_events", - "resume_roce_pause_events", "rx_pcs_symbol_err", "rx_discard_bytes_cos0", "rx_discard_packets_cos0", @@ -315,14 +301,6 @@ class Thor2EthtoolStatistics(BaseModel): "rx_discard_packets_cos7", "rx_fec_uncorrectable_blocks", "rx_filter_miss", - "pfc_pri0_tx_transitions", - "pfc_pri1_tx_transitions", - "pfc_pri2_tx_transitions", - "pfc_pri3_tx_transitions", - "pfc_pri4_tx_transitions", - "pfc_pri5_tx_transitions", - "pfc_pri6_tx_transitions", - "pfc_pri7_tx_transitions", "hw_db_recov_dbs_dropped", "hw_db_recov_oo_drop_count", "lpbk_tx_discards", @@ -352,6 +330,28 @@ class Thor2EthtoolStatistics(BaseModel): "pfc_pri5_rx_transitions", "pfc_pri6_rx_transitions", "pfc_pri7_rx_transitions", + "tx_pause_frames", + "tx_pfc_frames", + "tx_pfc_ena_frames_pri0", + "tx_pfc_ena_frames_pri1", + "tx_pfc_ena_frames_pri2", + "tx_pfc_ena_frames_pri3", + "tx_pfc_ena_frames_pri4", + "tx_pfc_ena_frames_pri5", + "tx_pfc_ena_frames_pri6", + "tx_pfc_ena_frames_pri7", + "continuous_pause_events", + "resume_pause_events", + "continuous_roce_pause_events", + "resume_roce_pause_events", + "pfc_pri0_tx_transitions", + "pfc_pri1_tx_transitions", + "pfc_pri2_tx_transitions", + "pfc_pri3_tx_transitions", + "pfc_pri4_tx_transitions", + "pfc_pri5_tx_transitions", + "pfc_pri6_tx_transitions", + "pfc_pri7_tx_transitions", ] @@ -513,7 +513,6 @@ class Cx7EthtoolStatistics(BaseModel): "rx_oversize_pkts_phy", "rx_symbol_err_phy", "rx_unsupported_op_phy", - "tx_pause_ctrl_phy", "rx_discards_phy", "tx_discards_phy", "tx_errors_phy", @@ -536,24 +535,6 @@ class Cx7EthtoolStatistics(BaseModel): "rx_prio5_discards", "rx_prio6_discards", "rx_prio7_discards", - "tx_global_pause", - "tx_prio0_pause", - "tx_prio1_pause", - "tx_prio2_pause", - "tx_prio3_pause", - "tx_prio4_pause", - "tx_prio5_pause", - "tx_prio6_pause", - "tx_prio7_pause", - "tx_global_pause_duration", - "tx_prio0_pause_duration", - "tx_prio1_pause_duration", - "tx_prio2_pause_duration", - "tx_prio3_pause_duration", - "tx_prio4_pause_duration", - "tx_prio5_pause_duration", - "tx_prio6_pause_duration", - "tx_prio7_pause_duration", "tx_pause_storm_warning_events", "tx_pause_storm_error_events", "module_unplug", @@ -619,6 +600,25 @@ class Cx7EthtoolStatistics(BaseModel): "rx_prio5_pause_duration", "rx_prio6_pause_duration", "rx_prio7_pause_duration", + "tx_pause_ctrl_phy", + "tx_global_pause", + "tx_prio0_pause", + "tx_prio1_pause", + "tx_prio2_pause", + "tx_prio3_pause", + "tx_prio4_pause", + "tx_prio5_pause", + "tx_prio6_pause", + "tx_prio7_pause", + "tx_global_pause_duration", + "tx_prio0_pause_duration", + "tx_prio1_pause_duration", + "tx_prio2_pause_duration", + "tx_prio3_pause_duration", + "tx_prio4_pause_duration", + "tx_prio5_pause_duration", + "tx_prio6_pause_duration", + "tx_prio7_pause_duration", ] diff --git a/nodescraper/plugins/inband/network/network_analyzer.py b/nodescraper/plugins/inband/network/network_analyzer.py index 27280c37..400233f8 100644 --- a/nodescraper/plugins/inband/network/network_analyzer.py +++ b/nodescraper/plugins/inband/network/network_analyzer.py @@ -35,40 +35,26 @@ class NetworkAnalyzer(RegexAnalyzer[NetworkDataModel, NetworkAnalyzerArgs]): - """Check network statistics for errors (PFC and other network error counters).""" + """Check network statistics for errors.""" DATA_MODEL = NetworkDataModel - # Regex patterns for error fields checked from network statistics - ERROR_REGEX: list[ErrorRegex] = [ - ErrorRegex( - regex=re.compile(r"^tx_pfc_frames$"), - message="tx_pfc_frames is non-zero", - event_category=EventCategory.NETWORK, - ), - ErrorRegex( - regex=re.compile(r"^tx_pfc_ena_frames_pri\d+$"), - message="tx_pfc_ena_frames_pri* is non-zero", - event_category=EventCategory.NETWORK, - ), - ErrorRegex( - regex=re.compile(r"^pfc_pri\d+_tx_transitions$"), - message="pfc_pri*_tx_transitions is non-zero", - event_category=EventCategory.NETWORK, - ), - ] + # No built-in regex patterns: RDMA-scoped counter classification lives in the vendor + # ethtool models (ethtool_vendor.py). This list is only extended by user-supplied + # NetworkAnalyzerArgs.error_regex patterns. + ERROR_REGEX: list[ErrorRegex] = [] def analyze_data( self, data: NetworkDataModel, args: Optional[NetworkAnalyzerArgs] = None ) -> TaskResult: - """Analyze ethtool -S statistics: regex-based (per interface) and vendor-based (RDMA-scoped). + """Analyze ethtool -S statistics via RDMA-scoped vendor models and any user-supplied regex. Args: data: Network data model with ethtool_info and/or rdma_ethtool_statistics. args: Optional analyzer arguments with custom error regex support. Returns: - TaskResult with OK, WARNING (no data or vendor warning counters only), or ERROR. + TaskResult with OK, WARNING (no devices, or only warning-tier counters), or ERROR. """ if not data.ethtool_info and not data.rdma_ethtool_statistics: self.result.message = "No network devices found" @@ -79,10 +65,18 @@ def analyze_data( args = NetworkAnalyzerArgs() final_error_regex = self._convert_and_extend_error_regex(args.error_regex, self.ERROR_REGEX) + compiled_exclusions = [re.compile(pattern) for pattern in (args.exclusion_regex or [])] + skipped_devices: set[str] = set() regex_error = False + regex_warning = False for interface_name, ethtool_info in data.ethtool_info.items(): - errors_on_interface: list[tuple[str, int]] = [] + if compiled_exclusions and any( + pattern.search(interface_name) for pattern in compiled_exclusions + ): + skipped_devices.add(interface_name) + continue + matches_on_interface: list[tuple[str, int, EventPriority]] = [] for stat_name, stat_value in ethtool_info.statistics.items(): for error_regex_obj in final_error_regex: if error_regex_obj.regex.match(stat_name): @@ -92,27 +86,44 @@ def analyze_data( break if value > 0: - errors_on_interface.append((stat_name, value)) + matches_on_interface.append( + (stat_name, value, error_regex_obj.event_priority) + ) break - if errors_on_interface: - regex_error = True - error_names = [e[0] for e in errors_on_interface] - errors_data = {field: value for field, value in errors_on_interface} + if matches_on_interface: + has_error = any( + priority == EventPriority.ERROR for _, _, priority in matches_on_interface + ) + if has_error: + regex_error = True + else: + regex_warning = True + priority = EventPriority.ERROR if has_error else EventPriority.WARNING + severity = "error" if has_error else "warning" + match_names = [match[0] for match in matches_on_interface] + matches_data = {name: value for name, value, _ in matches_on_interface} self._log_event( category=EventCategory.NETWORK, - description=f"Network error detected on {interface_name}: [{', '.join(error_names)}]", + description=f"Network {severity} detected on {interface_name}: [{', '.join(match_names)}]", data={ "interface": interface_name, - "errors": errors_data, + "errors": matches_data, }, - priority=EventPriority.ERROR, + priority=priority, console_log=True, ) vendor_error = False vendor_warning = False for stat in data.rdma_ethtool_statistics: + if ( + stat.netdev + and compiled_exclusions + and any(pattern.search(stat.netdev) for pattern in compiled_exclusions) + ): + skipped_devices.add(stat.netdev) + continue if stat.vendor_statistics is None: continue @@ -150,10 +161,14 @@ def analyze_data( if regex_error or vendor_error: self.result.message = "Network errors detected in statistics" self.result.status = ExecutionStatus.ERROR - elif vendor_warning: - self.result.message = "Network vendor ethtool warning counters non-zero" + elif regex_warning or vendor_warning: + self.result.message = "Network warning counters non-zero in statistics" self.result.status = ExecutionStatus.WARNING else: self.result.message = "No network errors detected in statistics" self.result.status = ExecutionStatus.OK + + if skipped_devices: + self.result.message += f" ({len(skipped_devices)} skipped)" + return self.result diff --git a/nodescraper/plugins/inband/network/network_collector.py b/nodescraper/plugins/inband/network/network_collector.py index 7e5e4a39..0fcc1659 100644 --- a/nodescraper/plugins/inband/network/network_collector.py +++ b/nodescraper/plugins/inband/network/network_collector.py @@ -483,11 +483,17 @@ def _parse_ethtool_statistics(self, output: str, interface: str) -> Dict[str, st stats_dict[key.strip()] = value.strip() return stats_dict - def _collect_ethtool_info(self, interfaces: List[NetworkInterface]) -> Dict[str, EthtoolInfo]: + def _collect_ethtool_info( + self, + interfaces: List[NetworkInterface], + exclusions: Optional[List[re.Pattern]] = None, + ) -> Dict[str, EthtoolInfo]: """Collect ethtool information for all network interfaces. Args: interfaces: List of NetworkInterface objects to collect ethtool info for + exclusions: Compiled regex patterns; interfaces whose name matches any + pattern are skipped (ethtool is not run against them). Returns: Dictionary mapping interface name to EthtoolInfo @@ -495,6 +501,8 @@ def _collect_ethtool_info(self, interfaces: List[NetworkInterface]) -> Dict[str, ethtool_data = {} for iface in interfaces: + if exclusions and any(pattern.search(iface.name) for pattern in exclusions): + continue cmd = self.CMD_ETHTOOL_TEMPLATE.format(interface=iface.name) res_ethtool = self._run_sut_cmd(cmd, sudo=True) @@ -634,8 +642,15 @@ def _collect_rdma_scoped_ethtool_statistic( vendor_statistics=vendor_stats, ) - def _collect_rdma_scoped_ethtool(self) -> tuple[List[str], List[EthtoolStatistics]]: - """Collect ethtool -S for netdevs listed on RDMA links (error-scraper EthtoolCollector parity).""" + def _collect_rdma_scoped_ethtool( + self, exclusions: Optional[List[re.Pattern]] = None + ) -> tuple[List[str], List[EthtoolStatistics]]: + """Collect ethtool -S for netdevs listed on RDMA links (error-scraper EthtoolCollector parity). + + Args: + exclusions: Compiled regex patterns; netdevs whose name matches any pattern + are skipped (ethtool -S is not run against them). + """ netdev_list: List[str] = [] statistics_list: List[EthtoolStatistics] = [] @@ -657,6 +672,8 @@ def _collect_rdma_scoped_ethtool(self) -> tuple[List[str], List[EthtoolStatistic ifname = link.get("ifname") or "" if netdev: + if exclusions and any(pattern.search(netdev) for pattern in exclusions): + continue netdev_list.append(netdev) stat = self._collect_rdma_scoped_ethtool_statistic(netdev, ifname) if stat is not None: @@ -776,6 +793,11 @@ def collect_data( rdma_ethtool_netdevs: List[str] = [] rdma_ethtool_statistics: List[EthtoolStatistics] = [] + # Compile device-exclusion regex patterns (netdevs matching any pattern are skipped) + compiled_exclusions = [ + re.compile(pattern) for pattern in ((args.exclusion_regex if args else None) or []) + ] + # Check network connectivity if URL is provided if args and args.url: cmd = args.netprobe if args.netprobe else "ping" @@ -812,7 +834,7 @@ def collect_data( # Collect ethtool information for interfaces if interfaces: - ethtool_data = self._collect_ethtool_info(interfaces) + ethtool_data = self._collect_ethtool_info(interfaces, compiled_exclusions) self._log_event( category=EventCategory.NETWORK, description=f"Collected ethtool info for {len(ethtool_data)} interfaces", @@ -820,7 +842,9 @@ def collect_data( ) if self.system_info.os_family == OSFamily.LINUX: - rdma_ethtool_netdevs, rdma_ethtool_statistics = self._collect_rdma_scoped_ethtool() + rdma_ethtool_netdevs, rdma_ethtool_statistics = self._collect_rdma_scoped_ethtool( + compiled_exclusions + ) # Collect routing table res_route = self._run_sut_cmd(self.CMD_ROUTE) diff --git a/nodescraper/plugins/inband/rdma/analyzer_args.py b/nodescraper/plugins/inband/rdma/analyzer_args.py new file mode 100644 index 00000000..0507ed15 --- /dev/null +++ b/nodescraper/plugins/inband/rdma/analyzer_args.py @@ -0,0 +1,39 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import Optional + +from pydantic import Field + +from nodescraper.models import AnalyzerArgs + + +class RdmaAnalyzerArgs(AnalyzerArgs): + """Arguments for the RDMA analyzer.""" + + exclusion_regex: Optional[list[str]] = Field( + default=None, + description="Regex patterns matched against an interface netdev; matching interfaces are skipped.", + ) diff --git a/nodescraper/plugins/inband/rdma/rdma_analyzer.py b/nodescraper/plugins/inband/rdma/rdma_analyzer.py index 00f6977f..bc440214 100644 --- a/nodescraper/plugins/inband/rdma/rdma_analyzer.py +++ b/nodescraper/plugins/inband/rdma/rdma_analyzer.py @@ -23,21 +23,25 @@ # SOFTWARE. # ############################################################################### +import re from typing import Optional from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus from nodescraper.interfaces import DataAnalyzer from nodescraper.models import TaskResult +from .analyzer_args import RdmaAnalyzerArgs from .rdmadata import RdmaDataModel -class RdmaAnalyzer(DataAnalyzer[RdmaDataModel, None]): +class RdmaAnalyzer(DataAnalyzer[RdmaDataModel, RdmaAnalyzerArgs]): """Check RDMA statistics for errors (RoCE and other RDMA error counters).""" DATA_MODEL = RdmaDataModel - def analyze_data(self, data: RdmaDataModel, args: Optional[None] = None) -> TaskResult: + def analyze_data( + self, data: RdmaDataModel, args: Optional[RdmaAnalyzerArgs] = None + ) -> TaskResult: """Analyze RDMA statistics for non-zero error counters. Error and critical counter names come from each vendor's statistics model @@ -55,37 +59,55 @@ def analyze_data(self, data: RdmaDataModel, args: Optional[None] = None) -> Task self.result.status = ExecutionStatus.WARNING return self.result + if not args: + args = RdmaAnalyzerArgs() + + compiled_exclusions = [re.compile(pattern) for pattern in (args.exclusion_regex or [])] + error_state = False + skipped_count = 0 for stat in data.statistic_list: + # Skip this interface if its netdev matches any exclusion regex + if stat.netdev and any(pattern.search(stat.netdev) for pattern in compiled_exclusions): + skipped_count += 1 + continue + if stat.vendor_statistics is None: continue error_fields = stat.vendor_statistics.error_fields - critical_fields = stat.vendor_statistics.critial_error_fields + critical_fields = stat.vendor_statistics.critical_error_fields + detected_errors: dict[str, int] = {} + has_critical = False for error_field in error_fields + critical_fields: error_value = getattr(stat.vendor_statistics, error_field, None) - if error_value is not None and error_value > 0: - priority = ( - EventPriority.CRITICAL - if error_field in critical_fields - else EventPriority.ERROR - ) - self._log_event( - category=EventCategory.NETWORK, - description=f"RDMA error detected: {error_field}", - data={ - "interface": stat.ifname, - "port": stat.port, - "error_field": error_field, - "error_count": error_value, - }, - priority=priority, - console_log=True, - ) - error_state = True + detected_errors[error_field] = error_value + if error_field in critical_fields: + has_critical = True + + if not detected_errors: + continue + + priority = EventPriority.CRITICAL if has_critical else EventPriority.ERROR + error_summary = ", ".join( + f"{field}={value}" for field, value in detected_errors.items() + ) + self._log_event( + category=EventCategory.NETWORK, + description=f"RDMA errors detected on {stat.netdev or stat.ifname}: {error_summary}", + data={ + "netdev": stat.netdev, + "interface": stat.ifname, + "port": stat.port, + "errors": detected_errors, + }, + priority=priority, + console_log=True, + ) + error_state = True if error_state: self.result.message = "RDMA errors detected in statistics" @@ -93,4 +115,8 @@ def analyze_data(self, data: RdmaDataModel, args: Optional[None] = None) -> Task else: self.result.message = "No RDMA errors detected in statistics" self.result.status = ExecutionStatus.OK + + if skipped_count: + self.result.message += f" ({skipped_count} skipped)" + return self.result diff --git a/nodescraper/plugins/inband/rdma/rdma_collector.py b/nodescraper/plugins/inband/rdma/rdma_collector.py index 67f4073b..e1b40ebc 100644 --- a/nodescraper/plugins/inband/rdma/rdma_collector.py +++ b/nodescraper/plugins/inband/rdma/rdma_collector.py @@ -289,7 +289,7 @@ def _get_rdma_link(self) -> Optional[list[RdmaLink]]: data={"exception": get_exception_traceback(e)}, priority=EventPriority.WARNING, ) - return links + return None def collect_data(self, args: None = None) -> tuple[TaskResult, Optional[RdmaDataModel]]: """Collect RDMA statistics, link data, and device/link text output. @@ -301,6 +301,14 @@ def collect_data(self, args: None = None) -> tuple[TaskResult, Optional[RdmaData links = self._get_rdma_link() statistics = self._get_rdma_statistics() + # Cross-reference netdev from link data onto statistics (the + # 'rdma statistic' output does not include the netdev name). + if statistics and links: + netdev_map = {link.ifname: link.netdev for link in links if link.ifname is not None} + for stat in statistics: + if stat.ifname in netdev_map: + stat.netdev = netdev_map[stat.ifname] + dev_list: list[RdmaDevice] = [] res_rdma_dev = self._run_sut_cmd(self.CMD_RDMA_DEV) if res_rdma_dev.exit_code == 0: diff --git a/nodescraper/plugins/inband/rdma/rdma_plugin.py b/nodescraper/plugins/inband/rdma/rdma_plugin.py index fac85862..4bf5e8cc 100644 --- a/nodescraper/plugins/inband/rdma/rdma_plugin.py +++ b/nodescraper/plugins/inband/rdma/rdma_plugin.py @@ -25,14 +25,16 @@ ############################################################################### from nodescraper.base import InBandDataPlugin +from .analyzer_args import RdmaAnalyzerArgs from .rdma_analyzer import RdmaAnalyzer from .rdma_collector import RdmaCollector from .rdmadata import RdmaDataModel -class RdmaPlugin(InBandDataPlugin[RdmaDataModel, None, None]): +class RdmaPlugin(InBandDataPlugin[RdmaDataModel, None, RdmaAnalyzerArgs]): """Plugin for collection and analysis of RDMA statistics and link data.""" DATA_MODEL = RdmaDataModel COLLECTOR = RdmaCollector ANALYZER = RdmaAnalyzer + ANALYZER_ARGS = RdmaAnalyzerArgs diff --git a/nodescraper/plugins/inband/rdma/rdmadata.py b/nodescraper/plugins/inband/rdma/rdmadata.py index 965cb5b5..2859d325 100644 --- a/nodescraper/plugins/inband/rdma/rdmadata.py +++ b/nodescraper/plugins/inband/rdma/rdmadata.py @@ -131,7 +131,7 @@ class PollaraRdmaStatistics(BaseModel): "rx_rdma_mtu_discard_pkts", ] - critial_error_fields: ClassVar[list[str]] = [] + critical_error_fields: ClassVar[list[str]] = [] class Thor2RdmaStatistics(BaseModel): @@ -291,7 +291,7 @@ class Thor2RdmaStatistics(BaseModel): "rx_icrc_encapsulated", ] - critial_error_fields: ClassVar[list[str]] = [ + critical_error_fields: ClassVar[list[str]] = [ "unrecoverable_err", "res_tx_pci_err", "res_rx_pci_err", @@ -354,7 +354,7 @@ class Cx7RdmaStatistics(BaseModel): "rx_icrc_encapsulated", ] - critial_error_fields: ClassVar[list[str]] = [] + critical_error_fields: ClassVar[list[str]] = [] RdmaVendorStatistics = Union[PollaraRdmaStatistics, Thor2RdmaStatistics, Cx7RdmaStatistics] @@ -370,6 +370,7 @@ class Cx7RdmaStatistics(BaseModel): class RdmaStatistics(BaseModel): # Interface information ifname: Optional[str] = None + netdev: Optional[str] = None port: Optional[int] = None vendor_statistics: Optional[RdmaVendorStatistics] = None diff --git a/test/unit/plugin/test_network_analyzer.py b/test/unit/plugin/test_network_analyzer.py index 6b7aeff3..4fa6ea66 100644 --- a/test/unit/plugin/test_network_analyzer.py +++ b/test/unit/plugin/test_network_analyzer.py @@ -37,6 +37,16 @@ NetworkDataModel, ) +# Built-in regex defaults were removed; the vendor ethtool models own RDMA counter +# classification. These custom patterns exercise the user-supplied regex path. +CUSTOM_REGEX = [ + { + "regex": r"^custom_err_\d+$", + "message": "custom err counter non-zero", + "event_category": "NETWORK", + } +] + @pytest.fixture def network_analyzer(system_info): @@ -91,60 +101,70 @@ def test_no_errors_detected(network_analyzer, clean_network_model): assert len(result.events) == 0 -def test_single_error_detected(network_analyzer, clean_ethtool_info): - """Test with data containing a single error.""" - clean_ethtool_info.statistics["tx_pfc_frames"] = "5" - model = NetworkDataModel(ethtool_info={"eth0": clean_ethtool_info}) - result = network_analyzer.analyze_data(model) +def test_single_custom_match_detected(network_analyzer): + """A single non-zero counter matched by a user-supplied custom regex reports as ERROR.""" + ethtool = EthtoolInfo( + interface="eth0", + raw_output="dummy", + statistics={"custom_err_0": "5"}, + ) + model = NetworkDataModel(ethtool_info={"eth0": ethtool}) + result = network_analyzer.analyze_data( + model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) + ) assert result.status == ExecutionStatus.ERROR - assert "Network errors detected in statistics" in result.message + assert "errors detected" in result.message assert len(result.events) == 1 - assert result.events[0].description == "Network error detected on eth0: [tx_pfc_frames]" + assert result.events[0].description == "Network error detected on eth0: [custom_err_0]" assert result.events[0].priority == EventPriority.ERROR - assert result.events[0].data["errors"] == {"tx_pfc_frames": 5} + assert result.events[0].data["errors"] == {"custom_err_0": 5} assert result.events[0].data["interface"] == "eth0" -def test_multiple_errors_same_interface(network_analyzer, clean_ethtool_info): - """Test with data containing multiple errors on the same interface.""" - clean_ethtool_info.statistics["tx_pfc_frames"] = "10" - clean_ethtool_info.statistics["tx_pfc_ena_frames_pri0"] = "3" - clean_ethtool_info.statistics["pfc_pri2_tx_transitions"] = "7" - model = NetworkDataModel(ethtool_info={"eth0": clean_ethtool_info}) - result = network_analyzer.analyze_data(model) +def test_multiple_matches_same_interface(network_analyzer): + """Multiple non-zero counters on one interface produce a single grouped event.""" + ethtool = EthtoolInfo( + interface="eth0", + raw_output="dummy", + statistics={"custom_err_0": "10", "custom_err_1": "3", "custom_err_2": "7"}, + ) + model = NetworkDataModel(ethtool_info={"eth0": ethtool}) + result = network_analyzer.analyze_data( + model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) + ) assert result.status == ExecutionStatus.ERROR - assert "Network errors detected in statistics" in result.message + assert "errors detected" in result.message assert len(result.events) == 1 # one event per interface assert result.events[0].priority == EventPriority.ERROR - # Check all 3 errors are present + # Check all 3 counters are present assert len(result.events[0].data["errors"]) == 3 - assert result.events[0].data["errors"]["tx_pfc_frames"] == 10 - assert result.events[0].data["errors"]["tx_pfc_ena_frames_pri0"] == 3 - assert result.events[0].data["errors"]["pfc_pri2_tx_transitions"] == 7 + assert result.events[0].data["errors"]["custom_err_0"] == 10 + assert result.events[0].data["errors"]["custom_err_1"] == 3 + assert result.events[0].data["errors"]["custom_err_2"] == 7 -def test_multiple_interfaces_with_errors(network_analyzer): - """Test with errors across multiple interfaces.""" +def test_multiple_interfaces_with_matches(network_analyzer): + """Test with custom-regex matches across multiple interfaces.""" eth0 = EthtoolInfo( interface="eth0", raw_output="dummy", statistics={ - "tx_pfc_frames": "15", - "tx_pfc_ena_frames_pri1": "0", + "custom_err_0": "15", + "custom_err_1": "0", }, ) eth1 = EthtoolInfo( interface="eth1", raw_output="dummy", statistics={ - "pfc_pri3_tx_transitions": "8", + "custom_err_3": "8", }, ) eth2 = EthtoolInfo( interface="eth2", raw_output="dummy", statistics={ - "tx_pfc_ena_frames_pri7": "100", + "custom_err_7": "100", }, ) model = NetworkDataModel( @@ -154,7 +174,9 @@ def test_multiple_interfaces_with_errors(network_analyzer): "eth2": eth2, } ) - result = network_analyzer.analyze_data(model) + result = network_analyzer.analyze_data( + model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) + ) assert result.status == ExecutionStatus.ERROR assert len(result.events) == 3 interfaces = {event.data["interface"] for event in result.events} @@ -174,14 +196,14 @@ def test_rdma_ethtool_vendor_error_only(network_analyzer): stat = EthtoolStatistics( netdev="eth0", rdma_ifname="bnxt0", - vendor_statistics=Thor2EthtoolStatistics(tx_pfc_frames=4), + vendor_statistics=Thor2EthtoolStatistics(rx_fcs_err_frames=4), ) model = NetworkDataModel(ethtool_info={}, rdma_ethtool_statistics=[stat]) result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.ERROR assert "Network errors detected" in result.message assert len(result.events) == 1 - assert result.events[0].data["error_field"] == "tx_pfc_frames" + assert result.events[0].data["error_field"] == "rx_fcs_err_frames" assert result.events[0].data["error_count"] == 4 assert result.events[0].priority == EventPriority.ERROR @@ -202,6 +224,15 @@ def test_rdma_ethtool_vendor_warning_only(network_analyzer): assert result.events[0].priority == EventPriority.WARNING +def test_thor2_tx_pause_counters_are_warning_tier(): + """Sync with error-scraper: TX pause/PFC counters are warning-tier, not error-tier.""" + assert "tx_pfc_frames" in Thor2EthtoolStatistics.warning_fields + assert "tx_pfc_frames" not in Thor2EthtoolStatistics.error_fields + assert "tx_pause_frames" in Thor2EthtoolStatistics.warning_fields + # RX-side hard error counters remain error-tier + assert "rx_fcs_err_frames" in Thor2EthtoolStatistics.error_fields + + def test_rdma_ethtool_no_vendor_model_ok(network_analyzer): """RDMA ethtool row without parsed vendor statistics is ignored by vendor path.""" stat = EthtoolStatistics(netdev="eth0", rdma_ifname="unknown0", vendor_statistics=None) @@ -211,27 +242,27 @@ def test_rdma_ethtool_no_vendor_model_ok(network_analyzer): assert len(result.events) == 0 -def test_regex_patterns_priority_numbers(network_analyzer): - """Test that regex patterns match various priority numbers (0-7 and beyond).""" +def test_custom_regex_matches_multi_digit_suffixes(network_analyzer): + """Test that a user-supplied \\d+ pattern matches single- and double-digit suffixes.""" ethtool = EthtoolInfo( interface="eth0", raw_output="dummy", statistics={ - "tx_pfc_ena_frames_pri0": "1", - "tx_pfc_ena_frames_pri3": "2", - "tx_pfc_ena_frames_pri7": "3", - "tx_pfc_ena_frames_pri10": "4", # Test double-digit - "pfc_pri0_tx_transitions": "5", - "pfc_pri5_tx_transitions": "6", - "pfc_pri15_tx_transitions": "7", # Test double-digit + "custom_err_0": "1", + "custom_err_3": "2", + "custom_err_7": "3", + "custom_err_10": "4", # Test double-digit + "custom_err_15": "5", # Test double-digit }, ) model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data(model) + result = network_analyzer.analyze_data( + model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) + ) assert result.status == ExecutionStatus.ERROR assert len(result.events) == 1 - # All 7 errors should be detected - assert len(result.events[0].data["errors"]) == 7 + # All 5 counters should be detected + assert len(result.events[0].data["errors"]) == 5 def test_non_numeric_values_ignored(network_analyzer): @@ -240,18 +271,20 @@ def test_non_numeric_values_ignored(network_analyzer): interface="eth0", raw_output="dummy", statistics={ - "tx_pfc_frames": "N/A", # Non-numeric - "tx_pfc_ena_frames_pri0": "invalid", # Non-numeric - "pfc_pri1_tx_transitions": "5", # Valid error + "custom_err_0": "N/A", # Non-numeric + "custom_err_1": "invalid", # Non-numeric + "custom_err_2": "5", # Valid }, ) model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data(model) + result = network_analyzer.analyze_data( + model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) + ) assert result.status == ExecutionStatus.ERROR assert len(result.events) == 1 - # Only the valid numeric error should be reported + # Only the valid numeric counter should be reported assert len(result.events[0].data["errors"]) == 1 - assert result.events[0].data["errors"]["pfc_pri1_tx_transitions"] == 5 + assert result.events[0].data["errors"]["custom_err_2"] == 5 def test_zero_values_not_reported(network_analyzer): @@ -260,13 +293,15 @@ def test_zero_values_not_reported(network_analyzer): interface="eth0", raw_output="dummy", statistics={ - "tx_pfc_frames": "0", - "tx_pfc_ena_frames_pri0": "0", - "pfc_pri1_tx_transitions": "0", + "custom_err_0": "0", + "custom_err_1": "0", + "custom_err_2": "0", }, ) model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data(model) + result = network_analyzer.analyze_data( + model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) + ) assert result.status == ExecutionStatus.OK assert len(result.events) == 0 @@ -277,43 +312,45 @@ def test_non_matching_fields_ignored(network_analyzer): interface="eth0", raw_output="dummy", statistics={ - "rx_bytes": "999999999", # High value but not an error field - "tx_bytes": "888888888", # High value but not an error field - "some_random_counter": "12345", # Not an error field - "tx_pfc_frames": "5", # This SHOULD be detected + "rx_bytes": "999999999", # High value but not a matched field + "tx_bytes": "888888888", # High value but not a matched field + "some_random_counter": "12345", # Not a matched field + "custom_err_0": "5", # This SHOULD be detected }, ) model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data(model) + result = network_analyzer.analyze_data( + model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) + ) assert result.status == ExecutionStatus.ERROR assert len(result.events) == 1 - # Only tx_pfc_frames should be reported + # Only custom_err_0 should be reported assert len(result.events[0].data["errors"]) == 1 - assert "tx_pfc_frames" in result.events[0].data["errors"] + assert "custom_err_0" in result.events[0].data["errors"] def test_mixed_interfaces_with_and_without_errors(network_analyzer): - """Test with some interfaces having errors and others clean.""" + """Test with some interfaces having matches and others clean.""" eth0_error = EthtoolInfo( interface="eth0", raw_output="dummy", statistics={ - "tx_pfc_frames": "10", + "custom_err_0": "10", }, ) eth1_clean = EthtoolInfo( interface="eth1", raw_output="dummy", statistics={ - "tx_pfc_frames": "0", - "tx_pfc_ena_frames_pri0": "0", + "custom_err_0": "0", + "custom_err_1": "0", }, ) eth2_error = EthtoolInfo( interface="eth2", raw_output="dummy", statistics={ - "pfc_pri5_tx_transitions": "20", + "custom_err_5": "20", }, ) model = NetworkDataModel( @@ -323,7 +360,9 @@ def test_mixed_interfaces_with_and_without_errors(network_analyzer): "eth2": eth2_error, } ) - result = network_analyzer.analyze_data(model) + result = network_analyzer.analyze_data( + model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) + ) assert result.status == ExecutionStatus.ERROR # Only 2 events (eth0 and eth2), eth1 should not generate an event assert len(result.events) == 2 @@ -332,13 +371,13 @@ def test_mixed_interfaces_with_and_without_errors(network_analyzer): def test_custom_error_regex_detected(network_analyzer): - """Test that custom regex in analyzer args is applied in addition to defaults.""" + """Test that user-supplied custom regex patterns are applied (no built-in defaults).""" ethtool = EthtoolInfo( interface="eth0", raw_output="dummy", statistics={ "custom_tx_drops": "9", # Matched via custom regex only - "tx_pfc_frames": "0", + "tx_pfc_frames": "0", # No built-in pattern; ignored unless RDMA vendor-scoped }, ) model = NetworkDataModel(ethtool_info={"eth0": ethtool}) @@ -358,3 +397,84 @@ def test_custom_error_regex_detected(network_analyzer): assert len(result.events) == 1 assert result.events[0].data["interface"] == "eth0" assert result.events[0].data["errors"] == {"custom_tx_drops": 9} + + +def test_custom_warning_priority_regex(network_analyzer): + """A user-supplied WARNING-priority custom pattern yields WARNING status via the regex path.""" + ethtool = EthtoolInfo( + interface="eth0", + raw_output="dummy", + statistics={"custom_warn": "3"}, + ) + model = NetworkDataModel(ethtool_info={"eth0": ethtool}) + args = NetworkAnalyzerArgs( + error_regex=[ + { + "regex": r"^custom_warn$", + "message": "custom warn counter", + "event_category": "NETWORK", + "event_priority": "WARNING", + } + ] + ) + + result = network_analyzer.analyze_data(model, args=args) + + assert result.status == ExecutionStatus.WARNING + assert "warning counters" in result.message + assert len(result.events) == 1 + assert result.events[0].priority == EventPriority.WARNING + assert result.events[0].description == "Network warning detected on eth0: [custom_warn]" + assert result.events[0].data["errors"] == {"custom_warn": 3} + + +def test_exclusion_regex_skips_vendor_netdev(network_analyzer): + """A netdev matching exclusion_regex is skipped on the vendor path; count noted in message.""" + stat = EthtoolStatistics( + netdev="eth0", + rdma_ifname="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(rx_fcs_err_frames=4), + ) + model = NetworkDataModel(ethtool_info={}, rdma_ethtool_statistics=[stat]) + args = NetworkAnalyzerArgs(exclusion_regex=[r"^eth0$"]) + result = network_analyzer.analyze_data(model, args=args) + assert result.status == ExecutionStatus.OK + assert len(result.events) == 0 + assert "1 skipped" in result.message + + +def test_exclusion_regex_skips_regex_path_interface(network_analyzer): + """An interface matching exclusion_regex is skipped on the user-supplied regex path.""" + ethtool = EthtoolInfo( + interface="eth0", + raw_output="dummy", + statistics={"custom_err_0": "5"}, + ) + model = NetworkDataModel(ethtool_info={"eth0": ethtool}) + args = NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX, exclusion_regex=[r"^eth0$"]) + result = network_analyzer.analyze_data(model, args=args) + assert result.status == ExecutionStatus.OK + assert len(result.events) == 0 + assert "1 skipped" in result.message + + +def test_exclusion_regex_partial_match_mixed(network_analyzer): + """exclusion_regex uses search(): a substring pattern skips only matching netdevs.""" + stat_skip = EthtoolStatistics( + netdev="bnxt_eth2", + rdma_ifname="bnxt0", + vendor_statistics=Thor2EthtoolStatistics(rx_fcs_err_frames=4), + ) + stat_keep = EthtoolStatistics( + netdev="mlx_eth3", + rdma_ifname="mlx5_0", + vendor_statistics=Thor2EthtoolStatistics(rx_fcs_err_frames=7), + ) + model = NetworkDataModel(ethtool_info={}, rdma_ethtool_statistics=[stat_skip, stat_keep]) + args = NetworkAnalyzerArgs(exclusion_regex=[r"bnxt"]) + result = network_analyzer.analyze_data(model, args=args) + # bnxt_eth2 skipped, mlx_eth3 still flagged as ERROR + assert result.status == ExecutionStatus.ERROR + assert len(result.events) == 1 + assert result.events[0].data["netdev"] == "mlx_eth3" + assert "1 skipped" in result.message diff --git a/test/unit/plugin/test_rdma_analyzer.py b/test/unit/plugin/test_rdma_analyzer.py index ada67c04..718c5f0c 100644 --- a/test/unit/plugin/test_rdma_analyzer.py +++ b/test/unit/plugin/test_rdma_analyzer.py @@ -30,6 +30,7 @@ import pytest from nodescraper.enums import EventPriority, ExecutionStatus +from nodescraper.plugins.inband.rdma.analyzer_args import RdmaAnalyzerArgs from nodescraper.plugins.inband.rdma.rdma_analyzer import RdmaAnalyzer from nodescraper.plugins.inband.rdma.rdmadata import ( VENDOR_PREFIX_MAP, @@ -103,9 +104,9 @@ def test_single_error_detected(rdma_analyzer, example_stat_dicts): assert result.status == ExecutionStatus.ERROR assert "RDMA errors detected in statistics" in result.message assert len(result.events) == 1 - assert result.events[0].description == "RDMA error detected: req_rx_pkt_seq_err" + assert "req_rx_pkt_seq_err=5" in result.events[0].description assert result.events[0].priority == EventPriority.ERROR - assert result.events[0].data["error_count"] == 5 + assert result.events[0].data["errors"]["req_rx_pkt_seq_err"] == 5 assert result.events[0].data["interface"] == "ionic_0" @@ -118,9 +119,17 @@ def test_multiple_errors_detected(rdma_analyzer, example_stat_dicts): result = rdma_analyzer.analyze_data(model) assert result.status == ExecutionStatus.ERROR assert "RDMA errors detected in statistics" in result.message - assert len(result.events) == 3 + # Errors on the same interface are aggregated into a single event: + # ionic_0 (2 counters) + mlx5_0 (1 counter) -> 2 events + assert len(result.events) == 2 for event in result.events: assert event.priority == EventPriority.ERROR + events_by_iface = {event.data["interface"]: event for event in result.events} + assert events_by_iface["ionic_0"].data["errors"] == { + "req_rx_rmt_acc_err": 10, + "req_tx_loc_oper_err": 3, + } + assert events_by_iface["mlx5_0"].data["errors"] == {"packet_seq_err": 7} def test_critical_error_detected(rdma_analyzer): @@ -138,9 +147,10 @@ def test_critical_error_detected(rdma_analyzer): result = rdma_analyzer.analyze_data(model) assert result.status == ExecutionStatus.ERROR assert "RDMA errors detected in statistics" in result.message - assert len(result.events) == 2 - critical_events = [e for e in result.events if e.priority == EventPriority.CRITICAL] - assert len(critical_events) == 2 + # One event per interface; escalated to CRITICAL because critical counters are set. + assert len(result.events) == 1 + assert result.events[0].priority == EventPriority.CRITICAL + assert result.events[0].data["errors"] == {"unrecoverable_err": 1, "res_tx_pci_err": 2} def test_empty_statistics(rdma_analyzer): @@ -184,7 +194,7 @@ def test_all_error_types(rdma_analyzer): model = RdmaDataModel(statistic_list=stats) result = rdma_analyzer.analyze_data(model) assert result.status == ExecutionStatus.ERROR - assert len(result.events) == 3 + assert len(result.events) == 2 interfaces = {event.data["interface"] for event in result.events} assert interfaces == {"ionic_test", "mlx5_test"} @@ -310,3 +320,60 @@ def test_rdma_link_multiple_interfaces(rdma_analyzer, clean_stats): result = rdma_analyzer.analyze_data(model) assert result.status == ExecutionStatus.OK assert len(result.events) == 0 + + +def test_netdev_used_in_event(rdma_analyzer): + stats = [ + RdmaStatistics( + ifname="ionic_0", + netdev="benic8p1", + port=1, + vendor_statistics=PollaraRdmaStatistics(req_rx_pkt_seq_err=4), + ) + ] + model = RdmaDataModel(statistic_list=stats) + result = rdma_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.ERROR + assert len(result.events) == 1 + assert result.events[0].data["netdev"] == "benic8p1" + assert "benic8p1" in result.events[0].description + + +def test_exclusion_regex_skips_interface(rdma_analyzer): + stats = [ + RdmaStatistics( + ifname="ionic_0", + netdev="benic8p1", + port=1, + vendor_statistics=PollaraRdmaStatistics(req_rx_pkt_seq_err=4), + ), + RdmaStatistics( + ifname="mlx5_0", + netdev="benic9p1", + port=1, + vendor_statistics=Cx7RdmaStatistics(packet_seq_err=2), + ), + ] + model = RdmaDataModel(statistic_list=stats) + result = rdma_analyzer.analyze_data(model, RdmaAnalyzerArgs(exclusion_regex=["benic8"])) + assert result.status == ExecutionStatus.ERROR + # ionic_0 (netdev benic8p1) is excluded; only the mlx5_0 error is reported. + assert len(result.events) == 1 + assert result.events[0].data["interface"] == "mlx5_0" + assert "1 skipped" in result.message + + +def test_exclusion_regex_all_skipped(rdma_analyzer): + stats = [ + RdmaStatistics( + ifname="ionic_0", + netdev="benic8p1", + port=1, + vendor_statistics=PollaraRdmaStatistics(req_rx_pkt_seq_err=4), + ) + ] + model = RdmaDataModel(statistic_list=stats) + result = rdma_analyzer.analyze_data(model, RdmaAnalyzerArgs(exclusion_regex=["benic8p1"])) + assert result.status == ExecutionStatus.OK + assert len(result.events) == 0 + assert "1 skipped" in result.message diff --git a/test/unit/plugin/test_rdma_collector.py b/test/unit/plugin/test_rdma_collector.py index eb687c54..38d162ca 100644 --- a/test/unit/plugin/test_rdma_collector.py +++ b/test/unit/plugin/test_rdma_collector.py @@ -78,6 +78,8 @@ def test_collect_success(collector, conn_mock, rdma_link_output, rdma_statistic_ # Full link fixture has 4 ionic links assert len(data.link_list) == 4 assert data.link_list[0].ifname == "ionic_0" + # netdev is cross-referenced from link data onto statistics + assert data.statistic_list[0].netdev == "benic8p1" def test_collect_both_commands_fail(collector, conn_mock): From b6c59dea303064cafc2f3c0b5fa0e2bb64a49e83 Mon Sep 17 00:00:00 2001 From: sunnyhe2 Date: Mon, 6 Jul 2026 14:07:21 -0700 Subject: [PATCH 2/9] fix exclusion regex --- .../plugins/inband/network/analyzer_args.py | 4 -- .../inband/network/network_analyzer.py | 44 ++++++------- .../inband/network/network_collector.py | 63 +++++++++---------- .../plugins/inband/rdma/collector_args.py | 31 +++++++++ .../plugins/inband/rdma/rdma_analyzer.py | 28 ++++++--- .../plugins/inband/rdma/rdma_collector.py | 7 ++- .../plugins/inband/rdma/rdma_plugin.py | 4 +- test/unit/plugin/test_network_analyzer.py | 52 --------------- test/unit/plugin/test_rdma_analyzer.py | 4 +- 9 files changed, 113 insertions(+), 124 deletions(-) create mode 100644 nodescraper/plugins/inband/rdma/collector_args.py diff --git a/nodescraper/plugins/inband/network/analyzer_args.py b/nodescraper/plugins/inband/network/analyzer_args.py index d2eda7fa..f2e63047 100644 --- a/nodescraper/plugins/inband/network/analyzer_args.py +++ b/nodescraper/plugins/inband/network/analyzer_args.py @@ -38,7 +38,3 @@ class NetworkAnalyzerArgs(AnalyzerArgs): default=None, description="Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern.", ) - exclusion_regex: Optional[list[str]] = Field( - default=None, - description="Regex patterns matched (search) against netdev/interface names; any match is skipped (not analyzed).", - ) diff --git a/nodescraper/plugins/inband/network/network_analyzer.py b/nodescraper/plugins/inband/network/network_analyzer.py index 400233f8..2967df59 100644 --- a/nodescraper/plugins/inband/network/network_analyzer.py +++ b/nodescraper/plugins/inband/network/network_analyzer.py @@ -23,7 +23,6 @@ # SOFTWARE. # ############################################################################### -import re from typing import Optional from nodescraper.base.regexanalyzer import ErrorRegex, RegexAnalyzer @@ -65,17 +64,10 @@ def analyze_data( args = NetworkAnalyzerArgs() final_error_regex = self._convert_and_extend_error_regex(args.error_regex, self.ERROR_REGEX) - compiled_exclusions = [re.compile(pattern) for pattern in (args.exclusion_regex or [])] - skipped_devices: set[str] = set() regex_error = False regex_warning = False for interface_name, ethtool_info in data.ethtool_info.items(): - if compiled_exclusions and any( - pattern.search(interface_name) for pattern in compiled_exclusions - ): - skipped_devices.add(interface_name) - continue matches_on_interface: list[tuple[str, int, EventPriority]] = [] for stat_name, stat_value in ethtool_info.statistics.items(): for error_regex_obj in final_error_regex: @@ -116,14 +108,9 @@ def analyze_data( vendor_error = False vendor_warning = False + vendor_error_fields: set[str] = set() + vendor_warning_fields: set[str] = set() for stat in data.rdma_ethtool_statistics: - if ( - stat.netdev - and compiled_exclusions - and any(pattern.search(stat.netdev) for pattern in compiled_exclusions) - ): - skipped_devices.add(stat.netdev) - continue if stat.vendor_statistics is None: continue @@ -138,13 +125,22 @@ def analyze_data( priority = EventPriority.WARNING if is_warning_tier else EventPriority.ERROR if is_warning_tier: vendor_warning = True + vendor_warning_fields.add(field_name) else: vendor_error = True + vendor_error_fields.add(field_name) + # Use a single grouped description per severity so the run summary + # collapses every occurrence into one "Ethtool warning detected" / + # "Ethtool error detected" entry instead of one line per field. The + # specific field is still preserved in the event data below and in + # the consolidated console line emitted after the loop. desc = ( - f"Ethtool warning detected: {field_name}" - if is_warning_tier - else f"Ethtool error detected: {field_name}" + "Ethtool warning detected" if is_warning_tier else "Ethtool error detected" ) + # Per-field events are still recorded for the run summary and event + # log, but console logging is suppressed here to avoid repeating the + # same message once per device. A single consolidated line is emitted + # after the loop instead. self._log_event( category=EventCategory.NETWORK, description=desc, @@ -155,9 +151,16 @@ def analyze_data( "error_count": error_value, }, priority=priority, - console_log=True, + console_log=False, ) + if vendor_error: + self.logger.error("Ethtool error detected: %s", ", ".join(sorted(vendor_error_fields))) + if vendor_warning: + self.logger.warning( + "Ethtool warning detected: %s", ", ".join(sorted(vendor_warning_fields)) + ) + if regex_error or vendor_error: self.result.message = "Network errors detected in statistics" self.result.status = ExecutionStatus.ERROR @@ -168,7 +171,4 @@ def analyze_data( self.result.message = "No network errors detected in statistics" self.result.status = ExecutionStatus.OK - if skipped_devices: - self.result.message += f" ({len(skipped_devices)} skipped)" - return self.result diff --git a/nodescraper/plugins/inband/network/network_collector.py b/nodescraper/plugins/inband/network/network_collector.py index 0fcc1659..b960e574 100644 --- a/nodescraper/plugins/inband/network/network_collector.py +++ b/nodescraper/plugins/inband/network/network_collector.py @@ -30,7 +30,6 @@ from pydantic import ValidationError from nodescraper.base import InBandDataCollector -from nodescraper.connection.inband import TextFileArtifact from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily from nodescraper.models import TaskResult from nodescraper.utils import get_exception_traceback @@ -487,7 +486,7 @@ def _collect_ethtool_info( self, interfaces: List[NetworkInterface], exclusions: Optional[List[re.Pattern]] = None, - ) -> Dict[str, EthtoolInfo]: + ) -> Tuple[Dict[str, EthtoolInfo], set[str]]: """Collect ethtool information for all network interfaces. Args: @@ -496,31 +495,21 @@ def _collect_ethtool_info( pattern are skipped (ethtool is not run against them). Returns: - Dictionary mapping interface name to EthtoolInfo + Tuple of (dictionary mapping interface name to EthtoolInfo, set of + interface names skipped due to an exclusion regex) """ ethtool_data = {} + skipped: set[str] = set() for iface in interfaces: if exclusions and any(pattern.search(iface.name) for pattern in exclusions): + skipped.add(iface.name) continue cmd = self.CMD_ETHTOOL_TEMPLATE.format(interface=iface.name) res_ethtool = self._run_sut_cmd(cmd, sudo=True) if res_ethtool.exit_code == 0: ethtool_info = self._parse_ethtool(iface.name, res_ethtool.stdout) - # Collect ethtool -S (statistics) for error/health analysis - cmd_s = self.CMD_ETHTOOL_S_TEMPLATE.format(interface=iface.name) - res_ethtool_s = self._run_sut_cmd(cmd_s, sudo=True) - if res_ethtool_s.exit_code == 0 and res_ethtool_s.stdout: - ethtool_info.statistics = self._parse_ethtool_statistics( - res_ethtool_s.stdout, iface.name - ) - self.result.artifacts.append( - TextFileArtifact( - filename=f"{iface.name}.log", - contents=res_ethtool_s.stdout, - ) - ) ethtool_data[iface.name] = ethtool_info self._log_event( category=EventCategory.NETWORK, @@ -535,7 +524,7 @@ def _collect_ethtool_info( priority=EventPriority.WARNING, ) - return ethtool_data + return ethtool_data, skipped def _collect_rdma_link_json(self) -> Optional[list[dict]]: """Parse JSON from `rdma link -j`. Returns None on failure, [] when no links.""" @@ -578,7 +567,7 @@ def _collect_rdma_scoped_ethtool_statistic( self, netdev: str, ifname: str ) -> Optional[EthtoolStatistics]: """Run `ethtool -S` for netdev and attach vendor-parsed stats (prefix from RDMA ifname).""" - cmd_s = f"ethtool -S {netdev}" + cmd_s = self.CMD_ETHTOOL_S_TEMPLATE.format(interface=netdev) res = self._run_sut_cmd(cmd_s, sudo=True) if res.exit_code != 0: self._log_event( @@ -594,13 +583,6 @@ def _collect_rdma_scoped_ethtool_statistic( ) return None - if res.stdout: - self.result.artifacts.append( - TextFileArtifact( - filename=f"rdma-ethtool-{netdev}.log", - contents=res.stdout, - ) - ) stats_dict = self._parse_ethtool_statistics(res.stdout, netdev) vendor_stats: Optional[VendorEthtoolStatisticsModel] = None @@ -644,19 +626,24 @@ def _collect_rdma_scoped_ethtool_statistic( def _collect_rdma_scoped_ethtool( self, exclusions: Optional[List[re.Pattern]] = None - ) -> tuple[List[str], List[EthtoolStatistics]]: + ) -> tuple[List[str], List[EthtoolStatistics], set[str]]: """Collect ethtool -S for netdevs listed on RDMA links (error-scraper EthtoolCollector parity). Args: exclusions: Compiled regex patterns; netdevs whose name matches any pattern are skipped (ethtool -S is not run against them). + + Returns: + Tuple of (netdev list, statistics list, set of netdev names skipped due to + an exclusion regex) """ netdev_list: List[str] = [] statistics_list: List[EthtoolStatistics] = [] + skipped: set[str] = set() link_data = self._collect_rdma_link_json() if link_data is None: - return netdev_list, statistics_list + return netdev_list, statistics_list, skipped for link in link_data: if not isinstance(link, dict): @@ -673,6 +660,7 @@ def _collect_rdma_scoped_ethtool( if netdev: if exclusions and any(pattern.search(netdev) for pattern in exclusions): + skipped.add(netdev) continue netdev_list.append(netdev) stat = self._collect_rdma_scoped_ethtool_statistic(netdev, ifname) @@ -689,7 +677,7 @@ def _collect_rdma_scoped_ethtool( priority=EventPriority.INFO, ) - return netdev_list, statistics_list + return netdev_list, statistics_list, skipped def _collect_lldp_info(self) -> None: """Collect LLDP information using lldpcli and lldpctl commands.""" @@ -788,10 +776,11 @@ def collect_data( routes = [] rules = [] neighbors = [] - ethtool_data = {} + ethtool_data: Dict[str, EthtoolInfo] = {} network_accessible: Optional[bool] = None rdma_ethtool_netdevs: List[str] = [] rdma_ethtool_statistics: List[EthtoolStatistics] = [] + skipped_devices: set[str] = set() # Compile device-exclusion regex patterns (netdevs matching any pattern are skipped) compiled_exclusions = [ @@ -834,7 +823,10 @@ def collect_data( # Collect ethtool information for interfaces if interfaces: - ethtool_data = self._collect_ethtool_info(interfaces, compiled_exclusions) + ethtool_data, ethtool_skipped = self._collect_ethtool_info( + interfaces, compiled_exclusions + ) + skipped_devices |= ethtool_skipped self._log_event( category=EventCategory.NETWORK, description=f"Collected ethtool info for {len(ethtool_data)} interfaces", @@ -842,9 +834,12 @@ def collect_data( ) if self.system_info.os_family == OSFamily.LINUX: - rdma_ethtool_netdevs, rdma_ethtool_statistics = self._collect_rdma_scoped_ethtool( - compiled_exclusions - ) + ( + rdma_ethtool_netdevs, + rdma_ethtool_statistics, + rdma_skipped, + ) = self._collect_rdma_scoped_ethtool(compiled_exclusions) + skipped_devices |= rdma_skipped # Collect routing table res_route = self._run_sut_cmd(self.CMD_ROUTE) @@ -901,6 +896,8 @@ def collect_data( self._collect_lldp_info() self.result.message = "Network data collected successfully" + if skipped_devices: + self.result.message += f" ({len(skipped_devices)} skipped)" network_data = NetworkDataModel( interfaces=interfaces, diff --git a/nodescraper/plugins/inband/rdma/collector_args.py b/nodescraper/plugins/inband/rdma/collector_args.py new file mode 100644 index 00000000..3a7162dc --- /dev/null +++ b/nodescraper/plugins/inband/rdma/collector_args.py @@ -0,0 +1,31 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### + +from nodescraper.models import CollectorArgs + + +class RdmaCollectorArgs(CollectorArgs): + """Collection arguments for the RDMA collector.""" diff --git a/nodescraper/plugins/inband/rdma/rdma_analyzer.py b/nodescraper/plugins/inband/rdma/rdma_analyzer.py index bc440214..ba48c20e 100644 --- a/nodescraper/plugins/inband/rdma/rdma_analyzer.py +++ b/nodescraper/plugins/inband/rdma/rdma_analyzer.py @@ -64,7 +64,10 @@ def analyze_data( compiled_exclusions = [re.compile(pattern) for pattern in (args.exclusion_regex or [])] - error_state = False + error_detected = False + critical_detected = False + error_fields_seen: set[str] = set() + critical_fields_seen: set[str] = set() skipped_count = 0 for stat in data.statistic_list: @@ -87,17 +90,20 @@ def analyze_data( detected_errors[error_field] = error_value if error_field in critical_fields: has_critical = True + critical_detected = True + critical_fields_seen.add(error_field) + else: + error_detected = True + error_fields_seen.add(error_field) if not detected_errors: continue priority = EventPriority.CRITICAL if has_critical else EventPriority.ERROR - error_summary = ", ".join( - f"{field}={value}" for field, value in detected_errors.items() - ) + desc = "RDMA critical error detected" if has_critical else "RDMA error detected" self._log_event( category=EventCategory.NETWORK, - description=f"RDMA errors detected on {stat.netdev or stat.ifname}: {error_summary}", + description=desc, data={ "netdev": stat.netdev, "interface": stat.ifname, @@ -105,11 +111,17 @@ def analyze_data( "errors": detected_errors, }, priority=priority, - console_log=True, + console_log=False, + ) + + if critical_detected: + self.logger.critical( + "RDMA critical error detected: %s", ", ".join(sorted(critical_fields_seen)) ) - error_state = True + if error_detected: + self.logger.error("RDMA error detected: %s", ", ".join(sorted(error_fields_seen))) - if error_state: + if error_detected or critical_detected: self.result.message = "RDMA errors detected in statistics" self.result.status = ExecutionStatus.ERROR else: diff --git a/nodescraper/plugins/inband/rdma/rdma_collector.py b/nodescraper/plugins/inband/rdma/rdma_collector.py index e1b40ebc..fb841cd5 100644 --- a/nodescraper/plugins/inband/rdma/rdma_collector.py +++ b/nodescraper/plugins/inband/rdma/rdma_collector.py @@ -34,6 +34,7 @@ from nodescraper.models import TaskResult from nodescraper.utils import get_exception_traceback +from .collector_args import RdmaCollectorArgs from .rdmadata import ( VENDOR_PREFIX_MAP, RdmaDataModel, @@ -45,7 +46,7 @@ ) -class RdmaCollector(InBandDataCollector[RdmaDataModel, None]): +class RdmaCollector(InBandDataCollector[RdmaDataModel, RdmaCollectorArgs]): """Collect RDMA status and statistics via rdma link and rdma statistic commands.""" DATA_MODEL = RdmaDataModel @@ -291,7 +292,9 @@ def _get_rdma_link(self) -> Optional[list[RdmaLink]]: ) return None - def collect_data(self, args: None = None) -> tuple[TaskResult, Optional[RdmaDataModel]]: + def collect_data( + self, args: Optional[RdmaCollectorArgs] = None + ) -> tuple[TaskResult, Optional[RdmaDataModel]]: """Collect RDMA statistics, link data, and device/link text output. Returns: diff --git a/nodescraper/plugins/inband/rdma/rdma_plugin.py b/nodescraper/plugins/inband/rdma/rdma_plugin.py index 4bf5e8cc..0224d47f 100644 --- a/nodescraper/plugins/inband/rdma/rdma_plugin.py +++ b/nodescraper/plugins/inband/rdma/rdma_plugin.py @@ -26,15 +26,17 @@ from nodescraper.base import InBandDataPlugin from .analyzer_args import RdmaAnalyzerArgs +from .collector_args import RdmaCollectorArgs from .rdma_analyzer import RdmaAnalyzer from .rdma_collector import RdmaCollector from .rdmadata import RdmaDataModel -class RdmaPlugin(InBandDataPlugin[RdmaDataModel, None, RdmaAnalyzerArgs]): +class RdmaPlugin(InBandDataPlugin[RdmaDataModel, RdmaCollectorArgs, RdmaAnalyzerArgs]): """Plugin for collection and analysis of RDMA statistics and link data.""" DATA_MODEL = RdmaDataModel COLLECTOR = RdmaCollector + COLLECTOR_ARGS = RdmaCollectorArgs ANALYZER = RdmaAnalyzer ANALYZER_ARGS = RdmaAnalyzerArgs diff --git a/test/unit/plugin/test_network_analyzer.py b/test/unit/plugin/test_network_analyzer.py index 4fa6ea66..832125e5 100644 --- a/test/unit/plugin/test_network_analyzer.py +++ b/test/unit/plugin/test_network_analyzer.py @@ -426,55 +426,3 @@ def test_custom_warning_priority_regex(network_analyzer): assert result.events[0].priority == EventPriority.WARNING assert result.events[0].description == "Network warning detected on eth0: [custom_warn]" assert result.events[0].data["errors"] == {"custom_warn": 3} - - -def test_exclusion_regex_skips_vendor_netdev(network_analyzer): - """A netdev matching exclusion_regex is skipped on the vendor path; count noted in message.""" - stat = EthtoolStatistics( - netdev="eth0", - rdma_ifname="bnxt0", - vendor_statistics=Thor2EthtoolStatistics(rx_fcs_err_frames=4), - ) - model = NetworkDataModel(ethtool_info={}, rdma_ethtool_statistics=[stat]) - args = NetworkAnalyzerArgs(exclusion_regex=[r"^eth0$"]) - result = network_analyzer.analyze_data(model, args=args) - assert result.status == ExecutionStatus.OK - assert len(result.events) == 0 - assert "1 skipped" in result.message - - -def test_exclusion_regex_skips_regex_path_interface(network_analyzer): - """An interface matching exclusion_regex is skipped on the user-supplied regex path.""" - ethtool = EthtoolInfo( - interface="eth0", - raw_output="dummy", - statistics={"custom_err_0": "5"}, - ) - model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - args = NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX, exclusion_regex=[r"^eth0$"]) - result = network_analyzer.analyze_data(model, args=args) - assert result.status == ExecutionStatus.OK - assert len(result.events) == 0 - assert "1 skipped" in result.message - - -def test_exclusion_regex_partial_match_mixed(network_analyzer): - """exclusion_regex uses search(): a substring pattern skips only matching netdevs.""" - stat_skip = EthtoolStatistics( - netdev="bnxt_eth2", - rdma_ifname="bnxt0", - vendor_statistics=Thor2EthtoolStatistics(rx_fcs_err_frames=4), - ) - stat_keep = EthtoolStatistics( - netdev="mlx_eth3", - rdma_ifname="mlx5_0", - vendor_statistics=Thor2EthtoolStatistics(rx_fcs_err_frames=7), - ) - model = NetworkDataModel(ethtool_info={}, rdma_ethtool_statistics=[stat_skip, stat_keep]) - args = NetworkAnalyzerArgs(exclusion_regex=[r"bnxt"]) - result = network_analyzer.analyze_data(model, args=args) - # bnxt_eth2 skipped, mlx_eth3 still flagged as ERROR - assert result.status == ExecutionStatus.ERROR - assert len(result.events) == 1 - assert result.events[0].data["netdev"] == "mlx_eth3" - assert "1 skipped" in result.message diff --git a/test/unit/plugin/test_rdma_analyzer.py b/test/unit/plugin/test_rdma_analyzer.py index 718c5f0c..1a3804a5 100644 --- a/test/unit/plugin/test_rdma_analyzer.py +++ b/test/unit/plugin/test_rdma_analyzer.py @@ -104,7 +104,7 @@ def test_single_error_detected(rdma_analyzer, example_stat_dicts): assert result.status == ExecutionStatus.ERROR assert "RDMA errors detected in statistics" in result.message assert len(result.events) == 1 - assert "req_rx_pkt_seq_err=5" in result.events[0].description + assert result.events[0].description == "RDMA error detected" assert result.events[0].priority == EventPriority.ERROR assert result.events[0].data["errors"]["req_rx_pkt_seq_err"] == 5 assert result.events[0].data["interface"] == "ionic_0" @@ -336,7 +336,7 @@ def test_netdev_used_in_event(rdma_analyzer): assert result.status == ExecutionStatus.ERROR assert len(result.events) == 1 assert result.events[0].data["netdev"] == "benic8p1" - assert "benic8p1" in result.events[0].description + assert result.events[0].description == "RDMA error detected" def test_exclusion_regex_skips_interface(rdma_analyzer): From f44d70d097b060735784f7c33400b768ee7c0391 Mon Sep 17 00:00:00 2001 From: sunnyhe2 Date: Mon, 6 Jul 2026 15:38:44 -0700 Subject: [PATCH 3/9] switch error field add --- nodescraper/command_artifact_html.py | 59 ++++++++++++ .../scale_out_arista/scaleoutaristadata.py | 80 ++++++++-------- .../switch/scale_out_dell/scaleoutdelldata.py | 94 +++++++++---------- .../inband/switch/switch_analyzer_base.py | 21 ++++- test/unit/plugin/test_switch_analyzer_base.py | 81 ++++++++++++++++ 5 files changed, 246 insertions(+), 89 deletions(-) diff --git a/nodescraper/command_artifact_html.py b/nodescraper/command_artifact_html.py index 5bf3c5be..18e93606 100644 --- a/nodescraper/command_artifact_html.py +++ b/nodescraper/command_artifact_html.py @@ -121,6 +121,17 @@ } pre.stderr { color: var(--fail-fg); } .no-results { color: var(--muted); padding: 24px; text-align: center; display: none; } + .copy-btn { + flex: 0 0 auto; + display: inline-flex; align-items: center; justify-content: center; + background: var(--bg); color: var(--muted); + border: 1px solid var(--border); border-radius: 6px; + padding: 5px 7px; cursor: pointer; line-height: 0; + transition: color .15s ease, border-color .15s ease, background .15s ease; + } + .copy-btn:hover { background: var(--panel-hover); border-color: var(--accent); color: var(--text); } + .copy-btn.copied { color: var(--ok-fg); border-color: var(--ok-fg); } + .copy-btn svg { display: block; } @@ -143,6 +154,9 @@ ▸ {command} exit {exit_code} +
{stdout_block} @@ -176,6 +190,51 @@ document.getElementById('collapseAll').addEventListener('click', () => { items.forEach(d => d.open = false); }); + + const copyButtons = Array.from(document.querySelectorAll('.copy-btn')); + const COPY_SVG = copyButtons.length ? copyButtons[0].innerHTML : ''; + const CHECK_SVG = ''; + + function showCopied(btn) { + btn.classList.add('copied'); + btn.innerHTML = CHECK_SVG; + if (btn._resetTimer) clearTimeout(btn._resetTimer); + btn._resetTimer = setTimeout(() => { + btn.classList.remove('copied'); + btn.innerHTML = COPY_SVG; + }, 1200); + } + + function fallbackCopy(text, btn) { + const ta = document.createElement('textarea'); + ta.value = text; + ta.style.position = 'fixed'; + ta.style.top = '-1000px'; + ta.style.opacity = '0'; + document.body.appendChild(ta); + ta.focus(); + ta.select(); + try { document.execCommand('copy'); showCopied(btn); } catch (e) {} + document.body.removeChild(ta); + } + + copyButtons.forEach(btn => { + btn.addEventListener('click', (e) => { + e.preventDefault(); + e.stopPropagation(); + const card = btn.closest('details.cmd'); + const parts = []; + card.querySelectorAll('pre').forEach(pre => { + if (!pre.classList.contains('empty')) parts.push(pre.textContent); + }); + const text = parts.join('\\n'); + if (navigator.clipboard && navigator.clipboard.writeText) { + navigator.clipboard.writeText(text).then(() => showCopied(btn)).catch(() => fallbackCopy(text, btn)); + } else { + fallbackCopy(text, btn); + } + }); + }); diff --git a/nodescraper/plugins/inband/switch/scale_out_arista/scaleoutaristadata.py b/nodescraper/plugins/inband/switch/scale_out_arista/scaleoutaristadata.py index 1c48a8da..041a8c0c 100644 --- a/nodescraper/plugins/inband/switch/scale_out_arista/scaleoutaristadata.py +++ b/nodescraper/plugins/inband/switch/scale_out_arista/scaleoutaristadata.py @@ -24,7 +24,7 @@ # ############################################################################### -from typing import ClassVar, Dict, List, Optional, Union +from typing import ClassVar, Dict, List, Optional from pydantic import BaseModel, ConfigDict from pydantic.alias_generators import to_camel @@ -98,8 +98,8 @@ class FanConfiguration(BaseModel): speed_hw_override: Optional[bool] = None speed_stable: Optional[bool] = None - error_fields: ClassVar[dict[str, str]] = { - "status": "ok", + error_fields: ClassVar[dict[str, List[str]]] = { + "status": ["ok"], } @@ -124,9 +124,9 @@ class AristaSystemEnv(BaseModel): power_supply_slots: Optional[List[FanConfiguration]] = None fan_tray_slots: Optional[List[FanConfiguration]] = None - error_fields: ClassVar[dict[str, Union[str, bool]]] = { - "system_status": "coolingOk", - "fans_status": "fanAlarmOk", + error_fields: ClassVar[dict[str, List[str]]] = { + "system_status": ["coolingOk"], + "fans_status": ["fanAlarmOk"], } @@ -139,9 +139,9 @@ class VlanInformation(BaseModel): interface_mode: Optional[str] = None interface_forwarding_model: Optional[str] = None - error_fields: ClassVar[dict[str, str]] = { - "interface_mode": "routed", - "interface_forwarding_model": "routed", + error_fields: ClassVar[dict[str, List[str]]] = { + "interface_mode": ["routed"], + "interface_forwarding_model": ["routed"], } @@ -160,10 +160,10 @@ class AristaPortStatus(BaseModel): line_protocol_status: Optional[str] = None interface_damped: Optional[bool] = None - error_fields: ClassVar[dict[str, str]] = { - "link_status": "connected", - "duplex": "duplexFull", - "line_protocol_status": "up", + error_fields: ClassVar[dict[str, List[str]]] = { + "link_status": ["connected"], + "duplex": ["duplexFull"], + "line_protocol_status": ["up"], } @@ -180,14 +180,14 @@ class AristaCountersErrors(BaseModel): alignment_errors: Optional[int] = None symbol_errors: Optional[int] = None - error_fields: ClassVar[dict[str, str]] = { - "in_errors": "0", - "frame_too_longs": "0", - "out_errors": "0", - "frame_too_shorts": "0", - "fcs_errors": "0", - "alignment_errors": "0", - "symbol_errors": "0", + error_fields: ClassVar[dict[str, List[str]]] = { + "in_errors": ["0"], + "frame_too_longs": ["0"], + "out_errors": ["0"], + "frame_too_shorts": ["0"], + "fcs_errors": ["0"], + "alignment_errors": ["0"], + "symbol_errors": ["0"], } @@ -259,10 +259,10 @@ class AristaDroppedPacketCounters(BaseModel): out_uc_dropped_pkts: Optional[int] = None out_mc_dropped_pkts: Optional[int] = None - warning_fields: ClassVar[dict[str, str]] = { - "in_dropped_pkts": "0", - "out_uc_dropped_pkts": "0", - "out_mc_dropped_pkts": "0", + warning_fields: ClassVar[dict[str, List[str]]] = { + "in_dropped_pkts": ["0"], + "out_uc_dropped_pkts": ["0"], + "out_mc_dropped_pkts": ["0"], } @@ -275,10 +275,10 @@ class AristaDropPrecedenceCounters(BaseModel): dp1_dropped_pkts: Optional[int] = None dp2_dropped_pkts: Optional[int] = None - error_fields: ClassVar[dict[str, str]] = { - "dp0_dropped_pkts": "0", - "dp1_dropped_pkts": "0", - "dp2_dropped_pkts": "0", + error_fields: ClassVar[dict[str, List[str]]] = { + "dp0_dropped_pkts": ["0"], + "dp1_dropped_pkts": ["0"], + "dp2_dropped_pkts": ["0"], } @@ -293,9 +293,9 @@ class AristaPerQueueCounters(BaseModel): pkts_drop: Optional[int] = None bytes_drop: Optional[int] = None - warning_fields: ClassVar[dict[str, str]] = { - "pkts_drop": "0", - "bytes_drop": "0", + warning_fields: ClassVar[dict[str, List[str]]] = { + "pkts_drop": ["0"], + "bytes_drop": ["0"], } @@ -311,9 +311,9 @@ class AristaPauseFrameCounters(BaseModel): tx_pause: Optional[int] = None rx_pause: Optional[int] = None - error_fields: ClassVar[dict[str, str]] = { - "tx_pause": "0", - "rx_pause": "0", + error_fields: ClassVar[dict[str, List[str]]] = { + "tx_pause": ["0"], + "rx_pause": ["0"], } @@ -325,8 +325,8 @@ class AristaEcnCounters(BaseModel): txq: Optional[str] = None marked_packets: Optional[str] = None - warning_fields: ClassVar[dict[str, str]] = { - "marked_packets": "0", + warning_fields: ClassVar[dict[str, List[str]]] = { + "marked_packets": ["0", "-"], } @@ -338,9 +338,9 @@ class AristaPfcCounters(BaseModel): rx_frames: Optional[int] = None tx_frames: Optional[int] = None - warning_fields: ClassVar[dict[str, str]] = { - "rx_frames": "0", - "tx_frames": "0", + warning_fields: ClassVar[dict[str, List[str]]] = { + "rx_frames": ["0"], + "tx_frames": ["0"], } diff --git a/nodescraper/plugins/inband/switch/scale_out_dell/scaleoutdelldata.py b/nodescraper/plugins/inband/switch/scale_out_dell/scaleoutdelldata.py index 5c6af271..d7739f1a 100644 --- a/nodescraper/plugins/inband/switch/scale_out_dell/scaleoutdelldata.py +++ b/nodescraper/plugins/inband/switch/scale_out_dell/scaleoutdelldata.py @@ -44,9 +44,9 @@ class DellArpEntry(BaseModel): type: Optional[str] = None action: Optional[str] = None - error_fields: ClassVar[dict[str, str]] = { - "address": "NOT_NULL", - "hardware_address": "NOT_NULL", + error_fields: ClassVar[dict[str, List[str]]] = { + "address": ["NOT_NULL"], + "hardware_address": ["NOT_NULL"], } @@ -62,8 +62,8 @@ class DellRouteEntry(BaseModel): distance_metric: Optional[str] = None last_update: Optional[str] = None - error_fields: ClassVar[dict[str, str]] = { - "destination": "NOT_NULL", + error_fields: ClassVar[dict[str, List[str]]] = { + "destination": ["NOT_NULL"], } @@ -81,8 +81,8 @@ class DellInterfaceStatus(BaseModel): mtu: Optional[int] = None alternate_name: Optional[str] = None - error_fields: ClassVar[dict[str, str]] = { - "oper": "up", + error_fields: ClassVar[dict[str, List[str]]] = { + "oper": ["up"], } @@ -112,17 +112,17 @@ class DellInterfaceCounters(BaseModel): tx_drp: Optional[int] = None tx_oversize: Optional[int] = None - error_fields: ClassVar[dict[str, str]] = { - "state": "U", - "rx_err": "0", - "rx_oversize": "0", - "tx_err": "0", - "tx_oversize": "0", + error_fields: ClassVar[dict[str, List[str]]] = { + "state": ["U"], + "rx_err": ["0"], + "rx_oversize": ["0"], + "tx_err": ["0"], + "tx_oversize": ["0"], } - warning_fields: ClassVar[dict[str, str]] = { - "rx_drp": "0", - "tx_drp": "0", + warning_fields: ClassVar[dict[str, List[str]]] = { + "rx_drp": ["0"], + "tx_drp": ["0"], } @@ -168,14 +168,14 @@ class DellInterfaceDetailCounters(BaseModel): time_since_counters_last_cleared: Optional[str] = None - error_fields: ClassVar[dict[str, str]] = { - "packets_received_9217_16383_octets": "0", - "packets_transmitted_9217_16383_octets": "0", - "jabbers_received": "0", - "fragments_received": "0", - "undersize_received": "0", - "overruns_received": "0", - "crc_errors_received": "0", + error_fields: ClassVar[dict[str, List[str]]] = { + "packets_received_9217_16383_octets": ["0"], + "packets_transmitted_9217_16383_octets": ["0"], + "jabbers_received": ["0"], + "fragments_received": ["0"], + "undersize_received": ["0"], + "overruns_received": ["0"], + "crc_errors_received": ["0"], } @@ -193,9 +193,9 @@ class DellQueueCounter(BaseModel): drop_pkts: Optional[int] = None drop_bytes: Optional[int] = None - warning_fields: ClassVar[dict[str, str]] = { - "drop_pkts": "0", - "drop_bytes": "0", + warning_fields: ClassVar[dict[str, List[str]]] = { + "drop_pkts": ["0"], + "drop_bytes": ["0"], } @@ -217,15 +217,15 @@ class DellPfcStatistics(BaseModel): pfc6: Optional[int] = None pfc7: Optional[int] = None - warning_fields: ClassVar[dict[str, str]] = { - "pfc0": "0", - "pfc1": "0", - "pfc2": "0", - "pfc3": "0", - "pfc4": "0", - "pfc5": "0", - "pfc6": "0", - "pfc7": "0", + warning_fields: ClassVar[dict[str, List[str]]] = { + "pfc0": ["0"], + "pfc1": ["0"], + "pfc2": ["0"], + "pfc3": ["0"], + "pfc4": ["0"], + "pfc5": ["0"], + "pfc6": ["0"], + "pfc7": ["0"], } @@ -247,17 +247,17 @@ class DellPfcWatchdogQueueStats(BaseModel): rx_last_ok: Optional[int] = None rx_last_drop: Optional[int] = None - warning_fields: ClassVar[dict[str, str]] = { - "storms_detected": "0", - "storms_restored": "0", - "transmitted_ok": "0", - "transmitted_drop": "0", - "received_ok": "0", - "received_drop": "0", - "tx_last_ok": "0", - "tx_last_drop": "0", - "rx_last_ok": "0", - "rx_last_drop": "0", + warning_fields: ClassVar[dict[str, List[str]]] = { + "storms_detected": ["0"], + "storms_restored": ["0"], + "transmitted_ok": ["0"], + "transmitted_drop": ["0"], + "received_ok": ["0"], + "received_drop": ["0"], + "tx_last_ok": ["0"], + "tx_last_drop": ["0"], + "rx_last_ok": ["0"], + "rx_last_drop": ["0"], } diff --git a/nodescraper/plugins/inband/switch/switch_analyzer_base.py b/nodescraper/plugins/inband/switch/switch_analyzer_base.py index bc0a5ba6..4b4d7eae 100644 --- a/nodescraper/plugins/inband/switch/switch_analyzer_base.py +++ b/nodescraper/plugins/inband/switch/switch_analyzer_base.py @@ -100,8 +100,14 @@ def _model_is_analyzed(model_cls: Type[BaseModel]) -> bool: def _values_match(actual: Any, expected: Any) -> bool: - """Compare an actual model value to an expected value.""" + """Compare an actual model value to an expected value. + When ``expected`` is a list or tuple of candidate values the comparison + succeeds if ``actual`` matches any candidate (logical OR). + """ + + if isinstance(expected, (list, tuple)): + return any(_values_match(actual, candidate) for candidate in expected) if isinstance(expected, str) and expected == "NOT_NULL": if actual is None: return False @@ -111,6 +117,17 @@ def _values_match(actual: Any, expected: Any) -> bool: return str(actual) == str(expected) +def _expects_not_null(expected: Any) -> bool: + """Return True if ``expected`` requires a non-null actual value. + + Supports ``expected`` being a list/tuple of candidate values. + """ + + if isinstance(expected, (list, tuple)): + return any(_expects_not_null(item) for item in expected) + return isinstance(expected, str) and expected == "NOT_NULL" + + class SwitchAnalyzerBase(Generic[TSwitchData]): """Shared scaffolding for vendor-specific switch analyzers. @@ -498,7 +515,7 @@ def _check_fields( if not hasattr(model, field_name): continue actual = getattr(model, field_name) - if actual is None and not (isinstance(expected, str) and expected == "NOT_NULL"): + if actual is None and not _expects_not_null(expected): continue if _values_match(actual, expected): continue diff --git a/test/unit/plugin/test_switch_analyzer_base.py b/test/unit/plugin/test_switch_analyzer_base.py index 98a750ab..9685b592 100644 --- a/test/unit/plugin/test_switch_analyzer_base.py +++ b/test/unit/plugin/test_switch_analyzer_base.py @@ -44,6 +44,10 @@ DellPortData, ScaleOutDellDataModel, ) +from nodescraper.plugins.inband.switch.switch_analyzer_base import ( + _expects_not_null, + _values_match, +) @pytest.fixture @@ -169,3 +173,80 @@ def test_analyzer_messages_distinguish_errors_and_warnings(analyzer): result = analyzer.analyze_data(data) assert result.status == ExecutionStatus.ERROR assert result.message.startswith("Arista errors and warnings detected") + + +@pytest.mark.parametrize( + "actual, expected, matches", + [ + ("U", ["U", "S"], True), + ("S", ["U", "S"], True), + ("D", ["U", "S"], False), + ("U", ["U"], True), + ("D", ["U"], False), + ("anything", ["NOT_NULL"], True), + (None, ["NOT_NULL"], False), + ], +) +def test_values_match_supports_list_or(actual, expected, matches): + assert _values_match(actual, expected) is matches + + +def test_expects_not_null_detects_nested_not_null(): + assert _expects_not_null(["U", "NOT_NULL"]) is True + assert _expects_not_null(["U", "S"]) is False + assert _expects_not_null(["NOT_NULL"]) is True + + +def test_error_warning_field_values_are_lists(): + assert DellInterfaceCounters.error_fields["state"] == ["U"] + assert DellInterfaceCounters.warning_fields["rx_drp"] == ["0"] + assert AristaPortStatus.error_fields["link_status"] == ["connected"] + assert all(isinstance(value, list) for value in DellInterfaceCounters.error_fields.values()) + + +def test_dell_state_list_or_passes_on_any_match(dell_analyzer, monkeypatch): + monkeypatch.setitem(DellInterfaceCounters.error_fields, "state", ["U", "S"]) + data = ScaleOutDellDataModel( + port={ + "Eth1/1/1": DellPortData( + interface_counters=DellInterfaceCounters( + state="S", + rx_err=0, + rx_oversize=0, + tx_err=0, + tx_oversize=0, + rx_drp=0, + tx_drp=0, + ), + ), + } + ) + + result = dell_analyzer.analyze_data(data) + + assert result.status != ExecutionStatus.ERROR + assert not any("state" in event.description for event in result.events) + + +def test_dell_state_list_or_errors_when_no_match(dell_analyzer, monkeypatch): + monkeypatch.setitem(DellInterfaceCounters.error_fields, "state", ["U", "S"]) + data = ScaleOutDellDataModel( + port={ + "Eth1/1/1": DellPortData( + interface_counters=DellInterfaceCounters( + state="D", + rx_err=0, + rx_oversize=0, + tx_err=0, + tx_oversize=0, + rx_drp=0, + tx_drp=0, + ), + ), + } + ) + + result = dell_analyzer.analyze_data(data) + + assert result.status == ExecutionStatus.ERROR + assert any("state" in event.description for event in result.events) From e9c1c9f4e94fc44f7de65c04cbfe68a79c303521 Mon Sep 17 00:00:00 2001 From: sunnyhe2 Date: Tue, 7 Jul 2026 10:30:04 -0700 Subject: [PATCH 4/9] removed empty collector arg --- .../plugins/inband/rdma/collector_args.py | 31 ------------------- .../plugins/inband/rdma/rdma_collector.py | 7 ++--- .../plugins/inband/rdma/rdma_plugin.py | 4 +-- 3 files changed, 3 insertions(+), 39 deletions(-) delete mode 100644 nodescraper/plugins/inband/rdma/collector_args.py diff --git a/nodescraper/plugins/inband/rdma/collector_args.py b/nodescraper/plugins/inband/rdma/collector_args.py deleted file mode 100644 index 3a7162dc..00000000 --- a/nodescraper/plugins/inband/rdma/collector_args.py +++ /dev/null @@ -1,31 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### - -from nodescraper.models import CollectorArgs - - -class RdmaCollectorArgs(CollectorArgs): - """Collection arguments for the RDMA collector.""" diff --git a/nodescraper/plugins/inband/rdma/rdma_collector.py b/nodescraper/plugins/inband/rdma/rdma_collector.py index fb841cd5..740f5998 100644 --- a/nodescraper/plugins/inband/rdma/rdma_collector.py +++ b/nodescraper/plugins/inband/rdma/rdma_collector.py @@ -34,7 +34,6 @@ from nodescraper.models import TaskResult from nodescraper.utils import get_exception_traceback -from .collector_args import RdmaCollectorArgs from .rdmadata import ( VENDOR_PREFIX_MAP, RdmaDataModel, @@ -46,7 +45,7 @@ ) -class RdmaCollector(InBandDataCollector[RdmaDataModel, RdmaCollectorArgs]): +class RdmaCollector(InBandDataCollector[RdmaDataModel, None]): """Collect RDMA status and statistics via rdma link and rdma statistic commands.""" DATA_MODEL = RdmaDataModel @@ -292,9 +291,7 @@ def _get_rdma_link(self) -> Optional[list[RdmaLink]]: ) return None - def collect_data( - self, args: Optional[RdmaCollectorArgs] = None - ) -> tuple[TaskResult, Optional[RdmaDataModel]]: + def collect_data(self, args=None) -> tuple[TaskResult, Optional[RdmaDataModel]]: """Collect RDMA statistics, link data, and device/link text output. Returns: diff --git a/nodescraper/plugins/inband/rdma/rdma_plugin.py b/nodescraper/plugins/inband/rdma/rdma_plugin.py index 0224d47f..4bf5e8cc 100644 --- a/nodescraper/plugins/inband/rdma/rdma_plugin.py +++ b/nodescraper/plugins/inband/rdma/rdma_plugin.py @@ -26,17 +26,15 @@ from nodescraper.base import InBandDataPlugin from .analyzer_args import RdmaAnalyzerArgs -from .collector_args import RdmaCollectorArgs from .rdma_analyzer import RdmaAnalyzer from .rdma_collector import RdmaCollector from .rdmadata import RdmaDataModel -class RdmaPlugin(InBandDataPlugin[RdmaDataModel, RdmaCollectorArgs, RdmaAnalyzerArgs]): +class RdmaPlugin(InBandDataPlugin[RdmaDataModel, None, RdmaAnalyzerArgs]): """Plugin for collection and analysis of RDMA statistics and link data.""" DATA_MODEL = RdmaDataModel COLLECTOR = RdmaCollector - COLLECTOR_ARGS = RdmaCollectorArgs ANALYZER = RdmaAnalyzer ANALYZER_ARGS = RdmaAnalyzerArgs From d7cfa61ce24e005ea22b68ad07ee08afec20e040 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 7 Jul 2026 19:11:33 +0000 Subject: [PATCH 5/9] docs: Update plugin documentation [automated] --- docs/PLUGIN_DOC.md | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/docs/PLUGIN_DOC.md b/docs/PLUGIN_DOC.md index deebb1a6..0e870d02 100644 --- a/docs/PLUGIN_DOC.md +++ b/docs/PLUGIN_DOC.md @@ -17,14 +17,14 @@ | KernelPlugin | sh -c 'uname -a'
sh -c 'cat /proc/sys/kernel/numa_balancing'
wmic os get Version /Value | **Analyzer Args:**
- `exp_kernel`: Union[str, list] — Expected kernel version string(s) to match (e.g. from uname -a).
- `exp_numa`: Optional[int] — Expected value for kernel.numa_balancing (e.g. 0 or 1).
- `regex_match`: bool — If True, match exp_kernel as regex; otherwise exact match. | - | [KernelDataModel](#KernelDataModel-Model) | [KernelCollector](#Collector-Class-KernelCollector) | [KernelAnalyzer](#Data-Analyzer-Class-KernelAnalyzer) | | KernelModulePlugin | cat /proc/modules
modinfo amdgpu
wmic os get Version /Value | **Analyzer Args:**
- `kernel_modules`: dict[str, dict] — Expected kernel module name -> {version, etc.}. Analyzer checks collected modules match.
- `regex_filter`: list[str] — List of regex patterns to filter which collected modules are checked (default: amd). | - | [KernelModuleDataModel](#KernelModuleDataModel-Model) | [KernelModuleCollector](#Collector-Class-KernelModuleCollector) | [KernelModuleAnalyzer](#Data-Analyzer-Class-KernelModuleAnalyzer) | | MemoryPlugin | free -b
lsmem
numactl -H
wmic OS get FreePhysicalMemory /Value; wmic ComputerSystem get TotalPhysicalMemory /Value | **Analyzer Args:**
- `ratio`: float — Required free-memory ratio (0-1). Analysis fails if free/total < ratio.
- `memory_threshold`: str — Minimum free memory required (e.g. '30Gi', '1T'). Used when ratio is not sufficient. | - | [MemoryDataModel](#MemoryDataModel-Model) | [MemoryCollector](#Collector-Class-MemoryCollector) | [MemoryAnalyzer](#Data-Analyzer-Class-MemoryAnalyzer) | -| NetworkPlugin | ip addr show
curl
ethtool -S {interface}
ethtool {interface}
lldpcli show neighbor
lldpctl
ip neighbor show
ping
rdma link -j
ip route show
ip rule show
wget | **Built-in Regexes:**
- tx_pfc_frames is non-zero: `^tx_pfc_frames$`
- tx_pfc_ena_frames_pri* is non-zero: `^tx_pfc_ena_frames_pri\d+$`
- pfc_pri*_tx_transitions is non-zero: `^pfc_pri\d+_tx_transitions$`
**Analyzer Args:**
- `error_regex`: Union[list[nodescraper.base.regexanalyzer.ErrorRegex], list[dict], NoneType] — Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `url`: Optional[str] — Optional URL to probe for network connectivity (used with netprobe).
- `netprobe`: Optional[Literal['ping', 'wget', 'curl']] — Tool to use for network connectivity probe: ping, wget, or curl. | [NetworkDataModel](#NetworkDataModel-Model) | [NetworkCollector](#Collector-Class-NetworkCollector) | [NetworkAnalyzer](#Data-Analyzer-Class-NetworkAnalyzer) | +| NetworkPlugin | ip addr show
curl
ethtool -S {interface}
ethtool {interface}
lldpcli show neighbor
lldpctl
ip neighbor show
ping
rdma link -j
ip route show
ip rule show
wget | **Analyzer Args:**
- `error_regex`: Union[list[nodescraper.base.regexanalyzer.ErrorRegex], list[dict], NoneType] — Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `url`: Optional[str] — Optional URL to probe for network connectivity (used with netprobe).
- `netprobe`: Optional[Literal['ping', 'wget', 'curl']] — Tool to use for network connectivity probe: ping, wget, or curl.
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched (search) against netdev/interface names; any match is skipped (ethtool not run against it). | [NetworkDataModel](#NetworkDataModel-Model) | [NetworkCollector](#Collector-Class-NetworkCollector) | [NetworkAnalyzer](#Data-Analyzer-Class-NetworkAnalyzer) | | NicPlugin | niccli --listdev
niccli --list
niccli --list_devices
niccli -dev {device_num} nvm -getoption pcie_relaxed_ordering
niccli --dev {device_num} nvm --getoption pcie_relaxed_ordering
niccli -dev {device_num} nvm -getoption performance_profile
niccli --dev {device_num} nvm --getoption performance_profile
niccli -dev {device_num} nvm -getoption support_rdma -scope 0
niccli -dev {device_num} getqos
niccli --dev {device_num} nvm --getoption support_rdma --scope 0
niccli --dev {device_num} qos --ets --show
niccli --version
nicctl show card
nicctl --version
nicctl show card flash partition --json
nicctl show card interrupts --json
nicctl show card logs --non-persistent
nicctl show card logs --boot-fault
nicctl show card logs --persistent
nicctl show card profile --json
nicctl show card time --json
nicctl show card statistics packet-buffer summary --json
nicctl show lif statistics --json
nicctl show lif internal queue-to-ud-pinning
nicctl show pipeline internal anomalies
nicctl show pipeline internal rsq-ring
nicctl show pipeline internal statistics memory
nicctl show port fsm
nicctl show port transceiver --json
nicctl show port statistics --json
nicctl show port internal mac
nicctl show qos headroom --json
nicctl show rdma queue --json
nicctl show rdma queue-pair --detail --json
nicctl show version firmware
nicctl show dcqcn
nicctl show environment
nicctl show lif
nicctl show pcie ats
nicctl show port
nicctl show qos
nicctl show rdma statistics
nicctl show version host-software
nicctl show dcqcn --card {card_id} --json
nicctl show card hardware-config --card {card_id} | **Analyzer Args:**
- `expected_values`: Optional[Dict[str, Dict[str, Any]]] — Per-command expected checks keyed by canonical key (see command_to_canonical_key).
- `performance_profile_expected`: str — Expected Broadcom performance_profile value (case-insensitive). Default RoCE.
- `support_rdma_disabled_values`: List[str] — Values that indicate RDMA is not supported (case-insensitive).
- `pcie_relaxed_ordering_expected`: str — Expected Broadcom pcie_relaxed_ordering value (e.g. 'Relaxed ordering = enabled'); checked case-insensitively. Defaul...
- `expected_qos_prio_map`: Optional[Dict[Any, Any]] — Expected priority-to-TC map (e.g. {0: 0, 1: 1}; keys may be int or str in config). Checked per device when set.
- `expected_qos_pfc_enabled`: Optional[int] — Expected PFC enabled value (0/1 or bitmask). Checked per device when set.
- `expected_qos_tsa_map`: Optional[Dict[Any, Any]] — Expected TSA map for ETS (e.g. {0: 'ets', 1: 'strict'}; keys may be int or str in config). Checked per device when set.
- `expected_qos_tc_bandwidth`: Optional[List[int]] — Expected TC bandwidth percentages. Checked per device when set.
- `require_qos_consistent_across_adapters`: bool — When True and no expected_qos_* are set, require all adapters to have the same prio_map, pfc_enabled, and tsa_map.
- `nicctl_log_error_regex`: Optional[List[Dict[str, Any]]] — Optional list of error patterns for nicctl show card logs. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `commands`: Optional[List[str]] — Optional list of niccli/nicctl commands to run. When None, default command set is used.
- `use_sudo_niccli`: bool — If True, run niccli commands with sudo when required.
- `use_sudo_nicctl`: bool — If True, run nicctl commands with sudo when required. | [NicDataModel](#NicDataModel-Model) | [NicCollector](#Collector-Class-NicCollector) | [NicAnalyzer](#Data-Analyzer-Class-NicAnalyzer) | | NvmePlugin | nvme smart-log {dev}
nvme error-log {dev} --log-entries=256
nvme id-ctrl {dev}
nvme id-ns {dev}{ns}
nvme fw-log {dev}
nvme self-test-log {dev}
nvme get-log {dev} --log-id=6 --log-len=512
nvme telemetry-log {dev} --output-file={dev}_{f_name}
nvme list -o json | - | - | [NvmeDataModel](#NvmeDataModel-Model) | [NvmeCollector](#Collector-Class-NvmeCollector) | - | | OsPlugin | sh -c '( lsb_release -ds || (cat /etc/*release | grep PRETTY_NAME) || uname -om ) 2>/dev/null | head -n1'
cat /etc/*release | grep VERSION_ID
wmic os get Version /value
wmic os get Caption /Value | **Analyzer Args:**
- `exp_os`: Union[str, list] — Expected OS name/version string(s) to match (e.g. from lsb_release or /etc/os-release).
- `exact_match`: bool — If True, require exact match for exp_os; otherwise substring match. | - | [OsDataModel](#OsDataModel-Model) | [OsCollector](#Collector-Class-OsCollector) | [OsAnalyzer](#Data-Analyzer-Class-OsAnalyzer) | | PackagePlugin | dnf list --installed
dpkg-query -W
pacman -Q
cat /etc/*release
wmic product get name,version | **Analyzer Args:**
- `exp_package_ver`: Dict[str, Optional[str]] — Map package name -> expected version (None = any version). Checked against installed packages.
- `regex_match`: bool — If True, match package versions with regex; otherwise exact or prefix match.
- `rocm_regex`: Optional[str] — Optional regex to identify ROCm package version (used when enable_rocm_regex is True).
- `enable_rocm_regex`: bool — If True, use rocm_regex (or default pattern) to extract ROCm version for checks. | - | [PackageDataModel](#PackageDataModel-Model) | [PackageCollector](#Collector-Class-PackageCollector) | [PackageAnalyzer](#Data-Analyzer-Class-PackageAnalyzer) | | PciePlugin | lspci -d {vendor_id}: -nn
lspci -x
lspci -xxxx
lspci -PP
lspci -PP -d {vendor_id}:{dev_id}
lspci -PP -D -d {vendor_id}:{dev_id}
lspci -PP -D
lspci -vvv
lspci -vvvt | **Analyzer Args:**
- `exp_speed`: int — Expected PCIe link speed (generation 1–5).
- `exp_width`: int — Expected PCIe link width in lanes (1–16).
- `exp_sriov_count`: int — Expected SR-IOV virtual function count.
- `exp_gpu_count_override`: Optional[int] — Override expected GPU count for validation.
- `exp_max_payload_size`: Union[Dict[int, int], int, NoneType] — Expected max payload size: int for all devices, or dict keyed by device ID.
- `exp_max_rd_req_size`: Union[Dict[int, int], int, NoneType] — Expected max read request size: int for all devices, or dict keyed by device ID.
- `exp_ten_bit_tag_req_en`: Union[Dict[int, int], int, NoneType] — Expected 10-bit tag request enable: int for all devices, or dict keyed by device ID. | - | [PcieDataModel](#PcieDataModel-Model) | [PcieCollector](#Collector-Class-PcieCollector) | [PcieAnalyzer](#Data-Analyzer-Class-PcieAnalyzer) | | ProcessPlugin | top -b -n 1
rocm-smi --showpids
top -b -n 1 -o %CPU | **Analyzer Args:**
- `max_kfd_processes`: int — Maximum allowed number of KFD (Kernel Fusion Driver) processes; 0 disables the check.
- `max_cpu_usage`: float — Maximum allowed CPU usage (percent) for process checks. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `top_n_process`: int — Number of top processes by CPU usage to collect (e.g. for top -b -n 1 -o %%CPU). | [ProcessDataModel](#ProcessDataModel-Model) | [ProcessCollector](#Collector-Class-ProcessCollector) | [ProcessAnalyzer](#Data-Analyzer-Class-ProcessAnalyzer) | -| RdmaPlugin | rdma link -j
rdma dev
rdma link
rdma statistic -j | - | - | [RdmaDataModel](#RdmaDataModel-Model) | [RdmaCollector](#Collector-Class-RdmaCollector) | [RdmaAnalyzer](#Data-Analyzer-Class-RdmaAnalyzer) | +| RdmaPlugin | rdma link -j
rdma dev
rdma link
rdma statistic -j | **Analyzer Args:**
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched against an interface netdev; matching interfaces are skipped. | - | [RdmaDataModel](#RdmaDataModel-Model) | [RdmaCollector](#Collector-Class-RdmaCollector) | [RdmaAnalyzer](#Data-Analyzer-Class-RdmaAnalyzer) | | RegexSearchPlugin | - | Runs RegexSearchAnalyzer: user-defined patterns via analysis_args.error_regex (same shape as Dmesg).
Emits regex match events with optional per-file source in the description when scanning directories.
**Analyzer Args:**
- `error_regex`: Optional[list[dict[str, Any]]] — Regex patterns to search for; each dict may include regex (str), message, event_category, event_priority (same as Dme...
- `interval_to_collapse_event`: int — Seconds within which repeated events are collapsed into one.
- `num_timestamps`: int — Number of timestamps to include per event in output. | - | [RegexSearchData](#RegexSearchData-Model) | - | [RegexSearchAnalyzer](#Data-Analyzer-Class-RegexSearchAnalyzer) | | RocmPlugin | {rocm_path}/opencl/bin/*/clinfo
env | grep -Ei 'rocm|hsa|hip|mpi|openmp|ucx|miopen'
ls /sys/class/kfd/kfd/proc/
grep -i -E 'rocm' /etc/ld.so.conf.d/*
{rocm_path}/bin/rocminfo
ls -v -d {rocm_path}*
ls -v -d {rocm_path}-[3-7]* | tail -1
ldconfig -p | grep -i -E 'rocm'
grep . -H -r -i {rocm_path}/.info/* | **Analyzer Args:**
- `exp_rocm`: Union[str, list] — Expected ROCm version string(s) to match (e.g. from rocminfo).
- `exp_rocm_latest`: str — Expected 'latest' ROCm path or version string for versioned installs.
- `exp_rocm_sub_versions`: dict[str, Union[str, list]] — Map sub-version name (e.g. version_rocm) to expected string or list of allowed strings. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `rocm_path`: str — Base path to ROCm installation (e.g. /opt/rocm). Used for rocminfo, clinfo, and version discovery. | [RocmDataModel](#RocmDataModel-Model) | [RocmCollector](#Collector-Class-RocmCollector) | [RocmAnalyzer](#Data-Analyzer-Class-RocmAnalyzer) | | StoragePlugin | sh -c 'df -lH -B1 | grep -v 'boot''
wmic LogicalDisk Where DriveType="3" Get DeviceId,Size,FreeSpace | - | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `skip_sudo`: bool — If True, do not use sudo when running df and related storage commands. | [StorageDataModel](#StorageDataModel-Model) | [StorageCollector](#Collector-Class-StorageCollector) | [StorageAnalyzer](#Data-Analyzer-Class-StorageAnalyzer) | @@ -2134,7 +2134,7 @@ Check memory usage is within the maximum allowed used memory ### Description -Check network statistics for errors (PFC and other network error counters). +Check network statistics for errors. **Bases**: ['RegexAnalyzer'] @@ -2142,18 +2142,7 @@ Check network statistics for errors (PFC and other network error counters). ### Class Variables -- **ERROR_REGEX**: `[ - regex=re.compile('^tx_pfc_frames$') message='tx_pfc_frames is non-zero' event_category= event_priority=, - regex=re.compile('^tx_pfc_ena_frames_pri\\d+$') message='tx_pfc_ena_frames_pri* is non-zero' event_category= event_priority=, - regex=re.compile('^pfc_pri\\d+_tx_transitions$') message='pfc_pri*_tx_transitions is non-zero' event_category= event_priority= -]` - -### Regex Patterns - -- **Built-in Regexes:** -- - tx_pfc_frames is non-zero: `^tx_pfc_frames$` -- - tx_pfc_ena_frames_pri* is non-zero: `^tx_pfc_ena_frames_pri\d+$` -- - pfc_pri*_tx_transitions is non-zero: `^pfc_pri\d+_tx_transitions$` +- **ERROR_REGEX**: `[]` ## Data Analyzer Class NicAnalyzer @@ -2601,6 +2590,20 @@ Arguments for PCIe analyzer - **max_kfd_processes**: `int` — Maximum allowed number of KFD (Kernel Fusion Driver) processes; 0 disables the check. - **max_cpu_usage**: `float` — Maximum allowed CPU usage (percent) for process checks. +## Analyzer Args Class RdmaAnalyzerArgs + +### Description + +Arguments for the RDMA analyzer. + +**Bases**: ['AnalyzerArgs'] + +**Link to code**: [analyzer_args.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/rdma/analyzer_args.py) + +### Annotations / fields + +- **exclusion_regex**: `Optional[list[str]]` — Regex patterns matched against an interface netdev; matching interfaces are skipped. + ## Analyzer Args Class RegexSearchAnalyzerArgs ### Description From abe4dc5660b7677970b1b5b5a4b5943317d9e042 Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Tue, 7 Jul 2026 15:41:41 -0500 Subject: [PATCH 6/9] ignorning base class from docs --- docs/PLUGIN_DOC.md | 1 - docs/generate_plugin_doc_bundle.py | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/PLUGIN_DOC.md b/docs/PLUGIN_DOC.md index deebb1a6..0c7d33c1 100644 --- a/docs/PLUGIN_DOC.md +++ b/docs/PLUGIN_DOC.md @@ -44,7 +44,6 @@ | RedfishEndpointPlugin | Redfish GET: explicit paths from collection_args.uris (parallel when max_workers>1).
Optional paged GET following the Members collection OData nextLink field when follow_next_link is true.
Redfish GET tree: when discover_tree is true, walks from api_root using OData resource id links and Members navigation (depth and endpoint caps from collection_args). | For each entry in analysis_args.checks, reads JSON paths in collected responses and compares values to constraints (eq, min/max, anyOf, regex, etc.).
URI key "*" runs checks against every collected response body.
**Analyzer Args:**
- `checks`: dict[str, dict[str, Union[int, float, str, bool, dict[str, Any]]]] — Map: URI or '*' -> { property_path: constraint }. URI keys must match a key in the collected responses (exact match).... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `uris`: list[str] — Redfish URIs to GET. Ignored when discover_tree is True.
- `discover_tree`: bool — If True, discover endpoints from the BMC Redfish tree (service root and links) instead of using uris.
- `tree_max_depth`: int — When discover_tree is True: max traversal depth (1=service root only, 2=root + collections, 3=+ members).
- `tree_max_endpoints`: int — When discover_tree is True: max endpoints to discover (0=no limit).
- `max_workers`: int — Max concurrent GETs (1=sequential). Use >1 for async endpoint fetches.
- `follow_next_link`: bool — If True, follow Redfish Members collection OData nextLink pagination for each URI and merge all pages into a single r...
- `max_pages`: int — When follow_next_link is True: safety cap on the number of pages to follow per URI (default 200). | [RedfishEndpointDataModel](#RedfishEndpointDataModel-Model) | [RedfishEndpointCollector](#Collector-Class-RedfishEndpointCollector) | [RedfishEndpointAnalyzer](#Data-Analyzer-Class-RedfishEndpointAnalyzer) | | RedfishOemDiagPlugin | Redfish LogService.CollectDiagnosticData for each entry in collection_args.oem_diagnostic_types (collection_args.log_service_path selects the LogService).
Optional binary archives under the plugin log path when log_path is set. | Summarizes success/failure per OEM diagnostic type from collected results.
When analysis_args.require_all_success is true, fails the run if any type failed collection.
**Analyzer Args:**
- `require_all_success`: bool — If True, analysis fails when any OEM type collection failed. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `log_service_path`: str — Redfish path to the LogService (e.g. DiagLogs).
- `oem_diagnostic_types_allowable`: Optional[list[str]] — Allowable OEM diagnostic types for this architecture/BMC. When set, used for validation and as default for oem_diagno...
- `oem_diagnostic_types`: list[str] — OEM diagnostic types to collect. When empty and oem_diagnostic_types_allowable is set, defaults to that list.
- `task_timeout_s`: int — Max seconds to wait for each BMC task. | [RedfishOemDiagDataModel](#RedfishOemDiagDataModel-Model) | [RedfishOemDiagCollector](#Collector-Class-RedfishOemDiagCollector) | [RedfishOemDiagAnalyzer](#Data-Analyzer-Class-RedfishOemDiagAnalyzer) | | ServiceabilityPluginMI3XX | - | **Analyzer Args:**
- `hub_python_module`: Optional[str] — Import path for the hub module (class implements hub_analyze_method); hub_options forwards kwargs.
- `hub_display_name`: Optional[str] — Optional label for analyzer status messages.
- `afid_sag_path`: Optional[str] — Path to hub config (e.g. AFID_SAG.json); passed as hub_init_path_kwarg.
- `hub_init_path_kwarg`: str — Hub __init__ keyword that receives afid_sag_path.
- `hub_analyze_method`: str — Hub method called with rf_events first (default get_service_info).
- `skip_hub`: bool — If True, only build afid_events without running the service hub.
- `cper_decode_module`: Optional[str] — Module import path for CPER decoding when events include CPER attachments.
- `cper_decode_method`: str — Callable on cper_decode_module: file-like CPER in, (return_code, decode_dict) out.
- `hub_options`: Optional[dict[str, Any]] — Extra kwargs for hub __init__ and analyze; collected cper_data overrides cper_data key.
- `from_ac_cycle`: int — from_ac_cycle kwarg for the hub analyze call (merged after hub_options).
- `from_date`: Optional[str] — Optional from_date for the hub analyze call (merged after hub_options).
- `designation_serials`: Optional[dict[str, str]] — Optional designation_serials for the hub analyze call (merged after hub_options).
- `suppress_service_actions`: Optional[list[str]] — Optional suppress_service_actions for the hub analyze call (merged after hub_options). | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors...
- `uri`: Optional[str] — Optional alias for ``rf_event_log_uri``. When both ``uri`` and ``rf_event_log_uri`` are explicitly set to non-empty v...
- `rf_event_log_uri`: str — Redfish URI for the event log ``Entries`` collection.
- `rf_chassis_devices`: Optional[List[str]] — Chassis designations for Assembly GETs; required with ``rf_assembly_uri_template``.
- `rf_assembly_uri_template`: Optional[str] — Redfish URI template containing ``{device}`` for each chassis Assembly resource.
- `rf_firmware_bundle_uri`: Optional[str] — Redfish URI for firmware bundle inventory when subclasses extract component details.
- `follow_next_link`: bool — If True, follow Members@odata.nextLink up to max_pages; else single GET.
- `max_pages`: int — Safety cap on the number of pages when following event log pagination.
- `top`: Optional[int] — Most recent N entries via $skip after count probe; None collects full window.
- `reference_time`: Optional[str] — Optional ISO-8601 date or date-time used with time_operator (e.g. 2026-05-17 or 2026-05-17T13:01:00).
- `time_operator`: Optional[Literal['>', '>=', '<', '<=', '==']] — Comparison operator applied when reference_time is set. | [ServiceabilityDataModel](#ServiceabilityDataModel-Model) | [MI3XXCollector](#Collector-Class-MI3XXCollector) | [MI3XXAnalyzer](#Data-Analyzer-Class-MI3XXAnalyzer) | -| ServiceabilityPluginBase | - | - | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Arista collectors... | [ServiceabilityDataModel](#ServiceabilityDataModel-Model) | [ServiceabilityCollectorBase](#Collector-Class-ServiceabilityCollectorBase) | - | # Collectors diff --git a/docs/generate_plugin_doc_bundle.py b/docs/generate_plugin_doc_bundle.py index cd9897b0..75c4d412 100644 --- a/docs/generate_plugin_doc_bundle.py +++ b/docs/generate_plugin_doc_bundle.py @@ -49,6 +49,8 @@ # ``nodescraper.plugins`` itself does not match every module starting with that string. PLUGIN_MODULE_PREFIX = f"{PACKAGE_PLUGINS_ROOT}." DEFAULT_PACKAGES = (PACKAGE_PLUGINS_ROOT,) +# OOB plugin stubs with no concrete COLLECTOR/ANALYZER; document subclasses only. +PLUGIN_DOC_IGNORE_CLASSES = frozenset({"ServiceabilityPluginBase"}) def get_attr(obj: Any, name: str, default: Any = None) -> Any: @@ -230,6 +232,8 @@ def plugins_for_package_prefix(base_classes: Iterable[type], package_prefix: str continue if not is_concrete_plugin_class(cls): continue + if cls.__name__ in PLUGIN_DOC_IGNORE_CLASSES: + continue found.append(cls) return found From a6dfd68ca96665a51176f452d16bbdb369ade576 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 8 Jul 2026 00:22:35 +0000 Subject: [PATCH 7/9] docs: Update plugin documentation [automated] --- docs/PLUGIN_DOC.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/docs/PLUGIN_DOC.md b/docs/PLUGIN_DOC.md index 755a44f6..fe6612ae 100644 --- a/docs/PLUGIN_DOC.md +++ b/docs/PLUGIN_DOC.md @@ -1313,20 +1313,6 @@ Collect MI3XX BMC Redfish data: event log members (with pagination), firmware in ServiceabilityDataModel -## Collector Class ServiceabilityCollectorBase - -### Description - -OOB Redfish collection skeleton; subclasses implement filtering, CPER handling, and JSON parsing. - -**Bases**: ['RedfishDataCollector', 'Generic'] - -**Link to code**: [serviceability_collector.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/serviceability/serviceability_collector.py) - -### Provides Data - -ServiceabilityDataModel - # Data Models ## GenericCollectionDataModel Model From 385b7a452ec47890f6b8b115a90ef0e483aaf13f Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Wed, 8 Jul 2026 11:12:32 -0500 Subject: [PATCH 8/9] updates --- nodescraper/base/regexanalyzer.py | 28 ++++ .../plugins/inband/dmesg/analyzer_args.py | 4 +- .../plugins/inband/dmesg/dmesg_analyzer.py | 11 +- nodescraper/plugins/inband/dmesg/mce_utils.py | 97 ++++++++++++-- test/unit/plugin/test_dmesg_analyzer.py | 116 +++++++++++++++++ test/unit/plugin/test_mce_utils.py | 123 +++++++++++++++++- 6 files changed, 362 insertions(+), 17 deletions(-) diff --git a/nodescraper/base/regexanalyzer.py b/nodescraper/base/regexanalyzer.py index 1fab952d..1671db7d 100644 --- a/nodescraper/base/regexanalyzer.py +++ b/nodescraper/base/regexanalyzer.py @@ -138,6 +138,24 @@ def _line_at_match_position(self, content: str, match_start: int) -> str: line_end = len(content) return content[line_start:line_end] + def _line_index_at_position(self, content: str, position: int) -> int: + if position <= 0: + return 0 + return content.count("\n", 0, position) + + def _match_intersects_skip_lines( + self, + content: str, + match_start: int, + match_end: int, + skip_line_indices: frozenset[int], + ) -> bool: + if not skip_line_indices: + return False + start_line = self._line_index_at_position(content, match_start) + end_line = self._line_index_at_position(content, max(match_start, match_end - 1)) + return any(line_idx in skip_line_indices for line_idx in range(start_line, end_line + 1)) + def _should_ignore_regex_match( self, content: str, @@ -245,6 +263,7 @@ def check_all_regexes( num_timestamps: int = 3, interval_to_collapse_event: int = 60, ignore_match_rules: Optional[Sequence[ParsedIgnoreMatchRule]] = None, + skip_line_indices: Optional[frozenset[int]] = None, ) -> list[RegexEvent]: """Iterate over all ERROR_REGEX and check content for any matches @@ -262,6 +281,7 @@ def check_all_regexes( num_timestamps (int, optional): maximum number of timestamps to keep for each event. Defaults to 3. interval_to_collapse_event (int, optional): time interval in seconds to collapse events. Defaults to 60. ignore_match_rules (Optional[Sequence[ParsedIgnoreMatchRule]], optional): Parsed skip rules. Defaults to None. + skip_line_indices (Optional[frozenset[int]], optional): Line indices to skip entirely. Defaults to None. Returns: list[RegexEvent]: list of regex event objects @@ -296,10 +316,18 @@ def _is_within_interval(new_timestamp_str: str, existing_timestamps: list[str]) return False skip_rules = list(ignore_match_rules) if ignore_match_rules else [] + suppressed_lines = skip_line_indices or frozenset() for error_regex_obj in error_regex: for match_obj in error_regex_obj.regex.finditer(content): raw_match = match_obj.group(0) + if self._match_intersects_skip_lines( + content, + match_obj.start(), + match_obj.end(), + suppressed_lines, + ): + continue if self._should_ignore_regex_match( content, match_obj.start(), diff --git a/nodescraper/plugins/inband/dmesg/analyzer_args.py b/nodescraper/plugins/inband/dmesg/analyzer_args.py index ca2294f6..4b2a3797 100644 --- a/nodescraper/plugins/inband/dmesg/analyzer_args.py +++ b/nodescraper/plugins/inband/dmesg/analyzer_args.py @@ -76,6 +76,8 @@ class DmesgAnalyzerArgs(TimeRangeAnalysisArgs): "Rules that skip regex matches during analysis. Each rule may use line_regex, " "match_regex, message, and/or mce_banks. Within a rule all specified fields must " "match; any matching rule suppresses the hit. mce_banks accepts bank ids and " - 'inclusive ranges such as "60-63".' + 'inclusive ranges such as "60-63". When a rule matches an MCA bank, only ' + "[Hardware Error]: lines in that incident block are suppressed; interleaved " + "non-MCE dmesg lines in the same block are not." ), ) diff --git a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py index 65a613f9..58a0a2ce 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py @@ -35,7 +35,12 @@ from .analyzer_args import DmesgAnalyzerArgs from .dmesgdata import DmesgData -from .mce_utils import parse_correctable_mce_counts, parse_uncorrectable_mce_counts +from .mce_utils import ( + hardware_error_block_line_indices, + ignored_mce_block_line_indices, + parse_correctable_mce_counts, + parse_uncorrectable_mce_counts, +) class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): @@ -714,6 +719,8 @@ def analyze_data( dmesg_content = data.dmesg_content ignore_match_rules, ignore_mce_banks = parse_ignore_match_rules(args.ignore_match_rules) + ignored_mce_block_lines = ignored_mce_block_line_indices(dmesg_content, ignore_mce_banks) + hardware_error_block_lines = hardware_error_block_line_indices(dmesg_content) known_err_events = self.check_all_regexes( content=dmesg_content, @@ -722,6 +729,7 @@ def analyze_data( num_timestamps=args.num_timestamps, interval_to_collapse_event=args.interval_to_collapse_event, ignore_match_rules=ignore_match_rules, + skip_line_indices=ignored_mce_block_lines, ) if args.exclude_category: known_err_events = [ @@ -752,6 +760,7 @@ def analyze_data( num_timestamps=args.num_timestamps, interval_to_collapse_event=args.interval_to_collapse_event, ignore_match_rules=ignore_match_rules, + skip_line_indices=hardware_error_block_lines, ) for err_event in err_events: diff --git a/nodescraper/plugins/inband/dmesg/mce_utils.py b/nodescraper/plugins/inband/dmesg/mce_utils.py index 9d2b0692..6f515fef 100644 --- a/nodescraper/plugins/inband/dmesg/mce_utils.py +++ b/nodescraper/plugins/inband/dmesg/mce_utils.py @@ -24,9 +24,16 @@ # ############################################################################### import re -from typing import FrozenSet, Optional +from typing import FrozenSet, Optional, Sequence -from nodescraper.base.match_ignore import extract_mce_bank_from_line +from nodescraper.base.match_ignore import extract_mce_banks_from_text + +_MCE_PRIMARY_START_RE = re.compile( + r"\[Hardware Error\]:\s*(?:Corrected error|Uncorrected error|Machine check events logged)", + re.IGNORECASE, +) +_MCE_STATUS_START_RE = re.compile(r"\bMC\d+_STATUS\[", re.IGNORECASE) +_MCE_DETAIL_LINE_RE = re.compile(r"\[Hardware Error\]:") _CORRECTABLE_SUMMARY_RE = re.compile( r"(?P\d+)\s+correctable hardware errors detected in total in (?P\w+) block" @@ -97,6 +104,76 @@ def _gpu_index_for_bdf(bdf: str, bdf_order: list[str]) -> int: return bdf_order.index(bdf) +def _is_mce_primary_starter(line: str) -> bool: + return _MCE_PRIMARY_START_RE.search(line) is not None + + +def _is_mce_block_starter(line: str, *, in_block: bool) -> bool: + """Return True when line opens a new MCE incident block.""" + if _is_mce_primary_starter(line): + return True + if not in_block and _MCE_STATUS_START_RE.search(line) is not None: + return True + return False + + +def _is_mce_detail_line(line: str) -> bool: + return _MCE_DETAIL_LINE_RE.search(line) is not None + + +def _mce_detail_line_indices_in_range(lines: Sequence[str], start: int, end: int) -> set[int]: + return {index for index in range(start, end) if _is_mce_detail_line(lines[index])} + + +def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, int]]: + """Return (start, end) line index ranges for MCE incident blocks. + + A block begins at a primary MCE header (Corrected/Uncorrected/Machine check logged) + or at the first MCn_STATUS line when not already inside a block. The block then + includes subsequent lines until the next primary header or EOF. Blank lines, warn/err + noise, and other non-MCE dmesg lines between detail entries belong to the same block. + """ + blocks: list[tuple[int, int]] = [] + index = 0 + total = len(lines) + while index < total: + if not _is_mce_block_starter(lines[index], in_block=False): + index += 1 + continue + start = index + index += 1 + while index < total: + if _is_mce_block_starter(lines[index], in_block=True): + break + index += 1 + blocks.append((start, index)) + return blocks + + +def hardware_error_block_line_indices(content: str) -> frozenset[int]: + """Return [Hardware Error]: line indices that belong to an MCE incident block.""" + lines = content.splitlines() + suppressed: set[int] = set() + for start, end in iter_hardware_error_block_ranges(lines): + suppressed.update(_mce_detail_line_indices_in_range(lines, start, end)) + return frozenset(suppressed) + + +def ignored_mce_block_line_indices(content: str, ignore_banks: FrozenSet[int]) -> frozenset[int]: + """Return [Hardware Error]: line indices for MCE blocks containing any ignored MCA bank.""" + if not ignore_banks: + return frozenset() + lines = content.splitlines() + suppressed: set[int] = set() + for start, end in iter_hardware_error_block_ranges(lines): + block_banks: set[int] = set() + for line in lines[start:end]: + block_banks.update(extract_mce_banks_from_text(line)) + if block_banks & ignore_banks: + suppressed.update(_mce_detail_line_indices_in_range(lines, start, end)) + return frozenset(suppressed) + + def parse_correctable_mce_counts( content: str, ignore_banks: Optional[FrozenSet[int]] = None, @@ -109,8 +186,11 @@ def parse_correctable_mce_counts( counts: dict[str, int] = {} gpu_bdf_order: list[str] = [] ignored = ignore_banks or frozenset() + ignored_block_lines = ignored_mce_block_line_indices(content, ignored) - for line in content.splitlines(): + for line_no, line in enumerate(content.splitlines()): + if line_no in ignored_block_lines: + continue gpu_match = _GPU_CORRECTABLE_RE.search(line) if gpu_match: bdf = gpu_match.group("bdf") @@ -134,9 +214,6 @@ def parse_correctable_mce_counts( status_match = _MCE_CE_STATUS_RE.search(line) if status_match: - bank = extract_mce_bank_from_line(line) - if bank is not None and bank in ignored: - continue part = ( _normalize_cpu_label(status_match.group("cpu")) if status_match.group("cpu") @@ -155,8 +232,11 @@ def parse_uncorrectable_mce_counts( counts: dict[str, int] = {} gpu_bdf_order: list[str] = [] ignored = ignore_banks or frozenset() + ignored_block_lines = ignored_mce_block_line_indices(content, ignored) - for line in content.splitlines(): + for line_no, line in enumerate(content.splitlines()): + if line_no in ignored_block_lines: + continue gpu_match = _GPU_UNCORRECTABLE_RE.search(line) if gpu_match: bdf = gpu_match.group("bdf") @@ -176,9 +256,6 @@ def parse_uncorrectable_mce_counts( status_match = _MCE_UC_STATUS_RE.search(line) if status_match: - bank = extract_mce_bank_from_line(line) - if bank is not None and bank in ignored: - continue part = ( _normalize_cpu_label(status_match.group("cpu")) if status_match.group("cpu") diff --git a/test/unit/plugin/test_dmesg_analyzer.py b/test/unit/plugin/test_dmesg_analyzer.py index d4697f9e..0de2ae21 100644 --- a/test/unit/plugin/test_dmesg_analyzer.py +++ b/test/unit/plugin/test_dmesg_analyzer.py @@ -1126,10 +1126,15 @@ def test_ignore_match_rules_skips_matching_lines(system_info): def test_ignore_match_rules_mce_banks_and_threshold(system_info): dmesg_content = ( + "kern :err : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" "kern :err : 2038-01-19T00:00:00,000000+00:00 " "[Hardware Error]: Machine Check: CPU0 MC1_STATUS[0xcafe|CE|Misc]: 0x0\n" "kern :err : 2038-01-19T00:00:01,000000+00:00 " "[Hardware Error]: Machine Check: CPU0 MC2_STATUS[0xfeed|CE|Misc]: 0x0\n" + "\n" + "kern :err : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" "kern :err : 2038-01-19T00:00:02,000000+00:00 " "[Hardware Error]: Machine Check: CPU0 MC5_STATUS[0xbeef|CE|Misc]: 0x0\n" ) @@ -1160,6 +1165,8 @@ def test_ignore_match_rules_mce_bank_range(system_info): "kern :err : 2038-01-19T00:00:01,000000+00:00 " "[Hardware Error]: Machine Check: CPU0 MC7_STATUS[0x2|CE|Misc]: 0x0\n" "kern :err : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :err : 2038-01-19T00:00:02,000000+00:00 " "[Hardware Error]: Machine Check: CPU0 MC9_STATUS[0x3|CE|Misc]: 0x0\n" ) @@ -1210,3 +1217,112 @@ def test_ignore_match_rules_scoped_by_message(system_info): by_desc = {event.description: event for event in res.events} assert "Dummy Error Alpha" not in by_desc assert "Dummy Error Beta" in by_desc + + +def test_hardware_error_block_not_reported_as_unknown(system_info): + dmesg_content = ( + "kern :info : 2038-01-19T00:00:00,000000+00:00 " + "mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 " + "amdgpu 0000:de:ad.0: amdgpu: dummy notice inside block\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" + "kern :emerg : 2038-01-19T00:00:06,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :err : 2038-01-19T00:00:10,000000+00:00 unrelated plugin failure\n" + ) + + analyzer = DmesgAnalyzer(system_info=system_info) + res = analyzer.analyze_data( + DmesgData(dmesg_content=dmesg_content), + args=DmesgAnalyzerArgs( + check_unknown_dmesg_errors=True, + ignore_match_rules=[{"mce_banks": ["60-63"]}], + ), + ) + + descriptions = {event.description for event in res.events} + assert "MCE Corrected Error" not in descriptions + assert "RAS Corrected Error" not in descriptions + unknown_events = [event for event in res.events if event.description == "Unknown dmesg error"] + assert len(unknown_events) == 1 + assert unknown_events[0].data["match_content"] == "unrelated plugin failure" + + +def test_hardware_error_block_reports_mce_without_ignore(system_info): + dmesg_content = ( + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + ) + + analyzer = DmesgAnalyzer(system_info=system_info) + res = analyzer.analyze_data( + DmesgData(dmesg_content=dmesg_content), + args=DmesgAnalyzerArgs(check_unknown_dmesg_errors=True), + ) + + descriptions = {event.description for event in res.events} + assert "Unknown dmesg error" not in descriptions + assert "MCE Corrected Error" in descriptions or "RAS Corrected Error" in descriptions + + +def test_mce_interleave_pattern_suppresses_block_with_ignored_banks(system_info): + """Dummy excerpt modeled on dmesg_mce_interleave.log: warn/blank lines inside MCE blocks.""" + dmesg_content = ( + "kern :info : 2038-01-19T00:00:00,000000+00:00 " + "mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :warn : 2038-01-19T00:00:03,000000+00:00 " + "workqueue: dummy_mce_worker hogged CPU for >10000us 5 times, consider switching to WQ_UNBOUND\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" + "\n" + "kern :emerg : 2038-01-19T00:00:06,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :info : 2038-01-19T00:00:07,000000+00:00 " + "mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2038-01-19T00:00:08,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:09,000000+00:00 " + "[Hardware Error]: CPU:24 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xbbb\n" + "kern :emerg : 2038-01-19T00:00:10,000000+00:00 [Hardware Error]: PPIN: 0xcccccccccccccccc\n" + "kern :err : 2038-01-19T00:00:11,000000+00:00 dummy harness fault outside mce blocks\n" + ) + + analyzer = DmesgAnalyzer(system_info=system_info) + res = analyzer.analyze_data( + DmesgData(dmesg_content=dmesg_content), + args=DmesgAnalyzerArgs( + check_unknown_dmesg_errors=True, + mce_threshold=1, + ignore_match_rules=[{"mce_banks": ["60-63"]}], + error_regex=[ + { + "regex": r"dummy_mce_worker hogged CPU", + "message": "Dummy Workqueue Hog", + "event_category": "SW_DRIVER", + } + ], + ), + ) + + descriptions = {event.description for event in res.events} + assert "MCE Corrected Error" not in descriptions + assert "RAS Corrected Error" not in descriptions + assert not any("mce_threshold" in event.data for event in res.events) + assert "Dummy Workqueue Hog" in descriptions + unknown_events = [event for event in res.events if event.description == "Unknown dmesg error"] + assert len(unknown_events) == 1 + assert unknown_events[0].data["match_content"] == "dummy harness fault outside mce blocks" diff --git a/test/unit/plugin/test_mce_utils.py b/test/unit/plugin/test_mce_utils.py index c8fd20f2..cb9fb538 100644 --- a/test/unit/plugin/test_mce_utils.py +++ b/test/unit/plugin/test_mce_utils.py @@ -24,6 +24,9 @@ # ############################################################################### from nodescraper.plugins.inband.dmesg.mce_utils import ( + hardware_error_block_line_indices, + ignored_mce_block_line_indices, + iter_hardware_error_block_ranges, parse_correctable_mce_counts, parse_uncorrectable_mce_counts, ) @@ -31,9 +34,9 @@ def test_parse_correctable_mce_counts_cpu_summary_and_status(): content = ( - "kern :warn : 2024-06-11T14:30:00,123456+00:00 " + "kern :warn : 2038-01-19T00:00:00,000000+00:00 " "mce: 3 correctable hardware errors detected in total in mc0 block on CPU1\n" - "kern :warn : 2024-06-11T14:30:02,222222+00:00 " + "kern :warn : 2038-01-19T00:00:01,000000+00:00 " "[Hardware Error]: CPU0 MC2_STATUS[0x0|CE|]: 0xabc\n" ) @@ -44,8 +47,8 @@ def test_parse_correctable_mce_counts_cpu_summary_and_status(): def test_parse_correctable_mce_counts_gpu_summary(): content = ( - "kern :err : 2024-10-07T10:17:15,145363-04:00 " - "amdgpu 0000:c1:00.0: amdgpu: socket: 4, die: 0 " + "kern :err : 2038-01-19T00:00:00,000000+00:00 " + "amdgpu 0000:de:ad.0: amdgpu: socket: 4, die: 0 " "3 correctable hardware errors detected in total in gfx block\n" ) @@ -75,9 +78,119 @@ def test_parse_correctable_mce_counts_skips_ignored_banks(): counts = parse_correctable_mce_counts(content, ignore_banks=frozenset({1, 2})) + assert counts == {} + + +def test_parse_correctable_mce_counts_skips_ignored_banks_in_separate_block(): + content = ( + "[Hardware Error]: CPU0 MC1_STATUS[0x0|CE|]: 0x1\n" + "[Hardware Error]: CPU0 MC2_STATUS[0x0|CE|]: 0x2\n" + "\n" + "[Hardware Error]: CPU0 MC5_STATUS[0x0|CE|]: 0x3\n" + ) + + counts = parse_correctable_mce_counts(content, ignore_banks=frozenset({1, 2})) + assert counts == {"CPU0": 1} +def test_hardware_error_block_line_indices(): + content = ( + "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" + "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" + "kern :info : ts unrelated line\n" + ) + + assert hardware_error_block_line_indices(content) == frozenset({0, 1, 2}) + + +def test_hardware_error_block_splits_info_preamble_from_emerg_details(): + content = ( + "kern :info : 2038-01-19T00:00:00,000000+00:00 " + "mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" + "kern :emerg : 2038-01-19T00:00:06,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + ) + lines = content.splitlines() + + assert iter_hardware_error_block_ranges(lines) == [(0, 1), (1, 6)] + assert hardware_error_block_line_indices(content) == frozenset({0, 1, 2, 3, 4, 5}) + assert ignored_mce_block_line_indices(content, frozenset({60})) == frozenset({1, 2, 3, 4, 5}) + + +def test_hardware_error_block_includes_interleaved_non_mce_lines(): + content = ( + "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" + "kern :emerg : ts amdgpu 0000:de:ad.0: amdgpu: unrelated reset notice\n" + "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" + "kern :emerg : ts [Hardware Error]: cache level: L3/GEN\n" + ) + + assert hardware_error_block_line_indices(content) == frozenset({0, 1, 3, 4}) + assert ignored_mce_block_line_indices(content, frozenset({60})) == frozenset({0, 1, 3, 4}) + + +def test_ignored_mce_block_line_indices(): + content = ( + "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" + "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" + "\n" + "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" + "kern :err : ts [Hardware Error]: CPU0 MC5_STATUS[0x0|CE|]: 0x3\n" + ) + + ignored = ignored_mce_block_line_indices(content, frozenset({60})) + + assert ignored == frozenset({0, 1, 2}) + assert iter_hardware_error_block_ranges(content.splitlines()) == [(0, 4), (4, 6)] + + +def test_mce_block_includes_blank_line_and_warn_interleave(): + """Dummy excerpt: warn/workqueue noise and blank lines inside one MCE block.""" + content = ( + "kern :info : 2038-01-19T00:00:00,000000+00:00 " + "mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :warn : 2038-01-19T00:00:03,000000+00:00 " + "workqueue: dummy_mce_worker hogged CPU for >10000us 5 times, consider switching to WQ_UNBOUND\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" + "\n" + "kern :emerg : 2038-01-19T00:00:06,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :info : 2038-01-19T00:00:07,000000+00:00 " + "mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2038-01-19T00:00:08,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:09,000000+00:00 " + "[Hardware Error]: CPU:24 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xbbb\n" + "kern :emerg : 2038-01-19T00:00:10,000000+00:00 [Hardware Error]: PPIN: 0xcccccccccccccccc\n" + ) + lines = content.splitlines() + + assert iter_hardware_error_block_ranges(lines) == [(0, 1), (1, 9), (9, 10), (10, 13)] + assert hardware_error_block_line_indices(content) == frozenset({0, 1, 2, 4, 5, 7, 8, 9, 10, 11}) + assert ignored_mce_block_line_indices(content, frozenset({60})) == frozenset( + {1, 2, 4, 5, 7, 10, 11} + ) + assert 3 not in ignored_mce_block_line_indices(content, frozenset({60})) + assert 6 not in ignored_mce_block_line_indices(content, frozenset({60})) + + def test_parse_correctable_mce_counts_cpu_colon_status(): content = ( "kern :err : 2038-01-19T00:00:00,000000+00:00 " @@ -94,7 +207,7 @@ def test_parse_correctable_mce_counts_both_cpu_formats(): "[Hardware Error]: CPU0 MC1_STATUS[0x0|CE|]: 0x1\n" "kern :err : 2038-01-19T00:00:00,000000+00:00 " "[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xabc\n" - "kern :warn : 2024-06-11T14:30:00,123456+00:00 " + "kern :warn : 2038-01-19T00:00:02,000000+00:00 " "mce: 2 correctable hardware errors detected in total in mc0 block on CPU:1\n" ) From ccf8d2be21315af7ccdef9ec40fb62939713984c Mon Sep 17 00:00:00 2001 From: Alexandra Bara Date: Wed, 8 Jul 2026 13:47:57 -0500 Subject: [PATCH 9/9] fix for mce block jitter --- nodescraper/base/regexanalyzer.py | 23 ++++++-- .../plugins/inband/dmesg/dmesg_analyzer.py | 6 +-- nodescraper/plugins/inband/dmesg/mce_utils.py | 53 +++++++++++++++++-- test/unit/plugin/test_mce_utils.py | 4 +- 4 files changed, 75 insertions(+), 11 deletions(-) diff --git a/nodescraper/base/regexanalyzer.py b/nodescraper/base/regexanalyzer.py index 1671db7d..1d7adf61 100644 --- a/nodescraper/base/regexanalyzer.py +++ b/nodescraper/base/regexanalyzer.py @@ -156,6 +156,16 @@ def _match_intersects_skip_lines( end_line = self._line_index_at_position(content, max(match_start, match_end - 1)) return any(line_idx in skip_line_indices for line_idx in range(start_line, end_line + 1)) + def _match_on_skipped_line( + self, + content: str, + match_start: int, + skip_line_indices: frozenset[int], + ) -> bool: + if not skip_line_indices: + return False + return self._line_index_at_position(content, match_start) in skip_line_indices + def _should_ignore_regex_match( self, content: str, @@ -319,14 +329,18 @@ def _is_within_interval(new_timestamp_str: str, existing_timestamps: list[str]) suppressed_lines = skip_line_indices or frozenset() for error_regex_obj in error_regex: - for match_obj in error_regex_obj.regex.finditer(content): + search_from = 0 + while search_from <= len(content): + match_obj = error_regex_obj.regex.search(content, search_from) + if match_obj is None: + break raw_match = match_obj.group(0) - if self._match_intersects_skip_lines( + if self._match_on_skipped_line( content, match_obj.start(), - match_obj.end(), suppressed_lines, ): + search_from = match_obj.start() + 1 continue if self._should_ignore_regex_match( content, @@ -335,6 +349,7 @@ def _is_within_interval(new_timestamp_str: str, existing_timestamps: list[str]) error_regex_obj.message, skip_rules, ): + search_from = match_obj.start() + 1 continue # Extract timestamp from the line where match occurs @@ -388,6 +403,8 @@ def _is_within_interval(new_timestamp_str: str, existing_timestamps: list[str]) regex_event_list.append(new_event) + search_from = match_obj.end() + all_events = list(regex_map.values()) if group else regex_event_list # Prune timestamp lists to keep only first N and last N timestamps diff --git a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py index 58a0a2ce..9c7b11b2 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py @@ -36,8 +36,8 @@ from .analyzer_args import DmesgAnalyzerArgs from .dmesgdata import DmesgData from .mce_utils import ( - hardware_error_block_line_indices, ignored_mce_block_line_indices, + mce_block_all_line_indices, parse_correctable_mce_counts, parse_uncorrectable_mce_counts, ) @@ -720,7 +720,7 @@ def analyze_data( ignore_match_rules, ignore_mce_banks = parse_ignore_match_rules(args.ignore_match_rules) ignored_mce_block_lines = ignored_mce_block_line_indices(dmesg_content, ignore_mce_banks) - hardware_error_block_lines = hardware_error_block_line_indices(dmesg_content) + mce_block_lines = mce_block_all_line_indices(dmesg_content) known_err_events = self.check_all_regexes( content=dmesg_content, @@ -760,7 +760,7 @@ def analyze_data( num_timestamps=args.num_timestamps, interval_to_collapse_event=args.interval_to_collapse_event, ignore_match_rules=ignore_match_rules, - skip_line_indices=hardware_error_block_lines, + skip_line_indices=mce_block_lines, ) for err_event in err_events: diff --git a/nodescraper/plugins/inband/dmesg/mce_utils.py b/nodescraper/plugins/inband/dmesg/mce_utils.py index 6f515fef..b08e3a83 100644 --- a/nodescraper/plugins/inband/dmesg/mce_utils.py +++ b/nodescraper/plugins/inband/dmesg/mce_utils.py @@ -125,13 +125,43 @@ def _mce_detail_line_indices_in_range(lines: Sequence[str], start: int, end: int return {index for index in range(start, end) if _is_mce_detail_line(lines[index])} +def _next_non_blank_line_index(lines: Sequence[str], index: int) -> Optional[int]: + """Return the next non-blank line index after index, or None when none remain.""" + for candidate in range(index + 1, len(lines)): + if lines[candidate].strip(): + return candidate + return None + + +def _is_mce_status_only_starter(line: str) -> bool: + """Return True when line opens a block via MCn_STATUS without a primary header.""" + return not _is_mce_primary_starter(line) and _MCE_STATUS_START_RE.search(line) is not None + + +def _has_mce_detail_line_ahead(lines: Sequence[str], start_index: int) -> bool: + """Return True when another [Hardware Error]: line appears before the next incident.""" + for idx in range(start_index + 1, len(lines)): + line = lines[idx] + if not line.strip(): + continue + if _is_mce_primary_starter(line): + return False + if _is_mce_status_only_starter(line): + return False + if _is_mce_detail_line(line): + return True + return False + + def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, int]]: """Return (start, end) line index ranges for MCE incident blocks. A block begins at a primary MCE header (Corrected/Uncorrected/Machine check logged) or at the first MCn_STATUS line when not already inside a block. The block then - includes subsequent lines until the next primary header or EOF. Blank lines, warn/err - noise, and other non-MCE dmesg lines between detail entries belong to the same block. + includes subsequent lines until the next primary header, a blank line before a bare + MCn_STATUS starter, trailing non-MCE lines with no further [Hardware Error]: detail, + or EOF. Blank lines, warn/err noise, and other non-MCE dmesg lines between detail + entries belong to the same block. """ blocks: list[tuple[int, int]] = [] index = 0 @@ -143,7 +173,15 @@ def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, in start = index index += 1 while index < total: - if _is_mce_block_starter(lines[index], in_block=True): + if not lines[index].strip(): + next_line = _next_non_blank_line_index(lines, index) + if next_line is not None and _is_mce_status_only_starter(lines[next_line]): + break + elif _is_mce_block_starter(lines[index], in_block=True): + break + elif not _is_mce_detail_line(lines[index]) and not _has_mce_detail_line_ahead( + lines, index + ): break index += 1 blocks.append((start, index)) @@ -159,6 +197,15 @@ def hardware_error_block_line_indices(content: str) -> frozenset[int]: return frozenset(suppressed) +def mce_block_all_line_indices(content: str) -> frozenset[int]: + """Return every line index that belongs to an MCE incident block.""" + lines = content.splitlines() + suppressed: set[int] = set() + for start, end in iter_hardware_error_block_ranges(lines): + suppressed.update(range(start, end)) + return frozenset(suppressed) + + def ignored_mce_block_line_indices(content: str, ignore_banks: FrozenSet[int]) -> frozenset[int]: """Return [Hardware Error]: line indices for MCE blocks containing any ignored MCA bank.""" if not ignore_banks: diff --git a/test/unit/plugin/test_mce_utils.py b/test/unit/plugin/test_mce_utils.py index cb9fb538..53a58f00 100644 --- a/test/unit/plugin/test_mce_utils.py +++ b/test/unit/plugin/test_mce_utils.py @@ -182,10 +182,10 @@ def test_mce_block_includes_blank_line_and_warn_interleave(): ) lines = content.splitlines() - assert iter_hardware_error_block_ranges(lines) == [(0, 1), (1, 9), (9, 10), (10, 13)] + assert iter_hardware_error_block_ranges(lines) == [(0, 1), (1, 8), (8, 9), (9, 12)] assert hardware_error_block_line_indices(content) == frozenset({0, 1, 2, 4, 5, 7, 8, 9, 10, 11}) assert ignored_mce_block_line_indices(content, frozenset({60})) == frozenset( - {1, 2, 4, 5, 7, 10, 11} + {1, 2, 4, 5, 7, 9, 10, 11} ) assert 3 not in ignored_mce_block_line_indices(content, frozenset({60})) assert 6 not in ignored_mce_block_line_indices(content, frozenset({60}))