DmesgPlugin update: MCE block ignore

nodescraper/base/regexanalyzer.py

@@ -138,6 +138,34 @@ def _line_at_match_position(self, content: str, match_start: int) -> str:
             line_end = len(content)
         return content[line_start:line_end]
 
+    def _line_index_at_position(self, content: str, position: int) -> int:
+        if position <= 0:
+            return 0
+        return content.count("\n", 0, position)
+
+    def _match_intersects_skip_lines(
+        self,
+        content: str,
+        match_start: int,
+        match_end: int,
+        skip_line_indices: frozenset[int],
+    ) -> bool:
+        if not skip_line_indices:
+            return False
+        start_line = self._line_index_at_position(content, match_start)
+        end_line = self._line_index_at_position(content, max(match_start, match_end - 1))
+        return any(line_idx in skip_line_indices for line_idx in range(start_line, end_line + 1))
+
+    def _match_on_skipped_line(
+        self,
+        content: str,
+        match_start: int,
+        skip_line_indices: frozenset[int],
+    ) -> bool:
+        if not skip_line_indices:
+            return False
+        return self._line_index_at_position(content, match_start) in skip_line_indices
+
     def _should_ignore_regex_match(
         self,
         content: str,
@@ -245,6 +273,7 @@ def check_all_regexes(
         num_timestamps: int = 3,
         interval_to_collapse_event: int = 60,
         ignore_match_rules: Optional[Sequence[ParsedIgnoreMatchRule]] = None,
+        skip_line_indices: Optional[frozenset[int]] = None,
     ) -> list[RegexEvent]:
         """Iterate over all ERROR_REGEX and check content for any matches
 
@@ -262,6 +291,7 @@ def check_all_regexes(
             num_timestamps (int, optional): maximum number of timestamps to keep for each event. Defaults to 3.
             interval_to_collapse_event (int, optional): time interval in seconds to collapse events. Defaults to 60.
             ignore_match_rules (Optional[Sequence[ParsedIgnoreMatchRule]], optional): Parsed skip rules. Defaults to None.
+            skip_line_indices (Optional[frozenset[int]], optional): Line indices to skip entirely. Defaults to None.
 
         Returns:
             list[RegexEvent]: list of regex event objects
@@ -296,17 +326,30 @@ def _is_within_interval(new_timestamp_str: str, existing_timestamps: list[str])
             return False
 
         skip_rules = list(ignore_match_rules) if ignore_match_rules else []
+        suppressed_lines = skip_line_indices or frozenset()
 
         for error_regex_obj in error_regex:
-            for match_obj in error_regex_obj.regex.finditer(content):
+            search_from = 0
+            while search_from <= len(content):
+                match_obj = error_regex_obj.regex.search(content, search_from)
+                if match_obj is None:
+                    break
                 raw_match = match_obj.group(0)
+                if self._match_on_skipped_line(
+                    content,
+                    match_obj.start(),
+                    suppressed_lines,
+                ):
+                    search_from = match_obj.start() + 1
+                    continue
                 if self._should_ignore_regex_match(
                     content,
                     match_obj.start(),
                     raw_match,
                     error_regex_obj.message,
                     skip_rules,
                 ):
+                    search_from = match_obj.start() + 1
                     continue
 
                 # Extract timestamp from the line where match occurs
@@ -360,6 +403,8 @@ def _is_within_interval(new_timestamp_str: str, existing_timestamps: list[str])
 
                     regex_event_list.append(new_event)
 
+                search_from = match_obj.end()
+
         all_events = list(regex_map.values()) if group else regex_event_list
 
         # Prune timestamp lists to keep only first N and last N timestamps

nodescraper/plugins/inband/dmesg/analyzer_args.py

@@ -76,6 +76,8 @@ class DmesgAnalyzerArgs(TimeRangeAnalysisArgs):
             "Rules that skip regex matches during analysis. Each rule may use line_regex, "
             "match_regex, message, and/or mce_banks. Within a rule all specified fields must "
             "match; any matching rule suppresses the hit. mce_banks accepts bank ids and "
-            'inclusive ranges such as "60-63".'
+            'inclusive ranges such as "60-63". When a rule matches an MCA bank, only '
+            "[Hardware Error]: lines in that incident block are suppressed; interleaved "
+            "non-MCE dmesg lines in the same block are not."
         ),
     )

nodescraper/plugins/inband/dmesg/dmesg_analyzer.py

@@ -35,7 +35,12 @@
 
 from .analyzer_args import DmesgAnalyzerArgs
 from .dmesgdata import DmesgData
-from .mce_utils import parse_correctable_mce_counts, parse_uncorrectable_mce_counts
+from .mce_utils import (
+    ignored_mce_block_line_indices,
+    mce_block_all_line_indices,
+    parse_correctable_mce_counts,
+    parse_uncorrectable_mce_counts,
+)
 
 
 class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]):
@@ -714,6 +719,8 @@ def analyze_data(
             dmesg_content = data.dmesg_content
 
         ignore_match_rules, ignore_mce_banks = parse_ignore_match_rules(args.ignore_match_rules)
+        ignored_mce_block_lines = ignored_mce_block_line_indices(dmesg_content, ignore_mce_banks)
+        mce_block_lines = mce_block_all_line_indices(dmesg_content)
 
         known_err_events = self.check_all_regexes(
             content=dmesg_content,
@@ -722,6 +729,7 @@ def analyze_data(
             num_timestamps=args.num_timestamps,
             interval_to_collapse_event=args.interval_to_collapse_event,
             ignore_match_rules=ignore_match_rules,
+            skip_line_indices=ignored_mce_block_lines,
         )
         if args.exclude_category:
             known_err_events = [
@@ -752,6 +760,7 @@ def analyze_data(
                 num_timestamps=args.num_timestamps,
                 interval_to_collapse_event=args.interval_to_collapse_event,
                 ignore_match_rules=ignore_match_rules,
+                skip_line_indices=mce_block_lines,
             )
 
             for err_event in err_events:

nodescraper/plugins/inband/dmesg/mce_utils.py

@@ -24,9 +24,16 @@
 #
 ###############################################################################
 import re
-from typing import FrozenSet, Optional
+from typing import FrozenSet, Optional, Sequence
 
-from nodescraper.base.match_ignore import extract_mce_bank_from_line
+from nodescraper.base.match_ignore import extract_mce_banks_from_text
+
+_MCE_PRIMARY_START_RE = re.compile(
+    r"\[Hardware Error\]:\s*(?:Corrected error|Uncorrected error|Machine check events logged)",
+    re.IGNORECASE,
+)
+_MCE_STATUS_START_RE = re.compile(r"\bMC\d+_STATUS\[", re.IGNORECASE)
+_MCE_DETAIL_LINE_RE = re.compile(r"\[Hardware Error\]:")
 
 _CORRECTABLE_SUMMARY_RE = re.compile(
     r"(?P<count>\d+)\s+correctable hardware errors detected in total in (?P<block>\w+) block"
@@ -97,6 +104,123 @@ def _gpu_index_for_bdf(bdf: str, bdf_order: list[str]) -> int:
     return bdf_order.index(bdf)
 
 
+def _is_mce_primary_starter(line: str) -> bool:
+    return _MCE_PRIMARY_START_RE.search(line) is not None
+
+
+def _is_mce_block_starter(line: str, *, in_block: bool) -> bool:
+    """Return True when line opens a new MCE incident block."""
+    if _is_mce_primary_starter(line):
+        return True
+    if not in_block and _MCE_STATUS_START_RE.search(line) is not None:
+        return True
+    return False
+
+
+def _is_mce_detail_line(line: str) -> bool:
+    return _MCE_DETAIL_LINE_RE.search(line) is not None
+
+
+def _mce_detail_line_indices_in_range(lines: Sequence[str], start: int, end: int) -> set[int]:
+    return {index for index in range(start, end) if _is_mce_detail_line(lines[index])}
+
+
+def _next_non_blank_line_index(lines: Sequence[str], index: int) -> Optional[int]:
+    """Return the next non-blank line index after index, or None when none remain."""
+    for candidate in range(index + 1, len(lines)):
+        if lines[candidate].strip():
+            return candidate
+    return None
+
+
+def _is_mce_status_only_starter(line: str) -> bool:
+    """Return True when line opens a block via MCn_STATUS without a primary header."""
+    return not _is_mce_primary_starter(line) and _MCE_STATUS_START_RE.search(line) is not None
+
+
+def _has_mce_detail_line_ahead(lines: Sequence[str], start_index: int) -> bool:
+    """Return True when another [Hardware Error]: line appears before the next incident."""
+    for idx in range(start_index + 1, len(lines)):
+        line = lines[idx]
+        if not line.strip():
+            continue
+        if _is_mce_primary_starter(line):
+            return False
+        if _is_mce_status_only_starter(line):
+            return False
+        if _is_mce_detail_line(line):
+            return True
+    return False
+
+
+def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, int]]:
+    """Return (start, end) line index ranges for MCE incident blocks.
+
+    A block begins at a primary MCE header (Corrected/Uncorrected/Machine check logged)
+    or at the first MCn_STATUS line when not already inside a block. The block then
+    includes subsequent lines until the next primary header, a blank line before a bare
+    MCn_STATUS starter, trailing non-MCE lines with no further [Hardware Error]: detail,
+    or EOF. Blank lines, warn/err noise, and other non-MCE dmesg lines between detail
+    entries belong to the same block.
+    """
+    blocks: list[tuple[int, int]] = []
+    index = 0
+    total = len(lines)
+    while index < total:
+        if not _is_mce_block_starter(lines[index], in_block=False):
+            index += 1
+            continue
+        start = index
+        index += 1
+        while index < total:
+            if not lines[index].strip():
+                next_line = _next_non_blank_line_index(lines, index)
+                if next_line is not None and _is_mce_status_only_starter(lines[next_line]):
+                    break
+            elif _is_mce_block_starter(lines[index], in_block=True):
+                break
+            elif not _is_mce_detail_line(lines[index]) and not _has_mce_detail_line_ahead(
+                lines, index
+            ):
+                break
+            index += 1
+        blocks.append((start, index))
+    return blocks
+
+
+def hardware_error_block_line_indices(content: str) -> frozenset[int]:
+    """Return [Hardware Error]: line indices that belong to an MCE incident block."""
+    lines = content.splitlines()
+    suppressed: set[int] = set()
+    for start, end in iter_hardware_error_block_ranges(lines):
+        suppressed.update(_mce_detail_line_indices_in_range(lines, start, end))
+    return frozenset(suppressed)
+
+
+def mce_block_all_line_indices(content: str) -> frozenset[int]:
+    """Return every line index that belongs to an MCE incident block."""
+    lines = content.splitlines()
+    suppressed: set[int] = set()
+    for start, end in iter_hardware_error_block_ranges(lines):
+        suppressed.update(range(start, end))
+    return frozenset(suppressed)
+
+
+def ignored_mce_block_line_indices(content: str, ignore_banks: FrozenSet[int]) -> frozenset[int]:
+    """Return [Hardware Error]: line indices for MCE blocks containing any ignored MCA bank."""
+    if not ignore_banks:
+        return frozenset()
+    lines = content.splitlines()
+    suppressed: set[int] = set()
+    for start, end in iter_hardware_error_block_ranges(lines):
+        block_banks: set[int] = set()
+        for line in lines[start:end]:
+            block_banks.update(extract_mce_banks_from_text(line))
+        if block_banks & ignore_banks:
+            suppressed.update(_mce_detail_line_indices_in_range(lines, start, end))
+    return frozenset(suppressed)
+
+
 def parse_correctable_mce_counts(
     content: str,
     ignore_banks: Optional[FrozenSet[int]] = None,
@@ -109,8 +233,11 @@ def parse_correctable_mce_counts(
     counts: dict[str, int] = {}
     gpu_bdf_order: list[str] = []
     ignored = ignore_banks or frozenset()
+    ignored_block_lines = ignored_mce_block_line_indices(content, ignored)
 
-    for line in content.splitlines():
+    for line_no, line in enumerate(content.splitlines()):
+        if line_no in ignored_block_lines:
+            continue
         gpu_match = _GPU_CORRECTABLE_RE.search(line)
         if gpu_match:
             bdf = gpu_match.group("bdf")
@@ -134,9 +261,6 @@ def parse_correctable_mce_counts(
 
         status_match = _MCE_CE_STATUS_RE.search(line)
         if status_match:
-            bank = extract_mce_bank_from_line(line)
-            if bank is not None and bank in ignored:
-                continue
             part = (
                 _normalize_cpu_label(status_match.group("cpu"))
                 if status_match.group("cpu")
@@ -155,8 +279,11 @@ def parse_uncorrectable_mce_counts(
     counts: dict[str, int] = {}
     gpu_bdf_order: list[str] = []
     ignored = ignore_banks or frozenset()
+    ignored_block_lines = ignored_mce_block_line_indices(content, ignored)
 
-    for line in content.splitlines():
+    for line_no, line in enumerate(content.splitlines()):
+        if line_no in ignored_block_lines:
+            continue
         gpu_match = _GPU_UNCORRECTABLE_RE.search(line)
         if gpu_match:
             bdf = gpu_match.group("bdf")
@@ -176,9 +303,6 @@ def parse_uncorrectable_mce_counts(
 
         status_match = _MCE_UC_STATUS_RE.search(line)
         if status_match:
-            bank = extract_mce_bank_from_line(line)
-            if bank is not None and bank in ignored:
-                continue
             part = (
                 _normalize_cpu_label(status_match.group("cpu"))
                 if status_match.group("cpu")