diff --git a/nodescraper/base/regexanalyzer.py b/nodescraper/base/regexanalyzer.py index 1fab952d..1d7adf61 100644 --- a/nodescraper/base/regexanalyzer.py +++ b/nodescraper/base/regexanalyzer.py @@ -138,6 +138,34 @@ def _line_at_match_position(self, content: str, match_start: int) -> str: line_end = len(content) return content[line_start:line_end] + def _line_index_at_position(self, content: str, position: int) -> int: + if position <= 0: + return 0 + return content.count("\n", 0, position) + + def _match_intersects_skip_lines( + self, + content: str, + match_start: int, + match_end: int, + skip_line_indices: frozenset[int], + ) -> bool: + if not skip_line_indices: + return False + start_line = self._line_index_at_position(content, match_start) + end_line = self._line_index_at_position(content, max(match_start, match_end - 1)) + return any(line_idx in skip_line_indices for line_idx in range(start_line, end_line + 1)) + + def _match_on_skipped_line( + self, + content: str, + match_start: int, + skip_line_indices: frozenset[int], + ) -> bool: + if not skip_line_indices: + return False + return self._line_index_at_position(content, match_start) in skip_line_indices + def _should_ignore_regex_match( self, content: str, @@ -245,6 +273,7 @@ def check_all_regexes( num_timestamps: int = 3, interval_to_collapse_event: int = 60, ignore_match_rules: Optional[Sequence[ParsedIgnoreMatchRule]] = None, + skip_line_indices: Optional[frozenset[int]] = None, ) -> list[RegexEvent]: """Iterate over all ERROR_REGEX and check content for any matches @@ -262,6 +291,7 @@ def check_all_regexes( num_timestamps (int, optional): maximum number of timestamps to keep for each event. Defaults to 3. interval_to_collapse_event (int, optional): time interval in seconds to collapse events. Defaults to 60. ignore_match_rules (Optional[Sequence[ParsedIgnoreMatchRule]], optional): Parsed skip rules. Defaults to None. + skip_line_indices (Optional[frozenset[int]], optional): Line indices to skip entirely. Defaults to None. Returns: list[RegexEvent]: list of regex event objects @@ -296,10 +326,22 @@ def _is_within_interval(new_timestamp_str: str, existing_timestamps: list[str]) return False skip_rules = list(ignore_match_rules) if ignore_match_rules else [] + suppressed_lines = skip_line_indices or frozenset() for error_regex_obj in error_regex: - for match_obj in error_regex_obj.regex.finditer(content): + search_from = 0 + while search_from <= len(content): + match_obj = error_regex_obj.regex.search(content, search_from) + if match_obj is None: + break raw_match = match_obj.group(0) + if self._match_on_skipped_line( + content, + match_obj.start(), + suppressed_lines, + ): + search_from = match_obj.start() + 1 + continue if self._should_ignore_regex_match( content, match_obj.start(), @@ -307,6 +349,7 @@ def _is_within_interval(new_timestamp_str: str, existing_timestamps: list[str]) error_regex_obj.message, skip_rules, ): + search_from = match_obj.start() + 1 continue # Extract timestamp from the line where match occurs @@ -360,6 +403,8 @@ def _is_within_interval(new_timestamp_str: str, existing_timestamps: list[str]) regex_event_list.append(new_event) + search_from = match_obj.end() + all_events = list(regex_map.values()) if group else regex_event_list # Prune timestamp lists to keep only first N and last N timestamps diff --git a/nodescraper/plugins/inband/dmesg/analyzer_args.py b/nodescraper/plugins/inband/dmesg/analyzer_args.py index ca2294f6..4b2a3797 100644 --- a/nodescraper/plugins/inband/dmesg/analyzer_args.py +++ b/nodescraper/plugins/inband/dmesg/analyzer_args.py @@ -76,6 +76,8 @@ class DmesgAnalyzerArgs(TimeRangeAnalysisArgs): "Rules that skip regex matches during analysis. Each rule may use line_regex, " "match_regex, message, and/or mce_banks. Within a rule all specified fields must " "match; any matching rule suppresses the hit. mce_banks accepts bank ids and " - 'inclusive ranges such as "60-63".' + 'inclusive ranges such as "60-63". When a rule matches an MCA bank, only ' + "[Hardware Error]: lines in that incident block are suppressed; interleaved " + "non-MCE dmesg lines in the same block are not." ), ) diff --git a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py index 65a613f9..9c7b11b2 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py @@ -35,7 +35,12 @@ from .analyzer_args import DmesgAnalyzerArgs from .dmesgdata import DmesgData -from .mce_utils import parse_correctable_mce_counts, parse_uncorrectable_mce_counts +from .mce_utils import ( + ignored_mce_block_line_indices, + mce_block_all_line_indices, + parse_correctable_mce_counts, + parse_uncorrectable_mce_counts, +) class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): @@ -714,6 +719,8 @@ def analyze_data( dmesg_content = data.dmesg_content ignore_match_rules, ignore_mce_banks = parse_ignore_match_rules(args.ignore_match_rules) + ignored_mce_block_lines = ignored_mce_block_line_indices(dmesg_content, ignore_mce_banks) + mce_block_lines = mce_block_all_line_indices(dmesg_content) known_err_events = self.check_all_regexes( content=dmesg_content, @@ -722,6 +729,7 @@ def analyze_data( num_timestamps=args.num_timestamps, interval_to_collapse_event=args.interval_to_collapse_event, ignore_match_rules=ignore_match_rules, + skip_line_indices=ignored_mce_block_lines, ) if args.exclude_category: known_err_events = [ @@ -752,6 +760,7 @@ def analyze_data( num_timestamps=args.num_timestamps, interval_to_collapse_event=args.interval_to_collapse_event, ignore_match_rules=ignore_match_rules, + skip_line_indices=mce_block_lines, ) for err_event in err_events: diff --git a/nodescraper/plugins/inband/dmesg/mce_utils.py b/nodescraper/plugins/inband/dmesg/mce_utils.py index 9d2b0692..b08e3a83 100644 --- a/nodescraper/plugins/inband/dmesg/mce_utils.py +++ b/nodescraper/plugins/inband/dmesg/mce_utils.py @@ -24,9 +24,16 @@ # ############################################################################### import re -from typing import FrozenSet, Optional +from typing import FrozenSet, Optional, Sequence -from nodescraper.base.match_ignore import extract_mce_bank_from_line +from nodescraper.base.match_ignore import extract_mce_banks_from_text + +_MCE_PRIMARY_START_RE = re.compile( + r"\[Hardware Error\]:\s*(?:Corrected error|Uncorrected error|Machine check events logged)", + re.IGNORECASE, +) +_MCE_STATUS_START_RE = re.compile(r"\bMC\d+_STATUS\[", re.IGNORECASE) +_MCE_DETAIL_LINE_RE = re.compile(r"\[Hardware Error\]:") _CORRECTABLE_SUMMARY_RE = re.compile( r"(?P\d+)\s+correctable hardware errors detected in total in (?P\w+) block" @@ -97,6 +104,123 @@ def _gpu_index_for_bdf(bdf: str, bdf_order: list[str]) -> int: return bdf_order.index(bdf) +def _is_mce_primary_starter(line: str) -> bool: + return _MCE_PRIMARY_START_RE.search(line) is not None + + +def _is_mce_block_starter(line: str, *, in_block: bool) -> bool: + """Return True when line opens a new MCE incident block.""" + if _is_mce_primary_starter(line): + return True + if not in_block and _MCE_STATUS_START_RE.search(line) is not None: + return True + return False + + +def _is_mce_detail_line(line: str) -> bool: + return _MCE_DETAIL_LINE_RE.search(line) is not None + + +def _mce_detail_line_indices_in_range(lines: Sequence[str], start: int, end: int) -> set[int]: + return {index for index in range(start, end) if _is_mce_detail_line(lines[index])} + + +def _next_non_blank_line_index(lines: Sequence[str], index: int) -> Optional[int]: + """Return the next non-blank line index after index, or None when none remain.""" + for candidate in range(index + 1, len(lines)): + if lines[candidate].strip(): + return candidate + return None + + +def _is_mce_status_only_starter(line: str) -> bool: + """Return True when line opens a block via MCn_STATUS without a primary header.""" + return not _is_mce_primary_starter(line) and _MCE_STATUS_START_RE.search(line) is not None + + +def _has_mce_detail_line_ahead(lines: Sequence[str], start_index: int) -> bool: + """Return True when another [Hardware Error]: line appears before the next incident.""" + for idx in range(start_index + 1, len(lines)): + line = lines[idx] + if not line.strip(): + continue + if _is_mce_primary_starter(line): + return False + if _is_mce_status_only_starter(line): + return False + if _is_mce_detail_line(line): + return True + return False + + +def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, int]]: + """Return (start, end) line index ranges for MCE incident blocks. + + A block begins at a primary MCE header (Corrected/Uncorrected/Machine check logged) + or at the first MCn_STATUS line when not already inside a block. The block then + includes subsequent lines until the next primary header, a blank line before a bare + MCn_STATUS starter, trailing non-MCE lines with no further [Hardware Error]: detail, + or EOF. Blank lines, warn/err noise, and other non-MCE dmesg lines between detail + entries belong to the same block. + """ + blocks: list[tuple[int, int]] = [] + index = 0 + total = len(lines) + while index < total: + if not _is_mce_block_starter(lines[index], in_block=False): + index += 1 + continue + start = index + index += 1 + while index < total: + if not lines[index].strip(): + next_line = _next_non_blank_line_index(lines, index) + if next_line is not None and _is_mce_status_only_starter(lines[next_line]): + break + elif _is_mce_block_starter(lines[index], in_block=True): + break + elif not _is_mce_detail_line(lines[index]) and not _has_mce_detail_line_ahead( + lines, index + ): + break + index += 1 + blocks.append((start, index)) + return blocks + + +def hardware_error_block_line_indices(content: str) -> frozenset[int]: + """Return [Hardware Error]: line indices that belong to an MCE incident block.""" + lines = content.splitlines() + suppressed: set[int] = set() + for start, end in iter_hardware_error_block_ranges(lines): + suppressed.update(_mce_detail_line_indices_in_range(lines, start, end)) + return frozenset(suppressed) + + +def mce_block_all_line_indices(content: str) -> frozenset[int]: + """Return every line index that belongs to an MCE incident block.""" + lines = content.splitlines() + suppressed: set[int] = set() + for start, end in iter_hardware_error_block_ranges(lines): + suppressed.update(range(start, end)) + return frozenset(suppressed) + + +def ignored_mce_block_line_indices(content: str, ignore_banks: FrozenSet[int]) -> frozenset[int]: + """Return [Hardware Error]: line indices for MCE blocks containing any ignored MCA bank.""" + if not ignore_banks: + return frozenset() + lines = content.splitlines() + suppressed: set[int] = set() + for start, end in iter_hardware_error_block_ranges(lines): + block_banks: set[int] = set() + for line in lines[start:end]: + block_banks.update(extract_mce_banks_from_text(line)) + if block_banks & ignore_banks: + suppressed.update(_mce_detail_line_indices_in_range(lines, start, end)) + return frozenset(suppressed) + + def parse_correctable_mce_counts( content: str, ignore_banks: Optional[FrozenSet[int]] = None, @@ -109,8 +233,11 @@ def parse_correctable_mce_counts( counts: dict[str, int] = {} gpu_bdf_order: list[str] = [] ignored = ignore_banks or frozenset() + ignored_block_lines = ignored_mce_block_line_indices(content, ignored) - for line in content.splitlines(): + for line_no, line in enumerate(content.splitlines()): + if line_no in ignored_block_lines: + continue gpu_match = _GPU_CORRECTABLE_RE.search(line) if gpu_match: bdf = gpu_match.group("bdf") @@ -134,9 +261,6 @@ def parse_correctable_mce_counts( status_match = _MCE_CE_STATUS_RE.search(line) if status_match: - bank = extract_mce_bank_from_line(line) - if bank is not None and bank in ignored: - continue part = ( _normalize_cpu_label(status_match.group("cpu")) if status_match.group("cpu") @@ -155,8 +279,11 @@ def parse_uncorrectable_mce_counts( counts: dict[str, int] = {} gpu_bdf_order: list[str] = [] ignored = ignore_banks or frozenset() + ignored_block_lines = ignored_mce_block_line_indices(content, ignored) - for line in content.splitlines(): + for line_no, line in enumerate(content.splitlines()): + if line_no in ignored_block_lines: + continue gpu_match = _GPU_UNCORRECTABLE_RE.search(line) if gpu_match: bdf = gpu_match.group("bdf") @@ -176,9 +303,6 @@ def parse_uncorrectable_mce_counts( status_match = _MCE_UC_STATUS_RE.search(line) if status_match: - bank = extract_mce_bank_from_line(line) - if bank is not None and bank in ignored: - continue part = ( _normalize_cpu_label(status_match.group("cpu")) if status_match.group("cpu") diff --git a/test/unit/plugin/test_dmesg_analyzer.py b/test/unit/plugin/test_dmesg_analyzer.py index d4697f9e..0de2ae21 100644 --- a/test/unit/plugin/test_dmesg_analyzer.py +++ b/test/unit/plugin/test_dmesg_analyzer.py @@ -1126,10 +1126,15 @@ def test_ignore_match_rules_skips_matching_lines(system_info): def test_ignore_match_rules_mce_banks_and_threshold(system_info): dmesg_content = ( + "kern :err : 2038-01-19T00:00:00,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" "kern :err : 2038-01-19T00:00:00,000000+00:00 " "[Hardware Error]: Machine Check: CPU0 MC1_STATUS[0xcafe|CE|Misc]: 0x0\n" "kern :err : 2038-01-19T00:00:01,000000+00:00 " "[Hardware Error]: Machine Check: CPU0 MC2_STATUS[0xfeed|CE|Misc]: 0x0\n" + "\n" + "kern :err : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" "kern :err : 2038-01-19T00:00:02,000000+00:00 " "[Hardware Error]: Machine Check: CPU0 MC5_STATUS[0xbeef|CE|Misc]: 0x0\n" ) @@ -1160,6 +1165,8 @@ def test_ignore_match_rules_mce_bank_range(system_info): "kern :err : 2038-01-19T00:00:01,000000+00:00 " "[Hardware Error]: Machine Check: CPU0 MC7_STATUS[0x2|CE|Misc]: 0x0\n" "kern :err : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :err : 2038-01-19T00:00:02,000000+00:00 " "[Hardware Error]: Machine Check: CPU0 MC9_STATUS[0x3|CE|Misc]: 0x0\n" ) @@ -1210,3 +1217,112 @@ def test_ignore_match_rules_scoped_by_message(system_info): by_desc = {event.description: event for event in res.events} assert "Dummy Error Alpha" not in by_desc assert "Dummy Error Beta" in by_desc + + +def test_hardware_error_block_not_reported_as_unknown(system_info): + dmesg_content = ( + "kern :info : 2038-01-19T00:00:00,000000+00:00 " + "mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:03,000000+00:00 " + "amdgpu 0000:de:ad.0: amdgpu: dummy notice inside block\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" + "kern :emerg : 2038-01-19T00:00:06,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :err : 2038-01-19T00:00:10,000000+00:00 unrelated plugin failure\n" + ) + + analyzer = DmesgAnalyzer(system_info=system_info) + res = analyzer.analyze_data( + DmesgData(dmesg_content=dmesg_content), + args=DmesgAnalyzerArgs( + check_unknown_dmesg_errors=True, + ignore_match_rules=[{"mce_banks": ["60-63"]}], + ), + ) + + descriptions = {event.description for event in res.events} + assert "MCE Corrected Error" not in descriptions + assert "RAS Corrected Error" not in descriptions + unknown_events = [event for event in res.events if event.description == "Unknown dmesg error"] + assert len(unknown_events) == 1 + assert unknown_events[0].data["match_content"] == "unrelated plugin failure" + + +def test_hardware_error_block_reports_mce_without_ignore(system_info): + dmesg_content = ( + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + ) + + analyzer = DmesgAnalyzer(system_info=system_info) + res = analyzer.analyze_data( + DmesgData(dmesg_content=dmesg_content), + args=DmesgAnalyzerArgs(check_unknown_dmesg_errors=True), + ) + + descriptions = {event.description for event in res.events} + assert "Unknown dmesg error" not in descriptions + assert "MCE Corrected Error" in descriptions or "RAS Corrected Error" in descriptions + + +def test_mce_interleave_pattern_suppresses_block_with_ignored_banks(system_info): + """Dummy excerpt modeled on dmesg_mce_interleave.log: warn/blank lines inside MCE blocks.""" + dmesg_content = ( + "kern :info : 2038-01-19T00:00:00,000000+00:00 " + "mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :warn : 2038-01-19T00:00:03,000000+00:00 " + "workqueue: dummy_mce_worker hogged CPU for >10000us 5 times, consider switching to WQ_UNBOUND\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" + "\n" + "kern :emerg : 2038-01-19T00:00:06,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :info : 2038-01-19T00:00:07,000000+00:00 " + "mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2038-01-19T00:00:08,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:09,000000+00:00 " + "[Hardware Error]: CPU:24 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xbbb\n" + "kern :emerg : 2038-01-19T00:00:10,000000+00:00 [Hardware Error]: PPIN: 0xcccccccccccccccc\n" + "kern :err : 2038-01-19T00:00:11,000000+00:00 dummy harness fault outside mce blocks\n" + ) + + analyzer = DmesgAnalyzer(system_info=system_info) + res = analyzer.analyze_data( + DmesgData(dmesg_content=dmesg_content), + args=DmesgAnalyzerArgs( + check_unknown_dmesg_errors=True, + mce_threshold=1, + ignore_match_rules=[{"mce_banks": ["60-63"]}], + error_regex=[ + { + "regex": r"dummy_mce_worker hogged CPU", + "message": "Dummy Workqueue Hog", + "event_category": "SW_DRIVER", + } + ], + ), + ) + + descriptions = {event.description for event in res.events} + assert "MCE Corrected Error" not in descriptions + assert "RAS Corrected Error" not in descriptions + assert not any("mce_threshold" in event.data for event in res.events) + assert "Dummy Workqueue Hog" in descriptions + unknown_events = [event for event in res.events if event.description == "Unknown dmesg error"] + assert len(unknown_events) == 1 + assert unknown_events[0].data["match_content"] == "dummy harness fault outside mce blocks" diff --git a/test/unit/plugin/test_mce_utils.py b/test/unit/plugin/test_mce_utils.py index c8fd20f2..53a58f00 100644 --- a/test/unit/plugin/test_mce_utils.py +++ b/test/unit/plugin/test_mce_utils.py @@ -24,6 +24,9 @@ # ############################################################################### from nodescraper.plugins.inband.dmesg.mce_utils import ( + hardware_error_block_line_indices, + ignored_mce_block_line_indices, + iter_hardware_error_block_ranges, parse_correctable_mce_counts, parse_uncorrectable_mce_counts, ) @@ -31,9 +34,9 @@ def test_parse_correctable_mce_counts_cpu_summary_and_status(): content = ( - "kern :warn : 2024-06-11T14:30:00,123456+00:00 " + "kern :warn : 2038-01-19T00:00:00,000000+00:00 " "mce: 3 correctable hardware errors detected in total in mc0 block on CPU1\n" - "kern :warn : 2024-06-11T14:30:02,222222+00:00 " + "kern :warn : 2038-01-19T00:00:01,000000+00:00 " "[Hardware Error]: CPU0 MC2_STATUS[0x0|CE|]: 0xabc\n" ) @@ -44,8 +47,8 @@ def test_parse_correctable_mce_counts_cpu_summary_and_status(): def test_parse_correctable_mce_counts_gpu_summary(): content = ( - "kern :err : 2024-10-07T10:17:15,145363-04:00 " - "amdgpu 0000:c1:00.0: amdgpu: socket: 4, die: 0 " + "kern :err : 2038-01-19T00:00:00,000000+00:00 " + "amdgpu 0000:de:ad.0: amdgpu: socket: 4, die: 0 " "3 correctable hardware errors detected in total in gfx block\n" ) @@ -75,9 +78,119 @@ def test_parse_correctable_mce_counts_skips_ignored_banks(): counts = parse_correctable_mce_counts(content, ignore_banks=frozenset({1, 2})) + assert counts == {} + + +def test_parse_correctable_mce_counts_skips_ignored_banks_in_separate_block(): + content = ( + "[Hardware Error]: CPU0 MC1_STATUS[0x0|CE|]: 0x1\n" + "[Hardware Error]: CPU0 MC2_STATUS[0x0|CE|]: 0x2\n" + "\n" + "[Hardware Error]: CPU0 MC5_STATUS[0x0|CE|]: 0x3\n" + ) + + counts = parse_correctable_mce_counts(content, ignore_banks=frozenset({1, 2})) + assert counts == {"CPU0": 1} +def test_hardware_error_block_line_indices(): + content = ( + "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" + "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" + "kern :info : ts unrelated line\n" + ) + + assert hardware_error_block_line_indices(content) == frozenset({0, 1, 2}) + + +def test_hardware_error_block_splits_info_preamble_from_emerg_details(): + content = ( + "kern :info : 2038-01-19T00:00:00,000000+00:00 " + "mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" + "kern :emerg : 2038-01-19T00:00:06,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + ) + lines = content.splitlines() + + assert iter_hardware_error_block_ranges(lines) == [(0, 1), (1, 6)] + assert hardware_error_block_line_indices(content) == frozenset({0, 1, 2, 3, 4, 5}) + assert ignored_mce_block_line_indices(content, frozenset({60})) == frozenset({1, 2, 3, 4, 5}) + + +def test_hardware_error_block_includes_interleaved_non_mce_lines(): + content = ( + "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" + "kern :emerg : ts amdgpu 0000:de:ad.0: amdgpu: unrelated reset notice\n" + "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" + "kern :emerg : ts [Hardware Error]: cache level: L3/GEN\n" + ) + + assert hardware_error_block_line_indices(content) == frozenset({0, 1, 3, 4}) + assert ignored_mce_block_line_indices(content, frozenset({60})) == frozenset({0, 1, 3, 4}) + + +def test_ignored_mce_block_line_indices(): + content = ( + "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : ts [Hardware Error]: CPU:12 MC60_STATUS[Over|CE|MiscV]: 0x1\n" + "kern :emerg : ts [Hardware Error]: PPIN: 0xabc\n" + "\n" + "kern :emerg : ts [Hardware Error]: Corrected error, no action required.\n" + "kern :err : ts [Hardware Error]: CPU0 MC5_STATUS[0x0|CE|]: 0x3\n" + ) + + ignored = ignored_mce_block_line_indices(content, frozenset({60})) + + assert ignored == frozenset({0, 1, 2}) + assert iter_hardware_error_block_ranges(content.splitlines()) == [(0, 4), (4, 6)] + + +def test_mce_block_includes_blank_line_and_warn_interleave(): + """Dummy excerpt: warn/workqueue noise and blank lines inside one MCE block.""" + content = ( + "kern :info : 2038-01-19T00:00:00,000000+00:00 " + "mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2038-01-19T00:00:01,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:02,000000+00:00 " + "[Hardware Error]: CPU:12 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n" + "kern :warn : 2038-01-19T00:00:03,000000+00:00 " + "workqueue: dummy_mce_worker hogged CPU for >10000us 5 times, consider switching to WQ_UNBOUND\n" + "kern :emerg : 2038-01-19T00:00:04,000000+00:00 [Hardware Error]: PPIN: 0xbbbbbbbbbbbbbbbb\n" + "kern :emerg : 2038-01-19T00:00:05,000000+00:00 " + "[Hardware Error]: IPID: 0x0000000000000001, Syndrome: 0x0000000000000001\n" + "\n" + "kern :emerg : 2038-01-19T00:00:06,000000+00:00 " + "[Hardware Error]: cache level: L3/GEN, mem/io: IO, mem-tx: GEN, part-proc: SRC (no timeout)\n" + "kern :info : 2038-01-19T00:00:07,000000+00:00 " + "mce: [Hardware Error]: Machine check events logged\n" + "kern :emerg : 2038-01-19T00:00:08,000000+00:00 " + "[Hardware Error]: Corrected error, no action required.\n" + "kern :emerg : 2038-01-19T00:00:09,000000+00:00 " + "[Hardware Error]: CPU:24 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xbbb\n" + "kern :emerg : 2038-01-19T00:00:10,000000+00:00 [Hardware Error]: PPIN: 0xcccccccccccccccc\n" + ) + lines = content.splitlines() + + assert iter_hardware_error_block_ranges(lines) == [(0, 1), (1, 8), (8, 9), (9, 12)] + assert hardware_error_block_line_indices(content) == frozenset({0, 1, 2, 4, 5, 7, 8, 9, 10, 11}) + assert ignored_mce_block_line_indices(content, frozenset({60})) == frozenset( + {1, 2, 4, 5, 7, 9, 10, 11} + ) + assert 3 not in ignored_mce_block_line_indices(content, frozenset({60})) + assert 6 not in ignored_mce_block_line_indices(content, frozenset({60})) + + def test_parse_correctable_mce_counts_cpu_colon_status(): content = ( "kern :err : 2038-01-19T00:00:00,000000+00:00 " @@ -94,7 +207,7 @@ def test_parse_correctable_mce_counts_both_cpu_formats(): "[Hardware Error]: CPU0 MC1_STATUS[0x0|CE|]: 0x1\n" "kern :err : 2038-01-19T00:00:00,000000+00:00 " "[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[-|CE|Misc]: 0xabc\n" - "kern :warn : 2024-06-11T14:30:00,123456+00:00 " + "kern :warn : 2038-01-19T00:00:02,000000+00:00 " "mce: 2 correctable hardware errors detected in total in mc0 block on CPU:1\n" )