Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions nodescraper/plugins/inband/dmesg/dmesg_analyzer.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,13 @@
from .analyzer_args import DmesgAnalyzerArgs
from .dmesgdata import DmesgData
from .mce_utils import (
compile_mce_ce_status_regex,
compile_mce_uc_status_regex,
ignored_mce_block_line_indices,
mce_block_all_line_indices,
parse_correctable_mce_counts,
parse_uncorrectable_mce_counts,
trim_mce_status_match_content,
)


Expand Down Expand Up @@ -343,17 +346,13 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]):
event_category=EventCategory.RAS,
),
ErrorRegex(
regex=re.compile(
r"\[Hardware Error\]:.+MC\d+_STATUS\[[^\]]*\|CE\|[^\]]*\].*(?:\n.*){0,5}"
),
regex=compile_mce_ce_status_regex(),
message="MCE Corrected Error",
event_category=EventCategory.RAS,
event_priority=EventPriority.WARNING,
),
ErrorRegex(
regex=re.compile(
r"\[Hardware Error\]:.+MC\d+_STATUS\[[^\]]*\|UC\|[^\]]*\].*(?:\n.*){0,5}"
),
regex=compile_mce_uc_status_regex(),
message="MCE Uncorrected Error",
event_category=EventCategory.RAS,
),
Expand Down Expand Up @@ -731,6 +730,11 @@ def analyze_data(
ignore_match_rules=ignore_match_rules,
skip_line_indices=ignored_mce_block_lines,
)
for event in known_err_events:
if event.description in ("MCE Corrected Error", "MCE Uncorrected Error"):
event.data["match_content"] = trim_mce_status_match_content(
event.data["match_content"]
)
if args.exclude_category:
known_err_events = [
event for event in known_err_events if event.category not in args.exclude_category
Expand Down
41 changes: 40 additions & 1 deletion nodescraper/plugins/inband/dmesg/mce_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
#
###############################################################################
import re
from typing import FrozenSet, Optional, Sequence
from typing import FrozenSet, Optional, Sequence, Union

from nodescraper.base.match_ignore import extract_mce_banks_from_text

Expand Down Expand Up @@ -68,6 +68,45 @@
re.IGNORECASE,
)

_MCE_CE_STATUS_LINE_RE = re.compile(
r"\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|CE\|[^\]]*\][^\n]*",
re.IGNORECASE,
)

_MCE_UC_STATUS_LINE_RE = re.compile(
r"\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|UC\|[^\]]*\][^\n]*",
re.IGNORECASE,
)


def compile_mce_ce_status_regex() -> re.Pattern[str]:
"""Return a single-line regex for corrected MCn_STATUS hardware error rows."""
return _MCE_CE_STATUS_LINE_RE


def compile_mce_uc_status_regex() -> re.Pattern[str]:
"""Return a single-line regex for uncorrected MCn_STATUS hardware error rows."""
return _MCE_UC_STATUS_LINE_RE


def trim_mce_status_match_content(match: Union[str, list[str]]) -> str:
"""Keep only the MCn_STATUS [Hardware Error] row in match_content."""
if isinstance(match, list):
for item in match:
trimmed = trim_mce_status_match_content(item)
if _MCE_STATUS_START_RE.search(trimmed):
return trimmed
return match[0] if match else ""

for line in str(match).splitlines():
ce_match = _MCE_CE_STATUS_LINE_RE.search(line)
if ce_match:
return ce_match.group(0)
uc_match = _MCE_UC_STATUS_LINE_RE.search(line)
if uc_match:
return uc_match.group(0)
return str(match)


def _normalize_cpu_label(cpu: str) -> str:
return cpu.replace(":", "")
Expand Down
38 changes: 38 additions & 0 deletions test/unit/plugin/test_dmesg_analyzer.py
Original file line number Diff line number Diff line change
Expand Up @@ -1326,3 +1326,41 @@ def test_mce_interleave_pattern_suppresses_block_with_ignored_banks(system_info)
unknown_events = [event for event in res.events if event.description == "Unknown dmesg error"]
assert len(unknown_events) == 1
assert unknown_events[0].data["match_content"] == "dummy harness fault outside mce blocks"


def test_mce_match_content_is_single_status_line(system_info):
"""Adjacent incidents: events.json match_content must be one MCn_STATUS row."""
dmesg_content = (
"kern :emerg : 2038-01-19T00:00:00,000000+00:00 "
"[Hardware Error]: Corrected error, no action required.\n"
"kern :emerg : 2038-01-19T00:00:01,000000+00:00 "
"[Hardware Error]: CPU:72 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xaaa\n"
"kern :emerg : 2038-01-19T00:00:02,000000+00:00 "
"[Hardware Error]: Corrected error, no action required.\n"
"kern :emerg : 2038-01-19T00:00:03,000000+00:00 "
"[Hardware Error]: CPU:29 (00:00:0) MC49_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xbbb\n"
"kern :emerg : 2038-01-19T00:00:04,000000+00:00 "
"[Hardware Error]: Corrected error, no action required.\n"
"kern :emerg : 2038-01-19T00:00:05,000000+00:00 "
"[Hardware Error]: CPU:8 (00:00:0) MC60_STATUS[Over|CE|MiscV|-|-|-|SyndV|UECC|-|-|-]: 0xccc\n"
)

analyzer = DmesgAnalyzer(system_info=system_info)
res = analyzer.analyze_data(
DmesgData(dmesg_content=dmesg_content),
args=DmesgAnalyzerArgs(
check_unknown_dmesg_errors=False,
mce_threshold=1,
ignore_match_rules=[{"mce_banks": ["60-63"]}],
),
)

mce_events = [event for event in res.events if event.description == "MCE Corrected Error"]
assert len(mce_events) == 1
match_content = str(mce_events[0].data["match_content"])
assert match_content.startswith("[Hardware Error]:")
assert "MC49_STATUS" in match_content
assert "MC60_STATUS" not in match_content
assert "CPU:29" in match_content
assert "CPU:8" not in match_content
assert "\n" not in match_content
15 changes: 15 additions & 0 deletions test/unit/plugin/test_mce_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
iter_hardware_error_block_ranges,
parse_correctable_mce_counts,
parse_uncorrectable_mce_counts,
trim_mce_status_match_content,
)


Expand Down Expand Up @@ -191,6 +192,20 @@ def test_mce_block_includes_blank_line_and_warn_interleave():
assert 6 not in ignored_mce_block_line_indices(content, frozenset({60}))


def test_trim_mce_status_match_content_keeps_status_row_only():
multiline = (
"[Hardware Error]: CPU:29 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb\n"
"[Hardware Error]: Corrected error, no action required.\n"
"[Hardware Error]: CPU:8 (00:00:0) MC60_STATUS[Over|CE|MiscV]: 0xccc\n"
)

trimmed = trim_mce_status_match_content(multiline)

assert trimmed == ("[Hardware Error]: CPU:29 (00:00:0) MC49_STATUS[Over|CE|MiscV]: 0xbbb")
assert "MC60_STATUS" not in trimmed
assert "\n" not in trimmed


def test_parse_correctable_mce_counts_cpu_colon_status():
content = (
"kern :err : 2038-01-19T00:00:00,000000+00:00 "
Expand Down